Debian Codegraph @

.cargo/vendor/av-scenechange/src/asm/arm/64/satd.S

raw
1 /* Copyright (c) 2022-2023, The rav1e contributors. All rights reserved
2 *
3 * This source code is subject to the terms of the BSD 2 Clause License and
4 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
5 * was not distributed with this source code in the LICENSE file, you can
6 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
7 * Media Patent License 1.0 was not distributed with this source code in the
8 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
9 */
10
11 #include "src/asm/arm/asm.S"
12 #include "util.S"
13
14 .macro butterfly r0, r1, r2, r3, t=8h
15 add \r0\().\t, \r2\().\t, \r3\().\t
16 sub \r1\().\t, \r2\().\t, \r3\().\t
17 .endm
18
19 .macro butterflyw r0, r1, r2, r3, r4, r5
20 sxtl \r0\().4s, \r4\().4h
21 sxtl2 \r2\().4s, \r4\().8h
22 ssubw \r1\().4s, \r0\().4s, \r5\().4h
23 ssubw2 \r3\().4s, \r2\().4s, \r5\().8h
24 saddw \r0\().4s, \r0\().4s, \r5\().4h
25 saddw2 \r2\().4s, \r2\().4s, \r5\().8h
26 .endm
27
28 .macro interleave r0, r1, r2, r3
29 zip1 \r0\().8h, \r2\().8h, \r3\().8h
30 zip2 \r1\().8h, \r2\().8h, \r3\().8h
31 .endm
32
33 .macro interleave_pairs r0, r1, r2, r3
34 zip1 \r0\().4s, \r2\().4s, \r3\().4s
35 zip2 \r1\().4s, \r2\().4s, \r3\().4s
36 .endm
37
38 .macro interleave_quads r0, r1, r2, r3
39 zip1 \r0\().2d, \r2\().2d, \r3\().2d
40 zip2 \r1\().2d, \r2\().2d, \r3\().2d
41 .endm
42
43 .macro normalize_4
44 add w0, w0, 2
45 lsr w0, w0, 2
46 .endm
47
48 .macro normalize_8
49 add w0, w0, 4
50 lsr w0, w0, 3
51 .endm
52
53 // x0: src: *const u8,
54 // x1: src_stride: isize,
55 // x2: dst: *const u8,
56 // x3: dst_stride: isize,
57 function satd4x4_neon, export=1
58 #define src x0
59 #define src_stride x1
60 #define dst x2
61 #define dst_stride x3
62
63 ldr s0, [src]
64 ldr s1, [dst]
65
66 // subtract; cast to 16-bit
67 usubl v0.8h, v0.8b, v1.8b
68
69 ldr s1, [src, src_stride]
70 ldr s2, [dst, dst_stride]
71
72 usubl v1.8h, v1.8b, v2.8b
73
74 // stride * 2
75 lsl x8, src_stride, 1
76 lsl x9, dst_stride, 1
77
78 ldr s2, [src, x8]
79 ldr s3, [dst, x9]
80
81 usubl v2.8h, v2.8b, v3.8b
82
83 // stride * 3
84 add x8, src_stride, src_stride, lsl 1
85 add x9, dst_stride, dst_stride, lsl 1
86
87 ldr s3, [src, x8]
88 ldr s4, [dst, x9]
89
90 usubl v3.8h, v3.8b, v4.8b
91
92 // pack rows 0-2, 1-3
93 mov v0.d[1], v2.d[0]
94 mov v1.d[1], v3.d[0]
95
96 // Horizontal transform
97
98 // v0 0 1 2 3 8 9 10 11
99 // v1 4 5 6 7 12 13 14 15
100
101 butterfly v2, v3, v0, v1
102
103 // v2 [0+4][1+5][2+6][3+7] [8+12][9+13][10+14][11+15]
104 // v3 [0-4][1-5][2-6][3-7] [8-12][9-13][10-14][11-15]
105
106 interleave v0, v1, v2, v3
107
108 // v0 [ 0+4][ 0-4][ 1+5][ 1-5] [2 + 6][2 - 6][3 + 7][3 - 7]
109 // v1 [8+12][8-12][9+13][9-13] [10+14][10-14][11+15][11-15]
110
111 butterfly v2, v3, v0, v1
112
113 // v2 [0+4+8+12][0-4+8-12][1+5+9+13][1-5+9-13] [2+6+10+14][2-6+10-14][3+7+11+15][3-7+11-15]
114 // v3 [0+4-8-12][0-4-8+12][1+5-9-13][1-5-9+13] [2+6-10-14][2-6-10+14][3+7-11-15][3-7-11+15]
115
116 interleave_pairs v0, v1, v2, v3
117
118 // Vertical transform
119
120 butterfly v2, v3, v0, v1
121
122 interleave v0, v1, v2, v3
123
124 butterfly v2, v3, v0, v1
125
126 // sum up transform
127 abs v2.8h, v2.8h
128 abs v3.8h, v3.8h
129
130 add v0.8h, v2.8h, v3.8h
131
132 addv h0, v0.8h
133 fmov w0, s0
134 normalize_4
135 ret
136
137 #undef src
138 #undef src_stride
139 #undef dst
140 #undef dst_stride
141 endfunc
142
143 .macro DOUBLE_HADAMARD_4X4 hbd=0
144 // Horizontal transform
145
146 butterfly v2, v3, v0, v1
147 butterfly v6, v7, v4, v5
148
149 interleave v0, v1, v2, v3
150 interleave v4, v5, v6, v7
151
152 butterfly v2, v3, v0, v1
153 butterfly v6, v7, v4, v5
154
155 interleave_pairs v0, v1, v2, v3
156 interleave_pairs v4, v5, v6, v7
157
158 // Vertical transform
159
160 butterfly v2, v3, v0, v1
161 butterfly v6, v7, v4, v5
162
163 interleave v0, v1, v2, v3
164 interleave v4, v5, v6, v7
165
166 .if \hbd == 0
167 butterfly v2, v3, v0, v1
168 butterfly v6, v7, v4, v5
169 .else
170 butterflyw v2, v3, v16, v17, v0, v1
171 butterflyw v6, v7, v18, v19, v4, v5
172 .endif
173 .endm
174
175 .macro SUM_DOUBLE_HADAMARD_4X4
176 abs v2.8h, v2.8h
177 abs v3.8h, v3.8h
178 abs v6.8h, v6.8h
179 abs v7.8h, v7.8h
180
181 add v0.8h, v2.8h, v3.8h
182 add v1.8h, v6.8h, v7.8h
183 add v0.8h, v0.8h, v1.8h
184
185 addv h0, v0.8h
186 fmov w0, s0
187 normalize_4
188 .endm
189
190 function satd8x4_neon, export=1
191 #define src x0
192 #define src_stride x1
193 #define dst x2
194 #define dst_stride x3
195
196 // load 8 pixel row
197 ldr d0, [src]
198 ldr d1, [dst]
199
200 usubl v0.8h, v0.8b, v1.8b
201
202 ldr d1, [src, src_stride]
203 ldr d2, [dst, dst_stride]
204
205 usubl v1.8h, v1.8b, v2.8b
206
207 lsl x8, src_stride, 1
208 lsl x9, dst_stride, 1
209
210 ldr d2, [src, x8]
211 ldr d3, [dst, x9]
212
213 usubl v2.8h, v2.8b, v3.8b
214
215 // stride * 3
216 add x8, src_stride, src_stride, lsl 1
217 add x9, dst_stride, dst_stride, lsl 1
218
219 ldr d3, [src, x8]
220 ldr d4, [dst, x9]
221
222 usubl v3.8h, v3.8b, v4.8b
223
224 // extract top 64 bits out of register
225 // (4 x 16 bits = 64)
226
227 ext v4.16b, v0.16b, v0.16b, 8
228 ext v5.16b, v1.16b, v1.16b, 8
229
230 // pack rows 0-2, 1-3 (set 1)
231 mov v0.d[1], v2.d[0]
232 mov v1.d[1], v3.d[0]
233
234 // pack rows 0-2, 1-3 (set 2)
235 mov v4.d[1], v2.d[1]
236 mov v5.d[1], v3.d[1]
237
238 // v2-3 temp registers for first 4x4 block//
239 // 6-7 for second block
240
241 DOUBLE_HADAMARD_4X4
242
243 SUM_DOUBLE_HADAMARD_4X4
244 ret
245
246 #undef src
247 #undef src_stride
248 #undef dst
249 #undef dst_stride
250 endfunc
251
252 .macro load_row n0, n1, src, dst, src_stride, dst_stride, should_add=1
253 ldr s\n0, [\src]
254 ldr s\n1, [\dst]
255
256 usubl v\n0\().8h, v\n0\().8b, v\n1\().8b
257
258 .if \should_add != 0
259 add \src, \src, \src_stride
260 add \dst, \dst, \dst_stride
261 .endif
262 .endm
263
264 .macro load_row2 n0, n1, src, dst, src_stride, dst_stride
265 ldr s\n0, [\src, \src_stride]
266 ldr s\n1, [\dst, \dst_stride]
267
268 usubl v\n0\().8h, v\n0\().8b, v\n1\().8b
269 .endm
270
271 function satd4x8_neon, export=1
272 #define src x0
273 #define src_stride x1
274 #define dst x2
275 #define dst_stride x3
276
277 // 0 * stride
278 load_row 0, 1, src, dst, src_stride, dst_stride, 0
279 // 1 * stride
280 load_row2 1, 2, src, dst, src_stride, dst_stride
281 add src, src, src_stride, lsl 1
282 add dst, dst, dst_stride, lsl 1
283
284 // pattern repeats
285 load_row 2, 3, src, dst, src_stride, dst_stride, 0
286 load_row2 3, 4, src, dst, src_stride, dst_stride
287 add src, src, src_stride, lsl 1
288 add dst, dst, dst_stride, lsl 1
289
290 load_row 4, 5, src, dst, src_stride, dst_stride, 0
291 load_row2 5, 6, src, dst, src_stride, dst_stride
292 add src, src, src_stride, lsl 1
293 add dst, dst, dst_stride, lsl 1
294
295 load_row 6, 7, src, dst, src_stride, dst_stride, 0
296 load_row2 7, 8, src, dst, src_stride, dst_stride
297
298 // pack rows
299 mov v0.d[1], v2.d[0]
300 mov v1.d[1], v3.d[0]
301
302 mov v4.d[1], v6.d[0]
303 mov v5.d[1], v7.d[0]
304
305 DOUBLE_HADAMARD_4X4
306
307 SUM_DOUBLE_HADAMARD_4X4
308 ret
309
310 #undef src
311 #undef src_stride
312 #undef dst
313 #undef dst_stride
314 endfunc
315
316 function satd16x4_neon, export=1
317 #define src x0
318 #define src_stride x1
319 #define dst x2
320 #define dst_stride x3
321
322 #define ROW1 v0
323 #define ROW2 v1
324 #define TMP1 v2
325 #define TMP2 v3
326
327 #define ROW3 v4
328 #define ROW4 v5
329 #define TMP3 v6
330 #define TMP4 v7
331
332 #define ROW5 v16
333 #define ROW6 v17
334 #define TMP5 v20
335 #define TMP6 v21
336
337 #define ROW7 v18
338 #define ROW8 v19
339 #define TMP7 v22
340 #define TMP8 v23
341
342 // load 16 pixel row
343 ldr q0, [src]
344 ldr q1, [dst]
345
346 usubl2 v16.8h, v0.16b, v1.16b
347 usubl v0.8h, v0.8b, v1.8b
348
349 ldr q1, [src, src_stride]
350 ldr q2, [dst, dst_stride]
351
352 usubl2 v17.8h, v1.16b, v2.16b
353 usubl v1.8h, v1.8b, v2.8b
354
355 lsl x8, src_stride, 1
356 lsl x9, dst_stride, 1
357
358 ldr q2, [src, x8]
359 ldr q3, [dst, x9]
360
361 usubl2 v6.8h, v2.16b, v3.16b
362 usubl v2.8h, v2.8b, v3.8b
363
364 // stride * 3
365 add x8, src_stride, src_stride, lsl 1
366 add x9, dst_stride, dst_stride, lsl 1
367
368 ldr q3, [src, x8]
369 ldr q4, [dst, x9]
370
371 usubl2 v7.8h, v3.16b, v4.16b
372 usubl v3.8h, v3.8b, v4.8b
373
374 // swap high/low 64 bits
375 ext v4.16b, v0.16b, v0.16b, 8
376 ext v5.16b, v1.16b, v1.16b, 8
377
378 mov v0.d[1], v2.d[0]
379 mov v1.d[1], v3.d[0]
380
381 ext v18.16b, v16.16b, v16.16b, 8
382 ext v19.16b, v17.16b, v17.16b, 8
383
384 mov v16.d[1], v6.d[0]
385 mov v17.d[1], v7.d[0]
386 // 2-3 free
387
388 mov v4.d[1], v2.d[1]
389 mov v5.d[1], v3.d[1]
390 // 6-7 free
391 mov v18.d[1], v6.d[1]
392 mov v19.d[1], v7.d[1]
393
394 // 0,1 2,3
395 // 4,5 6,7
396 // 16,17 20,21
397 // 18,19 22,23
398
399 // quadruple 4x4 hadamard
400
401 butterfly TMP1, TMP2, ROW1, ROW2
402 butterfly TMP3, TMP4, ROW3, ROW4
403 butterfly TMP5, TMP6, ROW5, ROW6
404 butterfly TMP7, TMP8, ROW7, ROW8
405
406 interleave ROW1, ROW2, TMP1, TMP2
407 interleave ROW3, ROW4, TMP3, TMP4
408 interleave ROW5, ROW6, TMP5, TMP6
409 interleave ROW7, ROW8, TMP7, TMP8
410
411 butterfly TMP1, TMP2, ROW1, ROW2
412 butterfly TMP3, TMP4, ROW3, ROW4
413 butterfly TMP5, TMP6, ROW5, ROW6
414 butterfly TMP7, TMP8, ROW7, ROW8
415
416 interleave_pairs ROW1, ROW2, TMP1, TMP2
417 interleave_pairs ROW3, ROW4, TMP3, TMP4
418 interleave_pairs ROW5, ROW6, TMP5, TMP6
419 interleave_pairs ROW7, ROW8, TMP7, TMP8
420
421 butterfly TMP1, TMP2, ROW1, ROW2
422 butterfly TMP3, TMP4, ROW3, ROW4
423 butterfly TMP5, TMP6, ROW5, ROW6
424 butterfly TMP7, TMP8, ROW7, ROW8
425
426 interleave ROW1, ROW2, TMP1, TMP2
427 interleave ROW3, ROW4, TMP3, TMP4
428 interleave ROW5, ROW6, TMP5, TMP6
429 interleave ROW7, ROW8, TMP7, TMP8
430
431 butterfly TMP1, TMP2, ROW1, ROW2
432 butterfly TMP3, TMP4, ROW3, ROW4
433 butterfly TMP5, TMP6, ROW5, ROW6
434 butterfly TMP7, TMP8, ROW7, ROW8
435
436 // absolute value of transform coefficients
437 abs TMP1.8h, TMP1.8h
438 abs TMP2.8h, TMP2.8h
439 abs TMP3.8h, TMP3.8h
440 abs TMP4.8h, TMP4.8h
441 abs TMP5.8h, TMP5.8h
442 abs TMP6.8h, TMP6.8h
443 abs TMP7.8h, TMP7.8h
444 abs TMP8.8h, TMP8.8h
445
446 // stage 1 sum
447 add TMP1.8h, TMP1.8h, TMP5.8h
448 add TMP2.8h, TMP2.8h, TMP6.8h
449 add TMP3.8h, TMP3.8h, TMP7.8h
450 add TMP4.8h, TMP4.8h, TMP8.8h
451
452 // stage 2 sum
453 add TMP1.8h, TMP1.8h, TMP3.8h
454 add TMP2.8h, TMP2.8h, TMP4.8h
455 add v0.8h, TMP1.8h, TMP2.8h
456
457 addv h0, v0.8h
458 fmov w0, s0
459 normalize_4
460 ret
461
462 #undef src
463 #undef src_stride
464 #undef dst
465 #undef dst_stride
466
467 #undef ROW1
468 #undef TMP1
469 #undef ROW2
470 #undef TMP2
471 #undef ROW3
472 #undef TMP3
473 #undef ROW4
474 #undef TMP4
475 #undef ROW5
476 #undef TMP5
477 #undef ROW6
478 #undef TMP6
479 #undef ROW7
480 #undef TMP7
481 #undef ROW8
482 #undef TMP8
483 endfunc
484
485 function satd4x16_neon, export=1
486 #define src x0
487 #define src_stride x1
488 #define dst x2
489 #define dst_stride x3
490
491 load_row 0, 1, src, dst, src_stride, dst_stride, 0
492 load_row2 1, 2, src, dst, src_stride, dst_stride
493 add src, src, src_stride, lsl 1
494 add dst, dst, dst_stride, lsl 1
495
496 load_row 2, 3, src, dst, src_stride, dst_stride, 0
497 load_row2 3, 4, src, dst, src_stride, dst_stride
498 add src, src, src_stride, lsl 1
499 add dst, dst, dst_stride, lsl 1
500
501 load_row 4, 5, src, dst, src_stride, dst_stride, 0
502 load_row2 5, 6, src, dst, src_stride, dst_stride
503 add src, src, src_stride, lsl 1
504 add dst, dst, dst_stride, lsl 1
505
506 load_row 6, 7, src, dst, src_stride, dst_stride, 0
507 load_row2 7, 16, src, dst, src_stride, dst_stride
508 add src, src, src_stride, lsl 1
509 add dst, dst, dst_stride, lsl 1
510
511 load_row 16, 17, src, dst, src_stride, dst_stride, 0
512 load_row2 17, 18, src, dst, src_stride, dst_stride
513 add src, src, src_stride, lsl 1
514 add dst, dst, dst_stride, lsl 1
515
516 load_row 18, 19, src, dst, src_stride, dst_stride, 0
517 load_row2 19, 20, src, dst, src_stride, dst_stride
518 add src, src, src_stride, lsl 1
519 add dst, dst, dst_stride, lsl 1
520
521 load_row 20, 21, src, dst, src_stride, dst_stride, 0
522 load_row2 21, 22, src, dst, src_stride, dst_stride
523 add src, src, src_stride, lsl 1
524 add dst, dst, dst_stride, lsl 1
525
526 load_row 22, 23, src, dst, src_stride, dst_stride, 0
527 load_row2 23, 24, src, dst, src_stride, dst_stride
528
529 // pack rows
530 mov v0.d[1], v2.d[0]
531 mov v1.d[1], v3.d[0]
532
533 mov v4.d[1], v6.d[0]
534 mov v5.d[1], v7.d[0]
535
536 mov v16.d[1], v18.d[0]
537 mov v17.d[1], v19.d[0]
538
539 mov v20.d[1], v22.d[0]
540 mov v21.d[1], v23.d[0]
541
542 butterfly v2, v3, v0, v1
543 butterfly v6, v7, v4, v5
544 butterfly v18, v19, v16, v17
545 butterfly v22, v23, v20, v21
546
547 interleave v0, v1, v2, v3
548 interleave v4, v5, v6, v7
549 interleave v16, v17, v18, v19
550 interleave v20, v21, v22, v23
551
552 butterfly v2, v3, v0, v1
553 butterfly v6, v7, v4, v5
554 butterfly v18, v19, v16, v17
555 butterfly v22, v23, v20, v21
556
557 interleave_pairs v0, v1, v2, v3
558 interleave_pairs v4, v5, v6, v7
559 interleave_pairs v16, v17, v18, v19
560 interleave_pairs v20, v21, v22, v23
561
562 butterfly v2, v3, v0, v1
563 butterfly v6, v7, v4, v5
564 butterfly v18, v19, v16, v17
565 butterfly v22, v23, v20, v21
566
567 interleave v0, v1, v2, v3
568 interleave v4, v5, v6, v7
569 interleave v16, v17, v18, v19
570 interleave v20, v21, v22, v23
571
572 butterfly v2, v3, v0, v1
573 butterfly v6, v7, v4, v5
574 butterfly v18, v19, v16, v17
575 butterfly v22, v23, v20, v21
576
577 abs v2.8h, v2.8h
578 abs v3.8h, v3.8h
579 abs v6.8h, v6.8h
580 abs v7.8h, v7.8h
581 abs v18.8h, v18.8h
582 abs v19.8h, v19.8h
583 abs v22.8h, v22.8h
584 abs v23.8h, v23.8h
585
586 add v2.8h, v2.8h, v3.8h
587 add v6.8h, v6.8h, v7.8h
588 add v18.8h, v18.8h, v19.8h
589 add v22.8h, v22.8h, v23.8h
590
591 add v2.8h, v2.8h, v6.8h
592 add v18.8h, v18.8h, v22.8h
593 add v0.8h, v2.8h, v18.8h
594
595 addv h0, v0.8h
596 fmov w0, s0
597 normalize_4
598 ret
599
600 #undef src
601 #undef src_stride
602 #undef dst
603 #undef dst_stride
604 endfunc
605
606 .macro load_rows n0, n1, n2, src, dst, src_stride, dst_stride, n3=0, n4=0
607 .if \n3 == 0
608 ldr d\n0, [\src]
609 ldr d\n1, [\dst]
610 .else
611 ldr q\n0, [\src]
612 ldr q\n1, [\dst]
613 usubl2 v\n3\().8h, v\n0\().16b, v\n1\().16b
614 .endif
615 usubl v\n0\().8h, v\n0\().8b, v\n1\().8b
616
617 .if \n4 == 0
618 ldr d\n1, [\src, \src_stride]
619 ldr d\n2, [\dst, \dst_stride]
620 .else
621 ldr q\n1, [\src, \src_stride]
622 ldr q\n2, [\dst, \dst_stride]
623 usubl2 v\n4\().8h, v\n1\().16b, v\n2\().16b
624 .endif
625 usubl v\n1\().8h, v\n1\().8b, v\n2\().8b
626
627 add \src, \src, \src_stride, lsl 1
628 add \dst, \dst, \dst_stride, lsl 1
629 .endm
630
631 .macro HADAMARD_8X8_H \
632 a0 a1 a2 a3 a4 a5 a6 a7 \
633 b0 b1 b2 b3 b4 b5 b6 b7
634
635 // Horizontal transform
636
637 butterfly v\b0, v\b1, v\a0, v\a1
638 butterfly v\b2, v\b3, v\a2, v\a3
639 butterfly v\b4, v\b5, v\a4, v\a5
640 butterfly v\b6, v\b7, v\a6, v\a7
641
642 interleave v\a0, v\a1, v\b0, v\b1
643 interleave v\a2, v\a3, v\b2, v\b3
644 interleave v\a4, v\a5, v\b4, v\b5
645 interleave v\a6, v\a7, v\b6, v\b7
646
647 butterfly v\b0, v\b2, v\a0, v\a2
648 butterfly v\b1, v\b3, v\a1, v\a3
649 butterfly v\b4, v\b6, v\a4, v\a6
650 butterfly v\b5, v\b7, v\a5, v\a7
651
652 interleave_pairs v\a0, v\a2, v\b0, v\b2
653 interleave_pairs v\a1, v\a3, v\b1, v\b3
654 interleave_pairs v\a4, v\a6, v\b4, v\b6
655 interleave_pairs v\a5, v\a7, v\b5, v\b7
656
657 butterfly v\b0, v\b4, v\a0, v\a4
658 butterfly v\b1, v\b5, v\a1, v\a5
659 butterfly v\b2, v\b6, v\a2, v\a6
660 butterfly v\b3, v\b7, v\a3, v\a7
661
662 interleave_quads v\a0, v\a4, v\b0, v\b4
663 interleave_quads v\a1, v\a5, v\b1, v\b5
664 interleave_quads v\a2, v\a6, v\b2, v\b6
665 interleave_quads v\a3, v\a7, v\b3, v\b7
666 .endm
667
668 .macro HADAMARD_8X8_V \
669 a0 a1 a2 a3 a4 a5 a6 a7 \
670 b0 b1 b2 b3 b4 b5 b6 b7
671
672 // Vertical transform
673
674 butterfly v\b0, v\b1, v\a0, v\a1
675 butterfly v\b2, v\b3, v\a2, v\a3
676 butterfly v\b4, v\b5, v\a4, v\a5
677 butterfly v\b6, v\b7, v\a6, v\a7
678
679 butterfly v\a0, v\a2, v\b0, v\b2
680 butterfly v\a1, v\a3, v\b1, v\b3
681 butterfly v\a4, v\a6, v\b4, v\b6
682 butterfly v\a5, v\a7, v\b5, v\b7
683
684 butterfly v\b0, v\b4, v\a0, v\a4
685 butterfly v\b1, v\b5, v\a1, v\a5
686 butterfly v\b2, v\b6, v\a2, v\a6
687 butterfly v\b3, v\b7, v\a3, v\a7
688 .endm
689
690 .macro SUM_HADAMARD_8X8 \
691 a0 a1 a2 a3 a4 a5 a6 a7 \
692 b0 b1 b2 b3 b4 b5 b6 b7
693
694 // absolute value of transform coefficients
695 abs v\b0\().8h, v\b0\().8h
696 abs v\b1\().8h, v\b1\().8h
697 abs v\b2\().8h, v\b2\().8h
698 abs v\b3\().8h, v\b3\().8h
699 abs v\b4\().8h, v\b4\().8h
700 abs v\b5\().8h, v\b5\().8h
701 abs v\b6\().8h, v\b6\().8h
702 abs v\b7\().8h, v\b7\().8h
703
704 // stage 1 sum
705 sxtl v\a0\().4s, v\b0\().4h
706 sxtl v\a1\().4s, v\b1\().4h
707 sxtl v\a2\().4s, v\b2\().4h
708 sxtl v\a3\().4s, v\b3\().4h
709 saddw2 v\a0\().4s, v\a0\().4s, v\b0\().8h
710 saddw2 v\a1\().4s, v\a1\().4s, v\b1\().8h
711 saddw2 v\a2\().4s, v\a2\().4s, v\b2\().8h
712 saddw2 v\a3\().4s, v\a3\().4s, v\b3\().8h
713 saddw v\a0\().4s, v\a0\().4s, v\b4\().4h
714 saddw2 v\a1\().4s, v\a1\().4s, v\b4\().8h
715 saddw v\a2\().4s, v\a2\().4s, v\b5\().4h
716 saddw2 v\a3\().4s, v\a3\().4s, v\b5\().8h
717 saddw v\a0\().4s, v\a0\().4s, v\b6\().4h
718 saddw2 v\a1\().4s, v\a1\().4s, v\b6\().8h
719 saddw v\a2\().4s, v\a2\().4s, v\b7\().4h
720 saddw2 v\a3\().4s, v\a3\().4s, v\b7\().8h
721
722 // stage 2 sum
723 add v\a0\().4s, v\a0\().4s, v\a1\().4s
724 add v\a2\().4s, v\a2\().4s, v\a3\().4s
725
726 // stage 3 sum
727 add v0.4s, v\a0\().4s, v\a2\().4s
728 addv s0, v0.4s
729 .endm
730
731 .macro SATD_8X8 \
732 a0 a1 a2 a3 a4 a5 a6 a7 \
733 b0 b1 b2 b3 b4 b5 b6 b7
734
735 HADAMARD_8X8_H \
736 \a0, \a1, \a2, \a3, \a4, \a5, \a6, \a7, \
737 \b0, \b1, \b2, \b3, \b4, \b5, \b6, \b7
738
739 HADAMARD_8X8_V \
740 \a0, \a1, \a2, \a3, \a4, \a5, \a6, \a7, \
741 \b0, \b1, \b2, \b3, \b4, \b5, \b6, \b7
742
743 SUM_HADAMARD_8X8 \
744 \a0, \a1, \a2, \a3, \a4, \a5, \a6, \a7, \
745 \b0, \b1, \b2, \b3, \b4, \b5, \b6, \b7
746 .endm
747
748
749 function satd8x8_neon, export=1
750 #define src x0
751 #define src_stride x1
752 #define dst x2
753 #define dst_stride x3
754
755 #define subtotal w9
756 #define total w10
757 #define height w13
758
759 mov height, 8
760 mov total, wzr
761
762 // 0, 1; 2, 3
763 // 4, 5; 6, 7
764 // 16, 17; 20, 21
765 // 18, 19; 22, 23
766
767 L(satd_w8):
768 load_rows 0, 1, 2, src, dst, src_stride, dst_stride
769 load_rows 4, 5, 6, src, dst, src_stride, dst_stride
770 load_rows 16, 17, 20, src, dst, src_stride, dst_stride
771 load_rows 18, 19, 22, src, dst, src_stride, dst_stride
772
773 SATD_8X8 \
774 0, 1, 4, 5, 16, 17, 18, 19, \
775 2, 3, 6, 7, 20, 21, 22, 23
776
777 fmov subtotal, s0
778 add total, subtotal, total
779
780 subs height, height, #8
781 bne L(satd_w8)
782
783 mov w0, total
784 normalize_8
785 ret
786
787 #undef src
788 #undef src_stride
789 #undef dst
790 #undef dst_stride
791
792 #undef subtotal
793 #undef total
794 #undef height
795 endfunc
796
797 .macro DOUBLE_HADAMARD_8X8 \
798 a0 a1 a2 a3 a4 a5 a6 a7 \
799 b0 b1 b2 b3 b4 b5 b6 b7 \
800 c0 c1 c2 c3 c4 c5 c6 c7
801
802 // Horizontal transform
803
804 butterfly v\b0, v\b1, v\a0, v\a1
805 butterfly v\b2, v\b3, v\a2, v\a3
806 butterfly v\b4, v\b5, v\a4, v\a5
807 butterfly v\b6, v\b7, v\a6, v\a7
808 butterfly v\a0, v\a1, v\c0, v\c1
809 butterfly v\a2, v\a3, v\c2, v\c3
810 butterfly v\a4, v\a5, v\c4, v\c5
811 butterfly v\a6, v\a7, v\c6, v\c7
812
813 interleave v\c0, v\c1, v\b0, v\b1
814 interleave v\c2, v\c3, v\b2, v\b3
815 interleave v\c4, v\c5, v\b4, v\b5
816 interleave v\c6, v\c7, v\b6, v\b7
817 interleave v\b0, v\b1, v\a0, v\a1
818 interleave v\b2, v\b3, v\a2, v\a3
819 interleave v\b4, v\b5, v\a4, v\a5
820 interleave v\b6, v\b7, v\a6, v\a7
821
822 butterfly v\a0, v\a2, v\c0, v\c2
823 butterfly v\a1, v\a3, v\c1, v\c3
824 butterfly v\a4, v\a6, v\c4, v\c6
825 butterfly v\a5, v\a7, v\c5, v\c7
826 butterfly v\c0, v\c2, v\b0, v\b2
827 butterfly v\c1, v\c3, v\b1, v\b3
828 butterfly v\c4, v\c6, v\b4, v\b6
829 butterfly v\c5, v\c7, v\b5, v\b7
830
831 interleave_pairs v\b0, v\b2, v\a0, v\a2
832 interleave_pairs v\b1, v\b3, v\a1, v\a3
833 interleave_pairs v\b4, v\b6, v\a4, v\a6
834 interleave_pairs v\b5, v\b7, v\a5, v\a7
835 interleave_pairs v\a0, v\a2, v\c0, v\c2
836 interleave_pairs v\a1, v\a3, v\c1, v\c3
837 interleave_pairs v\a4, v\a6, v\c4, v\c6
838 interleave_pairs v\a5, v\a7, v\c5, v\c7
839
840 butterfly v\c0, v\c4, v\b0, v\b4
841 butterfly v\c1, v\c5, v\b1, v\b5
842 butterfly v\c2, v\c6, v\b2, v\b6
843 butterfly v\c3, v\c7, v\b3, v\b7
844 butterfly v\b0, v\b4, v\a0, v\a4
845 butterfly v\b1, v\b5, v\a1, v\a5
846 butterfly v\b2, v\b6, v\a2, v\a6
847 butterfly v\b3, v\b7, v\a3, v\a7
848
849 interleave_quads v\a0, v\a4, v\c0, v\c4
850 interleave_quads v\a1, v\a5, v\c1, v\c5
851 interleave_quads v\a2, v\a6, v\c2, v\c6
852 interleave_quads v\a3, v\a7, v\c3, v\c7
853 interleave_quads v\c0, v\c4, v\b0, v\b4
854 interleave_quads v\c1, v\c5, v\b1, v\b5
855 interleave_quads v\c2, v\c6, v\b2, v\b6
856 interleave_quads v\c3, v\c7, v\b3, v\b7
857
858 // Vertical transform
859
860 butterfly v\b0, v\b1, v\a0, v\a1
861 butterfly v\b2, v\b3, v\a2, v\a3
862 butterfly v\b4, v\b5, v\a4, v\a5
863 butterfly v\b6, v\b7, v\a6, v\a7
864 butterfly v\a0, v\a1, v\c0, v\c1
865 butterfly v\a2, v\a3, v\c2, v\c3
866 butterfly v\a4, v\a5, v\c4, v\c5
867 butterfly v\a6, v\a7, v\c6, v\c7
868
869 butterfly v\c0, v\c2, v\b0, v\b2
870 butterfly v\c1, v\c3, v\b1, v\b3
871 butterfly v\c4, v\c6, v\b4, v\b6
872 butterfly v\c5, v\c7, v\b5, v\b7
873 butterfly v\b0, v\b2, v\a0, v\a2
874 butterfly v\b1, v\b3, v\a1, v\a3
875 butterfly v\b4, v\b6, v\a4, v\a6
876 butterfly v\b5, v\b7, v\a5, v\a7
877
878 butterfly v\a0, v\a4, v\c0, v\c4
879 butterfly v\a1, v\a5, v\c1, v\c5
880 butterfly v\a2, v\a6, v\c2, v\c6
881 butterfly v\a3, v\a7, v\c3, v\c7
882 butterfly v\c0, v\c4, v\b0, v\b4
883 butterfly v\c1, v\c5, v\b1, v\b5
884 butterfly v\c2, v\c6, v\b2, v\b6
885 butterfly v\c3, v\c7, v\b3, v\b7
886 .endm
887
888 .macro SUM_DOUBLE_HADAMARD_8X8 \
889 a0 a1 a2 a3 a4 a5 a6 a7 \
890 b0 b1 b2 b3 b4 b5 b6 b7 \
891 c0 c1 c2 c3 c4 c5 c6 c7
892
893 // absolute value of transform coefficients
894 abs v\a0\().8h, v\a0\().8h
895 abs v\a1\().8h, v\a1\().8h
896 abs v\a2\().8h, v\a2\().8h
897 abs v\a3\().8h, v\a3\().8h
898 abs v\a4\().8h, v\a4\().8h
899 abs v\a5\().8h, v\a5\().8h
900 abs v\a6\().8h, v\a6\().8h
901 abs v\a7\().8h, v\a7\().8h
902 abs v\c0\().8h, v\c0\().8h
903 abs v\c1\().8h, v\c1\().8h
904 abs v\c2\().8h, v\c2\().8h
905 abs v\c3\().8h, v\c3\().8h
906 abs v\c4\().8h, v\c4\().8h
907 abs v\c5\().8h, v\c5\().8h
908 abs v\c6\().8h, v\c6\().8h
909 abs v\c7\().8h, v\c7\().8h
910
911 // stage 1 sum
912 sxtl v\b0\().4s, v\a0\().4h
913 sxtl v\b1\().4s, v\a1\().4h
914 sxtl v\b2\().4s, v\a2\().4h
915 sxtl v\b3\().4s, v\a3\().4h
916 sxtl v\b4\().4s, v\a4\().4h
917 sxtl v\b5\().4s, v\a5\().4h
918 sxtl v\b6\().4s, v\a6\().4h
919 sxtl v\b7\().4s, v\a7\().4h
920 saddw2 v\b0\().4s, v\b0\().4s, v\a0\().8h
921 saddw2 v\b1\().4s, v\b1\().4s, v\a1\().8h
922 saddw2 v\b2\().4s, v\b2\().4s, v\a2\().8h
923 saddw2 v\b3\().4s, v\b3\().4s, v\a3\().8h
924 saddw2 v\b4\().4s, v\b4\().4s, v\a4\().8h
925 saddw2 v\b5\().4s, v\b5\().4s, v\a5\().8h
926 saddw2 v\b6\().4s, v\b6\().4s, v\a6\().8h
927 saddw2 v\b7\().4s, v\b7\().4s, v\a7\().8h
928 saddw v\b0\().4s, v\b0\().4s, v\c0\().4h
929 saddw2 v\b1\().4s, v\b1\().4s, v\c0\().8h
930 saddw v\b2\().4s, v\b2\().4s, v\c1\().4h
931 saddw2 v\b3\().4s, v\b3\().4s, v\c1\().8h
932 saddw v\b4\().4s, v\b4\().4s, v\c2\().4h
933 saddw2 v\b5\().4s, v\b5\().4s, v\c2\().8h
934 saddw v\b6\().4s, v\b6\().4s, v\c3\().4h
935 saddw2 v\b7\().4s, v\b7\().4s, v\c3\().8h
936 saddw v\b0\().4s, v\b0\().4s, v\c4\().4h
937 saddw2 v\b1\().4s, v\b1\().4s, v\c4\().8h
938 saddw v\b2\().4s, v\b2\().4s, v\c5\().4h
939 saddw2 v\b3\().4s, v\b3\().4s, v\c5\().8h
940 saddw v\b4\().4s, v\b4\().4s, v\c6\().4h
941 saddw2 v\b5\().4s, v\b5\().4s, v\c6\().8h
942 saddw v\b6\().4s, v\b6\().4s, v\c7\().4h
943 saddw2 v\b7\().4s, v\b7\().4s, v\c7\().8h
944
945 // stage 2 sum
946 add v\b0\().4s, v\b0\().4s, v\b1\().4s
947 add v\b2\().4s, v\b2\().4s, v\b3\().4s
948 add v\b4\().4s, v\b4\().4s, v\b5\().4s
949 add v\b6\().4s, v\b6\().4s, v\b7\().4s
950
951 // stage 3 sum
952 add v\b0\().4s, v\b0\().4s, v\b2\().4s
953 add v\b4\().4s, v\b4\().4s, v\b6\().4s
954
955 // stage 4 sum
956 add v0.4s, v\b0\().4s, v\b4\().4s
957 addv s0, v0.4s
958 .endm
959
960 function satd16x8_neon, export=1
961 #define src x0
962 #define src_stride x1
963 #define dst x2
964 #define dst_stride x3
965
966 #define subtotal w9
967 #define total w10
968 #define w_ext x11
969 #define w_bak w11
970 #define width w12
971 #define height w13
972
973 mov height, 8
974 mov width, 16
975 sxtw w_ext, width
976 mov total, wzr
977
978 // 0, 1; 2, 3; 24, 25
979 // 4, 5; 6, 7; 26, 27
980 // 16, 17; 20, 21; 28, 29
981 // 18, 19; 22, 23; 30, 31
982
983 L(satd_w16up):
984 load_rows 0, 1, 2, src, dst, src_stride, dst_stride, 24, 25
985 load_rows 4, 5, 6, src, dst, src_stride, dst_stride, 26, 27
986 load_rows 16, 17, 20, src, dst, src_stride, dst_stride, 28, 29
987 load_rows 18, 19, 22, src, dst, src_stride, dst_stride, 30, 31
988
989 DOUBLE_HADAMARD_8X8 \
990 0, 1, 4, 5, 16, 17, 18, 19, \
991 2, 3, 6, 7, 20, 21, 22, 23, \
992 24, 25, 26, 27, 28, 29, 30, 31
993
994 SUM_DOUBLE_HADAMARD_8X8 \
995 0, 1, 4, 5, 16, 17, 18, 19, \
996 2, 3, 6, 7, 20, 21, 22, 23, \
997 24, 25, 26, 27, 28, 29, 30, 31
998
999 fmov subtotal, s0
1000 add total, subtotal, total
1001
1002 sub src, src, src_stride, lsl 3
1003 sub dst, dst, dst_stride, lsl 3
1004 add src, src, #16
1005 add dst, dst, #16
1006 subs width, width, #16
1007 bne L(satd_w16up)
1008
1009 sub src, src, w_ext
1010 sub dst, dst, w_ext
1011 add src, src, src_stride, lsl 3
1012 add dst, dst, dst_stride, lsl 3
1013 subs height, height, #8
1014 mov width, w_bak
1015 bne L(satd_w16up)
1016
1017 mov w0, total
1018 normalize_8
1019 ret
1020
1021 #undef src
1022 #undef src_stride
1023 #undef dst
1024 #undef dst_stride
1025
1026 #undef w_ext
1027 #undef w_bak
1028 #undef subtotal
1029 #undef total
1030 #undef height
1031 #undef width
1032 endfunc
1033
1034 .macro satd_x8up width, height
1035 function satd\width\()x\height\()_neon, export=1
1036 mov w13, \height
1037 .if \width == 8
1038 mov w10, wzr
1039 b L(satd_w8)
1040 .else
1041 mov w12, \width
1042 sxtw x11, w12
1043 mov w10, wzr
1044 b L(satd_w16up)
1045 .endif
1046 endfunc
1047 .endm
1048
1049 satd_x8up 8, 16
1050 satd_x8up 8, 32
1051 satd_x8up 16, 16
1052 satd_x8up 16, 32
1053 satd_x8up 16, 64
1054 satd_x8up 32, 8
1055 satd_x8up 32, 16
1056 satd_x8up 32, 32
1057 satd_x8up 32, 64
1058 satd_x8up 64, 16
1059 satd_x8up 64, 32
1060 satd_x8up 64, 64
1061 satd_x8up 64, 128
1062 satd_x8up 128, 64
1063 satd_x8up 128, 128
1064
1065 .macro load_rows_hbd n0, n1, n2, src, dst, src_stride, dst_stride
1066 ldr q\n0, [\src]
1067 ldr q\n1, [\dst]
1068 sub v\n0\().8h, v\n0\().8h, v\n1\().8h
1069
1070 ldr q\n1, [\src, \src_stride]
1071 ldr q\n2, [\dst, \dst_stride]
1072 sub v\n1\().8h, v\n1\().8h, v\n2\().8h
1073
1074 add \src, \src, \src_stride, lsl 1
1075 add \dst, \dst, \dst_stride, lsl 1
1076 .endm
1077
1078 .macro HADAMARD_8X8_V_HBD \
1079 a0 a1 a2 a3 a4 a5 a6 a7 \
1080 b0 b1 b2 b3 b4 b5 b6 b7 \
1081 c0 c1 c2 c3 c4 c5 c6 c7
1082
1083 // Vertical transform
1084
1085 butterflyw v\b0, v\b1, v\c0, v\c1, v\a0, v\a1
1086 butterflyw v\b2, v\b3, v\c2, v\c3, v\a2, v\a3
1087 butterflyw v\b4, v\b5, v\c4, v\c5, v\a4, v\a5
1088 butterflyw v\b6, v\b7, v\c6, v\c7, v\a6, v\a7
1089
1090 butterfly v\a0, v\a2, v\b0, v\b2, 4s
1091 butterfly v\a1, v\a3, v\b1, v\b3, 4s
1092 butterfly v\a4, v\a6, v\b4, v\b6, 4s
1093 butterfly v\a5, v\a7, v\b5, v\b7, 4s
1094 butterfly v\b0, v\b2, v\c0, v\c2, 4s
1095 butterfly v\b1, v\b3, v\c1, v\c3, 4s
1096 butterfly v\b4, v\b6, v\c4, v\c6, 4s
1097 butterfly v\b5, v\b7, v\c5, v\c7, 4s
1098
1099 butterfly v\c0, v\c4, v\a0, v\a4, 4s
1100 butterfly v\c1, v\c5, v\a1, v\a5, 4s
1101 butterfly v\c2, v\c6, v\a2, v\a6, 4s
1102 butterfly v\c3, v\c7, v\a3, v\a7, 4s
1103 butterfly v\a0, v\a4, v\b0, v\b4, 4s
1104 butterfly v\a1, v\a5, v\b1, v\b5, 4s
1105 butterfly v\a2, v\a6, v\b2, v\b6, 4s
1106 butterfly v\a3, v\a7, v\b3, v\b7, 4s
1107 .endm
1108
1109 .macro SUM_HADAMARD_8X8_HBD \
1110 a0 a1 a2 a3 a4 a5 a6 a7 \
1111 c0 c1 c2 c3 c4 c5 c6 c7
1112
1113 // absolute value of transform coefficients
1114 abs v\a0\().4s, v\a0\().4s
1115 abs v\a1\().4s, v\a1\().4s
1116 abs v\a2\().4s, v\a2\().4s
1117 abs v\a3\().4s, v\a3\().4s
1118 abs v\a4\().4s, v\a4\().4s
1119 abs v\a5\().4s, v\a5\().4s
1120 abs v\a6\().4s, v\a6\().4s
1121 abs v\a7\().4s, v\a7\().4s
1122 abs v\c0\().4s, v\c0\().4s
1123 abs v\c1\().4s, v\c1\().4s
1124 abs v\c2\().4s, v\c2\().4s
1125 abs v\c3\().4s, v\c3\().4s
1126 abs v\c4\().4s, v\c4\().4s
1127 abs v\c5\().4s, v\c5\().4s
1128 abs v\c6\().4s, v\c6\().4s
1129 abs v\c7\().4s, v\c7\().4s
1130
1131 // stage 1 sum
1132 add v\a0\().4s, v\a0\().4s, v\a1\().4s
1133 add v\a2\().4s, v\a2\().4s, v\a3\().4s
1134 add v\a4\().4s, v\a4\().4s, v\a5\().4s
1135 add v\a6\().4s, v\a6\().4s, v\a7\().4s
1136 add v\c0\().4s, v\c0\().4s, v\c1\().4s
1137 add v\c2\().4s, v\c2\().4s, v\c3\().4s
1138 add v\c4\().4s, v\c4\().4s, v\c5\().4s
1139 add v\c6\().4s, v\c6\().4s, v\c7\().4s
1140
1141 // stage 2 sum
1142 add v\a0\().4s, v\a0\().4s, v\a2\().4s
1143 add v\a4\().4s, v\a4\().4s, v\a6\().4s
1144 add v\c0\().4s, v\c0\().4s, v\c2\().4s
1145 add v\c4\().4s, v\c4\().4s, v\c6\().4s
1146
1147 // stage 3 sum
1148 add v\a0\().4s, v\a0\().4s, v\a4\().4s
1149 add v\c0\().4s, v\c0\().4s, v\c4\().4s
1150
1151 // stage 4 sum
1152 add v0.4s, v\a0\().4s, v\c0\().4s
1153 addv s0, v0.4s
1154 .endm
1155
1156
1157 .macro SATD_8X8_HBD \
1158 a0 a1 a2 a3 a4 a5 a6 a7 \
1159 b0 b1 b2 b3 b4 b5 b6 b7 \
1160 c0 c1 c2 c3 c4 c5 c6 c7
1161
1162 HADAMARD_8X8_H \
1163 \a0, \a1, \a2, \a3, \a4, \a5, \a6, \a7, \
1164 \b0, \b1, \b2, \b3, \b4, \b5, \b6, \b7
1165
1166 HADAMARD_8X8_V_HBD \
1167 \a0, \a1, \a2, \a3, \a4, \a5, \a6, \a7, \
1168 \b0, \b1, \b2, \b3, \b4, \b5, \b6, \b7, \
1169 \c0, \c1, \c2, \c3, \c4, \c5, \c6, \c7
1170
1171 SUM_HADAMARD_8X8_HBD \
1172 \a0, \a1, \a2, \a3, \a4, \a5, \a6, \a7, \
1173 \c0, \c1, \c2, \c3, \c4, \c5, \c6, \c7
1174 .endm
1175
1176 function satd8x8_hbd_neon, export=1
1177 #define src x0
1178 #define src_stride x1
1179 #define dst x2
1180 #define dst_stride x3
1181
1182 #define subtotal w9
1183 #define total w10
1184 #define w_ext x11
1185 #define w_bak w11
1186 #define width w12
1187 #define height w13
1188
1189 mov height, 8
1190 mov width, 8
1191 sxtw w_ext, width
1192 mov total, wzr
1193
1194 // 0, 1; 2, 3; 24, 25
1195 // 4, 5; 6, 7; 26, 27
1196 // 16, 17; 20, 21; 28, 29
1197 // 18, 19; 22, 23; 30, 31
1198
1199 L(satd_w8up_hbd):
1200 load_rows_hbd 0, 1, 2, src, dst, src_stride, dst_stride
1201 load_rows_hbd 4, 5, 6, src, dst, src_stride, dst_stride
1202 load_rows_hbd 16, 17, 20, src, dst, src_stride, dst_stride
1203 load_rows_hbd 18, 19, 22, src, dst, src_stride, dst_stride
1204
1205 SATD_8X8_HBD \
1206 0, 1, 4, 5, 16, 17, 18, 19, \
1207 2, 3, 6, 7, 20, 21, 22, 23, \
1208 24, 25, 26, 27, 28, 29, 30, 31
1209
1210 fmov subtotal, s0
1211 add total, subtotal, total
1212
1213 sub src, src, src_stride, lsl 3
1214 sub dst, dst, dst_stride, lsl 3
1215 add src, src, #16
1216 add dst, dst, #16
1217 subs width, width, #8
1218 bne L(satd_w8up_hbd)
1219
1220 sub src, src, w_ext, lsl 1
1221 sub dst, dst, w_ext, lsl 1
1222 add src, src, src_stride, lsl 3
1223 add dst, dst, dst_stride, lsl 3
1224 subs height, height, #8
1225 mov width, w_bak
1226 bne L(satd_w8up_hbd)
1227
1228 mov w0, total
1229 normalize_8
1230 ret
1231
1232 #undef src
1233 #undef src_stride
1234 #undef dst
1235 #undef dst_stride
1236
1237 #undef w_ext
1238 #undef w_bak
1239 #undef subtotal
1240 #undef total
1241 #undef height
1242 #undef width
1243 endfunc
1244
1245 .macro satd_x8up_hbd width, height
1246 function satd\width\()x\height\()_hbd_neon, export=1
1247 mov w13, \height
1248 mov w12, \width
1249 sxtw x11, w12
1250 mov w10, wzr
1251 b L(satd_w8up_hbd)
1252 endfunc
1253 .endm
1254
1255 satd_x8up_hbd 8, 16
1256 satd_x8up_hbd 8, 32
1257 satd_x8up_hbd 16, 8
1258 satd_x8up_hbd 16, 16
1259 satd_x8up_hbd 16, 32
1260 satd_x8up_hbd 16, 64
1261 satd_x8up_hbd 32, 8
1262 satd_x8up_hbd 32, 16
1263 satd_x8up_hbd 32, 32
1264 satd_x8up_hbd 32, 64
1265 satd_x8up_hbd 64, 16
1266 satd_x8up_hbd 64, 32
1267 satd_x8up_hbd 64, 64
1268 satd_x8up_hbd 64, 128
1269 satd_x8up_hbd 128, 64
1270 satd_x8up_hbd 128, 128
1271
1272 // x0: src: *const u16,
1273 // x1: src_stride: isize,
1274 // x2: dst: *const u16,
1275 // x3: dst_stride: isize,
1276 function satd4x4_hbd_neon, export=1
1277 #define src x0
1278 #define src_stride x1
1279 #define dst x2
1280 #define dst_stride x3
1281
1282 #define subtotal w9
1283 #define total w10
1284 #define height w13
1285
1286 mov height, 4
1287 mov total, wzr
1288
1289 L(satd_w4_hbd):
1290 ldr d0, [src]
1291 ldr d1, [dst]
1292 sub v0.8h, v0.8h, v1.8h
1293
1294 ldr d1, [src, src_stride]
1295 ldr d2, [dst, dst_stride]
1296 sub v1.8h, v1.8h, v2.8h
1297
1298 add src, src, src_stride, lsl 1
1299 add dst, dst, dst_stride, lsl 1
1300
1301 ldr d2, [src]
1302 ldr d3, [dst]
1303 sub v2.8h, v2.8h, v3.8h
1304
1305 ldr d3, [src, src_stride]
1306 ldr d4, [dst, src_stride]
1307 sub v3.8h, v3.8h, v4.8h
1308
1309 add src, src, src_stride, lsl 1
1310 add dst, dst, dst_stride, lsl 1
1311
1312 // pack rows 0-2, 1-3
1313 mov v0.d[1], v2.d[0]
1314 mov v1.d[1], v3.d[0]
1315
1316 // Horizontal transform
1317 butterfly v2, v3, v0, v1
1318 interleave v0, v1, v2, v3
1319 butterfly v2, v3, v0, v1
1320 interleave_pairs v0, v1, v2, v3
1321 // Vertical transform
1322
1323 butterfly v2, v3, v0, v1
1324 interleave v0, v1, v2, v3
1325 butterflyw v2, v3, v4, v5, v0, v1
1326
1327 // absolute value of transform coefficients
1328 abs v2.4s, v2.4s
1329 abs v3.4s, v3.4s
1330 abs v4.4s, v4.4s
1331 abs v5.4s, v5.4s
1332
1333 // stage 1 sum
1334 add v2.4s, v2.4s, v3.4s
1335 add v4.4s, v4.4s, v5.4s
1336
1337 // stage 2 sum
1338 add v0.4s, v2.4s, v4.4s
1339 addv s0, v0.4s
1340
1341 fmov subtotal, s0
1342 add total, subtotal, total
1343
1344 subs height, height, #4
1345 bne L(satd_w4_hbd)
1346
1347 mov w0, total
1348 normalize_4
1349 ret
1350
1351 #undef src
1352 #undef src_stride
1353 #undef dst
1354 #undef dst_stride
1355
1356 #undef subtotal
1357 #undef total
1358 #undef height
1359 endfunc
1360
1361 function satd4x8_hbd_neon, export=1
1362 mov w13, 8
1363 mov w10, wzr
1364 b L(satd_w4_hbd)
1365 endfunc
1366
1367 function satd4x16_hbd_neon, export=1
1368 mov w13, 16
1369 mov w10, wzr
1370 b L(satd_w4_hbd)
1371 endfunc
1372
1373 .macro SUM_DOUBLE_HADAMARD_4X4_HBD \
1374 a0 a1 a2 a3 c0 c1 c2 c3
1375
1376 // absolute value of transform coefficients
1377 abs v\a0\().4s, v\a0\().4s
1378 abs v\a1\().4s, v\a1\().4s
1379 abs v\a2\().4s, v\a2\().4s
1380 abs v\a3\().4s, v\a3\().4s
1381 abs v\c0\().4s, v\c0\().4s
1382 abs v\c1\().4s, v\c1\().4s
1383 abs v\c2\().4s, v\c2\().4s
1384 abs v\c3\().4s, v\c3\().4s
1385
1386 // stage 1 sum
1387 add v\a0\().4s, v\a0\().4s, v\a1\().4s
1388 add v\a2\().4s, v\a2\().4s, v\a3\().4s
1389 add v\c0\().4s, v\c0\().4s, v\c1\().4s
1390 add v\c2\().4s, v\c2\().4s, v\c3\().4s
1391
1392 // stage 2 sum
1393 add v\a0\().4s, v\a0\().4s, v\a2\().4s
1394 add v\c0\().4s, v\c0\().4s, v\c2\().4s
1395
1396 // stage 3 sum
1397 add v0.4s, v\a0\().4s, v\c0\().4s
1398 addv s0, v0.4s
1399 .endm
1400
1401 function satd8x4_hbd_neon, export=1
1402 #define src x0
1403 #define src_stride x1
1404 #define dst x2
1405 #define dst_stride x3
1406
1407 #define subtotal w9
1408 #define total w10
1409 #define width w12
1410
1411 mov width, 8
1412 mov total, wzr
1413
1414 L(satd_h4_hbd):
1415 ldr q0, [src]
1416 ldr q1, [dst]
1417 sub v0.8h, v0.8h, v1.8h
1418
1419 ldr q1, [src, src_stride]
1420 ldr q2, [dst, dst_stride]
1421 sub v1.8h, v1.8h, v2.8h
1422
1423 lsl x8, src_stride, 1
1424 lsl x9, dst_stride, 1
1425
1426 ldr q2, [src, x8]
1427 ldr q3, [dst, x9]
1428 sub v2.8h, v2.8h, v3.8h
1429
1430 add x8, src_stride, src_stride, lsl 1
1431 add x9, dst_stride, dst_stride, lsl 1
1432
1433 ldr q3, [src, x8]
1434 ldr q4, [dst, x9]
1435 sub v3.8h, v3.8h, v4.8h
1436
1437 ext v4.16b, v0.16b, v0.16b, 8
1438 ext v5.16b, v1.16b, v1.16b, 8
1439 mov v0.d[1], v2.d[0]
1440 mov v1.d[1], v3.d[0]
1441 mov v4.d[1], v2.d[1]
1442 mov v5.d[1], v3.d[1]
1443
1444 DOUBLE_HADAMARD_4X4 hbd=1
1445 SUM_DOUBLE_HADAMARD_4X4_HBD 2, 3, 16, 17, 6, 7, 18, 19
1446
1447 fmov subtotal, s0
1448 add total, subtotal, total
1449
1450 add src, src, #16
1451 add dst, dst, #16
1452 subs width, width, #8
1453 bne L(satd_h4_hbd)
1454
1455 mov w0, total
1456 normalize_4
1457 ret
1458
1459 #undef src
1460 #undef src_stride
1461 #undef dst
1462 #undef dst_stride
1463
1464 #undef subtotal
1465 #undef total
1466 #undef width
1467 endfunc
1468
1469 function satd16x4_hbd_neon, export=1
1470 mov w12, 16
1471 mov w10, wzr
1472 b L(satd_h4_hbd)
1473 endfunc