Debian Codegraph @

.cargo/vendor/av-scenechange/src/asm/x86/satd16_avx2.asm

raw
1 ; Copyright (c) 2022, The rav1e contributors. All rights reserved
2 ;
3 ; This source code is subject to the terms of the BSD 2 Clause License and
4 ; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
5 ; was not distributed with this source code in the LICENSE file, you can
6 ; obtain it at www.aomedia.org/license/software. If the Alliance for Open
7 ; Media Patent License 1.0 was not distributed with this source code in the
8 ; PATENTS file, you can obtain it at www.aomedia.org/license/patent.
9
10 %include "config.asm"
11 %include "src/asm/x86/x86inc.asm"
12
13 %define m(x) mangle(private_prefix %+ _ %+ x %+ SUFFIX)
14
15 %if ARCH_X86_64
16
17 SECTION_RODATA 32
18
19 align 32
20 pw_1x16: times 16 dw 1
21
22 SECTION .text
23
24 %macro NORMALIZE4PT 0
25 add eax, 2
26 shr eax, 2
27 %endmacro
28
29 %macro NORMALIZE8PT 0
30 add eax, 4
31 shr eax, 3
32 %endmacro
33
34 ; Add and subtract registers
35 ;
36 ; Takes m0 and m1 as both input and output.
37 ; Requires m2 as a free register.
38 ;
39 ; If we start with this permutation:
40 ;
41 ; m0 0 1 2 3 4 5 6 7
42 ; m1 8 9 10 11 12 13 14 15
43 ;
44 ; Then the output will be as such:
45 ;
46 ; m0 [0+8][1+9][2+10][3+11] [4+12][5+13][6+14][7+15]
47 ; m1 [0-8][1-9][2-10][3-11] [4-12][5-13][6-14][7-15]
48 %macro BUTTERFLY 3
49 %define BIT_PRECISION %1
50 %define VEC_SIZE %2
51 ; use alternate registers 3,4,5
52 %define USE_ALT %3
53
54 %if USE_ALT == 1
55 SWAP 3, 0
56 SWAP 4, 1
57 SWAP 5, 2
58 %endif
59
60 %if VEC_SIZE == 32
61 %define V ym
62 %elif VEC_SIZE == 16
63 %define V xm
64 %endif
65
66 ; Use m2 as a temporary register, then swap
67 ; so that m0 and m1 contain the output.
68 %if BIT_PRECISION == 16
69 paddw V%+ 2, V%+ 0, V%+ 1
70 psubw V%+ 0, V%+ 1
71 %elif BIT_PRECISION == 32
72 paddd ym2, ym0, ym1
73 psubd ym0, ym1
74 %else
75 %error Incorrect precision specified (16 or 32 expected)
76 %endif
77
78 SWAP 2, 1, 0
79
80 %if USE_ALT == 1
81 SWAP 3, 0
82 SWAP 4, 1
83 SWAP 5, 2
84 %endif
85 %endmacro
86
87 ; Interleave packed rows together (in m0 and m1).
88 ; m2 should contain a free register.
89 ;
90 ; Macro argument takes size in bits of each element (where one
91 ; element is the difference between two original source pixels).
92 ;
93 ; If we start with this permutation:
94 ;
95 ; m0 0 1 2 3 4 5 6 7
96 ; m1 8 9 10 11 12 13 14 15
97 ;
98 ; Then, after INTERLEAVE, this will be the permutation:
99 ;
100 ; m0 0 8 1 9 2 10 3 11
101 ; m1 4 12 5 13 6 14 7 15
102 %macro INTERLEAVE 3
103 %define BIT_PRECISION %1
104 %define VEC_SIZE %2
105 %define USE_ALT %3
106
107 %if USE_ALT == 1
108 SWAP 3, 0
109 SWAP 4, 1
110 SWAP 5, 2
111 %endif
112
113 %if VEC_SIZE == 16
114 %define V xm
115 %elif VEC_SIZE == 32
116 %define V ym
117 %else
118 %error Invalid vector size (expected 16 or 32)
119 %endif
120
121 %if BIT_PRECISION == 16
122 punpcklwd V%+ 2, V%+ 0, V%+ 1
123 punpckhwd V%+ 0, V%+ 1
124 SWAP 2, 1, 0
125 %elif BIT_PRECISION == 32
126 punpckldq ym2, ym0, ym1
127 punpckhdq ym0, ym1
128 ; AVX2 shuffles operate over 128-bit halves of the full ymm register
129 ; in parallel, so these shuffles are required to fix up the permutation.
130 vperm2i128 ym1, ym2, ym0, 0x20
131 vperm2i128 ym0, ym2, ym0, 0x31
132 SWAP 0, 1
133 %else
134 %error Incorrect precision specified (16 or 32 expected)
135 %endif
136
137 %if USE_ALT == 1
138 SWAP 3, 0
139 SWAP 4, 1
140 SWAP 5, 2
141 %endif
142 %endmacro
143
144 ; Interleave pairs of 2 elements (in m0 and m1)
145 ; m2 should contain a free register.
146 %macro INTERLEAVE_PAIRS 3
147 %define BIT_PRECISION %1
148 %define VEC_SIZE %2
149 %define USE_ALT %3
150
151 %if USE_ALT == 1
152 SWAP 3, 0
153 SWAP 4, 1
154 SWAP 5, 2
155 %endif
156
157 %if VEC_SIZE == 16
158 %define V xm
159 %elif VEC_SIZE == 32
160 %define V ym
161 %else
162 %error Invalid vector size (expected 16 or 32)
163 %endif
164
165 %if BIT_PRECISION == 16
166 punpckldq V%+ 2, V%+ 0, V%+ 1
167 punpckhdq V%+ 0, V%+ 1
168 %elif BIT_PRECISION == 32
169 punpcklqdq ym2, ym0, ym1
170 punpckhqdq ym0, ym1
171 %else
172 %error Incorrect precision specified (16 or 32 expected)
173 %endif
174 SWAP 2, 1, 0
175
176 %if USE_ALT == 1
177 SWAP 3, 0
178 SWAP 4, 1
179 SWAP 5, 2
180 %endif
181 %endmacro
182
183 %macro HADAMARD_4X4_PACKED 2
184 %define BIT_PRECISION %1
185 ; Register size to use (in bytes)
186 %define VEC_SIZE %2
187
188 %if VEC_SIZE == 16
189 %define V xm
190 %elif VEC_SIZE == 32
191 %define V ym
192 %else
193 %error Invalid vector size (expected 16 or 32)
194 %endif
195
196 ; Starting registers:
197
198 ; m0 0 1 2 3
199 ; m1 4 5 6 7
200 ; m2 8 9 10 11
201 ; m3 12 13 14 15
202
203 ; Where each number represents an index of the
204 ; original block of differences.
205
206 ; Pack rows 0,2 and 1,3 into m0 and m1
207 %if BIT_PRECISION == 16
208 %if VEC_SIZE == 16
209 ; In this case, each row only has 64 bits, so we use
210 ; punpcklqdq only. The high 64 bits are always 0.
211 punpcklqdq xm0, xm2
212 punpcklqdq xm1, xm3
213 %elif VEC_SIZE == 32
214 ; The upper 128 bits of all input registers are zeroed
215 punpcklqdq m4, m0, m2
216 punpcklqdq m5, m1, m3
217 punpckhqdq m0, m0, m2
218 punpckhqdq m1, m1, m3
219 vinserti128 m0, m4, xm0, 1
220 vinserti128 m1, m5, xm1, 1
221 %endif
222 %elif BIT_PRECISION == 32
223 vinserti128 ym0, ym0, xm2, 1
224 vinserti128 ym1, ym1, xm3, 1
225 %else
226 %error Invalid bit precision (expected 16 or 32)
227 %endif
228
229 ; Now that we've packed rows 0-2 and 1-3 together,
230 ; this is our permutation:
231
232 ; m0 0 1 2 3 8 9 10 11
233 ; m1 4 5 6 7 12 13 14 15
234
235 ; For a 8x4 transform (with 16-bit coefficients), this pattern is
236 ; extended for each 128-bit half but for the second block, and thus
237 ; all comments also apply to the upper 128-bits for the 8x4 transform.
238
239 BUTTERFLY %1, %2, 0
240
241 ; m0 [0+4][1+5][2+6][3+7] [8+12][9+13][10+14][11+15]
242 ; m1 [0-4][1-5][2-6][3-7] [8-12][9-13][10-14][11-15]
243
244 INTERLEAVE %1, %2, 0
245
246 ; m0 [ 0+4][ 0-4][ 1+5][ 1-5] [2 + 6][2 - 6][3 + 7][3 - 7]
247 ; m1 [8+12][8-12][9+13][9-13] [10+14][10-14][11+15][11-15]
248
249 BUTTERFLY %1, %2, 0
250
251 ; m0 [0+4+8+12][0-4+8-12][1+5+9+13][1-5+9-13] [2+6+10+14][2-6+10-14][3+7+11+15][3-7+11-15]
252 ; m1 [0+4-8-12][0-4-8+12][1+5-9-13][1-5-9+13] [2+6-10-14][2-6-10+14][3+7-11-15][3-7-11+15]
253
254 ; for one row:
255 ; [0+1+2+3][0-1+2-3][0+1-2-3][0-1-2+3]
256 ; For the vertical transform, these are packed into a new column.
257
258 INTERLEAVE_PAIRS %1, %2, 0
259
260 ; p0 p1 p2 p3
261 ; m0 [0+4+ 8+12][0-4+ 8-12][0+4- 8-12][0-4- 8+12] [1+5+ 9+13][1-5+ 9-13][1+5- 9-13][1-5- 9+13]
262 ; m1 [2+6+10+14][2-6+10-14][2+6-10-14][2-6-10+14] [3+7+11+15][3-7+11-15][3+7-11-15][3-7-11+15]
263
264 ; According to this grid:
265
266 ; p0 q0 r0 s0
267 ; p1 q1 r1 s1
268 ; p2 q2 r2 s2
269 ; p3 q3 r3 s3
270
271 ; Horizontal transform; since the output is transposed from the original order,
272 ; we can do the same steps as the vertical transform and the result will be the same.
273 BUTTERFLY %1, %2, 0
274 INTERLEAVE %1, %2, 0
275 BUTTERFLY %1, %2, 0
276
277 ; Finished horizontal transform except for the last step (interleaving pairs),
278 ; which we skip, because after this we add up the absolute value of the
279 ; coefficients, which is a commutative operation (order does not matter).
280 %endmacro
281
282 ; Horizontal sum of mm register
283 ;
284 ; Inputs:
285 ; %1 = Element size in bits (16 or 32)
286 ; %2 = Size of input register in bytes (16 or 32)
287 ; You can e.g. pass 16 for this argument if you
288 ; only want to sum up the bottom 128-bits of a
289 ; ymm register.
290 ; %3 = Input register number
291 ; %4 = Temporary register number
292 ; %5 = Output register (e.g., eax)
293 %macro HSUM 5
294 %define E_SIZE %1
295 %define REG_SIZE %2
296 %define INPUT %3
297 %define TMP %4
298 %define OUTPUT %5
299
300 %if REG_SIZE == 16
301 %define V xm
302 %elif REG_SIZE == 32
303 %define V ym
304 %else
305 %error Invalid register size (expected 16 or 32)
306 %endif
307
308 %if E_SIZE == 16
309 ; Add adjacent pairs of 16-bit elements to produce 32-bit results,
310 ; then proceed with 32-bit sum
311 pmaddwd V%+INPUT, [pw_1x16]
312 %endif
313
314 %if mmsize == 32 && REG_SIZE == 32
315 ; Add upper half of ymm to xmm
316 vextracti128 xm%+TMP, ym%+INPUT, 1
317 paddd xm%+INPUT, xm%+TMP
318 %endif
319
320 ; Reduce 32-bit results
321 pshufd xm%+TMP, xm%+INPUT, q2323
322 paddd xm%+INPUT, xm%+TMP
323 pshufd xm%+TMP, xm%+INPUT, q1111
324 paddd xm%+INPUT, xm%+TMP
325 movd OUTPUT, xm%+INPUT
326 %endmacro
327
328 ; given m0-7, do butterfly as follows:
329 ; (m0, m1) = butterfly(m0, m1)
330 ; (m2, m3) = butterfly(m2, m3)
331 ; (m4, m5) = butterfly(m4, m5)
332 ; (m6, m7) = butterfly(m6, m7)
333 %macro BUTTERFLY_8X8 0
334 ; m8 is free
335 paddd m8, m0, m1
336 psubd m0, m1
337 SWAP 8, 1, 0
338
339 ; m8 is free
340 paddd m8, m2, m3
341 psubd m2, m3
342 SWAP 8, 3, 2
343
344 paddd m8, m4, m5
345 psubd m4, m5
346 SWAP 8, 5, 4
347
348 paddd m8, m6, m7
349 psubd m6, m7
350 SWAP 8, 7, 6
351 %endmacro
352
353 %macro HADAMARD_8X8_VERTICAL 0
354 BUTTERFLY_8X8
355 ; m0-7 contain a0-7
356
357 SWAP 2, 1
358 SWAP 6, 5
359
360 BUTTERFLY_8X8
361
362 SWAP 1, 4
363 SWAP 3, 6
364
365 BUTTERFLY_8X8
366
367 SWAP 2, 1
368 SWAP 2, 4
369 SWAP 3, 6
370 SWAP 5, 6
371 %endmacro
372
373 ; Transpose rows m0-7.
374 ; Output is also contained in m0-7.
375 ;
376 ; Uses m8, m10-15 as temporary registers (i.e. m9 is left unchanged.)
377 %macro TRANSPOSE8X8D 0
378 SWAP 9, 0
379 SWAP 10, 1
380 SWAP 11, 2
381 SWAP 12, 3
382 SWAP 13, 4
383 SWAP 14, 5
384 SWAP 15, 6
385 SWAP 2, 7
386
387 punpckldq m6, m9, m10
388 punpckldq m1, m11, m12
389 punpckhdq m8, m9, m10
390 punpckldq m4, m13, m14
391 punpckldq m9, m15, m2
392 vshufps m3, m6, m1, 0x4e
393 vpblendd m10, m6, m3, 0xcc
394 vshufps m6, m4, m9, 0x4e
395 punpckhdq m7, m11, m12
396 vpblendd m11, m4, m6, 0xcc
397 vpblendd m12, m3, m1, 0xcc
398 vperm2i128 m3, m10, m11, 0x20
399 punpckhdq m5, m13, m14
400 vpblendd m13, m6, m9, 0xcc
401 punpckhdq m4, m15, m2
402 vperm2i128 m2, m12, m13, 0x20
403 vshufps m14, m8, m7, 0x4e
404 vpblendd m15, m14, m7, 0xcc
405 vshufps m7, m5, m4, 0x4e
406 vpblendd m8, m8, m14, 0xcc
407 vpblendd m5, m5, m7, 0xcc
408 vperm2i128 m6, m8, m5, 0x20
409 vpblendd m4, m7, m4, 0xcc
410 vperm2i128 m7, m15, m4, 0x20
411 vperm2i128 m1, m10, m11, 0x31
412 vperm2i128 m9, m12, m13, 0x31
413 vperm2i128 m5, m8, m5, 0x31
414 vperm2i128 m4, m15, m4, 0x31
415
416 SWAP 0, 9
417
418 ; Output order is as follows:
419 ; 3 2 6 7 1 0 5 4
420
421 ; sort rows
422 SWAP 3, 0
423 ; 0 2 6 7 1 3 5 4
424 SWAP 1, 2
425 ; 0 1 6 7 2 3 5 4
426 SWAP 6, 2
427 ; 0 1 2 7 6 3 5 4
428 SWAP 7, 3
429 ; 0 1 2 3 6 7 5 4
430 SWAP 6, 4
431 ; 0 1 2 3 4 7 5 6
432 SWAP 7, 5
433 ; 0 1 2 3 4 5 7 6
434 SWAP 6, 7
435 ; 0 1 2 3 4 5 6 7
436 %endmacro
437
438 ; m0-7 as input; add coefficients to ymm9.
439 INIT_YMM avx2
440 cglobal satd_8x8_hbd_internal, 0, 0, 0, src, src_stride, dst, dst_stride, bdmax, \
441 src_stride3, dst_stride3
442 HADAMARD_8X8_VERTICAL
443
444 TRANSPOSE8X8D
445
446 HADAMARD_8X8_VERTICAL
447
448 REPX {pabsd x, x}, m0, m1, m2, m3, m4, m5, m6, m7
449
450 ; Add m0-7
451 paddd m0, m4
452 paddd m1, m5
453 paddd m2, m6
454 paddd m3, m7
455
456 paddd m0, m2
457 paddd m1, m3
458
459 paddd m0, m1
460 paddd ymm9, m0
461 ret
462
463 %macro LOAD_DIFF_8X8 0
464 movu xm0, [srcq + 0*src_strideq]
465 movu xm1, [srcq + 1*src_strideq]
466 movu xm2, [srcq + 2*src_strideq]
467 movu xm3, [srcq + src_stride3q ]
468 lea srcq, [srcq + 4*src_strideq]
469 movu xm4, [srcq + 0*src_strideq]
470 movu xm5, [srcq + 1*src_strideq]
471 movu xm6, [srcq + 2*src_strideq]
472 movu xm7, [srcq + src_stride3q ]
473
474 psubw xm0, [dstq + 0*dst_strideq]
475 psubw xm1, [dstq + 1*dst_strideq]
476 psubw xm2, [dstq + 2*dst_strideq]
477 psubw xm3, [dstq + dst_stride3q ]
478 lea dstq, [dstq + 4*dst_strideq]
479 psubw xm4, [dstq + 0*dst_strideq]
480 psubw xm5, [dstq + 1*dst_strideq]
481 psubw xm6, [dstq + 2*dst_strideq]
482 psubw xm7, [dstq + dst_stride3q ]
483
484 pmovsxwd m0, xm0
485 pmovsxwd m1, xm1
486 pmovsxwd m2, xm2
487 pmovsxwd m3, xm3
488 pmovsxwd m4, xm4
489 pmovsxwd m5, xm5
490 pmovsxwd m6, xm6
491 pmovsxwd m7, xm7
492 %endmacro
493
494 INIT_YMM avx2
495 cglobal satd_8x8_hbd, 5, 7, 16, src, src_stride, dst, dst_stride, bdmax, \
496 src_stride3, dst_stride3
497 lea src_stride3q, [3*src_strideq]
498 lea dst_stride3q, [3*dst_strideq]
499
500 LOAD_DIFF_8X8
501
502 ; m0-7 contain rows of 8x8 block to transform
503 ; with 32-bit coefficients
504
505 HADAMARD_8X8_VERTICAL
506
507 TRANSPOSE8X8D
508
509 HADAMARD_8X8_VERTICAL
510
511 REPX {pabsd x, x}, m0, m1, m2, m3, m4, m5, m6, m7
512
513 ; Add m0-7
514 paddd m0, m4
515 paddd m1, m5
516 paddd m2, m6
517 paddd m3, m7
518
519 paddd m0, m2
520 paddd m1, m3
521
522 paddd m0, m1
523
524 HSUM 32, 32, 0, 1, eax
525 NORMALIZE8PT
526 RET
527
528 INIT_YMM avx2
529 cglobal satd_4x4_hbd, 5, 7, 8, src, src_stride, dst, dst_stride, bdmax, \
530 src_stride3, dst_stride3
531 lea src_stride3q, [3*src_strideq]
532 lea dst_stride3q, [3*dst_strideq]
533
534 cmp bdmaxd, (1 << 10) - 1
535 jne .12bpc
536
537 ; Load src rows
538 movq xm0, [srcq + 0*src_strideq]
539 movq xm1, [srcq + 1*src_strideq]
540 movq xm2, [srcq + 2*src_strideq]
541 movq xm3, [srcq + src_stride3q ]
542
543 ; src -= dst
544 psubw xm0, [dstq + 0*dst_strideq]
545 psubw xm1, [dstq + 1*dst_strideq]
546 psubw xm2, [dstq + 2*dst_strideq]
547 psubw xm3, [dstq + dst_stride3q ]
548
549 HADAMARD_4X4_PACKED 16, 16
550
551 ; Sum up absolute value of transform coefficients
552 pabsw xm0, xm0
553 pabsw xm1, xm1
554 paddw xm0, xm1
555 HSUM 16, 16, 0, 1, eax
556 NORMALIZE4PT
557 RET
558 .12bpc:
559 ; this gives a nicer disassembly
560 RESET_MM_PERMUTATION
561
562 ; Load src rows
563 pmovzxwd xm0, [srcq + 0*src_strideq]
564 pmovzxwd xm1, [srcq + 1*src_strideq]
565 pmovzxwd xm2, [srcq + 2*src_strideq]
566 pmovzxwd xm3, [srcq + src_stride3q ]
567
568 ; Load dst rows
569 pmovzxwd xm4, [dstq + 0*dst_strideq]
570 pmovzxwd xm5, [dstq + 1*dst_strideq]
571 pmovzxwd xm6, [dstq + 2*dst_strideq]
572 pmovzxwd xm7, [dstq + dst_stride3q ]
573
574 ; src -= dst
575 psubd xm0, xm4
576 psubd xm1, xm5
577 psubd xm2, xm6
578 psubd xm3, xm7
579
580 HADAMARD_4X4_PACKED 32, 32
581
582 pabsd m0, m0
583 pabsd m1, m1
584 paddd m0, m1
585 HSUM 32, 32, 0, 1, eax
586 NORMALIZE4PT
587 RET
588
589 ; 32-bit input rows are in m0-3; result is in m0.
590 ; Uses m0-5 as temporary registers.
591 %macro HADAMARD_8X4_12BPC 0
592 vperm2i128 m4, m0, m2, 0x31
593 vperm2i128 m5, m1, m3, 0x31
594 vinserti128 m0, m0, xm2, 1
595 vinserti128 m1, m1, xm3, 1
596
597 ; Swap so m3,m4 are used as inputs.
598 SWAP 3, 4, 5
599
600 ; instead of using HADAMARD_4X4_PACKED twice, we interleave
601 ; 2 transforms operating over different registers for more
602 ; opportunity for instruction level parallelism.
603
604 BUTTERFLY 32, 32, 0
605 BUTTERFLY 32, 32, 1
606 INTERLEAVE 32, 32, 0
607 INTERLEAVE 32, 32, 1
608 BUTTERFLY 32, 32, 0
609 BUTTERFLY 32, 32, 1
610 INTERLEAVE_PAIRS 32, 32, 0
611 INTERLEAVE_PAIRS 32, 32, 1
612 BUTTERFLY 32, 32, 0
613 BUTTERFLY 32, 32, 1
614 INTERLEAVE 32, 32, 0
615 INTERLEAVE 32, 32, 1
616 BUTTERFLY 32, 32, 0
617 BUTTERFLY 32, 32, 1
618
619 pabsd m0, m0
620 pabsd m1, m1
621 pabsd m3, m3
622 pabsd m4, m4
623
624 paddd m0, m1
625 paddd m3, m4
626 paddd m0, m3
627 %endmacro
628
629 INIT_YMM avx2
630 cglobal satd_16x4_hbd, 5, 7, 12, src, src_stride, dst, dst_stride, bdmax, \
631 src_stride3, dst_stride3
632 lea src_stride3q, [3*src_strideq]
633 lea dst_stride3q, [3*dst_strideq]
634
635 cmp bdmaxd, (1 << 10) - 1
636 jne .12bpc
637
638 ; Load src rows
639 movu m0, [srcq + 0*src_strideq]
640 movu m1, [srcq + 1*src_strideq]
641 movu m2, [srcq + 2*src_strideq]
642 movu m3, [srcq + src_stride3q ]
643
644 ; src -= dst
645 psubw m0, [dstq + 0*dst_strideq]
646 psubw m1, [dstq + 1*dst_strideq]
647 psubw m2, [dstq + 2*dst_strideq]
648 psubw m3, [dstq + dst_stride3q ]
649
650 .10bpc_main:
651
652 ; Original permutation
653 ; m0 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
654 ; m1 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
655 ; m2 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47
656 ; m3 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63
657
658 ; Two registers perform 2 4x4 transforms in parallel
659
660 punpcklqdq m4, m0, m2
661 punpcklqdq m5, m1, m3
662
663 punpckhqdq m0, m0, m2
664 punpckhqdq m1, m1, m3
665
666 SWAP 4, 3
667 SWAP 5, 4
668
669 ; New permutation
670 ; m0 0 1 2 3 32 33 34 35 8 9 10 11 40 41 42 43
671 ; m1 16 17 18 19 48 49 50 51 24 25 26 27 56 57 58 59
672 ; m3 4 5 6 7 36 37 38 39 12 13 14 15 44 45 46 47
673 ; m4 20 21 22 23 52 53 54 55 28 29 30 31 60 61 62 63
674
675 BUTTERFLY 16, 32, 0
676 BUTTERFLY 16, 32, 1
677 INTERLEAVE 16, 32, 0
678 INTERLEAVE 16, 32, 1
679 BUTTERFLY 16, 32, 0
680 BUTTERFLY 16, 32, 1
681 INTERLEAVE_PAIRS 16, 32, 0
682 INTERLEAVE_PAIRS 16, 32, 1
683 BUTTERFLY 16, 32, 0
684 BUTTERFLY 16, 32, 1
685 INTERLEAVE 16, 32, 0
686 INTERLEAVE 16, 32, 1
687 BUTTERFLY 16, 32, 0
688 BUTTERFLY 16, 32, 1
689
690 pabsw m0, m0
691 pabsw m1, m1
692 pabsw m3, m3
693 pabsw m4, m4
694
695 paddw m0, m1
696 paddw m3, m4
697 paddw m0, m3
698
699 HSUM 16, 32, 0, 1, eax
700 NORMALIZE4PT
701 RET
702 .12bpc:
703 RESET_MM_PERMUTATION
704
705 mov bdmaxd, 2
706 pxor m6, m6
707 .12bpc_loop:
708 movu xm0, [srcq + 0*src_strideq]
709 movu xm1, [srcq + 1*src_strideq]
710 movu xm2, [srcq + 2*src_strideq]
711 movu xm3, [srcq + src_stride3q ]
712
713 psubw xm0, [dstq + 0*dst_strideq]
714 psubw xm1, [dstq + 1*dst_strideq]
715 psubw xm2, [dstq + 2*dst_strideq]
716 psubw xm3, [dstq + dst_stride3q ]
717
718 pmovsxwd m0, xm0
719 pmovsxwd m1, xm1
720 pmovsxwd m2, xm2
721 pmovsxwd m3, xm3
722
723 add srcq, 16
724 add dstq, 16
725
726 HADAMARD_8X4_12BPC
727 paddd m6, m0
728 dec bdmaxd
729 jnz .12bpc_loop
730
731 HSUM 32, 32, 6, 1, eax
732 NORMALIZE4PT
733 RET
734
735 INIT_YMM avx2
736 cglobal satd_4x16_hbd, 5, 7, 12, src, src_stride, dst, dst_stride, bdmax, \
737 src_stride3, dst_stride3
738 lea src_stride3q, [3*src_strideq]
739 lea dst_stride3q, [3*dst_strideq]
740
741 cmp bdmaxd, (1 << 10) - 1
742 jne .12bpc
743
744 ; BLOCK 1
745 movq xm0, [srcq + 0*src_strideq]
746 movq xm1, [srcq + 1*src_strideq]
747 movq xm2, [srcq + 2*src_strideq]
748 movq xm3, [srcq + src_stride3q ]
749 lea srcq, [srcq + 4*src_strideq]
750
751 psubw xm0, [dstq + 0*dst_strideq]
752 psubw xm1, [dstq + 1*dst_strideq]
753 psubw xm2, [dstq + 2*dst_strideq]
754 psubw xm3, [dstq + dst_stride3q ]
755 lea dstq, [dstq + 4*dst_strideq]
756
757 ; BLOCK 2
758 movq xm4, [srcq + 0*src_strideq]
759 movq xm5, [srcq + 1*src_strideq]
760 movq xm6, [srcq + 2*src_strideq]
761 movq xm7, [srcq + src_stride3q ]
762 lea srcq, [srcq + 4*src_strideq]
763
764 psubw xm4, [dstq + 0*dst_strideq]
765 psubw xm5, [dstq + 1*dst_strideq]
766 psubw xm6, [dstq + 2*dst_strideq]
767 psubw xm7, [dstq + dst_stride3q ]
768 lea dstq, [dstq + 4*dst_strideq]
769
770 vinserti128 m0, m0, xm4, 1
771 vinserti128 m1, m1, xm5, 1
772 vinserti128 m2, m2, xm6, 1
773 vinserti128 m3, m3, xm7, 1
774
775 ; BLOCK 3
776 movq xm4, [srcq + 0*src_strideq]
777 movq xm5, [srcq + 1*src_strideq]
778 movq xm6, [srcq + 2*src_strideq]
779 movq xm7, [srcq + src_stride3q ]
780 lea srcq, [srcq + 4*src_strideq]
781
782 psubw xm4, [dstq + 0*dst_strideq]
783 psubw xm5, [dstq + 1*dst_strideq]
784 psubw xm6, [dstq + 2*dst_strideq]
785 psubw xm7, [dstq + dst_stride3q ]
786 lea dstq, [dstq + 4*dst_strideq]
787
788 ; BLOCK 4
789 movq xm8, [srcq + 0*src_strideq]
790 movq xm9, [srcq + 1*src_strideq]
791 movq xm10, [srcq + 2*src_strideq]
792 movq xm11, [srcq + src_stride3q ]
793
794 psubw xm8, [dstq + 0*dst_strideq]
795 psubw xm9, [dstq + 1*dst_strideq]
796 psubw xm10, [dstq + 2*dst_strideq]
797 psubw xm11, [dstq + dst_stride3q ]
798
799 vinserti128 m4, m4, xm8, 1
800 vinserti128 m5, m5, xm9, 1
801 vinserti128 m6, m6, xm10, 1
802 vinserti128 m7, m7, xm11, 1
803
804 punpcklqdq m0, m0, m4
805 punpcklqdq m1, m1, m5
806 punpcklqdq m2, m2, m6
807 punpcklqdq m3, m3, m7
808
809 jmp m(satd_16x4_hbd).10bpc_main
810
811 .12bpc:
812 mov bdmaxd, 2
813 pxor m8, m8
814
815 .12bpc_loop:
816 ; BLOCK 1
817 movq xm0, [srcq + 0*src_strideq]
818 movq xm1, [srcq + 1*src_strideq]
819 movq xm2, [srcq + 2*src_strideq]
820 movq xm3, [srcq + src_stride3q ]
821 lea srcq, [srcq + 4*src_strideq]
822
823 psubw xm0, [dstq + 0*dst_strideq]
824 psubw xm1, [dstq + 1*dst_strideq]
825 psubw xm2, [dstq + 2*dst_strideq]
826 psubw xm3, [dstq + dst_stride3q ]
827 lea dstq, [dstq + 4*dst_strideq]
828
829 pmovsxwd xm0, xm0
830 pmovsxwd xm1, xm1
831 pmovsxwd xm2, xm2
832 pmovsxwd xm3, xm3
833
834 ; BLOCK 2
835 movq xm4, [srcq + 0*src_strideq]
836 movq xm5, [srcq + 1*src_strideq]
837 movq xm6, [srcq + 2*src_strideq]
838 movq xm7, [srcq + src_stride3q ]
839 lea srcq, [srcq + 4*src_strideq]
840
841 psubw xm4, [dstq + 0*dst_strideq]
842 psubw xm5, [dstq + 1*dst_strideq]
843 psubw xm6, [dstq + 2*dst_strideq]
844 psubw xm7, [dstq + dst_stride3q ]
845 lea dstq, [dstq + 4*dst_strideq]
846
847 pmovsxwd xm4, xm4
848 pmovsxwd xm5, xm5
849 pmovsxwd xm6, xm6
850 pmovsxwd xm7, xm7
851
852 vinserti128 m0, m0, xm4, 1
853 vinserti128 m1, m1, xm5, 1
854 vinserti128 m2, m2, xm6, 1
855 vinserti128 m3, m3, xm7, 1
856
857 HADAMARD_8X4_12BPC
858 paddd m8, m0
859 dec bdmaxd
860 jnz .12bpc_loop
861
862 HSUM 32, 32, 8, 0, eax
863 NORMALIZE4PT
864 RET
865
866 INIT_YMM avx2
867 cglobal satd_8x4_hbd, 5, 7, 12, src, src_stride, dst, dst_stride, bdmax, \
868 src_stride3, dst_stride3
869 lea src_stride3q, [3*src_strideq]
870 lea dst_stride3q, [3*dst_strideq]
871
872 cmp bdmaxd, (1 << 10) - 1
873 jne .12bpc
874
875 ; Load src rows
876 movu xm0, [srcq + 0*src_strideq]
877 movu xm1, [srcq + 1*src_strideq]
878 movu xm2, [srcq + 2*src_strideq]
879 movu xm3, [srcq + src_stride3q ]
880
881 ; src -= dst
882 psubw xm0, [dstq + 0*dst_strideq]
883 psubw xm1, [dstq + 1*dst_strideq]
884 psubw xm2, [dstq + 2*dst_strideq]
885 psubw xm3, [dstq + dst_stride3q ]
886
887 .10bpc_main:
888 HADAMARD_4X4_PACKED 16, 32
889
890 pabsw m0, m0
891 pabsw m1, m1
892 paddw m0, m1
893 HSUM 16, 32, 0, 1, eax
894 NORMALIZE4PT
895 RET
896 .12bpc:
897 RESET_MM_PERMUTATION
898
899 pmovzxwd m0, [srcq + 0*src_strideq]
900 pmovzxwd m1, [srcq + 1*src_strideq]
901 pmovzxwd m2, [srcq + 2*src_strideq]
902 pmovzxwd m3, [srcq + src_stride3q ]
903
904 pmovzxwd m4, [dstq + 0*dst_strideq]
905 pmovzxwd m5, [dstq + 1*dst_strideq]
906 pmovzxwd m6, [dstq + 2*dst_strideq]
907 pmovzxwd m7, [dstq + dst_stride3q ]
908
909 ; src -= dst
910 psubd m0, m4
911 psubd m1, m5
912 psubd m2, m6
913 psubd m3, m7
914
915 .12bpc_main:
916 HADAMARD_8X4_12BPC
917 HSUM 32, 32, 0, 1, eax
918 NORMALIZE4PT
919 RET
920
921 INIT_YMM avx2
922 cglobal satd_4x8_hbd, 5, 7, 12, src, src_stride, dst, dst_stride, bdmax, \
923 src_stride3, dst_stride3
924 lea src_stride3q, [3*src_strideq]
925 lea dst_stride3q, [3*dst_strideq]
926
927 cmp bdmaxd, (1 << 10) - 1
928 jne .12bpc
929
930 movq xm0, [srcq + 0*src_strideq]
931 movq xm1, [srcq + 1*src_strideq]
932 movq xm2, [srcq + 2*src_strideq]
933 movq xm3, [srcq + src_stride3q ]
934 lea srcq, [srcq + 4*src_strideq]
935 movq xm4, [srcq + 0*src_strideq]
936 movq xm5, [srcq + 1*src_strideq]
937 movq xm6, [srcq + 2*src_strideq]
938 movq xm7, [srcq + src_stride3q ]
939
940 ; This loads past the number of elements we are technically supposed
941 ; to read, however, this should still be safe, since at least one
942 ; valid element is in the memory address.
943 psubw xm0, [dstq + 0*dst_strideq]
944 psubw xm1, [dstq + 1*dst_strideq]
945 psubw xm2, [dstq + 2*dst_strideq]
946 psubw xm3, [dstq + dst_stride3q ]
947 lea dstq, [dstq + 4*dst_strideq]
948 psubw xm4, [dstq + 0*dst_strideq]
949 psubw xm5, [dstq + 1*dst_strideq]
950 psubw xm6, [dstq + 2*dst_strideq]
951 psubw xm7, [dstq + dst_stride3q ]
952
953 punpcklqdq xm0, xm0, xm4
954 punpcklqdq xm1, xm1, xm5
955 punpcklqdq xm2, xm2, xm6
956 punpcklqdq xm3, xm3, xm7
957
958 ; Jump to HADAMARD_4X4_PACKED in 8x4 satd, this saves us some binary size
959 ; by deduplicating the shared code.
960 jmp m(satd_8x4_hbd).10bpc_main
961 ; No return; we return in the other function.
962
963 .12bpc:
964 RESET_MM_PERMUTATION
965
966 pmovzxwd xm0, [srcq + 0*src_strideq]
967 pmovzxwd xm1, [srcq + 1*src_strideq]
968 pmovzxwd xm2, [srcq + 2*src_strideq]
969 pmovzxwd xm3, [srcq + src_stride3q ]
970 lea srcq, [srcq + 4*src_strideq]
971
972 pmovzxwd xm4, [dstq + 0*dst_strideq]
973 pmovzxwd xm5, [dstq + 1*dst_strideq]
974 pmovzxwd xm6, [dstq + 2*dst_strideq]
975 pmovzxwd xm7, [dstq + dst_stride3q ]
976 lea dstq, [dstq + 4*dst_strideq]
977
978 ; src -= dst
979 psubd xm0, xm4
980 psubd xm1, xm5
981 psubd xm2, xm6
982 psubd xm3, xm7
983
984 pmovzxwd xm4, [srcq + 0*src_strideq]
985 pmovzxwd xm5, [srcq + 1*src_strideq]
986 pmovzxwd xm6, [srcq + 2*src_strideq]
987 pmovzxwd xm7, [srcq + src_stride3q ]
988
989 pmovzxwd xm8, [dstq + 0*dst_strideq]
990 pmovzxwd xm9, [dstq + 1*dst_strideq]
991 pmovzxwd xm10, [dstq + 2*dst_strideq]
992 pmovzxwd xm11, [dstq + dst_stride3q ]
993
994 ; src -= dst (second block)
995 psubd xm4, xm8
996 psubd xm5, xm9
997 psubd xm6, xm10
998 psubd xm7, xm11
999
1000 vinserti128 m0, m0, xm4, 1
1001 vinserti128 m1, m1, xm5, 1
1002 vinserti128 m2, m2, xm6, 1
1003 vinserti128 m3, m3, xm7, 1
1004
1005 ; Jump to HADAMARD_4X4_PACKED in 8x4 satd, this saves us some binary size
1006 ; by deduplicating the shared code.
1007 jmp m(satd_8x4_hbd).12bpc_main
1008 ; No return; we return in the other function.
1009
1010
1011 ; <width>, <height>
1012 %macro SATD_NXM 2
1013
1014 INIT_YMM avx2
1015 cglobal satd_%1x%2_hbd, 5, 10, 16, src, src_stride, dst, dst_stride, bdmax, \
1016 src_stride3, dst_stride3, nsrc_stride4, ndst_stride4, rows
1017 lea nsrc_stride4q, [4*src_strideq]
1018 lea ndst_stride4q, [4*dst_strideq]
1019
1020 lea src_stride3q, [3*src_strideq]
1021 lea dst_stride3q, [3*dst_strideq]
1022
1023 neg nsrc_stride4q
1024 neg ndst_stride4q
1025
1026 pxor m9, m9
1027
1028 ; Height contains the number of rows.
1029 mov rowsd, %2/8
1030 .outer:
1031 mov bdmaxd, %1/8
1032
1033 ; Loop over blocks in same row.
1034 .loop:
1035 LOAD_DIFF_8X8
1036
1037 ; Fix up pointers and go to next block in same row.
1038 lea srcq, [srcq + nsrc_stride4q + 16]
1039 lea dstq, [dstq + ndst_stride4q + 16]
1040
1041 call m(satd_8x8_hbd_internal)
1042 dec bdmaxd
1043 jnz .loop
1044
1045 lea srcq, [srcq + 8*src_strideq - (%1*16)/8]
1046 lea dstq, [dstq + 8*dst_strideq - (%1*16)/8]
1047
1048 dec rowsd
1049 jnz .outer
1050
1051 HSUM 32, 32, 9, 0, eax
1052 NORMALIZE8PT
1053 RET
1054 %endmacro
1055
1056 %macro SATD_NX8 1
1057 INIT_YMM avx2
1058 cglobal satd_%1x8_hbd, 5, 9, 16, src, src_stride, dst, dst_stride, bdmax, \
1059 src_stride3, dst_stride3, nsrc_stride4, ndst_stride4
1060 lea nsrc_stride4q, [4*src_strideq]
1061 lea ndst_stride4q, [4*dst_strideq]
1062
1063 lea src_stride3q, [3*src_strideq]
1064 lea dst_stride3q, [3*dst_strideq]
1065
1066 neg nsrc_stride4q
1067 neg ndst_stride4q
1068
1069 pxor m9, m9
1070 mov bdmaxd, %1/8
1071
1072 .loop:
1073 LOAD_DIFF_8X8
1074
1075 lea srcq, [srcq + nsrc_stride4q + 16]
1076 lea dstq, [dstq + ndst_stride4q + 16]
1077
1078 call m(satd_8x8_hbd_internal)
1079 dec bdmaxd
1080 jnz .loop
1081
1082 HSUM 32, 32, 9, 0, eax
1083 NORMALIZE8PT
1084 RET
1085 %endmacro
1086
1087 %macro SATD_8XM 1
1088 INIT_YMM avx2
1089 cglobal satd_8x%1_hbd, 5, 7, 16, src, src_stride, dst, dst_stride, bdmax, \
1090 src_stride3, dst_stride3
1091 lea src_stride3q, [3*src_strideq]
1092 lea dst_stride3q, [3*dst_strideq]
1093
1094 pxor m9, m9
1095 mov bdmaxd, %1/8
1096
1097 .loop:
1098 LOAD_DIFF_8X8
1099
1100 lea srcq, [srcq + 4*src_strideq]
1101 lea dstq, [dstq + 4*dst_strideq]
1102
1103 call m(satd_8x8_hbd_internal)
1104 dec bdmaxd
1105 jnz .loop
1106
1107 HSUM 32, 32, 9, 0, eax
1108 NORMALIZE8PT
1109 RET
1110 %endmacro
1111
1112 SATD_NXM 16, 16
1113 SATD_NXM 32, 32
1114 SATD_NXM 64, 64
1115 SATD_NXM 128, 128
1116 SATD_NXM 16, 32
1117 SATD_NXM 16, 64
1118 SATD_NXM 32, 16
1119 SATD_NXM 32, 64
1120 SATD_NXM 64, 16
1121 SATD_NXM 64, 32
1122 SATD_NXM 64, 128
1123 SATD_NXM 128, 64
1124 SATD_NX8 16
1125 SATD_NX8 32
1126 SATD_8XM 16
1127 SATD_8XM 32
1128
1129 %endif ; ARCH_X86_64