| 1 |
; Copyright (c) 2022, The rav1e contributors. All rights reserved |
| 2 |
; |
| 3 |
; This source code is subject to the terms of the BSD 2 Clause License and |
| 4 |
; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License |
| 5 |
; was not distributed with this source code in the LICENSE file, you can |
| 6 |
; obtain it at www.aomedia.org/license/software. If the Alliance for Open |
| 7 |
; Media Patent License 1.0 was not distributed with this source code in the |
| 8 |
; PATENTS file, you can obtain it at www.aomedia.org/license/patent. |
| 9 |
|
| 10 |
%include "config.asm" |
| 11 |
%include "src/asm/x86/x86inc.asm" |
| 12 |
|
| 13 |
%define m(x) mangle(private_prefix %+ _ %+ x %+ SUFFIX) |
| 14 |
|
| 15 |
%if ARCH_X86_64 |
| 16 |
|
| 17 |
SECTION_RODATA 32 |
| 18 |
|
| 19 |
align 32 |
| 20 |
pw_1x16: times 16 dw 1 |
| 21 |
|
| 22 |
SECTION .text |
| 23 |
|
| 24 |
%macro NORMALIZE4PT 0 |
| 25 |
add eax, 2 |
| 26 |
shr eax, 2 |
| 27 |
%endmacro |
| 28 |
|
| 29 |
%macro NORMALIZE8PT 0 |
| 30 |
add eax, 4 |
| 31 |
shr eax, 3 |
| 32 |
%endmacro |
| 33 |
|
| 34 |
; Add and subtract registers |
| 35 |
; |
| 36 |
; Takes m0 and m1 as both input and output. |
| 37 |
; Requires m2 as a free register. |
| 38 |
; |
| 39 |
; If we start with this permutation: |
| 40 |
; |
| 41 |
; m0 0 1 2 3 4 5 6 7 |
| 42 |
; m1 8 9 10 11 12 13 14 15 |
| 43 |
; |
| 44 |
; Then the output will be as such: |
| 45 |
; |
| 46 |
; m0 [0+8][1+9][2+10][3+11] [4+12][5+13][6+14][7+15] |
| 47 |
; m1 [0-8][1-9][2-10][3-11] [4-12][5-13][6-14][7-15] |
| 48 |
%macro BUTTERFLY 3 |
| 49 |
%define BIT_PRECISION %1 |
| 50 |
%define VEC_SIZE %2 |
| 51 |
; use alternate registers 3,4,5 |
| 52 |
%define USE_ALT %3 |
| 53 |
|
| 54 |
%if USE_ALT == 1 |
| 55 |
SWAP 3, 0 |
| 56 |
SWAP 4, 1 |
| 57 |
SWAP 5, 2 |
| 58 |
%endif |
| 59 |
|
| 60 |
%if VEC_SIZE == 32 |
| 61 |
%define V ym |
| 62 |
%elif VEC_SIZE == 16 |
| 63 |
%define V xm |
| 64 |
%endif |
| 65 |
|
| 66 |
; Use m2 as a temporary register, then swap |
| 67 |
; so that m0 and m1 contain the output. |
| 68 |
%if BIT_PRECISION == 16 |
| 69 |
paddw V%+ 2, V%+ 0, V%+ 1 |
| 70 |
psubw V%+ 0, V%+ 1 |
| 71 |
%elif BIT_PRECISION == 32 |
| 72 |
paddd ym2, ym0, ym1 |
| 73 |
psubd ym0, ym1 |
| 74 |
%else |
| 75 |
%error Incorrect precision specified (16 or 32 expected) |
| 76 |
%endif |
| 77 |
|
| 78 |
SWAP 2, 1, 0 |
| 79 |
|
| 80 |
%if USE_ALT == 1 |
| 81 |
SWAP 3, 0 |
| 82 |
SWAP 4, 1 |
| 83 |
SWAP 5, 2 |
| 84 |
%endif |
| 85 |
%endmacro |
| 86 |
|
| 87 |
; Interleave packed rows together (in m0 and m1). |
| 88 |
; m2 should contain a free register. |
| 89 |
; |
| 90 |
; Macro argument takes size in bits of each element (where one |
| 91 |
; element is the difference between two original source pixels). |
| 92 |
; |
| 93 |
; If we start with this permutation: |
| 94 |
; |
| 95 |
; m0 0 1 2 3 4 5 6 7 |
| 96 |
; m1 8 9 10 11 12 13 14 15 |
| 97 |
; |
| 98 |
; Then, after INTERLEAVE, this will be the permutation: |
| 99 |
; |
| 100 |
; m0 0 8 1 9 2 10 3 11 |
| 101 |
; m1 4 12 5 13 6 14 7 15 |
| 102 |
%macro INTERLEAVE 3 |
| 103 |
%define BIT_PRECISION %1 |
| 104 |
%define VEC_SIZE %2 |
| 105 |
%define USE_ALT %3 |
| 106 |
|
| 107 |
%if USE_ALT == 1 |
| 108 |
SWAP 3, 0 |
| 109 |
SWAP 4, 1 |
| 110 |
SWAP 5, 2 |
| 111 |
%endif |
| 112 |
|
| 113 |
%if VEC_SIZE == 16 |
| 114 |
%define V xm |
| 115 |
%elif VEC_SIZE == 32 |
| 116 |
%define V ym |
| 117 |
%else |
| 118 |
%error Invalid vector size (expected 16 or 32) |
| 119 |
%endif |
| 120 |
|
| 121 |
%if BIT_PRECISION == 16 |
| 122 |
punpcklwd V%+ 2, V%+ 0, V%+ 1 |
| 123 |
punpckhwd V%+ 0, V%+ 1 |
| 124 |
SWAP 2, 1, 0 |
| 125 |
%elif BIT_PRECISION == 32 |
| 126 |
punpckldq ym2, ym0, ym1 |
| 127 |
punpckhdq ym0, ym1 |
| 128 |
; AVX2 shuffles operate over 128-bit halves of the full ymm register |
| 129 |
; in parallel, so these shuffles are required to fix up the permutation. |
| 130 |
vperm2i128 ym1, ym2, ym0, 0x20 |
| 131 |
vperm2i128 ym0, ym2, ym0, 0x31 |
| 132 |
SWAP 0, 1 |
| 133 |
%else |
| 134 |
%error Incorrect precision specified (16 or 32 expected) |
| 135 |
%endif |
| 136 |
|
| 137 |
%if USE_ALT == 1 |
| 138 |
SWAP 3, 0 |
| 139 |
SWAP 4, 1 |
| 140 |
SWAP 5, 2 |
| 141 |
%endif |
| 142 |
%endmacro |
| 143 |
|
| 144 |
; Interleave pairs of 2 elements (in m0 and m1) |
| 145 |
; m2 should contain a free register. |
| 146 |
%macro INTERLEAVE_PAIRS 3 |
| 147 |
%define BIT_PRECISION %1 |
| 148 |
%define VEC_SIZE %2 |
| 149 |
%define USE_ALT %3 |
| 150 |
|
| 151 |
%if USE_ALT == 1 |
| 152 |
SWAP 3, 0 |
| 153 |
SWAP 4, 1 |
| 154 |
SWAP 5, 2 |
| 155 |
%endif |
| 156 |
|
| 157 |
%if VEC_SIZE == 16 |
| 158 |
%define V xm |
| 159 |
%elif VEC_SIZE == 32 |
| 160 |
%define V ym |
| 161 |
%else |
| 162 |
%error Invalid vector size (expected 16 or 32) |
| 163 |
%endif |
| 164 |
|
| 165 |
%if BIT_PRECISION == 16 |
| 166 |
punpckldq V%+ 2, V%+ 0, V%+ 1 |
| 167 |
punpckhdq V%+ 0, V%+ 1 |
| 168 |
%elif BIT_PRECISION == 32 |
| 169 |
punpcklqdq ym2, ym0, ym1 |
| 170 |
punpckhqdq ym0, ym1 |
| 171 |
%else |
| 172 |
%error Incorrect precision specified (16 or 32 expected) |
| 173 |
%endif |
| 174 |
SWAP 2, 1, 0 |
| 175 |
|
| 176 |
%if USE_ALT == 1 |
| 177 |
SWAP 3, 0 |
| 178 |
SWAP 4, 1 |
| 179 |
SWAP 5, 2 |
| 180 |
%endif |
| 181 |
%endmacro |
| 182 |
|
| 183 |
%macro HADAMARD_4X4_PACKED 2 |
| 184 |
%define BIT_PRECISION %1 |
| 185 |
; Register size to use (in bytes) |
| 186 |
%define VEC_SIZE %2 |
| 187 |
|
| 188 |
%if VEC_SIZE == 16 |
| 189 |
%define V xm |
| 190 |
%elif VEC_SIZE == 32 |
| 191 |
%define V ym |
| 192 |
%else |
| 193 |
%error Invalid vector size (expected 16 or 32) |
| 194 |
%endif |
| 195 |
|
| 196 |
; Starting registers: |
| 197 |
|
| 198 |
; m0 0 1 2 3 |
| 199 |
; m1 4 5 6 7 |
| 200 |
; m2 8 9 10 11 |
| 201 |
; m3 12 13 14 15 |
| 202 |
|
| 203 |
; Where each number represents an index of the |
| 204 |
; original block of differences. |
| 205 |
|
| 206 |
; Pack rows 0,2 and 1,3 into m0 and m1 |
| 207 |
%if BIT_PRECISION == 16 |
| 208 |
%if VEC_SIZE == 16 |
| 209 |
; In this case, each row only has 64 bits, so we use |
| 210 |
; punpcklqdq only. The high 64 bits are always 0. |
| 211 |
punpcklqdq xm0, xm2 |
| 212 |
punpcklqdq xm1, xm3 |
| 213 |
%elif VEC_SIZE == 32 |
| 214 |
; The upper 128 bits of all input registers are zeroed |
| 215 |
punpcklqdq m4, m0, m2 |
| 216 |
punpcklqdq m5, m1, m3 |
| 217 |
punpckhqdq m0, m0, m2 |
| 218 |
punpckhqdq m1, m1, m3 |
| 219 |
vinserti128 m0, m4, xm0, 1 |
| 220 |
vinserti128 m1, m5, xm1, 1 |
| 221 |
%endif |
| 222 |
%elif BIT_PRECISION == 32 |
| 223 |
vinserti128 ym0, ym0, xm2, 1 |
| 224 |
vinserti128 ym1, ym1, xm3, 1 |
| 225 |
%else |
| 226 |
%error Invalid bit precision (expected 16 or 32) |
| 227 |
%endif |
| 228 |
|
| 229 |
; Now that we've packed rows 0-2 and 1-3 together, |
| 230 |
; this is our permutation: |
| 231 |
|
| 232 |
; m0 0 1 2 3 8 9 10 11 |
| 233 |
; m1 4 5 6 7 12 13 14 15 |
| 234 |
|
| 235 |
; For a 8x4 transform (with 16-bit coefficients), this pattern is |
| 236 |
; extended for each 128-bit half but for the second block, and thus |
| 237 |
; all comments also apply to the upper 128-bits for the 8x4 transform. |
| 238 |
|
| 239 |
BUTTERFLY %1, %2, 0 |
| 240 |
|
| 241 |
; m0 [0+4][1+5][2+6][3+7] [8+12][9+13][10+14][11+15] |
| 242 |
; m1 [0-4][1-5][2-6][3-7] [8-12][9-13][10-14][11-15] |
| 243 |
|
| 244 |
INTERLEAVE %1, %2, 0 |
| 245 |
|
| 246 |
; m0 [ 0+4][ 0-4][ 1+5][ 1-5] [2 + 6][2 - 6][3 + 7][3 - 7] |
| 247 |
; m1 [8+12][8-12][9+13][9-13] [10+14][10-14][11+15][11-15] |
| 248 |
|
| 249 |
BUTTERFLY %1, %2, 0 |
| 250 |
|
| 251 |
; m0 [0+4+8+12][0-4+8-12][1+5+9+13][1-5+9-13] [2+6+10+14][2-6+10-14][3+7+11+15][3-7+11-15] |
| 252 |
; m1 [0+4-8-12][0-4-8+12][1+5-9-13][1-5-9+13] [2+6-10-14][2-6-10+14][3+7-11-15][3-7-11+15] |
| 253 |
|
| 254 |
; for one row: |
| 255 |
; [0+1+2+3][0-1+2-3][0+1-2-3][0-1-2+3] |
| 256 |
; For the vertical transform, these are packed into a new column. |
| 257 |
|
| 258 |
INTERLEAVE_PAIRS %1, %2, 0 |
| 259 |
|
| 260 |
; p0 p1 p2 p3 |
| 261 |
; m0 [0+4+ 8+12][0-4+ 8-12][0+4- 8-12][0-4- 8+12] [1+5+ 9+13][1-5+ 9-13][1+5- 9-13][1-5- 9+13] |
| 262 |
; m1 [2+6+10+14][2-6+10-14][2+6-10-14][2-6-10+14] [3+7+11+15][3-7+11-15][3+7-11-15][3-7-11+15] |
| 263 |
|
| 264 |
; According to this grid: |
| 265 |
|
| 266 |
; p0 q0 r0 s0 |
| 267 |
; p1 q1 r1 s1 |
| 268 |
; p2 q2 r2 s2 |
| 269 |
; p3 q3 r3 s3 |
| 270 |
|
| 271 |
; Horizontal transform; since the output is transposed from the original order, |
| 272 |
; we can do the same steps as the vertical transform and the result will be the same. |
| 273 |
BUTTERFLY %1, %2, 0 |
| 274 |
INTERLEAVE %1, %2, 0 |
| 275 |
BUTTERFLY %1, %2, 0 |
| 276 |
|
| 277 |
; Finished horizontal transform except for the last step (interleaving pairs), |
| 278 |
; which we skip, because after this we add up the absolute value of the |
| 279 |
; coefficients, which is a commutative operation (order does not matter). |
| 280 |
%endmacro |
| 281 |
|
| 282 |
; Horizontal sum of mm register |
| 283 |
; |
| 284 |
; Inputs: |
| 285 |
; %1 = Element size in bits (16 or 32) |
| 286 |
; %2 = Size of input register in bytes (16 or 32) |
| 287 |
; You can e.g. pass 16 for this argument if you |
| 288 |
; only want to sum up the bottom 128-bits of a |
| 289 |
; ymm register. |
| 290 |
; %3 = Input register number |
| 291 |
; %4 = Temporary register number |
| 292 |
; %5 = Output register (e.g., eax) |
| 293 |
%macro HSUM 5 |
| 294 |
%define E_SIZE %1 |
| 295 |
%define REG_SIZE %2 |
| 296 |
%define INPUT %3 |
| 297 |
%define TMP %4 |
| 298 |
%define OUTPUT %5 |
| 299 |
|
| 300 |
%if REG_SIZE == 16 |
| 301 |
%define V xm |
| 302 |
%elif REG_SIZE == 32 |
| 303 |
%define V ym |
| 304 |
%else |
| 305 |
%error Invalid register size (expected 16 or 32) |
| 306 |
%endif |
| 307 |
|
| 308 |
%if E_SIZE == 16 |
| 309 |
; Add adjacent pairs of 16-bit elements to produce 32-bit results, |
| 310 |
; then proceed with 32-bit sum |
| 311 |
pmaddwd V%+INPUT, [pw_1x16] |
| 312 |
%endif |
| 313 |
|
| 314 |
%if mmsize == 32 && REG_SIZE == 32 |
| 315 |
; Add upper half of ymm to xmm |
| 316 |
vextracti128 xm%+TMP, ym%+INPUT, 1 |
| 317 |
paddd xm%+INPUT, xm%+TMP |
| 318 |
%endif |
| 319 |
|
| 320 |
; Reduce 32-bit results |
| 321 |
pshufd xm%+TMP, xm%+INPUT, q2323 |
| 322 |
paddd xm%+INPUT, xm%+TMP |
| 323 |
pshufd xm%+TMP, xm%+INPUT, q1111 |
| 324 |
paddd xm%+INPUT, xm%+TMP |
| 325 |
movd OUTPUT, xm%+INPUT |
| 326 |
%endmacro |
| 327 |
|
| 328 |
; given m0-7, do butterfly as follows: |
| 329 |
; (m0, m1) = butterfly(m0, m1) |
| 330 |
; (m2, m3) = butterfly(m2, m3) |
| 331 |
; (m4, m5) = butterfly(m4, m5) |
| 332 |
; (m6, m7) = butterfly(m6, m7) |
| 333 |
%macro BUTTERFLY_8X8 0 |
| 334 |
; m8 is free |
| 335 |
paddd m8, m0, m1 |
| 336 |
psubd m0, m1 |
| 337 |
SWAP 8, 1, 0 |
| 338 |
|
| 339 |
; m8 is free |
| 340 |
paddd m8, m2, m3 |
| 341 |
psubd m2, m3 |
| 342 |
SWAP 8, 3, 2 |
| 343 |
|
| 344 |
paddd m8, m4, m5 |
| 345 |
psubd m4, m5 |
| 346 |
SWAP 8, 5, 4 |
| 347 |
|
| 348 |
paddd m8, m6, m7 |
| 349 |
psubd m6, m7 |
| 350 |
SWAP 8, 7, 6 |
| 351 |
%endmacro |
| 352 |
|
| 353 |
%macro HADAMARD_8X8_VERTICAL 0 |
| 354 |
BUTTERFLY_8X8 |
| 355 |
; m0-7 contain a0-7 |
| 356 |
|
| 357 |
SWAP 2, 1 |
| 358 |
SWAP 6, 5 |
| 359 |
|
| 360 |
BUTTERFLY_8X8 |
| 361 |
|
| 362 |
SWAP 1, 4 |
| 363 |
SWAP 3, 6 |
| 364 |
|
| 365 |
BUTTERFLY_8X8 |
| 366 |
|
| 367 |
SWAP 2, 1 |
| 368 |
SWAP 2, 4 |
| 369 |
SWAP 3, 6 |
| 370 |
SWAP 5, 6 |
| 371 |
%endmacro |
| 372 |
|
| 373 |
; Transpose rows m0-7. |
| 374 |
; Output is also contained in m0-7. |
| 375 |
; |
| 376 |
; Uses m8, m10-15 as temporary registers (i.e. m9 is left unchanged.) |
| 377 |
%macro TRANSPOSE8X8D 0 |
| 378 |
SWAP 9, 0 |
| 379 |
SWAP 10, 1 |
| 380 |
SWAP 11, 2 |
| 381 |
SWAP 12, 3 |
| 382 |
SWAP 13, 4 |
| 383 |
SWAP 14, 5 |
| 384 |
SWAP 15, 6 |
| 385 |
SWAP 2, 7 |
| 386 |
|
| 387 |
punpckldq m6, m9, m10 |
| 388 |
punpckldq m1, m11, m12 |
| 389 |
punpckhdq m8, m9, m10 |
| 390 |
punpckldq m4, m13, m14 |
| 391 |
punpckldq m9, m15, m2 |
| 392 |
vshufps m3, m6, m1, 0x4e |
| 393 |
vpblendd m10, m6, m3, 0xcc |
| 394 |
vshufps m6, m4, m9, 0x4e |
| 395 |
punpckhdq m7, m11, m12 |
| 396 |
vpblendd m11, m4, m6, 0xcc |
| 397 |
vpblendd m12, m3, m1, 0xcc |
| 398 |
vperm2i128 m3, m10, m11, 0x20 |
| 399 |
punpckhdq m5, m13, m14 |
| 400 |
vpblendd m13, m6, m9, 0xcc |
| 401 |
punpckhdq m4, m15, m2 |
| 402 |
vperm2i128 m2, m12, m13, 0x20 |
| 403 |
vshufps m14, m8, m7, 0x4e |
| 404 |
vpblendd m15, m14, m7, 0xcc |
| 405 |
vshufps m7, m5, m4, 0x4e |
| 406 |
vpblendd m8, m8, m14, 0xcc |
| 407 |
vpblendd m5, m5, m7, 0xcc |
| 408 |
vperm2i128 m6, m8, m5, 0x20 |
| 409 |
vpblendd m4, m7, m4, 0xcc |
| 410 |
vperm2i128 m7, m15, m4, 0x20 |
| 411 |
vperm2i128 m1, m10, m11, 0x31 |
| 412 |
vperm2i128 m9, m12, m13, 0x31 |
| 413 |
vperm2i128 m5, m8, m5, 0x31 |
| 414 |
vperm2i128 m4, m15, m4, 0x31 |
| 415 |
|
| 416 |
SWAP 0, 9 |
| 417 |
|
| 418 |
; Output order is as follows: |
| 419 |
; 3 2 6 7 1 0 5 4 |
| 420 |
|
| 421 |
; sort rows |
| 422 |
SWAP 3, 0 |
| 423 |
; 0 2 6 7 1 3 5 4 |
| 424 |
SWAP 1, 2 |
| 425 |
; 0 1 6 7 2 3 5 4 |
| 426 |
SWAP 6, 2 |
| 427 |
; 0 1 2 7 6 3 5 4 |
| 428 |
SWAP 7, 3 |
| 429 |
; 0 1 2 3 6 7 5 4 |
| 430 |
SWAP 6, 4 |
| 431 |
; 0 1 2 3 4 7 5 6 |
| 432 |
SWAP 7, 5 |
| 433 |
; 0 1 2 3 4 5 7 6 |
| 434 |
SWAP 6, 7 |
| 435 |
; 0 1 2 3 4 5 6 7 |
| 436 |
%endmacro |
| 437 |
|
| 438 |
; m0-7 as input; add coefficients to ymm9. |
| 439 |
INIT_YMM avx2 |
| 440 |
cglobal satd_8x8_hbd_internal, 0, 0, 0, src, src_stride, dst, dst_stride, bdmax, \ |
| 441 |
src_stride3, dst_stride3 |
| 442 |
HADAMARD_8X8_VERTICAL |
| 443 |
|
| 444 |
TRANSPOSE8X8D |
| 445 |
|
| 446 |
HADAMARD_8X8_VERTICAL |
| 447 |
|
| 448 |
REPX {pabsd x, x}, m0, m1, m2, m3, m4, m5, m6, m7 |
| 449 |
|
| 450 |
; Add m0-7 |
| 451 |
paddd m0, m4 |
| 452 |
paddd m1, m5 |
| 453 |
paddd m2, m6 |
| 454 |
paddd m3, m7 |
| 455 |
|
| 456 |
paddd m0, m2 |
| 457 |
paddd m1, m3 |
| 458 |
|
| 459 |
paddd m0, m1 |
| 460 |
paddd ymm9, m0 |
| 461 |
ret |
| 462 |
|
| 463 |
%macro LOAD_DIFF_8X8 0 |
| 464 |
movu xm0, [srcq + 0*src_strideq] |
| 465 |
movu xm1, [srcq + 1*src_strideq] |
| 466 |
movu xm2, [srcq + 2*src_strideq] |
| 467 |
movu xm3, [srcq + src_stride3q ] |
| 468 |
lea srcq, [srcq + 4*src_strideq] |
| 469 |
movu xm4, [srcq + 0*src_strideq] |
| 470 |
movu xm5, [srcq + 1*src_strideq] |
| 471 |
movu xm6, [srcq + 2*src_strideq] |
| 472 |
movu xm7, [srcq + src_stride3q ] |
| 473 |
|
| 474 |
psubw xm0, [dstq + 0*dst_strideq] |
| 475 |
psubw xm1, [dstq + 1*dst_strideq] |
| 476 |
psubw xm2, [dstq + 2*dst_strideq] |
| 477 |
psubw xm3, [dstq + dst_stride3q ] |
| 478 |
lea dstq, [dstq + 4*dst_strideq] |
| 479 |
psubw xm4, [dstq + 0*dst_strideq] |
| 480 |
psubw xm5, [dstq + 1*dst_strideq] |
| 481 |
psubw xm6, [dstq + 2*dst_strideq] |
| 482 |
psubw xm7, [dstq + dst_stride3q ] |
| 483 |
|
| 484 |
pmovsxwd m0, xm0 |
| 485 |
pmovsxwd m1, xm1 |
| 486 |
pmovsxwd m2, xm2 |
| 487 |
pmovsxwd m3, xm3 |
| 488 |
pmovsxwd m4, xm4 |
| 489 |
pmovsxwd m5, xm5 |
| 490 |
pmovsxwd m6, xm6 |
| 491 |
pmovsxwd m7, xm7 |
| 492 |
%endmacro |
| 493 |
|
| 494 |
INIT_YMM avx2 |
| 495 |
cglobal satd_8x8_hbd, 5, 7, 16, src, src_stride, dst, dst_stride, bdmax, \ |
| 496 |
src_stride3, dst_stride3 |
| 497 |
lea src_stride3q, [3*src_strideq] |
| 498 |
lea dst_stride3q, [3*dst_strideq] |
| 499 |
|
| 500 |
LOAD_DIFF_8X8 |
| 501 |
|
| 502 |
; m0-7 contain rows of 8x8 block to transform |
| 503 |
; with 32-bit coefficients |
| 504 |
|
| 505 |
HADAMARD_8X8_VERTICAL |
| 506 |
|
| 507 |
TRANSPOSE8X8D |
| 508 |
|
| 509 |
HADAMARD_8X8_VERTICAL |
| 510 |
|
| 511 |
REPX {pabsd x, x}, m0, m1, m2, m3, m4, m5, m6, m7 |
| 512 |
|
| 513 |
; Add m0-7 |
| 514 |
paddd m0, m4 |
| 515 |
paddd m1, m5 |
| 516 |
paddd m2, m6 |
| 517 |
paddd m3, m7 |
| 518 |
|
| 519 |
paddd m0, m2 |
| 520 |
paddd m1, m3 |
| 521 |
|
| 522 |
paddd m0, m1 |
| 523 |
|
| 524 |
HSUM 32, 32, 0, 1, eax |
| 525 |
NORMALIZE8PT |
| 526 |
RET |
| 527 |
|
| 528 |
INIT_YMM avx2 |
| 529 |
cglobal satd_4x4_hbd, 5, 7, 8, src, src_stride, dst, dst_stride, bdmax, \ |
| 530 |
src_stride3, dst_stride3 |
| 531 |
lea src_stride3q, [3*src_strideq] |
| 532 |
lea dst_stride3q, [3*dst_strideq] |
| 533 |
|
| 534 |
cmp bdmaxd, (1 << 10) - 1 |
| 535 |
jne .12bpc |
| 536 |
|
| 537 |
; Load src rows |
| 538 |
movq xm0, [srcq + 0*src_strideq] |
| 539 |
movq xm1, [srcq + 1*src_strideq] |
| 540 |
movq xm2, [srcq + 2*src_strideq] |
| 541 |
movq xm3, [srcq + src_stride3q ] |
| 542 |
|
| 543 |
; src -= dst |
| 544 |
psubw xm0, [dstq + 0*dst_strideq] |
| 545 |
psubw xm1, [dstq + 1*dst_strideq] |
| 546 |
psubw xm2, [dstq + 2*dst_strideq] |
| 547 |
psubw xm3, [dstq + dst_stride3q ] |
| 548 |
|
| 549 |
HADAMARD_4X4_PACKED 16, 16 |
| 550 |
|
| 551 |
; Sum up absolute value of transform coefficients |
| 552 |
pabsw xm0, xm0 |
| 553 |
pabsw xm1, xm1 |
| 554 |
paddw xm0, xm1 |
| 555 |
HSUM 16, 16, 0, 1, eax |
| 556 |
NORMALIZE4PT |
| 557 |
RET |
| 558 |
.12bpc: |
| 559 |
; this gives a nicer disassembly |
| 560 |
RESET_MM_PERMUTATION |
| 561 |
|
| 562 |
; Load src rows |
| 563 |
pmovzxwd xm0, [srcq + 0*src_strideq] |
| 564 |
pmovzxwd xm1, [srcq + 1*src_strideq] |
| 565 |
pmovzxwd xm2, [srcq + 2*src_strideq] |
| 566 |
pmovzxwd xm3, [srcq + src_stride3q ] |
| 567 |
|
| 568 |
; Load dst rows |
| 569 |
pmovzxwd xm4, [dstq + 0*dst_strideq] |
| 570 |
pmovzxwd xm5, [dstq + 1*dst_strideq] |
| 571 |
pmovzxwd xm6, [dstq + 2*dst_strideq] |
| 572 |
pmovzxwd xm7, [dstq + dst_stride3q ] |
| 573 |
|
| 574 |
; src -= dst |
| 575 |
psubd xm0, xm4 |
| 576 |
psubd xm1, xm5 |
| 577 |
psubd xm2, xm6 |
| 578 |
psubd xm3, xm7 |
| 579 |
|
| 580 |
HADAMARD_4X4_PACKED 32, 32 |
| 581 |
|
| 582 |
pabsd m0, m0 |
| 583 |
pabsd m1, m1 |
| 584 |
paddd m0, m1 |
| 585 |
HSUM 32, 32, 0, 1, eax |
| 586 |
NORMALIZE4PT |
| 587 |
RET |
| 588 |
|
| 589 |
; 32-bit input rows are in m0-3; result is in m0. |
| 590 |
; Uses m0-5 as temporary registers. |
| 591 |
%macro HADAMARD_8X4_12BPC 0 |
| 592 |
vperm2i128 m4, m0, m2, 0x31 |
| 593 |
vperm2i128 m5, m1, m3, 0x31 |
| 594 |
vinserti128 m0, m0, xm2, 1 |
| 595 |
vinserti128 m1, m1, xm3, 1 |
| 596 |
|
| 597 |
; Swap so m3,m4 are used as inputs. |
| 598 |
SWAP 3, 4, 5 |
| 599 |
|
| 600 |
; instead of using HADAMARD_4X4_PACKED twice, we interleave |
| 601 |
; 2 transforms operating over different registers for more |
| 602 |
; opportunity for instruction level parallelism. |
| 603 |
|
| 604 |
BUTTERFLY 32, 32, 0 |
| 605 |
BUTTERFLY 32, 32, 1 |
| 606 |
INTERLEAVE 32, 32, 0 |
| 607 |
INTERLEAVE 32, 32, 1 |
| 608 |
BUTTERFLY 32, 32, 0 |
| 609 |
BUTTERFLY 32, 32, 1 |
| 610 |
INTERLEAVE_PAIRS 32, 32, 0 |
| 611 |
INTERLEAVE_PAIRS 32, 32, 1 |
| 612 |
BUTTERFLY 32, 32, 0 |
| 613 |
BUTTERFLY 32, 32, 1 |
| 614 |
INTERLEAVE 32, 32, 0 |
| 615 |
INTERLEAVE 32, 32, 1 |
| 616 |
BUTTERFLY 32, 32, 0 |
| 617 |
BUTTERFLY 32, 32, 1 |
| 618 |
|
| 619 |
pabsd m0, m0 |
| 620 |
pabsd m1, m1 |
| 621 |
pabsd m3, m3 |
| 622 |
pabsd m4, m4 |
| 623 |
|
| 624 |
paddd m0, m1 |
| 625 |
paddd m3, m4 |
| 626 |
paddd m0, m3 |
| 627 |
%endmacro |
| 628 |
|
| 629 |
INIT_YMM avx2 |
| 630 |
cglobal satd_16x4_hbd, 5, 7, 12, src, src_stride, dst, dst_stride, bdmax, \ |
| 631 |
src_stride3, dst_stride3 |
| 632 |
lea src_stride3q, [3*src_strideq] |
| 633 |
lea dst_stride3q, [3*dst_strideq] |
| 634 |
|
| 635 |
cmp bdmaxd, (1 << 10) - 1 |
| 636 |
jne .12bpc |
| 637 |
|
| 638 |
; Load src rows |
| 639 |
movu m0, [srcq + 0*src_strideq] |
| 640 |
movu m1, [srcq + 1*src_strideq] |
| 641 |
movu m2, [srcq + 2*src_strideq] |
| 642 |
movu m3, [srcq + src_stride3q ] |
| 643 |
|
| 644 |
; src -= dst |
| 645 |
psubw m0, [dstq + 0*dst_strideq] |
| 646 |
psubw m1, [dstq + 1*dst_strideq] |
| 647 |
psubw m2, [dstq + 2*dst_strideq] |
| 648 |
psubw m3, [dstq + dst_stride3q ] |
| 649 |
|
| 650 |
.10bpc_main: |
| 651 |
|
| 652 |
; Original permutation |
| 653 |
; m0 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 |
| 654 |
; m1 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 |
| 655 |
; m2 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 |
| 656 |
; m3 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 |
| 657 |
|
| 658 |
; Two registers perform 2 4x4 transforms in parallel |
| 659 |
|
| 660 |
punpcklqdq m4, m0, m2 |
| 661 |
punpcklqdq m5, m1, m3 |
| 662 |
|
| 663 |
punpckhqdq m0, m0, m2 |
| 664 |
punpckhqdq m1, m1, m3 |
| 665 |
|
| 666 |
SWAP 4, 3 |
| 667 |
SWAP 5, 4 |
| 668 |
|
| 669 |
; New permutation |
| 670 |
; m0 0 1 2 3 32 33 34 35 8 9 10 11 40 41 42 43 |
| 671 |
; m1 16 17 18 19 48 49 50 51 24 25 26 27 56 57 58 59 |
| 672 |
; m3 4 5 6 7 36 37 38 39 12 13 14 15 44 45 46 47 |
| 673 |
; m4 20 21 22 23 52 53 54 55 28 29 30 31 60 61 62 63 |
| 674 |
|
| 675 |
BUTTERFLY 16, 32, 0 |
| 676 |
BUTTERFLY 16, 32, 1 |
| 677 |
INTERLEAVE 16, 32, 0 |
| 678 |
INTERLEAVE 16, 32, 1 |
| 679 |
BUTTERFLY 16, 32, 0 |
| 680 |
BUTTERFLY 16, 32, 1 |
| 681 |
INTERLEAVE_PAIRS 16, 32, 0 |
| 682 |
INTERLEAVE_PAIRS 16, 32, 1 |
| 683 |
BUTTERFLY 16, 32, 0 |
| 684 |
BUTTERFLY 16, 32, 1 |
| 685 |
INTERLEAVE 16, 32, 0 |
| 686 |
INTERLEAVE 16, 32, 1 |
| 687 |
BUTTERFLY 16, 32, 0 |
| 688 |
BUTTERFLY 16, 32, 1 |
| 689 |
|
| 690 |
pabsw m0, m0 |
| 691 |
pabsw m1, m1 |
| 692 |
pabsw m3, m3 |
| 693 |
pabsw m4, m4 |
| 694 |
|
| 695 |
paddw m0, m1 |
| 696 |
paddw m3, m4 |
| 697 |
paddw m0, m3 |
| 698 |
|
| 699 |
HSUM 16, 32, 0, 1, eax |
| 700 |
NORMALIZE4PT |
| 701 |
RET |
| 702 |
.12bpc: |
| 703 |
RESET_MM_PERMUTATION |
| 704 |
|
| 705 |
mov bdmaxd, 2 |
| 706 |
pxor m6, m6 |
| 707 |
.12bpc_loop: |
| 708 |
movu xm0, [srcq + 0*src_strideq] |
| 709 |
movu xm1, [srcq + 1*src_strideq] |
| 710 |
movu xm2, [srcq + 2*src_strideq] |
| 711 |
movu xm3, [srcq + src_stride3q ] |
| 712 |
|
| 713 |
psubw xm0, [dstq + 0*dst_strideq] |
| 714 |
psubw xm1, [dstq + 1*dst_strideq] |
| 715 |
psubw xm2, [dstq + 2*dst_strideq] |
| 716 |
psubw xm3, [dstq + dst_stride3q ] |
| 717 |
|
| 718 |
pmovsxwd m0, xm0 |
| 719 |
pmovsxwd m1, xm1 |
| 720 |
pmovsxwd m2, xm2 |
| 721 |
pmovsxwd m3, xm3 |
| 722 |
|
| 723 |
add srcq, 16 |
| 724 |
add dstq, 16 |
| 725 |
|
| 726 |
HADAMARD_8X4_12BPC |
| 727 |
paddd m6, m0 |
| 728 |
dec bdmaxd |
| 729 |
jnz .12bpc_loop |
| 730 |
|
| 731 |
HSUM 32, 32, 6, 1, eax |
| 732 |
NORMALIZE4PT |
| 733 |
RET |
| 734 |
|
| 735 |
INIT_YMM avx2 |
| 736 |
cglobal satd_4x16_hbd, 5, 7, 12, src, src_stride, dst, dst_stride, bdmax, \ |
| 737 |
src_stride3, dst_stride3 |
| 738 |
lea src_stride3q, [3*src_strideq] |
| 739 |
lea dst_stride3q, [3*dst_strideq] |
| 740 |
|
| 741 |
cmp bdmaxd, (1 << 10) - 1 |
| 742 |
jne .12bpc |
| 743 |
|
| 744 |
; BLOCK 1 |
| 745 |
movq xm0, [srcq + 0*src_strideq] |
| 746 |
movq xm1, [srcq + 1*src_strideq] |
| 747 |
movq xm2, [srcq + 2*src_strideq] |
| 748 |
movq xm3, [srcq + src_stride3q ] |
| 749 |
lea srcq, [srcq + 4*src_strideq] |
| 750 |
|
| 751 |
psubw xm0, [dstq + 0*dst_strideq] |
| 752 |
psubw xm1, [dstq + 1*dst_strideq] |
| 753 |
psubw xm2, [dstq + 2*dst_strideq] |
| 754 |
psubw xm3, [dstq + dst_stride3q ] |
| 755 |
lea dstq, [dstq + 4*dst_strideq] |
| 756 |
|
| 757 |
; BLOCK 2 |
| 758 |
movq xm4, [srcq + 0*src_strideq] |
| 759 |
movq xm5, [srcq + 1*src_strideq] |
| 760 |
movq xm6, [srcq + 2*src_strideq] |
| 761 |
movq xm7, [srcq + src_stride3q ] |
| 762 |
lea srcq, [srcq + 4*src_strideq] |
| 763 |
|
| 764 |
psubw xm4, [dstq + 0*dst_strideq] |
| 765 |
psubw xm5, [dstq + 1*dst_strideq] |
| 766 |
psubw xm6, [dstq + 2*dst_strideq] |
| 767 |
psubw xm7, [dstq + dst_stride3q ] |
| 768 |
lea dstq, [dstq + 4*dst_strideq] |
| 769 |
|
| 770 |
vinserti128 m0, m0, xm4, 1 |
| 771 |
vinserti128 m1, m1, xm5, 1 |
| 772 |
vinserti128 m2, m2, xm6, 1 |
| 773 |
vinserti128 m3, m3, xm7, 1 |
| 774 |
|
| 775 |
; BLOCK 3 |
| 776 |
movq xm4, [srcq + 0*src_strideq] |
| 777 |
movq xm5, [srcq + 1*src_strideq] |
| 778 |
movq xm6, [srcq + 2*src_strideq] |
| 779 |
movq xm7, [srcq + src_stride3q ] |
| 780 |
lea srcq, [srcq + 4*src_strideq] |
| 781 |
|
| 782 |
psubw xm4, [dstq + 0*dst_strideq] |
| 783 |
psubw xm5, [dstq + 1*dst_strideq] |
| 784 |
psubw xm6, [dstq + 2*dst_strideq] |
| 785 |
psubw xm7, [dstq + dst_stride3q ] |
| 786 |
lea dstq, [dstq + 4*dst_strideq] |
| 787 |
|
| 788 |
; BLOCK 4 |
| 789 |
movq xm8, [srcq + 0*src_strideq] |
| 790 |
movq xm9, [srcq + 1*src_strideq] |
| 791 |
movq xm10, [srcq + 2*src_strideq] |
| 792 |
movq xm11, [srcq + src_stride3q ] |
| 793 |
|
| 794 |
psubw xm8, [dstq + 0*dst_strideq] |
| 795 |
psubw xm9, [dstq + 1*dst_strideq] |
| 796 |
psubw xm10, [dstq + 2*dst_strideq] |
| 797 |
psubw xm11, [dstq + dst_stride3q ] |
| 798 |
|
| 799 |
vinserti128 m4, m4, xm8, 1 |
| 800 |
vinserti128 m5, m5, xm9, 1 |
| 801 |
vinserti128 m6, m6, xm10, 1 |
| 802 |
vinserti128 m7, m7, xm11, 1 |
| 803 |
|
| 804 |
punpcklqdq m0, m0, m4 |
| 805 |
punpcklqdq m1, m1, m5 |
| 806 |
punpcklqdq m2, m2, m6 |
| 807 |
punpcklqdq m3, m3, m7 |
| 808 |
|
| 809 |
jmp m(satd_16x4_hbd).10bpc_main |
| 810 |
|
| 811 |
.12bpc: |
| 812 |
mov bdmaxd, 2 |
| 813 |
pxor m8, m8 |
| 814 |
|
| 815 |
.12bpc_loop: |
| 816 |
; BLOCK 1 |
| 817 |
movq xm0, [srcq + 0*src_strideq] |
| 818 |
movq xm1, [srcq + 1*src_strideq] |
| 819 |
movq xm2, [srcq + 2*src_strideq] |
| 820 |
movq xm3, [srcq + src_stride3q ] |
| 821 |
lea srcq, [srcq + 4*src_strideq] |
| 822 |
|
| 823 |
psubw xm0, [dstq + 0*dst_strideq] |
| 824 |
psubw xm1, [dstq + 1*dst_strideq] |
| 825 |
psubw xm2, [dstq + 2*dst_strideq] |
| 826 |
psubw xm3, [dstq + dst_stride3q ] |
| 827 |
lea dstq, [dstq + 4*dst_strideq] |
| 828 |
|
| 829 |
pmovsxwd xm0, xm0 |
| 830 |
pmovsxwd xm1, xm1 |
| 831 |
pmovsxwd xm2, xm2 |
| 832 |
pmovsxwd xm3, xm3 |
| 833 |
|
| 834 |
; BLOCK 2 |
| 835 |
movq xm4, [srcq + 0*src_strideq] |
| 836 |
movq xm5, [srcq + 1*src_strideq] |
| 837 |
movq xm6, [srcq + 2*src_strideq] |
| 838 |
movq xm7, [srcq + src_stride3q ] |
| 839 |
lea srcq, [srcq + 4*src_strideq] |
| 840 |
|
| 841 |
psubw xm4, [dstq + 0*dst_strideq] |
| 842 |
psubw xm5, [dstq + 1*dst_strideq] |
| 843 |
psubw xm6, [dstq + 2*dst_strideq] |
| 844 |
psubw xm7, [dstq + dst_stride3q ] |
| 845 |
lea dstq, [dstq + 4*dst_strideq] |
| 846 |
|
| 847 |
pmovsxwd xm4, xm4 |
| 848 |
pmovsxwd xm5, xm5 |
| 849 |
pmovsxwd xm6, xm6 |
| 850 |
pmovsxwd xm7, xm7 |
| 851 |
|
| 852 |
vinserti128 m0, m0, xm4, 1 |
| 853 |
vinserti128 m1, m1, xm5, 1 |
| 854 |
vinserti128 m2, m2, xm6, 1 |
| 855 |
vinserti128 m3, m3, xm7, 1 |
| 856 |
|
| 857 |
HADAMARD_8X4_12BPC |
| 858 |
paddd m8, m0 |
| 859 |
dec bdmaxd |
| 860 |
jnz .12bpc_loop |
| 861 |
|
| 862 |
HSUM 32, 32, 8, 0, eax |
| 863 |
NORMALIZE4PT |
| 864 |
RET |
| 865 |
|
| 866 |
INIT_YMM avx2 |
| 867 |
cglobal satd_8x4_hbd, 5, 7, 12, src, src_stride, dst, dst_stride, bdmax, \ |
| 868 |
src_stride3, dst_stride3 |
| 869 |
lea src_stride3q, [3*src_strideq] |
| 870 |
lea dst_stride3q, [3*dst_strideq] |
| 871 |
|
| 872 |
cmp bdmaxd, (1 << 10) - 1 |
| 873 |
jne .12bpc |
| 874 |
|
| 875 |
; Load src rows |
| 876 |
movu xm0, [srcq + 0*src_strideq] |
| 877 |
movu xm1, [srcq + 1*src_strideq] |
| 878 |
movu xm2, [srcq + 2*src_strideq] |
| 879 |
movu xm3, [srcq + src_stride3q ] |
| 880 |
|
| 881 |
; src -= dst |
| 882 |
psubw xm0, [dstq + 0*dst_strideq] |
| 883 |
psubw xm1, [dstq + 1*dst_strideq] |
| 884 |
psubw xm2, [dstq + 2*dst_strideq] |
| 885 |
psubw xm3, [dstq + dst_stride3q ] |
| 886 |
|
| 887 |
.10bpc_main: |
| 888 |
HADAMARD_4X4_PACKED 16, 32 |
| 889 |
|
| 890 |
pabsw m0, m0 |
| 891 |
pabsw m1, m1 |
| 892 |
paddw m0, m1 |
| 893 |
HSUM 16, 32, 0, 1, eax |
| 894 |
NORMALIZE4PT |
| 895 |
RET |
| 896 |
.12bpc: |
| 897 |
RESET_MM_PERMUTATION |
| 898 |
|
| 899 |
pmovzxwd m0, [srcq + 0*src_strideq] |
| 900 |
pmovzxwd m1, [srcq + 1*src_strideq] |
| 901 |
pmovzxwd m2, [srcq + 2*src_strideq] |
| 902 |
pmovzxwd m3, [srcq + src_stride3q ] |
| 903 |
|
| 904 |
pmovzxwd m4, [dstq + 0*dst_strideq] |
| 905 |
pmovzxwd m5, [dstq + 1*dst_strideq] |
| 906 |
pmovzxwd m6, [dstq + 2*dst_strideq] |
| 907 |
pmovzxwd m7, [dstq + dst_stride3q ] |
| 908 |
|
| 909 |
; src -= dst |
| 910 |
psubd m0, m4 |
| 911 |
psubd m1, m5 |
| 912 |
psubd m2, m6 |
| 913 |
psubd m3, m7 |
| 914 |
|
| 915 |
.12bpc_main: |
| 916 |
HADAMARD_8X4_12BPC |
| 917 |
HSUM 32, 32, 0, 1, eax |
| 918 |
NORMALIZE4PT |
| 919 |
RET |
| 920 |
|
| 921 |
INIT_YMM avx2 |
| 922 |
cglobal satd_4x8_hbd, 5, 7, 12, src, src_stride, dst, dst_stride, bdmax, \ |
| 923 |
src_stride3, dst_stride3 |
| 924 |
lea src_stride3q, [3*src_strideq] |
| 925 |
lea dst_stride3q, [3*dst_strideq] |
| 926 |
|
| 927 |
cmp bdmaxd, (1 << 10) - 1 |
| 928 |
jne .12bpc |
| 929 |
|
| 930 |
movq xm0, [srcq + 0*src_strideq] |
| 931 |
movq xm1, [srcq + 1*src_strideq] |
| 932 |
movq xm2, [srcq + 2*src_strideq] |
| 933 |
movq xm3, [srcq + src_stride3q ] |
| 934 |
lea srcq, [srcq + 4*src_strideq] |
| 935 |
movq xm4, [srcq + 0*src_strideq] |
| 936 |
movq xm5, [srcq + 1*src_strideq] |
| 937 |
movq xm6, [srcq + 2*src_strideq] |
| 938 |
movq xm7, [srcq + src_stride3q ] |
| 939 |
|
| 940 |
; This loads past the number of elements we are technically supposed |
| 941 |
; to read, however, this should still be safe, since at least one |
| 942 |
; valid element is in the memory address. |
| 943 |
psubw xm0, [dstq + 0*dst_strideq] |
| 944 |
psubw xm1, [dstq + 1*dst_strideq] |
| 945 |
psubw xm2, [dstq + 2*dst_strideq] |
| 946 |
psubw xm3, [dstq + dst_stride3q ] |
| 947 |
lea dstq, [dstq + 4*dst_strideq] |
| 948 |
psubw xm4, [dstq + 0*dst_strideq] |
| 949 |
psubw xm5, [dstq + 1*dst_strideq] |
| 950 |
psubw xm6, [dstq + 2*dst_strideq] |
| 951 |
psubw xm7, [dstq + dst_stride3q ] |
| 952 |
|
| 953 |
punpcklqdq xm0, xm0, xm4 |
| 954 |
punpcklqdq xm1, xm1, xm5 |
| 955 |
punpcklqdq xm2, xm2, xm6 |
| 956 |
punpcklqdq xm3, xm3, xm7 |
| 957 |
|
| 958 |
; Jump to HADAMARD_4X4_PACKED in 8x4 satd, this saves us some binary size |
| 959 |
; by deduplicating the shared code. |
| 960 |
jmp m(satd_8x4_hbd).10bpc_main |
| 961 |
; No return; we return in the other function. |
| 962 |
|
| 963 |
.12bpc: |
| 964 |
RESET_MM_PERMUTATION |
| 965 |
|
| 966 |
pmovzxwd xm0, [srcq + 0*src_strideq] |
| 967 |
pmovzxwd xm1, [srcq + 1*src_strideq] |
| 968 |
pmovzxwd xm2, [srcq + 2*src_strideq] |
| 969 |
pmovzxwd xm3, [srcq + src_stride3q ] |
| 970 |
lea srcq, [srcq + 4*src_strideq] |
| 971 |
|
| 972 |
pmovzxwd xm4, [dstq + 0*dst_strideq] |
| 973 |
pmovzxwd xm5, [dstq + 1*dst_strideq] |
| 974 |
pmovzxwd xm6, [dstq + 2*dst_strideq] |
| 975 |
pmovzxwd xm7, [dstq + dst_stride3q ] |
| 976 |
lea dstq, [dstq + 4*dst_strideq] |
| 977 |
|
| 978 |
; src -= dst |
| 979 |
psubd xm0, xm4 |
| 980 |
psubd xm1, xm5 |
| 981 |
psubd xm2, xm6 |
| 982 |
psubd xm3, xm7 |
| 983 |
|
| 984 |
pmovzxwd xm4, [srcq + 0*src_strideq] |
| 985 |
pmovzxwd xm5, [srcq + 1*src_strideq] |
| 986 |
pmovzxwd xm6, [srcq + 2*src_strideq] |
| 987 |
pmovzxwd xm7, [srcq + src_stride3q ] |
| 988 |
|
| 989 |
pmovzxwd xm8, [dstq + 0*dst_strideq] |
| 990 |
pmovzxwd xm9, [dstq + 1*dst_strideq] |
| 991 |
pmovzxwd xm10, [dstq + 2*dst_strideq] |
| 992 |
pmovzxwd xm11, [dstq + dst_stride3q ] |
| 993 |
|
| 994 |
; src -= dst (second block) |
| 995 |
psubd xm4, xm8 |
| 996 |
psubd xm5, xm9 |
| 997 |
psubd xm6, xm10 |
| 998 |
psubd xm7, xm11 |
| 999 |
|
| 1000 |
vinserti128 m0, m0, xm4, 1 |
| 1001 |
vinserti128 m1, m1, xm5, 1 |
| 1002 |
vinserti128 m2, m2, xm6, 1 |
| 1003 |
vinserti128 m3, m3, xm7, 1 |
| 1004 |
|
| 1005 |
; Jump to HADAMARD_4X4_PACKED in 8x4 satd, this saves us some binary size |
| 1006 |
; by deduplicating the shared code. |
| 1007 |
jmp m(satd_8x4_hbd).12bpc_main |
| 1008 |
; No return; we return in the other function. |
| 1009 |
|
| 1010 |
|
| 1011 |
; <width>, <height> |
| 1012 |
%macro SATD_NXM 2 |
| 1013 |
|
| 1014 |
INIT_YMM avx2 |
| 1015 |
cglobal satd_%1x%2_hbd, 5, 10, 16, src, src_stride, dst, dst_stride, bdmax, \ |
| 1016 |
src_stride3, dst_stride3, nsrc_stride4, ndst_stride4, rows |
| 1017 |
lea nsrc_stride4q, [4*src_strideq] |
| 1018 |
lea ndst_stride4q, [4*dst_strideq] |
| 1019 |
|
| 1020 |
lea src_stride3q, [3*src_strideq] |
| 1021 |
lea dst_stride3q, [3*dst_strideq] |
| 1022 |
|
| 1023 |
neg nsrc_stride4q |
| 1024 |
neg ndst_stride4q |
| 1025 |
|
| 1026 |
pxor m9, m9 |
| 1027 |
|
| 1028 |
; Height contains the number of rows. |
| 1029 |
mov rowsd, %2/8 |
| 1030 |
.outer: |
| 1031 |
mov bdmaxd, %1/8 |
| 1032 |
|
| 1033 |
; Loop over blocks in same row. |
| 1034 |
.loop: |
| 1035 |
LOAD_DIFF_8X8 |
| 1036 |
|
| 1037 |
; Fix up pointers and go to next block in same row. |
| 1038 |
lea srcq, [srcq + nsrc_stride4q + 16] |
| 1039 |
lea dstq, [dstq + ndst_stride4q + 16] |
| 1040 |
|
| 1041 |
call m(satd_8x8_hbd_internal) |
| 1042 |
dec bdmaxd |
| 1043 |
jnz .loop |
| 1044 |
|
| 1045 |
lea srcq, [srcq + 8*src_strideq - (%1*16)/8] |
| 1046 |
lea dstq, [dstq + 8*dst_strideq - (%1*16)/8] |
| 1047 |
|
| 1048 |
dec rowsd |
| 1049 |
jnz .outer |
| 1050 |
|
| 1051 |
HSUM 32, 32, 9, 0, eax |
| 1052 |
NORMALIZE8PT |
| 1053 |
RET |
| 1054 |
%endmacro |
| 1055 |
|
| 1056 |
%macro SATD_NX8 1 |
| 1057 |
INIT_YMM avx2 |
| 1058 |
cglobal satd_%1x8_hbd, 5, 9, 16, src, src_stride, dst, dst_stride, bdmax, \ |
| 1059 |
src_stride3, dst_stride3, nsrc_stride4, ndst_stride4 |
| 1060 |
lea nsrc_stride4q, [4*src_strideq] |
| 1061 |
lea ndst_stride4q, [4*dst_strideq] |
| 1062 |
|
| 1063 |
lea src_stride3q, [3*src_strideq] |
| 1064 |
lea dst_stride3q, [3*dst_strideq] |
| 1065 |
|
| 1066 |
neg nsrc_stride4q |
| 1067 |
neg ndst_stride4q |
| 1068 |
|
| 1069 |
pxor m9, m9 |
| 1070 |
mov bdmaxd, %1/8 |
| 1071 |
|
| 1072 |
.loop: |
| 1073 |
LOAD_DIFF_8X8 |
| 1074 |
|
| 1075 |
lea srcq, [srcq + nsrc_stride4q + 16] |
| 1076 |
lea dstq, [dstq + ndst_stride4q + 16] |
| 1077 |
|
| 1078 |
call m(satd_8x8_hbd_internal) |
| 1079 |
dec bdmaxd |
| 1080 |
jnz .loop |
| 1081 |
|
| 1082 |
HSUM 32, 32, 9, 0, eax |
| 1083 |
NORMALIZE8PT |
| 1084 |
RET |
| 1085 |
%endmacro |
| 1086 |
|
| 1087 |
%macro SATD_8XM 1 |
| 1088 |
INIT_YMM avx2 |
| 1089 |
cglobal satd_8x%1_hbd, 5, 7, 16, src, src_stride, dst, dst_stride, bdmax, \ |
| 1090 |
src_stride3, dst_stride3 |
| 1091 |
lea src_stride3q, [3*src_strideq] |
| 1092 |
lea dst_stride3q, [3*dst_strideq] |
| 1093 |
|
| 1094 |
pxor m9, m9 |
| 1095 |
mov bdmaxd, %1/8 |
| 1096 |
|
| 1097 |
.loop: |
| 1098 |
LOAD_DIFF_8X8 |
| 1099 |
|
| 1100 |
lea srcq, [srcq + 4*src_strideq] |
| 1101 |
lea dstq, [dstq + 4*dst_strideq] |
| 1102 |
|
| 1103 |
call m(satd_8x8_hbd_internal) |
| 1104 |
dec bdmaxd |
| 1105 |
jnz .loop |
| 1106 |
|
| 1107 |
HSUM 32, 32, 9, 0, eax |
| 1108 |
NORMALIZE8PT |
| 1109 |
RET |
| 1110 |
%endmacro |
| 1111 |
|
| 1112 |
SATD_NXM 16, 16 |
| 1113 |
SATD_NXM 32, 32 |
| 1114 |
SATD_NXM 64, 64 |
| 1115 |
SATD_NXM 128, 128 |
| 1116 |
SATD_NXM 16, 32 |
| 1117 |
SATD_NXM 16, 64 |
| 1118 |
SATD_NXM 32, 16 |
| 1119 |
SATD_NXM 32, 64 |
| 1120 |
SATD_NXM 64, 16 |
| 1121 |
SATD_NXM 64, 32 |
| 1122 |
SATD_NXM 64, 128 |
| 1123 |
SATD_NXM 128, 64 |
| 1124 |
SATD_NX8 16 |
| 1125 |
SATD_NX8 32 |
| 1126 |
SATD_8XM 16 |
| 1127 |
SATD_8XM 32 |
| 1128 |
|
| 1129 |
%endif ; ARCH_X86_64 |