| 1 |
; Copyright (c) 2022, The rav1e contributors. All rights reserved |
| 2 |
; |
| 3 |
; This source code is subject to the terms of the BSD 2 Clause License and |
| 4 |
; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License |
| 5 |
; was not distributed with this source code in the LICENSE file, you can |
| 6 |
; obtain it at www.aomedia.org/license/software. If the Alliance for Open |
| 7 |
; Media Patent License 1.0 was not distributed with this source code in the |
| 8 |
; PATENTS file, you can obtain it at www.aomedia.org/license/patent. |
| 9 |
|
| 10 |
%include "config.asm" |
| 11 |
%include "src/asm/x86/x86inc.asm" |
| 12 |
|
| 13 |
%if ARCH_X86_64 |
| 14 |
|
| 15 |
SECTION_RODATA |
| 16 |
|
| 17 |
align 32 |
| 18 |
mask_lut: db \ |
| 19 |
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \ |
| 20 |
-1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \ |
| 21 |
-1,-1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \ |
| 22 |
-1,-1,-1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \ |
| 23 |
-1,-1,-1,-1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \ |
| 24 |
-1,-1,-1,-1,-1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \ |
| 25 |
-1,-1,-1,-1,-1,-1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \ |
| 26 |
-1,-1,-1,-1,-1,-1,-1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \ |
| 27 |
-1,-1,-1,-1,-1,-1,-1,-1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \ |
| 28 |
-1,-1,-1,-1,-1,-1,-1,-1,-1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \ |
| 29 |
-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \ |
| 30 |
-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \ |
| 31 |
-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \ |
| 32 |
-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \ |
| 33 |
-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \ |
| 34 |
-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \ |
| 35 |
-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \ |
| 36 |
-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \ |
| 37 |
-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \ |
| 38 |
-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \ |
| 39 |
-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \ |
| 40 |
-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \ |
| 41 |
-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \ |
| 42 |
-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, 0, 0, 0, 0, 0, 0, 0, 0, 0, \ |
| 43 |
-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, 0, 0, 0, 0, 0, 0, 0, 0, \ |
| 44 |
-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, 0, 0, 0, 0, 0, 0, 0, \ |
| 45 |
-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, 0, 0, 0, 0, 0, 0, \ |
| 46 |
-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, 0, 0, 0, 0, 0, \ |
| 47 |
-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, 0, 0, 0, 0, \ |
| 48 |
-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, 0, 0, 0, \ |
| 49 |
-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, 0, 0, \ |
| 50 |
-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, 0, |
| 51 |
|
| 52 |
%macro JMP_TABLE 3-* |
| 53 |
%xdefine %%func mangle(private_prefix %+ _%1_%2) |
| 54 |
%xdefine %%table %1_%2_table |
| 55 |
%%table: |
| 56 |
%rep %0 - 2 |
| 57 |
dd (%%func %+ .%3) - (%%table) |
| 58 |
%rotate 1 |
| 59 |
%endrep |
| 60 |
%endmacro |
| 61 |
|
| 62 |
JMP_TABLE sad_plane_8bpc, avx2, vec0, vec1, vec2, vec3 |
| 63 |
JMP_TABLE sad_plane_8bpc, sse2, vec0, vec1, vec2, vec3 |
| 64 |
|
| 65 |
%use ifunc |
| 66 |
|
| 67 |
SECTION .text |
| 68 |
|
| 69 |
%macro SAD_PLANE_FN 0 |
| 70 |
cglobal sad_plane_8bpc, 5, 9, 9, p1, p2, stride, width, rows, \ |
| 71 |
resid_simd, resid, width_unrll, tmp0 |
| 72 |
mov resid_simdq, widthq |
| 73 |
mov residd, widthd |
| 74 |
and residd, mmsize - 1 |
| 75 |
and resid_simdq, -(mmsize) |
| 76 |
and widthq, -(4*mmsize) |
| 77 |
; LUT row size is always 32 regardless of mmsize (because the |
| 78 |
; start of the rows would be the same, so we reuse the same LUT) |
| 79 |
shl residd, ilog2(32) |
| 80 |
pxor xm0, xm0 |
| 81 |
pxor xm1, xm1 |
| 82 |
pxor xm2, xm2 |
| 83 |
pxor xm3, xm3 |
| 84 |
; load mask from lookup table into m8 |
| 85 |
lea tmp0q, [mask_lut] |
| 86 |
mova m8, [tmp0q + residq] |
| 87 |
|
| 88 |
DEFINE_ARGS p1, p2, stride, width, rows, \ |
| 89 |
resid_simd, resid, width_unrll, skip_ptr |
| 90 |
|
| 91 |
sub resid_simdq, widthq |
| 92 |
; need to divide by mmsize to load skip pointer |
| 93 |
shr resid_simdq, ilog2(mmsize) |
| 94 |
%if mmsize == 32 |
| 95 |
%define jmp_table sad_plane_8bpc_avx2_table |
| 96 |
%elif mmsize == 16 |
| 97 |
%define jmp_table sad_plane_8bpc_sse2_table |
| 98 |
%endif |
| 99 |
lea r6, [jmp_table] |
| 100 |
movsxd skip_ptrq, [r6 + 4*resid_simdq] |
| 101 |
add skip_ptrq, r6 |
| 102 |
|
| 103 |
; shift back (for residual to load correct number of bytes) |
| 104 |
shl resid_simdq, ilog2(mmsize) |
| 105 |
; set pointer to point after end of width of first row |
| 106 |
add p1q, widthq |
| 107 |
add p2q, widthq |
| 108 |
mov width_unrllq, widthq |
| 109 |
neg widthq |
| 110 |
.loop_row: |
| 111 |
test widthq, widthq |
| 112 |
jz .skip |
| 113 |
.loop: |
| 114 |
mova m4, [p1q + widthq + 0*mmsize] |
| 115 |
mova m5, [p1q + widthq + 1*mmsize] |
| 116 |
mova m6, [p1q + widthq + 2*mmsize] |
| 117 |
mova m7, [p1q + widthq + 3*mmsize] |
| 118 |
|
| 119 |
psadbw m4, m4, [p2q + widthq + 0*mmsize] |
| 120 |
psadbw m5, m5, [p2q + widthq + 1*mmsize] |
| 121 |
psadbw m6, m6, [p2q + widthq + 2*mmsize] |
| 122 |
psadbw m7, m7, [p2q + widthq + 3*mmsize] |
| 123 |
|
| 124 |
paddq m0, m4 |
| 125 |
paddq m1, m5 |
| 126 |
paddq m2, m6 |
| 127 |
paddq m3, m7 |
| 128 |
|
| 129 |
add widthq, 4*mmsize |
| 130 |
jnz .loop |
| 131 |
.skip: |
| 132 |
jmp skip_ptrq |
| 133 |
.vec3: |
| 134 |
mova m6, [p1q + 2*mmsize] |
| 135 |
psadbw m6, m6, [p2q + 2*mmsize] |
| 136 |
paddq m2, m6 |
| 137 |
.vec2: |
| 138 |
mova m5, [p1q + 1*mmsize] |
| 139 |
psadbw m5, m5, [p2q + 1*mmsize] |
| 140 |
paddq m1, m5 |
| 141 |
.vec1: |
| 142 |
mova m4, [p1q + 0*mmsize] |
| 143 |
psadbw m4, m4, [p2q + 0*mmsize] |
| 144 |
paddq m0, m4 |
| 145 |
.vec0: |
| 146 |
; skip residual element add if necessary |
| 147 |
test residd, residd |
| 148 |
jz .next_row |
| 149 |
; load residual elements and mask out elements past the width |
| 150 |
pand m4, m8, [p1q + resid_simdq] |
| 151 |
pand m5, m8, [p2q + resid_simdq] |
| 152 |
psadbw m4, m4, m5 |
| 153 |
paddq m2, m4 |
| 154 |
.next_row: |
| 155 |
; width is 0 after the unrolled loop, so subtracting is basically a mov + neg |
| 156 |
sub widthq, width_unrllq |
| 157 |
; since we started with p1+width, adding stride will get the |
| 158 |
; pointer at the end of the next row |
| 159 |
add p1q, strideq |
| 160 |
add p2q, strideq |
| 161 |
dec rowsd |
| 162 |
jnz .loop_row |
| 163 |
; final horizontal reduction |
| 164 |
paddq m2, m3 |
| 165 |
paddq m0, m1 |
| 166 |
paddq m0, m2 |
| 167 |
%if mmsize == 32 |
| 168 |
vextracti128 xm1, ym0, 1 |
| 169 |
paddq xm0, xm1 |
| 170 |
%endif |
| 171 |
pshufd xm1, xm0, q0032 |
| 172 |
paddq xm0, xm1 |
| 173 |
movq rax, xm0 |
| 174 |
RET |
| 175 |
%endmacro |
| 176 |
|
| 177 |
INIT_XMM sse2 |
| 178 |
SAD_PLANE_FN |
| 179 |
|
| 180 |
INIT_YMM avx2 |
| 181 |
SAD_PLANE_FN |
| 182 |
|
| 183 |
%endif ; ARCH_X86_64 |