| 1 |
; Copyright (c) 2022, The rav1e contributors. All rights reserved |
| 2 |
; |
| 3 |
; This source code is subject to the terms of the BSD 2 Clause License and |
| 4 |
; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License |
| 5 |
; was not distributed with this source code in the LICENSE file, you can |
| 6 |
; obtain it at www.aomedia.org/license/software. If the Alliance for Open |
| 7 |
; Media Patent License 1.0 was not distributed with this source code in the |
| 8 |
; PATENTS file, you can obtain it at www.aomedia.org/license/patent. |
| 9 |
|
| 10 |
%include "config.asm" |
| 11 |
%include "ext/x86/x86inc.asm" |
| 12 |
|
| 13 |
SECTION .text |
| 14 |
|
| 15 |
%if ARCH_X86_64 |
| 16 |
|
| 17 |
; m0: zero register |
| 18 |
; m1: src input |
| 19 |
; m2: dst input |
| 20 |
; m3 = sum(src_{i,j}) |
| 21 |
; m4 = sum(src_{i,j}^2) |
| 22 |
; m5 = sum(dst_{i,j}) |
| 23 |
; m6 = sum(dst_{i,j}^2) |
| 24 |
; m7 = sum(src_{i,j} * dst_{i,j}) |
| 25 |
; m8: tmp register |
| 26 |
%macro CDEF_DIST_W8_SSE2 0 |
| 27 |
psadbw m8, m1, m0 ; sum pixel values |
| 28 |
paddd m3, m8 ; accumulate |
| 29 |
punpcklbw m1, m0 ; convert to 16-bits |
| 30 |
pmaddwd m8, m1, m1 ; square and horz add |
| 31 |
paddd m4, m8 ; accumulate |
| 32 |
psadbw m8, m2, m0 ; same as above, but for dst |
| 33 |
paddd m5, m8 |
| 34 |
punpcklbw m2, m0 |
| 35 |
pmaddwd m8, m2, m2 |
| 36 |
paddd m6, m8 |
| 37 |
pmaddwd m8, m1, m2 ; src_{i,j} * dst_{i,j} (and horz add) |
| 38 |
paddd m7, m8 |
| 39 |
%endmacro |
| 40 |
|
| 41 |
; Refine sums into variances and sse |
| 42 |
; parameter: scale log2 relative to 8x8 |
| 43 |
%macro CDEF_DIST_REFINE_SSE2 1 |
| 44 |
; Compute [sum(src)^2, sum(dst)^2] |
| 45 |
punpckldq m3, m5 ; store sums in a single vector |
| 46 |
pcmpeqd m0, m0 ; -1 (for rounding) |
| 47 |
pmaddwd m3, m3 |
| 48 |
|
| 49 |
; Divide by area and round |
| 50 |
pslld m0, 5 - %1 |
| 51 |
psubd m3, m0 ; + (1 << (5 - %1)) |
| 52 |
psrld m3, 6 - %1 |
| 53 |
|
| 54 |
pshufd m0, m7, q3232 ; reduce sum(src * dst) |
| 55 |
punpckldq m1, m4, m6 ; reduce [sum(src^2), sum(dst^2)] |
| 56 |
paddd m7, m0 |
| 57 |
punpckhdq m4, m6 |
| 58 |
paddd m1, m4 |
| 59 |
paddd m7, m7 ; 2 * sum(src * dst) [Partially reduced; len 2] |
| 60 |
pshufd m0, m1, q3232 |
| 61 |
|
| 62 |
; Equivelent to: |
| 63 |
; paddd m1, m0 |
| 64 |
; psubd m0, m1, m7 ; sse = sum(src^2) + sum(dst^2) - sum(src * dst) |
| 65 |
; but with fewer dependancies |
| 66 |
psubd m7, m1 |
| 67 |
paddd m1, m0 ; [sum(src^2), sum(dst^2)] |
| 68 |
psubd m0, m7 ; sse (Needs reducing; len 2) |
| 69 |
|
| 70 |
psubd m1, m3 ; [src variance, dst variance] |
| 71 |
; Scale up the variances up to 8x8 |
| 72 |
; TODO: this can be handled inside ssim boost in the future |
| 73 |
%if %1 != 0 |
| 74 |
pslld m1, %1 |
| 75 |
%endif |
| 76 |
movq [ret_ptrq], m1 |
| 77 |
|
| 78 |
; Final reduce for sse |
| 79 |
pshuflw m2, m0, q3232 |
| 80 |
paddd m0, m2 |
| 81 |
movd [ret_ptrq+8], m0 |
| 82 |
%endmacro |
| 83 |
|
| 84 |
INIT_XMM sse2 |
| 85 |
cglobal cdef_dist_kernel_4x4, 5, 5, 9, \ |
| 86 |
src, src_stride, dst, dst_stride, ret_ptr |
| 87 |
pxor m0, m0 |
| 88 |
movd m1, [srcq] |
| 89 |
movd m2, [srcq+src_strideq] |
| 90 |
punpckldq m1, m2 |
| 91 |
movd m2, [dstq] |
| 92 |
movd m8, [dstq+dst_strideq] |
| 93 |
lea srcq, [srcq+2*src_strideq] |
| 94 |
lea dstq, [dstq+2*dst_strideq] |
| 95 |
punpckldq m2, m8 |
| 96 |
psadbw m3, m1, m0 |
| 97 |
punpcklbw m1, m0 |
| 98 |
pmaddwd m4, m1, m1 |
| 99 |
psadbw m5, m2, m0 |
| 100 |
punpcklbw m2, m0 |
| 101 |
pmaddwd m6, m2, m2 |
| 102 |
pmaddwd m7, m1, m2 |
| 103 |
|
| 104 |
movd m1, [srcq] |
| 105 |
movd m2, [srcq+src_strideq] |
| 106 |
punpckldq m1, m2 |
| 107 |
movd m2, [dstq] |
| 108 |
movd m8, [dstq+dst_strideq] |
| 109 |
punpckldq m2, m8 |
| 110 |
CDEF_DIST_W8_SSE2 |
| 111 |
|
| 112 |
CDEF_DIST_REFINE_SSE2 2 |
| 113 |
RET |
| 114 |
|
| 115 |
cglobal cdef_dist_kernel_4x8, 5, 7, 9, \ |
| 116 |
src, src_stride, dst, dst_stride, ret_ptr, src_stride3, dst_stride3 |
| 117 |
lea src_stride3q, [src_strideq*3] |
| 118 |
lea dst_stride3q, [dst_strideq*3] |
| 119 |
|
| 120 |
pxor m0, m0 |
| 121 |
movd m1, [srcq] |
| 122 |
movd m2, [srcq+src_strideq] |
| 123 |
punpckldq m1, m2 |
| 124 |
movd m2, [dstq] |
| 125 |
movd m8, [dstq+dst_strideq] |
| 126 |
punpckldq m2, m8 |
| 127 |
psadbw m3, m1, m0 |
| 128 |
punpcklbw m1, m0 |
| 129 |
pmaddwd m4, m1, m1 |
| 130 |
psadbw m5, m2, m0 |
| 131 |
punpcklbw m2, m0 |
| 132 |
pmaddwd m6, m2, m2 |
| 133 |
pmaddwd m7, m1, m2 |
| 134 |
|
| 135 |
movd m1, [srcq+2*src_strideq] |
| 136 |
movd m2, [srcq+src_stride3q] |
| 137 |
punpckldq m1, m2 |
| 138 |
movd m2, [dstq+2*dst_strideq] |
| 139 |
movd m8, [dstq+dst_stride3q] |
| 140 |
lea srcq, [srcq+4*src_strideq] |
| 141 |
lea dstq, [dstq+4*dst_strideq] |
| 142 |
punpckldq m2, m8 |
| 143 |
CDEF_DIST_W8_SSE2 |
| 144 |
movd m1, [srcq] |
| 145 |
movd m2, [srcq+src_strideq] |
| 146 |
punpckldq m1, m2 |
| 147 |
movd m2, [dstq] |
| 148 |
movd m8, [dstq+dst_strideq] |
| 149 |
punpckldq m2, m8 |
| 150 |
CDEF_DIST_W8_SSE2 |
| 151 |
movd m1, [srcq+2*src_strideq] |
| 152 |
movd m2, [srcq+src_stride3q] |
| 153 |
punpckldq m1, m2 |
| 154 |
movd m2, [dstq+2*dst_strideq] |
| 155 |
movd m8, [dstq+dst_stride3q] |
| 156 |
punpckldq m2, m8 |
| 157 |
CDEF_DIST_W8_SSE2 |
| 158 |
|
| 159 |
CDEF_DIST_REFINE_SSE2 1 |
| 160 |
RET |
| 161 |
|
| 162 |
cglobal cdef_dist_kernel_8x4, 5, 5, 9, \ |
| 163 |
src, src_stride, dst, dst_stride, ret_ptr |
| 164 |
pxor m0, m0 |
| 165 |
movq m1, [srcq] |
| 166 |
psadbw m3, m1, m0 |
| 167 |
punpcklbw m1, m0 |
| 168 |
pmaddwd m4, m1, m1 |
| 169 |
movq m2, [dstq] |
| 170 |
psadbw m5, m2, m0 |
| 171 |
punpcklbw m2, m0 |
| 172 |
pmaddwd m6, m2, m2 |
| 173 |
pmaddwd m7, m1, m2 |
| 174 |
|
| 175 |
movq m1, [srcq+src_strideq] |
| 176 |
movq m2, [dstq+dst_strideq] |
| 177 |
lea srcq, [srcq+2*src_strideq] |
| 178 |
lea dstq, [dstq+2*dst_strideq] |
| 179 |
CDEF_DIST_W8_SSE2 |
| 180 |
movq m1, [srcq] |
| 181 |
movq m2, [dstq] |
| 182 |
CDEF_DIST_W8_SSE2 |
| 183 |
movq m1, [srcq+src_strideq] |
| 184 |
movq m2, [dstq+dst_strideq] |
| 185 |
CDEF_DIST_W8_SSE2 |
| 186 |
|
| 187 |
CDEF_DIST_REFINE_SSE2 1 |
| 188 |
RET |
| 189 |
|
| 190 |
cglobal cdef_dist_kernel_8x8, 5, 7, 9, \ |
| 191 |
src, src_stride, dst, dst_stride, ret_ptr, src_stride3, dst_stride3 |
| 192 |
lea src_stride3q, [src_strideq*3] |
| 193 |
lea dst_stride3q, [dst_strideq*3] |
| 194 |
|
| 195 |
pxor m0, m0 |
| 196 |
movq m1, [srcq] |
| 197 |
psadbw m3, m1, m0 |
| 198 |
punpcklbw m1, m0 |
| 199 |
pmaddwd m4, m1, m1 |
| 200 |
movq m2, [dstq] |
| 201 |
psadbw m5, m2, m0 |
| 202 |
punpcklbw m2, m0 |
| 203 |
pmaddwd m6, m2, m2 |
| 204 |
pmaddwd m7, m1, m2 |
| 205 |
|
| 206 |
movq m1, [srcq+src_strideq] |
| 207 |
movq m2, [dstq+dst_strideq] |
| 208 |
CDEF_DIST_W8_SSE2 |
| 209 |
movq m1, [srcq+2*src_strideq] |
| 210 |
movq m2, [dstq+2*dst_strideq] |
| 211 |
CDEF_DIST_W8_SSE2 |
| 212 |
movq m1, [srcq+src_stride3q] |
| 213 |
movq m2, [dstq+dst_stride3q] |
| 214 |
CDEF_DIST_W8_SSE2 |
| 215 |
lea srcq, [srcq+src_strideq*4] |
| 216 |
lea dstq, [dstq+dst_strideq*4] |
| 217 |
|
| 218 |
movq m1, [srcq] |
| 219 |
movq m2, [dstq] |
| 220 |
CDEF_DIST_W8_SSE2 |
| 221 |
movq m1, [srcq+src_strideq] |
| 222 |
movq m2, [dstq+dst_strideq] |
| 223 |
CDEF_DIST_W8_SSE2 |
| 224 |
movq m1, [srcq+2*src_strideq] |
| 225 |
movq m2, [dstq+2*dst_strideq] |
| 226 |
CDEF_DIST_W8_SSE2 |
| 227 |
movq m1, [srcq+src_stride3q] |
| 228 |
movq m2, [dstq+dst_stride3q] |
| 229 |
CDEF_DIST_W8_SSE2 |
| 230 |
|
| 231 |
CDEF_DIST_REFINE_SSE2 0 |
| 232 |
RET |
| 233 |
%endif ; ARCH_X86_64 |