| 1 |
; |
| 2 |
; Copyright (c) 2016, Alliance for Open Media. All rights reserved |
| 3 |
; |
| 4 |
; This source code is subject to the terms of the BSD 2 Clause License and |
| 5 |
; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License |
| 6 |
; was not distributed with this source code in the LICENSE file, you can |
| 7 |
; obtain it at www.aomedia.org/license/software. If the Alliance for Open |
| 8 |
; Media Patent License 1.0 was not distributed with this source code in the |
| 9 |
; PATENTS file, you can obtain it at www.aomedia.org/license/patent. |
| 10 |
; |
| 11 |
|
| 12 |
; |
| 13 |
|
| 14 |
%include "config.asm" |
| 15 |
%include "ext/x86/x86inc.asm" |
| 16 |
|
| 17 |
SECTION .text |
| 18 |
|
| 19 |
%macro SAD_FN 4 |
| 20 |
%if %4 == 0 |
| 21 |
%if %3 == 5 |
| 22 |
cglobal sad%1x%2, 4, %3, 5, src, src_stride, ref, ref_stride, n_rows |
| 23 |
%else ; %3 == 7 |
| 24 |
cglobal sad%1x%2, 4, %3, 6, src, src_stride, ref, ref_stride, \ |
| 25 |
src_stride3, ref_stride3, n_rows |
| 26 |
%endif ; %3 == 5/7 |
| 27 |
%else ; avg |
| 28 |
%if %3 == 5 |
| 29 |
cglobal sad%1x%2_avg, 5, 1 + %3, 5, src, src_stride, ref, ref_stride, \ |
| 30 |
second_pred, n_rows |
| 31 |
%else ; %3 == 7 |
| 32 |
cglobal sad%1x%2_avg, 5, ARCH_X86_64 + %3, 6, src, src_stride, \ |
| 33 |
ref, ref_stride, \ |
| 34 |
second_pred, \ |
| 35 |
src_stride3, ref_stride3 |
| 36 |
%if ARCH_X86_64 |
| 37 |
%define n_rowsd r7d |
| 38 |
%else ; x86-32 |
| 39 |
%define n_rowsd dword r0m |
| 40 |
%endif ; x86-32/64 |
| 41 |
%endif ; %3 == 5/7 |
| 42 |
%endif ; avg/sad |
| 43 |
movsxdifnidn src_strideq, src_strided |
| 44 |
movsxdifnidn ref_strideq, ref_strided |
| 45 |
%if %3 == 7 |
| 46 |
lea src_stride3q, [src_strideq*3] |
| 47 |
lea ref_stride3q, [ref_strideq*3] |
| 48 |
%endif ; %3 == 7 |
| 49 |
%endmacro |
| 50 |
|
| 51 |
; unsigned int aom_sad128x128_sse2(uint8_t *src, int src_stride, |
| 52 |
; uint8_t *ref, int ref_stride); |
| 53 |
%macro SAD128XN 1-2 0 |
| 54 |
SAD_FN 128, %1, 5, %2 |
| 55 |
mov n_rowsd, %1 |
| 56 |
pxor m0, m0 |
| 57 |
|
| 58 |
.loop: |
| 59 |
movu m1, [refq] |
| 60 |
movu m2, [refq+16] |
| 61 |
movu m3, [refq+32] |
| 62 |
movu m4, [refq+48] |
| 63 |
%if %2 == 1 |
| 64 |
pavgb m1, [second_predq+mmsize*0] |
| 65 |
pavgb m2, [second_predq+mmsize*1] |
| 66 |
pavgb m3, [second_predq+mmsize*2] |
| 67 |
pavgb m4, [second_predq+mmsize*3] |
| 68 |
%endif |
| 69 |
psadbw m1, [srcq] |
| 70 |
psadbw m2, [srcq+16] |
| 71 |
psadbw m3, [srcq+32] |
| 72 |
psadbw m4, [srcq+48] |
| 73 |
|
| 74 |
paddd m1, m2 |
| 75 |
paddd m3, m4 |
| 76 |
paddd m0, m1 |
| 77 |
paddd m0, m3 |
| 78 |
|
| 79 |
movu m1, [refq+64] |
| 80 |
movu m2, [refq+80] |
| 81 |
movu m3, [refq+96] |
| 82 |
movu m4, [refq+112] |
| 83 |
%if %2 == 1 |
| 84 |
pavgb m1, [second_predq+mmsize*4] |
| 85 |
pavgb m2, [second_predq+mmsize*5] |
| 86 |
pavgb m3, [second_predq+mmsize*6] |
| 87 |
pavgb m4, [second_predq+mmsize*7] |
| 88 |
lea second_predq, [second_predq+mmsize*8] |
| 89 |
%endif |
| 90 |
psadbw m1, [srcq+64] |
| 91 |
psadbw m2, [srcq+80] |
| 92 |
psadbw m3, [srcq+96] |
| 93 |
psadbw m4, [srcq+112] |
| 94 |
|
| 95 |
add refq, ref_strideq |
| 96 |
add srcq, src_strideq |
| 97 |
|
| 98 |
paddd m1, m2 |
| 99 |
paddd m3, m4 |
| 100 |
paddd m0, m1 |
| 101 |
paddd m0, m3 |
| 102 |
|
| 103 |
sub n_rowsd, 1 |
| 104 |
jg .loop |
| 105 |
|
| 106 |
movhlps m1, m0 |
| 107 |
paddd m0, m1 |
| 108 |
movd eax, m0 |
| 109 |
RET |
| 110 |
%endmacro |
| 111 |
|
| 112 |
INIT_XMM sse2 |
| 113 |
SAD128XN 128 ; sad128x128_sse2 |
| 114 |
SAD128XN 128, 1 ; sad128x128_avg_sse2 |
| 115 |
SAD128XN 64 ; sad128x64_sse2 |
| 116 |
SAD128XN 64, 1 ; sad128x64_avg_sse2 |
| 117 |
|
| 118 |
|
| 119 |
; unsigned int aom_sad64x64_sse2(uint8_t *src, int src_stride, |
| 120 |
; uint8_t *ref, int ref_stride); |
| 121 |
%macro SAD64XN 1-2 0 |
| 122 |
SAD_FN 64, %1, 5, %2 |
| 123 |
mov n_rowsd, %1 |
| 124 |
pxor m0, m0 |
| 125 |
.loop: |
| 126 |
movu m1, [refq] |
| 127 |
movu m2, [refq+16] |
| 128 |
movu m3, [refq+32] |
| 129 |
movu m4, [refq+48] |
| 130 |
%if %2 == 1 |
| 131 |
pavgb m1, [second_predq+mmsize*0] |
| 132 |
pavgb m2, [second_predq+mmsize*1] |
| 133 |
pavgb m3, [second_predq+mmsize*2] |
| 134 |
pavgb m4, [second_predq+mmsize*3] |
| 135 |
lea second_predq, [second_predq+mmsize*4] |
| 136 |
%endif |
| 137 |
psadbw m1, [srcq] |
| 138 |
psadbw m2, [srcq+16] |
| 139 |
psadbw m3, [srcq+32] |
| 140 |
psadbw m4, [srcq+48] |
| 141 |
paddd m1, m2 |
| 142 |
paddd m3, m4 |
| 143 |
add refq, ref_strideq |
| 144 |
paddd m0, m1 |
| 145 |
add srcq, src_strideq |
| 146 |
paddd m0, m3 |
| 147 |
dec n_rowsd |
| 148 |
jg .loop |
| 149 |
|
| 150 |
movhlps m1, m0 |
| 151 |
paddd m0, m1 |
| 152 |
movd eax, m0 |
| 153 |
RET |
| 154 |
%endmacro |
| 155 |
|
| 156 |
INIT_XMM sse2 |
| 157 |
SAD64XN 128 ; sad64x128_sse2 |
| 158 |
SAD64XN 128, 1 ; sad64x128_avg_sse2 |
| 159 |
SAD64XN 64 ; sad64x64_sse2 |
| 160 |
SAD64XN 32 ; sad64x32_sse2 |
| 161 |
SAD64XN 64, 1 ; sad64x64_avg_sse2 |
| 162 |
SAD64XN 32, 1 ; sad64x32_avg_sse2 |
| 163 |
SAD64XN 16 ; sad64x16_sse2 |
| 164 |
SAD64XN 16, 1 ; sad64x16_avg_sse2 |
| 165 |
|
| 166 |
; unsigned int aom_sad32x32_sse2(uint8_t *src, int src_stride, |
| 167 |
; uint8_t *ref, int ref_stride); |
| 168 |
%macro SAD32XN 1-2 0 |
| 169 |
SAD_FN 32, %1, 5, %2 |
| 170 |
mov n_rowsd, %1/2 |
| 171 |
pxor m0, m0 |
| 172 |
.loop: |
| 173 |
movu m1, [refq] |
| 174 |
movu m2, [refq+16] |
| 175 |
movu m3, [refq+ref_strideq] |
| 176 |
movu m4, [refq+ref_strideq+16] |
| 177 |
%if %2 == 1 |
| 178 |
pavgb m1, [second_predq+mmsize*0] |
| 179 |
pavgb m2, [second_predq+mmsize*1] |
| 180 |
pavgb m3, [second_predq+mmsize*2] |
| 181 |
pavgb m4, [second_predq+mmsize*3] |
| 182 |
lea second_predq, [second_predq+mmsize*4] |
| 183 |
%endif |
| 184 |
psadbw m1, [srcq] |
| 185 |
psadbw m2, [srcq+16] |
| 186 |
psadbw m3, [srcq+src_strideq] |
| 187 |
psadbw m4, [srcq+src_strideq+16] |
| 188 |
paddd m1, m2 |
| 189 |
paddd m3, m4 |
| 190 |
lea refq, [refq+ref_strideq*2] |
| 191 |
paddd m0, m1 |
| 192 |
lea srcq, [srcq+src_strideq*2] |
| 193 |
paddd m0, m3 |
| 194 |
dec n_rowsd |
| 195 |
jg .loop |
| 196 |
|
| 197 |
movhlps m1, m0 |
| 198 |
paddd m0, m1 |
| 199 |
movd eax, m0 |
| 200 |
RET |
| 201 |
%endmacro |
| 202 |
|
| 203 |
INIT_XMM sse2 |
| 204 |
SAD32XN 64 ; sad32x64_sse2 |
| 205 |
SAD32XN 32 ; sad32x32_sse2 |
| 206 |
SAD32XN 16 ; sad32x16_sse2 |
| 207 |
SAD32XN 64, 1 ; sad32x64_avg_sse2 |
| 208 |
SAD32XN 32, 1 ; sad32x32_avg_sse2 |
| 209 |
SAD32XN 16, 1 ; sad32x16_avg_sse2 |
| 210 |
SAD32XN 8 ; sad_32x8_sse2 |
| 211 |
SAD32XN 8, 1 ; sad_32x8_avg_sse2 |
| 212 |
|
| 213 |
; unsigned int aom_sad16x{8,16}_sse2(uint8_t *src, int src_stride, |
| 214 |
; uint8_t *ref, int ref_stride); |
| 215 |
%macro SAD16XN 1-2 0 |
| 216 |
SAD_FN 16, %1, 7, %2 |
| 217 |
mov n_rowsd, %1/4 |
| 218 |
pxor m0, m0 |
| 219 |
|
| 220 |
.loop: |
| 221 |
movu m1, [refq] |
| 222 |
movu m2, [refq+ref_strideq] |
| 223 |
movu m3, [refq+ref_strideq*2] |
| 224 |
movu m4, [refq+ref_stride3q] |
| 225 |
%if %2 == 1 |
| 226 |
pavgb m1, [second_predq+mmsize*0] |
| 227 |
pavgb m2, [second_predq+mmsize*1] |
| 228 |
pavgb m3, [second_predq+mmsize*2] |
| 229 |
pavgb m4, [second_predq+mmsize*3] |
| 230 |
lea second_predq, [second_predq+mmsize*4] |
| 231 |
%endif |
| 232 |
psadbw m1, [srcq] |
| 233 |
psadbw m2, [srcq+src_strideq] |
| 234 |
psadbw m3, [srcq+src_strideq*2] |
| 235 |
psadbw m4, [srcq+src_stride3q] |
| 236 |
paddd m1, m2 |
| 237 |
paddd m3, m4 |
| 238 |
lea refq, [refq+ref_strideq*4] |
| 239 |
paddd m0, m1 |
| 240 |
lea srcq, [srcq+src_strideq*4] |
| 241 |
paddd m0, m3 |
| 242 |
dec n_rowsd |
| 243 |
jg .loop |
| 244 |
|
| 245 |
movhlps m1, m0 |
| 246 |
paddd m0, m1 |
| 247 |
movd eax, m0 |
| 248 |
RET |
| 249 |
%endmacro |
| 250 |
|
| 251 |
INIT_XMM sse2 |
| 252 |
SAD16XN 32 ; sad16x32_sse2 |
| 253 |
SAD16XN 16 ; sad16x16_sse2 |
| 254 |
SAD16XN 8 ; sad16x8_sse2 |
| 255 |
SAD16XN 32, 1 ; sad16x32_avg_sse2 |
| 256 |
SAD16XN 16, 1 ; sad16x16_avg_sse2 |
| 257 |
SAD16XN 8, 1 ; sad16x8_avg_sse2 |
| 258 |
SAD16XN 4 ; sad_16x4_sse2 |
| 259 |
SAD16XN 4, 1 ; sad_16x4_avg_sse2 |
| 260 |
SAD16XN 64 ; sad_16x64_sse2 |
| 261 |
SAD16XN 64, 1 ; sad_16x64_avg_sse2 |
| 262 |
|
| 263 |
; unsigned int aom_sad8x{8,16}_sse2(uint8_t *src, int src_stride, |
| 264 |
; uint8_t *ref, int ref_stride); |
| 265 |
%macro SAD8XN 1-2 0 |
| 266 |
SAD_FN 8, %1, 7, %2 |
| 267 |
mov n_rowsd, %1/4 |
| 268 |
pxor m0, m0 |
| 269 |
|
| 270 |
.loop: |
| 271 |
movh m1, [refq] |
| 272 |
movhps m1, [refq+ref_strideq] |
| 273 |
movh m2, [refq+ref_strideq*2] |
| 274 |
movhps m2, [refq+ref_stride3q] |
| 275 |
%if %2 == 1 |
| 276 |
pavgb m1, [second_predq+mmsize*0] |
| 277 |
pavgb m2, [second_predq+mmsize*1] |
| 278 |
lea second_predq, [second_predq+mmsize*2] |
| 279 |
%endif |
| 280 |
movh m3, [srcq] |
| 281 |
movhps m3, [srcq+src_strideq] |
| 282 |
movh m4, [srcq+src_strideq*2] |
| 283 |
movhps m4, [srcq+src_stride3q] |
| 284 |
psadbw m1, m3 |
| 285 |
psadbw m2, m4 |
| 286 |
lea refq, [refq+ref_strideq*4] |
| 287 |
paddd m0, m1 |
| 288 |
lea srcq, [srcq+src_strideq*4] |
| 289 |
paddd m0, m2 |
| 290 |
dec n_rowsd |
| 291 |
jg .loop |
| 292 |
|
| 293 |
movhlps m1, m0 |
| 294 |
paddd m0, m1 |
| 295 |
movd eax, m0 |
| 296 |
RET |
| 297 |
%endmacro |
| 298 |
|
| 299 |
INIT_XMM sse2 |
| 300 |
SAD8XN 16 ; sad8x16_sse2 |
| 301 |
SAD8XN 8 ; sad8x8_sse2 |
| 302 |
SAD8XN 4 ; sad8x4_sse2 |
| 303 |
SAD8XN 16, 1 ; sad8x16_avg_sse2 |
| 304 |
SAD8XN 8, 1 ; sad8x8_avg_sse2 |
| 305 |
SAD8XN 4, 1 ; sad8x4_avg_sse2 |
| 306 |
SAD8XN 32 ; sad_8x32_sse2 |
| 307 |
SAD8XN 32, 1 ; sad_8x32_avg_sse2 |
| 308 |
|
| 309 |
; unsigned int aom_sad4x{4, 8}_sse2(uint8_t *src, int src_stride, |
| 310 |
; uint8_t *ref, int ref_stride); |
| 311 |
%macro SAD4XN 1-2 0 |
| 312 |
SAD_FN 4, %1, 7, %2 |
| 313 |
mov n_rowsd, %1/4 |
| 314 |
pxor m0, m0 |
| 315 |
|
| 316 |
.loop: |
| 317 |
movd m1, [refq] |
| 318 |
movd m2, [refq+ref_strideq] |
| 319 |
movd m3, [refq+ref_strideq*2] |
| 320 |
movd m4, [refq+ref_stride3q] |
| 321 |
punpckldq m1, m2 |
| 322 |
punpckldq m3, m4 |
| 323 |
movlhps m1, m3 |
| 324 |
%if %2 == 1 |
| 325 |
pavgb m1, [second_predq+mmsize*0] |
| 326 |
lea second_predq, [second_predq+mmsize*1] |
| 327 |
%endif |
| 328 |
movd m2, [srcq] |
| 329 |
movd m5, [srcq+src_strideq] |
| 330 |
movd m4, [srcq+src_strideq*2] |
| 331 |
movd m3, [srcq+src_stride3q] |
| 332 |
punpckldq m2, m5 |
| 333 |
punpckldq m4, m3 |
| 334 |
movlhps m2, m4 |
| 335 |
psadbw m1, m2 |
| 336 |
lea refq, [refq+ref_strideq*4] |
| 337 |
paddd m0, m1 |
| 338 |
lea srcq, [srcq+src_strideq*4] |
| 339 |
dec n_rowsd |
| 340 |
jg .loop |
| 341 |
|
| 342 |
movhlps m1, m0 |
| 343 |
paddd m0, m1 |
| 344 |
movd eax, m0 |
| 345 |
RET |
| 346 |
%endmacro |
| 347 |
|
| 348 |
INIT_XMM sse2 |
| 349 |
SAD4XN 8 ; sad4x8_sse |
| 350 |
SAD4XN 4 ; sad4x4_sse |
| 351 |
SAD4XN 8, 1 ; sad4x8_avg_sse |
| 352 |
SAD4XN 4, 1 ; sad4x4_avg_sse |
| 353 |
SAD4XN 16 ; sad_4x16_sse2 |
| 354 |
SAD4XN 16, 1 ; sad_4x16_avg_sse2 |