| 1 |
; Copyright © 2018, VideoLAN and dav1d authors |
| 2 |
; Copyright © 2018, Two Orioles, LLC |
| 3 |
; All rights reserved. |
| 4 |
; |
| 5 |
; Redistribution and use in source and binary forms, with or without |
| 6 |
; modification, are permitted provided that the following conditions are met: |
| 7 |
; |
| 8 |
; 1. Redistributions of source code must retain the above copyright notice, this |
| 9 |
; list of conditions and the following disclaimer. |
| 10 |
; |
| 11 |
; 2. Redistributions in binary form must reproduce the above copyright notice, |
| 12 |
; this list of conditions and the following disclaimer in the documentation |
| 13 |
; and/or other materials provided with the distribution. |
| 14 |
; |
| 15 |
; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND |
| 16 |
; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
| 17 |
; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE |
| 18 |
; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR |
| 19 |
; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES |
| 20 |
; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
| 21 |
; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND |
| 22 |
; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 23 |
; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS |
| 24 |
; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 25 |
|
| 26 |
%include "config.asm" |
| 27 |
%include "ext/x86/x86inc.asm" |
| 28 |
|
| 29 |
%if ARCH_X86_64 |
| 30 |
|
| 31 |
%macro JMP_TABLE 2-* |
| 32 |
%xdefine %1_jmptable %%table |
| 33 |
%xdefine %%base mangle(private_prefix %+ _%1_avx2) |
| 34 |
%%table: |
| 35 |
%rep %0 - 1 |
| 36 |
dd %%base %+ .%2 - %%table |
| 37 |
%rotate 1 |
| 38 |
%endrep |
| 39 |
%endmacro |
| 40 |
|
| 41 |
%macro CDEF_FILTER_JMP_TABLE 1 |
| 42 |
JMP_TABLE cdef_filter_%1_8bpc, \ |
| 43 |
d6k0, d6k1, d7k0, d7k1, \ |
| 44 |
d0k0, d0k1, d1k0, d1k1, d2k0, d2k1, d3k0, d3k1, \ |
| 45 |
d4k0, d4k1, d5k0, d5k1, d6k0, d6k1, d7k0, d7k1, \ |
| 46 |
d0k0, d0k1, d1k0, d1k1 |
| 47 |
%endmacro |
| 48 |
|
| 49 |
SECTION_RODATA 32 |
| 50 |
|
| 51 |
pd_47130256: dd 4, 7, 1, 3, 0, 2, 5, 6 |
| 52 |
blend_4x4: dd 0x00, 0x80, 0x00, 0x00, 0x80, 0x80, 0x00, 0x00 |
| 53 |
dd 0x80, 0x00, 0x00 |
| 54 |
blend_4x8_0: dd 0x00, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80 |
| 55 |
blend_4x8_1: dd 0x00, 0x00, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80 |
| 56 |
dd 0x00, 0x00 |
| 57 |
blend_4x8_2: dd 0x0000, 0x8080, 0x8080, 0x8080, 0x8080, 0x8080, 0x8080, 0x8080 |
| 58 |
dd 0x0000 |
| 59 |
blend_4x8_3: dd 0x0000, 0x0000, 0x8080, 0x8080, 0x8080, 0x8080, 0x8080, 0x8080 |
| 60 |
dd 0x0000, 0x0000 |
| 61 |
blend_8x8_0: dq 0x00, 0x00, 0x80, 0x80, 0x80, 0x80 |
| 62 |
blend_8x8_1: dq 0x0000, 0x0000, 0x8080, 0x8080, 0x8080, 0x8080, 0x0000, 0x0000 |
| 63 |
div_table: dd 840, 420, 280, 210, 168, 140, 120, 105, 420, 210, 140, 105 |
| 64 |
shufw_6543210x:db 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1, 14, 15 |
| 65 |
shufb_lohi: db 0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15 |
| 66 |
pw_128: times 2 dw 128 |
| 67 |
pw_2048: times 2 dw 2048 |
| 68 |
tap_table: ; masks for 8 bit shifts |
| 69 |
db 0xFF, 0x7F, 0x3F, 0x1F, 0x0F, 0x07, 0x03, 0x01 |
| 70 |
; weights |
| 71 |
db 4, 2, 3, 3, 2, 1 |
| 72 |
db -1 * 16 + 1, -2 * 16 + 2 |
| 73 |
db 0 * 16 + 1, -1 * 16 + 2 |
| 74 |
db 0 * 16 + 1, 0 * 16 + 2 |
| 75 |
db 0 * 16 + 1, 1 * 16 + 2 |
| 76 |
db 1 * 16 + 1, 2 * 16 + 2 |
| 77 |
db 1 * 16 + 0, 2 * 16 + 1 |
| 78 |
db 1 * 16 + 0, 2 * 16 + 0 |
| 79 |
db 1 * 16 + 0, 2 * 16 - 1 |
| 80 |
; the last 6 are repeats of the first 6 so we don't need to & 7 |
| 81 |
db -1 * 16 + 1, -2 * 16 + 2 |
| 82 |
db 0 * 16 + 1, -1 * 16 + 2 |
| 83 |
db 0 * 16 + 1, 0 * 16 + 2 |
| 84 |
db 0 * 16 + 1, 1 * 16 + 2 |
| 85 |
db 1 * 16 + 1, 2 * 16 + 2 |
| 86 |
db 1 * 16 + 0, 2 * 16 + 1 |
| 87 |
|
| 88 |
CDEF_FILTER_JMP_TABLE 4x4 |
| 89 |
CDEF_FILTER_JMP_TABLE 4x8 |
| 90 |
CDEF_FILTER_JMP_TABLE 8x8 |
| 91 |
|
| 92 |
SECTION .text |
| 93 |
|
| 94 |
%macro PREP_REGS 2 ; w, h |
| 95 |
; off1/2/3[k] [6 total] from [tapq+12+(dir+0/2/6)*2+k] |
| 96 |
mov dird, r7m |
| 97 |
lea tableq, [cdef_filter_%1x%2_8bpc_jmptable] |
| 98 |
lea dirq, [tableq+dirq*2*4] |
| 99 |
%if %1 == 4 |
| 100 |
%if %2 == 4 |
| 101 |
DEFINE_ARGS dst, stride, left, top, bot, pri, sec, \ |
| 102 |
table, dir, dirjmp, stride3, k |
| 103 |
%else |
| 104 |
DEFINE_ARGS dst, stride, left, top, bot, pri, sec, \ |
| 105 |
table, dir, dirjmp, dst4, stride3, k |
| 106 |
lea dst4q, [dstq+strideq*4] |
| 107 |
%endif |
| 108 |
%else |
| 109 |
DEFINE_ARGS dst, stride, h, top1, bot, pri, sec, \ |
| 110 |
table, dir, dirjmp, top2, stride3, k |
| 111 |
mov hq, -8 |
| 112 |
lea top1q, [top1q+strideq*0] |
| 113 |
lea top2q, [top1q+strideq*1] |
| 114 |
%endif |
| 115 |
%if %1 == 4 |
| 116 |
lea stride3q, [strideq*3] |
| 117 |
%endif |
| 118 |
%endmacro |
| 119 |
|
| 120 |
%macro LOAD_BLOCK 2-3 0 ; w, h, init_min_max |
| 121 |
mov kd, 1 |
| 122 |
pxor m15, m15 ; sum |
| 123 |
%if %2 == 8 |
| 124 |
pxor m12, m12 |
| 125 |
%if %1 == 4 |
| 126 |
movd xm4, [dstq +strideq*0] |
| 127 |
movd xm6, [dstq +strideq*1] |
| 128 |
movd xm5, [dstq +strideq*2] |
| 129 |
movd xm7, [dstq +stride3q ] |
| 130 |
vinserti128 m4, [dst4q+strideq*0], 1 |
| 131 |
vinserti128 m6, [dst4q+strideq*1], 1 |
| 132 |
vinserti128 m5, [dst4q+strideq*2], 1 |
| 133 |
vinserti128 m7, [dst4q+stride3q ], 1 |
| 134 |
punpckldq m4, m6 |
| 135 |
punpckldq m5, m7 |
| 136 |
%else |
| 137 |
movq xm4, [dstq+strideq*0] |
| 138 |
movq xm5, [dstq+strideq*1] |
| 139 |
vinserti128 m4, [dstq+strideq*2], 1 |
| 140 |
vinserti128 m5, [dstq+stride3q ], 1 |
| 141 |
%endif |
| 142 |
punpcklqdq m4, m5 |
| 143 |
%else |
| 144 |
movd xm4, [dstq+strideq*0] |
| 145 |
movd xm5, [dstq+strideq*1] |
| 146 |
vinserti128 m4, [dstq+strideq*2], 1 |
| 147 |
vinserti128 m5, [dstq+stride3q ], 1 |
| 148 |
punpckldq m4, m5 |
| 149 |
%endif |
| 150 |
%if %3 == 1 |
| 151 |
mova m7, m4 ; min |
| 152 |
mova m8, m4 ; max |
| 153 |
%endif |
| 154 |
%endmacro |
| 155 |
|
| 156 |
%macro ACCUMULATE_TAP_BYTE 7-8 0 ; tap_offset, shift, mask, strength |
| 157 |
; mul_tap, w, h, clip |
| 158 |
; load p0/p1 |
| 159 |
movsxd dirjmpq, [dirq+kq*4+%1*2*4] |
| 160 |
add dirjmpq, tableq |
| 161 |
call dirjmpq |
| 162 |
|
| 163 |
%if %8 == 1 |
| 164 |
pmaxub m7, m5 |
| 165 |
pminub m8, m5 |
| 166 |
pmaxub m7, m6 |
| 167 |
pminub m8, m6 |
| 168 |
%endif |
| 169 |
|
| 170 |
; accumulate sum[m15] over p0/p1 |
| 171 |
%if %7 == 4 |
| 172 |
punpcklbw m5, m6 |
| 173 |
punpcklbw m6, m4, m4 |
| 174 |
psubusb m9, m5, m6 |
| 175 |
psubusb m5, m6, m5 |
| 176 |
por m9, m5 ; abs_diff_p01(p01 - px) |
| 177 |
pcmpeqb m5, m9 |
| 178 |
por m5, %5 |
| 179 |
psignb m6, %5, m5 |
| 180 |
psrlw m5, m9, %2 ; emulate 8-bit shift |
| 181 |
pand m5, %3 |
| 182 |
psubusb m5, %4, m5 |
| 183 |
pminub m5, m9 |
| 184 |
pmaddubsw m5, m6 |
| 185 |
paddw m15, m5 |
| 186 |
%else |
| 187 |
psubusb m9, m5, m4 |
| 188 |
psubusb m5, m4, m5 |
| 189 |
psubusb m11, m6, m4 |
| 190 |
psubusb m6, m4, m6 |
| 191 |
por m9, m5 ; abs_diff_p0(p0 - px) |
| 192 |
por m11, m6 ; abs_diff_p1(p1 - px) |
| 193 |
pcmpeqb m5, m9 |
| 194 |
pcmpeqb m6, m11 |
| 195 |
punpckhbw m10, m9, m11 |
| 196 |
punpcklbw m9, m11 |
| 197 |
por m5, %5 |
| 198 |
por m11, m6, %5 |
| 199 |
punpckhbw m6, m5, m11 |
| 200 |
punpcklbw m5, m11 |
| 201 |
psignb m11, %5, m6 |
| 202 |
psrlw m6, m10, %2 ; emulate 8-bit shift |
| 203 |
pand m6, %3 |
| 204 |
psubusb m6, %4, m6 |
| 205 |
pminub m6, m10 |
| 206 |
pmaddubsw m6, m11 |
| 207 |
paddw m12, m6 |
| 208 |
psignb m11, %5, m5 |
| 209 |
psrlw m5, m9, %2 ; emulate 8-bit shift |
| 210 |
pand m5, %3 |
| 211 |
psubusb m5, %4, m5 |
| 212 |
pminub m5, m9 |
| 213 |
pmaddubsw m5, m11 |
| 214 |
paddw m15, m5 |
| 215 |
%endif |
| 216 |
%endmacro |
| 217 |
|
| 218 |
%macro ADJUST_PIXEL 4-5 0 ; w, h, zero, pw_2048, clip |
| 219 |
%if %2 == 4 |
| 220 |
%if %5 == 1 |
| 221 |
punpcklbw m4, %3 |
| 222 |
%endif |
| 223 |
pcmpgtw %3, m15 |
| 224 |
paddw m15, %3 |
| 225 |
pmulhrsw m15, %4 |
| 226 |
%if %5 == 0 |
| 227 |
packsswb m15, m15 |
| 228 |
paddb m4, m15 |
| 229 |
%else |
| 230 |
paddw m4, m15 |
| 231 |
packuswb m4, m4 ; clip px in [0x0,0xff] |
| 232 |
pminub m4, m7 |
| 233 |
pmaxub m4, m8 |
| 234 |
%endif |
| 235 |
vextracti128 xm5, m4, 1 |
| 236 |
movd [dstq+strideq*0], xm4 |
| 237 |
movd [dstq+strideq*2], xm5 |
| 238 |
pextrd [dstq+strideq*1], xm4, 1 |
| 239 |
pextrd [dstq+stride3q ], xm5, 1 |
| 240 |
%else |
| 241 |
pcmpgtw m6, %3, m12 |
| 242 |
pcmpgtw m5, %3, m15 |
| 243 |
paddw m12, m6 |
| 244 |
paddw m15, m5 |
| 245 |
%if %5 == 1 |
| 246 |
punpckhbw m5, m4, %3 |
| 247 |
punpcklbw m4, %3 |
| 248 |
%endif |
| 249 |
pmulhrsw m12, %4 |
| 250 |
pmulhrsw m15, %4 |
| 251 |
%if %5 == 0 |
| 252 |
packsswb m15, m12 |
| 253 |
paddb m4, m15 |
| 254 |
%else |
| 255 |
paddw m5, m12 |
| 256 |
paddw m4, m15 |
| 257 |
packuswb m4, m5 ; clip px in [0x0,0xff] |
| 258 |
pminub m4, m7 |
| 259 |
pmaxub m4, m8 |
| 260 |
%endif |
| 261 |
vextracti128 xm5, m4, 1 |
| 262 |
%if %1 == 4 |
| 263 |
movd [dstq +strideq*0], xm4 |
| 264 |
movd [dst4q+strideq*0], xm5 |
| 265 |
pextrd [dstq +strideq*1], xm4, 1 |
| 266 |
pextrd [dst4q+strideq*1], xm5, 1 |
| 267 |
pextrd [dstq +strideq*2], xm4, 2 |
| 268 |
pextrd [dst4q+strideq*2], xm5, 2 |
| 269 |
pextrd [dstq +stride3q ], xm4, 3 |
| 270 |
pextrd [dst4q+stride3q ], xm5, 3 |
| 271 |
%else |
| 272 |
movq [dstq+strideq*0], xm4 |
| 273 |
movq [dstq+strideq*2], xm5 |
| 274 |
movhps [dstq+strideq*1], xm4 |
| 275 |
movhps [dstq+stride3q ], xm5 |
| 276 |
%endif |
| 277 |
%endif |
| 278 |
%endmacro |
| 279 |
|
| 280 |
%macro BORDER_PREP_REGS 2 ; w, h |
| 281 |
; off1/2/3[k] [6 total] from [tapq+12+(dir+0/2/6)*2+k] |
| 282 |
mov dird, r7m |
| 283 |
lea dirq, [tableq+dirq*2+14] |
| 284 |
%if %1*%2*2/mmsize > 1 |
| 285 |
%if %1 == 4 |
| 286 |
DEFINE_ARGS dst, stride, k, dir, stk, pri, sec, stride3, h, off |
| 287 |
%else |
| 288 |
DEFINE_ARGS dst, stride, k, dir, stk, pri, sec, h, off |
| 289 |
%endif |
| 290 |
mov hd, %1*%2*2/mmsize |
| 291 |
%else |
| 292 |
DEFINE_ARGS dst, stride, k, dir, stk, pri, sec, stride3, off |
| 293 |
%endif |
| 294 |
lea stkq, [px] |
| 295 |
pxor m11, m11 |
| 296 |
%endmacro |
| 297 |
|
| 298 |
%macro BORDER_LOAD_BLOCK 2-3 0 ; w, h, init_min_max |
| 299 |
mov kd, 1 |
| 300 |
%if %1 == 4 |
| 301 |
movq xm4, [stkq+32*0] |
| 302 |
movhps xm4, [stkq+32*1] |
| 303 |
movq xm5, [stkq+32*2] |
| 304 |
movhps xm5, [stkq+32*3] |
| 305 |
vinserti128 m4, xm5, 1 |
| 306 |
%else |
| 307 |
mova xm4, [stkq+32*0] ; px |
| 308 |
vinserti128 m4, [stkq+32*1], 1 |
| 309 |
%endif |
| 310 |
pxor m15, m15 ; sum |
| 311 |
%if %3 == 1 |
| 312 |
mova m7, m4 ; max |
| 313 |
mova m8, m4 ; min |
| 314 |
%endif |
| 315 |
%endmacro |
| 316 |
|
| 317 |
%macro ACCUMULATE_TAP_WORD 6-7 0 ; tap_offset, shift, mask, strength |
| 318 |
; mul_tap, w, clip |
| 319 |
; load p0/p1 |
| 320 |
movsx offq, byte [dirq+kq+%1] ; off1 |
| 321 |
%if %6 == 4 |
| 322 |
movq xm5, [stkq+offq*2+32*0] ; p0 |
| 323 |
movq xm6, [stkq+offq*2+32*2] |
| 324 |
movhps xm5, [stkq+offq*2+32*1] |
| 325 |
movhps xm6, [stkq+offq*2+32*3] |
| 326 |
vinserti128 m5, xm6, 1 |
| 327 |
%else |
| 328 |
movu xm5, [stkq+offq*2+32*0] ; p0 |
| 329 |
vinserti128 m5, [stkq+offq*2+32*1], 1 |
| 330 |
%endif |
| 331 |
neg offq ; -off1 |
| 332 |
%if %6 == 4 |
| 333 |
movq xm6, [stkq+offq*2+32*0] ; p1 |
| 334 |
movq xm9, [stkq+offq*2+32*2] |
| 335 |
movhps xm6, [stkq+offq*2+32*1] |
| 336 |
movhps xm9, [stkq+offq*2+32*3] |
| 337 |
vinserti128 m6, xm9, 1 |
| 338 |
%else |
| 339 |
movu xm6, [stkq+offq*2+32*0] ; p1 |
| 340 |
vinserti128 m6, [stkq+offq*2+32*1], 1 |
| 341 |
%endif |
| 342 |
%if %7 == 1 |
| 343 |
; out of bounds values are set to a value that is a both a large unsigned |
| 344 |
; value and a negative signed value. |
| 345 |
; use signed max and unsigned min to remove them |
| 346 |
pmaxsw m7, m5 ; max after p0 |
| 347 |
pminuw m8, m5 ; min after p0 |
| 348 |
pmaxsw m7, m6 ; max after p1 |
| 349 |
pminuw m8, m6 ; min after p1 |
| 350 |
%endif |
| 351 |
|
| 352 |
; accumulate sum[m15] over p0/p1 |
| 353 |
; calculate difference before converting |
| 354 |
psubw m5, m4 ; diff_p0(p0 - px) |
| 355 |
psubw m6, m4 ; diff_p1(p1 - px) |
| 356 |
|
| 357 |
; convert to 8-bits with signed saturation |
| 358 |
; saturating to large diffs has no impact on the results |
| 359 |
packsswb m5, m6 |
| 360 |
|
| 361 |
; group into pairs so we can accumulate using maddubsw |
| 362 |
pshufb m5, m12 |
| 363 |
pabsb m9, m5 |
| 364 |
psignb m10, %5, m5 |
| 365 |
psrlw m5, m9, %2 ; emulate 8-bit shift |
| 366 |
pand m5, %3 |
| 367 |
psubusb m5, %4, m5 |
| 368 |
|
| 369 |
; use unsigned min since abs diff can equal 0x80 |
| 370 |
pminub m5, m9 |
| 371 |
pmaddubsw m5, m10 |
| 372 |
paddw m15, m5 |
| 373 |
%endmacro |
| 374 |
|
| 375 |
%macro BORDER_ADJUST_PIXEL 2-3 0 ; w, pw_2048, clip |
| 376 |
pcmpgtw m9, m11, m15 |
| 377 |
paddw m15, m9 |
| 378 |
pmulhrsw m15, %2 |
| 379 |
paddw m4, m15 |
| 380 |
%if %3 == 1 |
| 381 |
pminsw m4, m7 |
| 382 |
pmaxsw m4, m8 |
| 383 |
%endif |
| 384 |
packuswb m4, m4 |
| 385 |
vextracti128 xm5, m4, 1 |
| 386 |
%if %1 == 4 |
| 387 |
movd [dstq+strideq*0], xm4 |
| 388 |
pextrd [dstq+strideq*1], xm4, 1 |
| 389 |
movd [dstq+strideq*2], xm5 |
| 390 |
pextrd [dstq+stride3q ], xm5, 1 |
| 391 |
%else |
| 392 |
movq [dstq+strideq*0], xm4 |
| 393 |
movq [dstq+strideq*1], xm5 |
| 394 |
%endif |
| 395 |
%endmacro |
| 396 |
|
| 397 |
%macro CDEF_FILTER 2 ; w, h |
| 398 |
INIT_YMM avx2 |
| 399 |
cglobal cdef_filter_%1x%2_8bpc, 5, 11, 0, dst, stride, left, top, bot, \ |
| 400 |
pri, sec, dir, damping, edge |
| 401 |
%assign stack_offset_entry stack_offset |
| 402 |
mov edged, edgem |
| 403 |
cmp edged, 0xf |
| 404 |
jne .border_block |
| 405 |
|
| 406 |
PUSH r11 |
| 407 |
PUSH r12 |
| 408 |
%if %2 == 4 |
| 409 |
%assign regs_used 13 |
| 410 |
ALLOC_STACK 0x60, 16 |
| 411 |
pmovzxbw xm0, [leftq+1] |
| 412 |
vpermq m0, m0, q0110 |
| 413 |
psrldq m1, m0, 4 |
| 414 |
vpalignr m2, m0, m0, 12 |
| 415 |
movu [rsp+0x10], m0 |
| 416 |
movu [rsp+0x28], m1 |
| 417 |
movu [rsp+0x40], m2 |
| 418 |
%elif %1 == 4 |
| 419 |
%assign regs_used 14 |
| 420 |
PUSH r13 |
| 421 |
ALLOC_STACK 8*2+%1*%2*1, 16 |
| 422 |
pmovzxwd m0, [leftq] |
| 423 |
mova [rsp+0x10], m0 |
| 424 |
%else |
| 425 |
%assign regs_used 15 |
| 426 |
PUSH r13 |
| 427 |
PUSH r14 |
| 428 |
ALLOC_STACK 8*4+%1*%2*2+32, 16 |
| 429 |
lea r11, [strideq*3] |
| 430 |
movu xm4, [dstq+strideq*2] |
| 431 |
pmovzxwq m0, [leftq+0] |
| 432 |
pmovzxwq m1, [leftq+8] |
| 433 |
vinserti128 m4, [dstq+r11], 1 |
| 434 |
pmovzxbd m2, [leftq+1] |
| 435 |
pmovzxbd m3, [leftq+9] |
| 436 |
mov [rsp+16], botq |
| 437 |
mova [rsp+0x20], m0 |
| 438 |
mova [rsp+0x40], m1 |
| 439 |
mova [rsp+0x60], m2 |
| 440 |
mova [rsp+0x80], m3 |
| 441 |
mova [rsp+0xa0], m4 |
| 442 |
lea botq, [dstq+strideq*4] |
| 443 |
%endif |
| 444 |
|
| 445 |
DEFINE_ARGS dst, stride, left, top, bot, pri, secdmp, zero, pridmp, damping |
| 446 |
mov dampingd, r8m |
| 447 |
xor zerod, zerod |
| 448 |
movifnidn prid, prim |
| 449 |
sub dampingd, 31 |
| 450 |
movifnidn secdmpd, secdmpm |
| 451 |
test prid, prid |
| 452 |
jz .sec_only |
| 453 |
movd xm0, prid |
| 454 |
lzcnt pridmpd, prid |
| 455 |
add pridmpd, dampingd |
| 456 |
cmovs pridmpd, zerod |
| 457 |
mov [rsp+0], pridmpq ; pri_shift |
| 458 |
test secdmpd, secdmpd |
| 459 |
jz .pri_only |
| 460 |
movd xm1, secdmpd |
| 461 |
lzcnt secdmpd, secdmpd |
| 462 |
add secdmpd, dampingd |
| 463 |
mov [rsp+8], secdmpq ; sec_shift |
| 464 |
|
| 465 |
DEFINE_ARGS dst, stride, left, top, bot, pri, secdmp, table, pridmp |
| 466 |
lea tableq, [tap_table] |
| 467 |
vpbroadcastb m13, [tableq+pridmpq] ; pri_shift_mask |
| 468 |
vpbroadcastb m14, [tableq+secdmpq] ; sec_shift_mask |
| 469 |
|
| 470 |
; pri/sec_taps[k] [4 total] |
| 471 |
DEFINE_ARGS dst, stride, left, top, bot, pri, sec, table, dir |
| 472 |
vpbroadcastb m0, xm0 ; pri_strength |
| 473 |
vpbroadcastb m1, xm1 ; sec_strength |
| 474 |
and prid, 1 |
| 475 |
lea priq, [tableq+priq*2+8] ; pri_taps |
| 476 |
lea secq, [tableq+12] ; sec_taps |
| 477 |
|
| 478 |
PREP_REGS %1, %2 |
| 479 |
%if %1*%2 > mmsize |
| 480 |
.v_loop: |
| 481 |
%endif |
| 482 |
LOAD_BLOCK %1, %2, 1 |
| 483 |
.k_loop: |
| 484 |
vpbroadcastb m2, [priq+kq] ; pri_taps |
| 485 |
vpbroadcastb m3, [secq+kq] ; sec_taps |
| 486 |
ACCUMULATE_TAP_BYTE 2, [rsp+0], m13, m0, m2, %1, %2, 1 ; dir + 0 |
| 487 |
ACCUMULATE_TAP_BYTE 4, [rsp+8], m14, m1, m3, %1, %2, 1 ; dir + 2 |
| 488 |
ACCUMULATE_TAP_BYTE 0, [rsp+8], m14, m1, m3, %1, %2, 1 ; dir - 2 |
| 489 |
dec kq |
| 490 |
jge .k_loop |
| 491 |
|
| 492 |
vpbroadcastd m10, [pw_2048] |
| 493 |
pxor m9, m9 |
| 494 |
ADJUST_PIXEL %1, %2, m9, m10, 1 |
| 495 |
%if %1*%2 > mmsize |
| 496 |
lea dstq, [dstq+strideq*4] |
| 497 |
lea top1q, [rsp+0xa0] |
| 498 |
lea top2q, [rsp+0xb0] |
| 499 |
mov botq, [rsp+16] |
| 500 |
add hq, 4 |
| 501 |
jl .v_loop |
| 502 |
%endif |
| 503 |
RET |
| 504 |
|
| 505 |
.pri_only: |
| 506 |
DEFINE_ARGS dst, stride, left, top, bot, pri, _, table, pridmp |
| 507 |
lea tableq, [tap_table] |
| 508 |
vpbroadcastb m13, [tableq+pridmpq] ; pri_shift_mask |
| 509 |
; pri/sec_taps[k] [4 total] |
| 510 |
DEFINE_ARGS dst, stride, left, top, bot, pri, _, table, dir |
| 511 |
vpbroadcastb m0, xm0 ; pri_strength |
| 512 |
and prid, 1 |
| 513 |
lea priq, [tableq+priq*2+8] ; pri_taps |
| 514 |
PREP_REGS %1, %2 |
| 515 |
vpbroadcastd m3, [pw_2048] |
| 516 |
pxor m1, m1 |
| 517 |
%if %1*%2 > mmsize |
| 518 |
.pri_v_loop: |
| 519 |
%endif |
| 520 |
LOAD_BLOCK %1, %2 |
| 521 |
.pri_k_loop: |
| 522 |
vpbroadcastb m2, [priq+kq] ; pri_taps |
| 523 |
ACCUMULATE_TAP_BYTE 2, [rsp+0], m13, m0, m2, %1, %2 ; dir + 0 |
| 524 |
dec kq |
| 525 |
jge .pri_k_loop |
| 526 |
ADJUST_PIXEL %1, %2, m1, m3 |
| 527 |
%if %1*%2 > mmsize |
| 528 |
lea dstq, [dstq+strideq*4] |
| 529 |
lea top1q, [rsp+0xa0] |
| 530 |
lea top2q, [rsp+0xb0] |
| 531 |
mov botq, [rsp+16] |
| 532 |
add hq, 4 |
| 533 |
jl .pri_v_loop |
| 534 |
%endif |
| 535 |
RET |
| 536 |
|
| 537 |
.sec_only: |
| 538 |
DEFINE_ARGS dst, stride, left, top, bot, _, secdmp, zero, _, damping |
| 539 |
movd xm1, secdmpd |
| 540 |
lzcnt secdmpd, secdmpd |
| 541 |
add secdmpd, dampingd |
| 542 |
mov [rsp+8], secdmpq ; sec_shift |
| 543 |
DEFINE_ARGS dst, stride, left, top, bot, _, secdmp, table |
| 544 |
lea tableq, [tap_table] |
| 545 |
vpbroadcastb m14, [tableq+secdmpq] ; sec_shift_mask |
| 546 |
; pri/sec_taps[k] [4 total] |
| 547 |
DEFINE_ARGS dst, stride, left, top, bot, _, sec, table, dir |
| 548 |
vpbroadcastb m1, xm1 ; sec_strength |
| 549 |
lea secq, [tableq+12] ; sec_taps |
| 550 |
PREP_REGS %1, %2 |
| 551 |
vpbroadcastd m2, [pw_2048] |
| 552 |
pxor m0, m0 |
| 553 |
%if %1*%2 > mmsize |
| 554 |
.sec_v_loop: |
| 555 |
%endif |
| 556 |
LOAD_BLOCK %1, %2 |
| 557 |
.sec_k_loop: |
| 558 |
vpbroadcastb m3, [secq+kq] ; sec_taps |
| 559 |
ACCUMULATE_TAP_BYTE 4, [rsp+8], m14, m1, m3, %1, %2 ; dir + 2 |
| 560 |
ACCUMULATE_TAP_BYTE 0, [rsp+8], m14, m1, m3, %1, %2 ; dir - 2 |
| 561 |
dec kq |
| 562 |
jge .sec_k_loop |
| 563 |
ADJUST_PIXEL %1, %2, m0, m2 |
| 564 |
%if %1*%2 > mmsize |
| 565 |
lea dstq, [dstq+strideq*4] |
| 566 |
lea top1q, [rsp+0xa0] |
| 567 |
lea top2q, [rsp+0xb0] |
| 568 |
mov botq, [rsp+16] |
| 569 |
add hq, 4 |
| 570 |
jl .sec_v_loop |
| 571 |
%endif |
| 572 |
RET |
| 573 |
|
| 574 |
.d0k0: |
| 575 |
%if %1 == 4 |
| 576 |
%if %2 == 4 |
| 577 |
vpbroadcastq m6, [dstq+strideq*1-1] |
| 578 |
vpbroadcastq m10, [dstq+strideq*2-1] |
| 579 |
movd xm5, [topq+strideq*1+1] |
| 580 |
movd xm9, [dstq+strideq*0+1] |
| 581 |
psrldq m11, m6, 2 |
| 582 |
psrldq m12, m10, 2 |
| 583 |
vinserti128 m6, [dstq+stride3q -1], 1 |
| 584 |
vinserti128 m10, [botq -1], 1 |
| 585 |
vpblendd m5, m11, 0x10 |
| 586 |
vpblendd m9, m12, 0x10 |
| 587 |
movu m11, [blend_4x4+16] |
| 588 |
punpckldq m6, m10 |
| 589 |
punpckldq m5, m9 |
| 590 |
vpblendvb m6, [rsp+gprsize+0x28], m11 |
| 591 |
%else |
| 592 |
movd xm5, [topq +strideq*1+1] |
| 593 |
movq xm6, [dstq +strideq*1-1] |
| 594 |
movq xm10, [dstq +stride3q -1] |
| 595 |
movq xm11, [dst4q+strideq*1-1] |
| 596 |
pinsrd xm5, [dstq +strideq*0+1], 1 |
| 597 |
movhps xm6, [dstq +strideq*2-1] |
| 598 |
movhps xm10, [dst4q+strideq*0-1] |
| 599 |
movhps xm11, [dst4q+strideq*2-1] |
| 600 |
psrldq xm9, xm6, 2 |
| 601 |
shufps xm5, xm9, q2010 ; -1 +0 +1 +2 |
| 602 |
shufps xm6, xm10, q2020 ; +1 +2 +3 +4 |
| 603 |
psrldq xm9, xm11, 2 |
| 604 |
psrldq xm10, 2 |
| 605 |
shufps xm10, xm9, q2020 ; +3 +4 +5 +6 |
| 606 |
movd xm9, [dst4q+stride3q -1] |
| 607 |
pinsrd xm9, [botq -1], 1 |
| 608 |
shufps xm11, xm9, q1020 ; +5 +6 +7 +8 |
| 609 |
pmovzxbw m9, [leftq+3] |
| 610 |
vinserti128 m6, xm11, 1 |
| 611 |
movu m11, [blend_4x8_0+4] |
| 612 |
vinserti128 m5, xm10, 1 |
| 613 |
vpblendvb m6, m9, m11 |
| 614 |
%endif |
| 615 |
%else |
| 616 |
lea r13, [blend_8x8_0+16] |
| 617 |
movq xm5, [top2q +1] |
| 618 |
vbroadcasti128 m10, [dstq+strideq*1-1] |
| 619 |
vbroadcasti128 m11, [dstq+strideq*2-1] |
| 620 |
movhps xm5, [dstq+strideq*0+1] |
| 621 |
vinserti128 m6, m10, [dstq+stride3q-1], 1 |
| 622 |
vinserti128 m9, m11, [botq -1], 1 |
| 623 |
psrldq m10, 2 |
| 624 |
psrldq m11, 2 |
| 625 |
punpcklqdq m6, m9 |
| 626 |
movu m9, [r13+hq*2*1+16*1] |
| 627 |
punpcklqdq m10, m11 |
| 628 |
vpblendd m5, m10, 0xF0 |
| 629 |
vpblendvb m6, [rsp+gprsize+0x60+hq*8+64+8*1], m9 |
| 630 |
%endif |
| 631 |
ret |
| 632 |
.d1k0: |
| 633 |
.d2k0: |
| 634 |
.d3k0: |
| 635 |
%if %1 == 4 |
| 636 |
%if %2 == 4 |
| 637 |
movq xm6, [dstq+strideq*0-1] |
| 638 |
movq xm9, [dstq+strideq*1-1] |
| 639 |
vinserti128 m6, [dstq+strideq*2-1], 1 |
| 640 |
vinserti128 m9, [dstq+stride3q -1], 1 |
| 641 |
movu m11, [rsp+gprsize+0x10] |
| 642 |
pcmpeqd m12, m12 |
| 643 |
psrldq m5, m6, 2 |
| 644 |
psrldq m10, m9, 2 |
| 645 |
psrld m12, 24 |
| 646 |
punpckldq m6, m9 |
| 647 |
punpckldq m5, m10 |
| 648 |
vpblendvb m6, m11, m12 |
| 649 |
%else |
| 650 |
movq xm6, [dstq +strideq*0-1] |
| 651 |
movq xm9, [dstq +strideq*2-1] |
| 652 |
movhps xm6, [dstq +strideq*1-1] |
| 653 |
movhps xm9, [dstq +stride3q -1] |
| 654 |
movq xm10, [dst4q+strideq*0-1] |
| 655 |
movhps xm10, [dst4q+strideq*1-1] |
| 656 |
psrldq xm5, xm6, 2 |
| 657 |
psrldq xm11, xm9, 2 |
| 658 |
shufps xm5, xm11, q2020 |
| 659 |
movq xm11, [dst4q+strideq*2-1] |
| 660 |
movhps xm11, [dst4q+stride3q -1] |
| 661 |
shufps xm6, xm9, q2020 |
| 662 |
shufps xm9, xm10, xm11, q2020 |
| 663 |
vinserti128 m6, xm9, 1 |
| 664 |
pmovzxbw m9, [leftq+1] |
| 665 |
psrldq xm10, 2 |
| 666 |
psrldq xm11, 2 |
| 667 |
shufps xm10, xm11, q2020 |
| 668 |
vpbroadcastd m11, [blend_4x8_0+4] |
| 669 |
vinserti128 m5, xm10, 1 |
| 670 |
vpblendvb m6, m9, m11 |
| 671 |
%endif |
| 672 |
%else |
| 673 |
movu xm5, [dstq+strideq*0-1] |
| 674 |
movu xm9, [dstq+strideq*1-1] |
| 675 |
vinserti128 m5, [dstq+strideq*2-1], 1 |
| 676 |
vinserti128 m9, [dstq+stride3q -1], 1 |
| 677 |
movu m10, [blend_8x8_0+16] |
| 678 |
punpcklqdq m6, m5, m9 |
| 679 |
vpblendvb m6, [rsp+gprsize+0x60+hq*8+64], m10 |
| 680 |
psrldq m5, 2 |
| 681 |
psrldq m9, 2 |
| 682 |
punpcklqdq m5, m9 |
| 683 |
%endif |
| 684 |
ret |
| 685 |
.d4k0: |
| 686 |
%if %1 == 4 |
| 687 |
%if %2 == 4 |
| 688 |
vpbroadcastq m10, [dstq+strideq*1-1] |
| 689 |
vpbroadcastq m11, [dstq+strideq*2-1] |
| 690 |
movd xm6, [topq+strideq*1-1] |
| 691 |
movd xm9, [dstq+strideq*0-1] |
| 692 |
psrldq m5, m10, 2 |
| 693 |
psrldq m12, m11, 2 |
| 694 |
vpblendd m6, m10, 0x10 |
| 695 |
vpblendd m9, m11, 0x10 |
| 696 |
movu m10, [blend_4x4] |
| 697 |
vinserti128 m5, [dstq+stride3q +1], 1 |
| 698 |
vinserti128 m12, [botq +1], 1 |
| 699 |
punpckldq m6, m9 |
| 700 |
punpckldq m5, m12 |
| 701 |
vpblendvb m6, [rsp+gprsize+0x40], m10 |
| 702 |
%else |
| 703 |
movd xm6, [topq +strideq*1-1] |
| 704 |
movq xm9, [dstq +strideq*1-1] |
| 705 |
movq xm10, [dstq +stride3q -1] |
| 706 |
movq xm11, [dst4q+strideq*1-1] |
| 707 |
pinsrd xm6, [dstq +strideq*0-1], 1 |
| 708 |
movhps xm9, [dstq +strideq*2-1] |
| 709 |
movhps xm10, [dst4q+strideq*0-1] |
| 710 |
movhps xm11, [dst4q+strideq*2-1] |
| 711 |
psrldq xm5, xm9, 2 |
| 712 |
shufps xm6, xm9, q2010 |
| 713 |
psrldq xm9, xm10, 2 |
| 714 |
shufps xm5, xm9, q2020 |
| 715 |
shufps xm10, xm11, q2020 |
| 716 |
movd xm9, [dst4q+stride3q +1] |
| 717 |
vinserti128 m6, xm10, 1 |
| 718 |
pinsrd xm9, [botq +1], 1 |
| 719 |
psrldq xm11, 2 |
| 720 |
pmovzxbw m10, [leftq-1] |
| 721 |
shufps xm11, xm9, q1020 |
| 722 |
movu m9, [blend_4x8_0] |
| 723 |
vinserti128 m5, xm11, 1 |
| 724 |
vpblendvb m6, m10, m9 |
| 725 |
%endif |
| 726 |
%else |
| 727 |
lea r13, [blend_8x8_0+8] |
| 728 |
movq xm6, [top2q -1] |
| 729 |
vbroadcasti128 m5, [dstq+strideq*1-1] |
| 730 |
vbroadcasti128 m9, [dstq+strideq*2-1] |
| 731 |
movhps xm6, [dstq+strideq*0-1] |
| 732 |
movu m11, [r13+hq*2*1+16*1] |
| 733 |
punpcklqdq m10, m5, m9 |
| 734 |
vinserti128 m5, [dstq+stride3q -1], 1 |
| 735 |
vinserti128 m9, [botq -1], 1 |
| 736 |
vpblendd m6, m10, 0xF0 |
| 737 |
vpblendvb m6, [rsp+gprsize+0x60+hq*8+64-8*1], m11 |
| 738 |
psrldq m5, 2 |
| 739 |
psrldq m9, 2 |
| 740 |
punpcklqdq m5, m9 |
| 741 |
%endif |
| 742 |
ret |
| 743 |
.d5k0: |
| 744 |
.d6k0: |
| 745 |
.d7k0: |
| 746 |
%if %1 == 4 |
| 747 |
%if %2 == 4 |
| 748 |
movd xm6, [topq+strideq*1 ] |
| 749 |
vpbroadcastd m5, [dstq+strideq*1 ] |
| 750 |
vpbroadcastd m9, [dstq+strideq*2 ] |
| 751 |
vpblendd xm6, [dstq+strideq*0-4], 0x2 |
| 752 |
vpblendd m5, m9, 0x22 |
| 753 |
vpblendd m6, m5, 0x30 |
| 754 |
vinserti128 m5, [dstq+stride3q ], 1 |
| 755 |
vpblendd m5, [botq -20], 0x20 |
| 756 |
%else |
| 757 |
movd xm6, [topq +strideq*1] |
| 758 |
movd xm5, [dstq +strideq*1] |
| 759 |
movd xm9, [dstq +stride3q ] |
| 760 |
movd xm10, [dst4q+strideq*1] |
| 761 |
movd xm11, [dst4q+stride3q ] |
| 762 |
pinsrd xm6, [dstq +strideq*0], 1 |
| 763 |
pinsrd xm5, [dstq +strideq*2], 1 |
| 764 |
pinsrd xm9, [dst4q+strideq*0], 1 |
| 765 |
pinsrd xm10, [dst4q+strideq*2], 1 |
| 766 |
pinsrd xm11, [botq ], 1 |
| 767 |
punpcklqdq xm6, xm5 |
| 768 |
punpcklqdq xm5, xm9 |
| 769 |
punpcklqdq xm9, xm10 |
| 770 |
punpcklqdq xm10, xm11 |
| 771 |
vinserti128 m6, xm9, 1 |
| 772 |
vinserti128 m5, xm10, 1 |
| 773 |
%endif |
| 774 |
%else |
| 775 |
movq xm6, [top2q ] |
| 776 |
movq xm5, [dstq+strideq*1] |
| 777 |
movq xm9, [dstq+stride3q ] |
| 778 |
movhps xm6, [dstq+strideq*0] |
| 779 |
movhps xm5, [dstq+strideq*2] |
| 780 |
movhps xm9, [botq ] |
| 781 |
vinserti128 m6, xm5, 1 |
| 782 |
vinserti128 m5, xm9, 1 |
| 783 |
%endif |
| 784 |
ret |
| 785 |
.d0k1: |
| 786 |
%if %1 == 4 |
| 787 |
%if %2 == 4 |
| 788 |
movd xm6, [dstq+strideq*2-2] |
| 789 |
movd xm9, [dstq+stride3q -2] |
| 790 |
movd xm5, [topq+strideq*0+2] |
| 791 |
movd xm10, [topq+strideq*1+2] |
| 792 |
pinsrw xm6, [leftq+4], 0 |
| 793 |
pinsrw xm9, [leftq+6], 0 |
| 794 |
vinserti128 m5, [dstq+strideq*0+2], 1 |
| 795 |
vinserti128 m10, [dstq+strideq*1+2], 1 |
| 796 |
vinserti128 m6, [botq+strideq*0-2], 1 |
| 797 |
vinserti128 m9, [botq+strideq*1-2], 1 |
| 798 |
punpckldq m5, m10 |
| 799 |
punpckldq m6, m9 |
| 800 |
%else |
| 801 |
movq xm6, [dstq +strideq*2-2] |
| 802 |
movd xm10, [dst4q+strideq*2-2] |
| 803 |
movd xm5, [topq +strideq*0+2] |
| 804 |
movq xm9, [dst4q+strideq*0-2] |
| 805 |
movhps xm6, [dstq +stride3q -2] |
| 806 |
pinsrw xm10, [dst4q+stride3q ], 3 |
| 807 |
pinsrd xm5, [topq +strideq*1+2], 1 |
| 808 |
movhps xm9, [dst4q+strideq*1-2] |
| 809 |
pinsrd xm10, [botq +strideq*0-2], 2 |
| 810 |
pinsrd xm5, [dstq +strideq*0+2], 2 |
| 811 |
pinsrd xm10, [botq +strideq*1-2], 3 |
| 812 |
pinsrd xm5, [dstq +strideq*1+2], 3 |
| 813 |
shufps xm11, xm6, xm9, q3131 |
| 814 |
shufps xm6, xm9, q2020 |
| 815 |
movu m9, [blend_4x8_3+8] |
| 816 |
vinserti128 m6, xm10, 1 |
| 817 |
vinserti128 m5, xm11, 1 |
| 818 |
vpblendvb m6, [rsp+gprsize+0x10+8], m9 |
| 819 |
%endif |
| 820 |
%else |
| 821 |
lea r13, [blend_8x8_1+16] |
| 822 |
movq xm6, [dstq+strideq*2-2] |
| 823 |
movq xm9, [dstq+stride3q -2] |
| 824 |
movq xm5, [top1q +2] |
| 825 |
movq xm10, [top2q +2] |
| 826 |
movu m11, [r13+hq*2*2+16*2] |
| 827 |
vinserti128 m6, [botq+strideq*0-2], 1 |
| 828 |
vinserti128 m9, [botq+strideq*1-2], 1 |
| 829 |
vinserti128 m5, [dstq+strideq*0+2], 1 |
| 830 |
vinserti128 m10, [dstq+strideq*1+2], 1 |
| 831 |
punpcklqdq m6, m9 |
| 832 |
punpcklqdq m5, m10 |
| 833 |
vpblendvb m6, [rsp+gprsize+0x20+hq*8+64+8*2], m11 |
| 834 |
%endif |
| 835 |
ret |
| 836 |
.d1k1: |
| 837 |
%if %1 == 4 |
| 838 |
%if %2 == 4 |
| 839 |
vpbroadcastq m6, [dstq+strideq*1-2] |
| 840 |
vpbroadcastq m9, [dstq+strideq*2-2] |
| 841 |
movd xm5, [topq+strideq*1+2] |
| 842 |
movd xm10, [dstq+strideq*0+2] |
| 843 |
psrldq m11, m6, 4 |
| 844 |
psrldq m12, m9, 4 |
| 845 |
vpblendd m5, m11, 0x10 |
| 846 |
movq xm11, [leftq+2] |
| 847 |
vinserti128 m6, [dstq+stride3q-2], 1 |
| 848 |
punpckldq xm11, xm11 |
| 849 |
vpblendd m10, m12, 0x10 |
| 850 |
pcmpeqd m12, m12 |
| 851 |
pmovzxwd m11, xm11 |
| 852 |
psrld m12, 16 |
| 853 |
punpckldq m6, m9 |
| 854 |
vpbroadcastd m9, [botq-2] |
| 855 |
vpblendvb m6, m11, m12 |
| 856 |
punpckldq m5, m10 |
| 857 |
vpblendd m6, m9, 0x20 |
| 858 |
%else |
| 859 |
movd xm5, [topq +strideq*1+2] |
| 860 |
movq xm6, [dstq +strideq*1-2] |
| 861 |
movq xm9, [dstq +stride3q -2] |
| 862 |
movq xm10, [dst4q+strideq*1-2] |
| 863 |
movd xm11, [dst4q+stride3q -2] |
| 864 |
pinsrd xm5, [dstq +strideq*0+2], 1 |
| 865 |
movhps xm6, [dstq +strideq*2-2] |
| 866 |
movhps xm9, [dst4q+strideq*0-2] |
| 867 |
movhps xm10, [dst4q+strideq*2-2] |
| 868 |
pinsrd xm11, [botq -2], 1 |
| 869 |
shufps xm5, xm6, q3110 |
| 870 |
shufps xm6, xm9, q2020 |
| 871 |
shufps xm9, xm10, q3131 |
| 872 |
shufps xm10, xm11, q1020 |
| 873 |
movu m11, [blend_4x8_2+4] |
| 874 |
vinserti128 m6, xm10, 1 |
| 875 |
vinserti128 m5, xm9, 1 |
| 876 |
vpblendvb m6, [rsp+gprsize+0x10+4], m11 |
| 877 |
%endif |
| 878 |
%else |
| 879 |
lea r13, [blend_8x8_1+16] |
| 880 |
movq xm5, [top2q +2] |
| 881 |
vbroadcasti128 m6, [dstq+strideq*1-2] |
| 882 |
vbroadcasti128 m9, [dstq+strideq*2-2] |
| 883 |
movhps xm5, [dstq+strideq*0+2] |
| 884 |
shufps m10, m6, m9, q2121 |
| 885 |
vinserti128 m6, [dstq+stride3q -2], 1 |
| 886 |
vinserti128 m9, [botq -2], 1 |
| 887 |
movu m11, [r13+hq*2*1+16*1] |
| 888 |
vpblendd m5, m10, 0xF0 |
| 889 |
punpcklqdq m6, m9 |
| 890 |
vpblendvb m6, [rsp+gprsize+0x20+hq*8+64+8*1], m11 |
| 891 |
%endif |
| 892 |
ret |
| 893 |
.d2k1: |
| 894 |
%if %1 == 4 |
| 895 |
%if %2 == 4 |
| 896 |
movq xm11, [leftq] |
| 897 |
movq xm6, [dstq+strideq*0-2] |
| 898 |
movq xm9, [dstq+strideq*1-2] |
| 899 |
vinserti128 m6, [dstq+strideq*2-2], 1 |
| 900 |
vinserti128 m9, [dstq+stride3q -2], 1 |
| 901 |
punpckldq xm11, xm11 |
| 902 |
psrldq m5, m6, 4 |
| 903 |
psrldq m10, m9, 4 |
| 904 |
pmovzxwd m11, xm11 |
| 905 |
punpckldq m6, m9 |
| 906 |
punpckldq m5, m10 |
| 907 |
pblendw m6, m11, 0x05 |
| 908 |
%else |
| 909 |
movq xm5, [dstq +strideq*0-2] |
| 910 |
movq xm9, [dstq +strideq*2-2] |
| 911 |
movq xm10, [dst4q+strideq*0-2] |
| 912 |
movq xm11, [dst4q+strideq*2-2] |
| 913 |
movhps xm5, [dstq +strideq*1-2] |
| 914 |
movhps xm9, [dstq +stride3q -2] |
| 915 |
movhps xm10, [dst4q+strideq*1-2] |
| 916 |
movhps xm11, [dst4q+stride3q -2] |
| 917 |
shufps xm6, xm5, xm9, q2020 |
| 918 |
shufps xm5, xm9, q3131 |
| 919 |
shufps xm9, xm10, xm11, q2020 |
| 920 |
shufps xm10, xm11, q3131 |
| 921 |
pmovzxwd m11, [leftq] |
| 922 |
vinserti128 m6, xm9, 1 |
| 923 |
vinserti128 m5, xm10, 1 |
| 924 |
pblendw m6, m11, 0x55 |
| 925 |
%endif |
| 926 |
%else |
| 927 |
mova m11, [rsp+gprsize+0x20+hq*8+64] |
| 928 |
movu xm5, [dstq+strideq*0-2] |
| 929 |
movu xm9, [dstq+strideq*1-2] |
| 930 |
vinserti128 m5, [dstq+strideq*2-2], 1 |
| 931 |
vinserti128 m9, [dstq+stride3q -2], 1 |
| 932 |
shufps m6, m5, m9, q1010 |
| 933 |
shufps m5, m9, q2121 |
| 934 |
pblendw m6, m11, 0x11 |
| 935 |
%endif |
| 936 |
ret |
| 937 |
.d3k1: |
| 938 |
%if %1 == 4 |
| 939 |
%if %2 == 4 |
| 940 |
vpbroadcastq m11, [dstq+strideq*1-2] |
| 941 |
vpbroadcastq m12, [dstq+strideq*2-2] |
| 942 |
movd xm6, [topq+strideq*1-2] |
| 943 |
movd xm9, [dstq+strideq*0-2] |
| 944 |
pblendw m11, [leftq-16+2], 0x01 |
| 945 |
pblendw m12, [leftq-16+4], 0x01 |
| 946 |
pinsrw xm9, [leftq- 0+0], 0 |
| 947 |
psrldq m5, m11, 4 |
| 948 |
psrldq m10, m12, 4 |
| 949 |
vinserti128 m5, [dstq+stride3q +2], 1 |
| 950 |
vinserti128 m10, [botq +2], 1 |
| 951 |
vpblendd m6, m11, 0x10 |
| 952 |
vpblendd m9, m12, 0x10 |
| 953 |
punpckldq m6, m9 |
| 954 |
punpckldq m5, m10 |
| 955 |
%else |
| 956 |
movd xm6, [topq +strideq*1-2] |
| 957 |
movq xm5, [dstq +strideq*1-2] |
| 958 |
movq xm9, [dstq +stride3q -2] |
| 959 |
movq xm10, [dst4q+strideq*1-2] |
| 960 |
movd xm11, [dst4q+stride3q +2] |
| 961 |
pinsrw xm6, [dstq +strideq*0 ], 3 |
| 962 |
movhps xm5, [dstq +strideq*2-2] |
| 963 |
movhps xm9, [dst4q+strideq*0-2] |
| 964 |
movhps xm10, [dst4q+strideq*2-2] |
| 965 |
pinsrd xm11, [botq +2], 1 |
| 966 |
shufps xm6, xm5, q2010 |
| 967 |
shufps xm5, xm9, q3131 |
| 968 |
shufps xm9, xm10, q2020 |
| 969 |
shufps xm10, xm11, q1031 |
| 970 |
movu m11, [blend_4x8_2] |
| 971 |
vinserti128 m6, xm9, 1 |
| 972 |
vinserti128 m5, xm10, 1 |
| 973 |
vpblendvb m6, [rsp+gprsize+0x10-4], m11 |
| 974 |
%endif |
| 975 |
%else |
| 976 |
lea r13, [blend_8x8_1+8] |
| 977 |
movq xm6, [top2q -2] |
| 978 |
vbroadcasti128 m5, [dstq+strideq*1-2] |
| 979 |
vbroadcasti128 m10, [dstq+strideq*2-2] |
| 980 |
movhps xm6, [dstq+strideq*0-2] |
| 981 |
punpcklqdq m9, m5, m10 |
| 982 |
vinserti128 m5, [dstq+stride3q -2], 1 |
| 983 |
vinserti128 m10, [botq -2], 1 |
| 984 |
movu m11, [r13+hq*2*1+16*1] |
| 985 |
vpblendd m6, m9, 0xF0 |
| 986 |
shufps m5, m10, q2121 |
| 987 |
vpblendvb m6, [rsp+gprsize+0x20+hq*8+64-8*1], m11 |
| 988 |
%endif |
| 989 |
ret |
| 990 |
.d4k1: |
| 991 |
%if %1 == 4 |
| 992 |
%if %2 == 4 |
| 993 |
vinserti128 m6, [dstq+strideq*0-2], 1 |
| 994 |
vinserti128 m9, [dstq+strideq*1-2], 1 |
| 995 |
movd xm5, [dstq+strideq*2+2] |
| 996 |
movd xm10, [dstq+stride3q +2] |
| 997 |
pblendw m6, [leftq-16+0], 0x01 |
| 998 |
pblendw m9, [leftq-16+2], 0x01 |
| 999 |
vinserti128 m5, [botq+strideq*0+2], 1 |
| 1000 |
vinserti128 m10, [botq+strideq*1+2], 1 |
| 1001 |
vpblendd m6, [topq+strideq*0-2], 0x01 |
| 1002 |
vpblendd m9, [topq+strideq*1-2], 0x01 |
| 1003 |
punpckldq m5, m10 |
| 1004 |
punpckldq m6, m9 |
| 1005 |
%else |
| 1006 |
movd xm6, [topq +strideq*0-2] |
| 1007 |
movq xm5, [dstq +strideq*2-2] |
| 1008 |
movq xm9, [dst4q+strideq*0-2] |
| 1009 |
movd xm10, [dst4q+strideq*2+2] |
| 1010 |
pinsrd xm6, [topq +strideq*1-2], 1 |
| 1011 |
movhps xm5, [dstq +stride3q -2] |
| 1012 |
movhps xm9, [dst4q+strideq*1-2] |
| 1013 |
pinsrd xm10, [dst4q+stride3q +2], 1 |
| 1014 |
pinsrd xm6, [dstq +strideq*0-2], 2 |
| 1015 |
pinsrd xm10, [botq +strideq*0+2], 2 |
| 1016 |
pinsrd xm6, [dstq +strideq*1-2], 3 |
| 1017 |
pinsrd xm10, [botq +strideq*1+2], 3 |
| 1018 |
shufps xm11, xm5, xm9, q2020 |
| 1019 |
shufps xm5, xm9, q3131 |
| 1020 |
movu m9, [blend_4x8_3] |
| 1021 |
vinserti128 m6, xm11, 1 |
| 1022 |
vinserti128 m5, xm10, 1 |
| 1023 |
vpblendvb m6, [rsp+gprsize+0x10-8], m9 |
| 1024 |
%endif |
| 1025 |
%else |
| 1026 |
lea r13, [blend_8x8_1] |
| 1027 |
movu m11, [r13+hq*2*2+16*2] |
| 1028 |
movq xm6, [top1q -2] |
| 1029 |
movq xm9, [top2q -2] |
| 1030 |
movq xm5, [dstq+strideq*2+2] |
| 1031 |
movq xm10, [dstq+stride3q +2] |
| 1032 |
vinserti128 m6, [dstq+strideq*0-2], 1 |
| 1033 |
vinserti128 m9, [dstq+strideq*1-2], 1 |
| 1034 |
vinserti128 m5, [botq+strideq*0+2], 1 |
| 1035 |
vinserti128 m10, [botq+strideq*1+2], 1 |
| 1036 |
punpcklqdq m6, m9 |
| 1037 |
vpblendvb m6, [rsp+gprsize+0x20+hq*8+64-8*2], m11 |
| 1038 |
punpcklqdq m5, m10 |
| 1039 |
%endif |
| 1040 |
ret |
| 1041 |
.d5k1: |
| 1042 |
%if %1 == 4 |
| 1043 |
%if %2 == 4 |
| 1044 |
movd xm6, [topq+strideq*0-1] |
| 1045 |
movd xm9, [topq+strideq*1-1] |
| 1046 |
movd xm5, [dstq+strideq*2+1] |
| 1047 |
movd xm10, [dstq+stride3q +1] |
| 1048 |
pcmpeqd m12, m12 |
| 1049 |
pmovzxbw m11, [leftq-8+1] |
| 1050 |
psrld m12, 24 |
| 1051 |
vinserti128 m6, [dstq+strideq*0-1], 1 |
| 1052 |
vinserti128 m9, [dstq+strideq*1-1], 1 |
| 1053 |
vinserti128 m5, [botq+strideq*0+1], 1 |
| 1054 |
vinserti128 m10, [botq+strideq*1+1], 1 |
| 1055 |
punpckldq m6, m9 |
| 1056 |
pxor m9, m9 |
| 1057 |
vpblendd m12, m9, 0x0F |
| 1058 |
punpckldq m5, m10 |
| 1059 |
vpblendvb m6, m11, m12 |
| 1060 |
%else |
| 1061 |
movd xm6, [topq +strideq*0-1] |
| 1062 |
movq xm5, [dstq +strideq*2-1] |
| 1063 |
movq xm9, [dst4q+strideq*0-1] |
| 1064 |
movd xm10, [dst4q+strideq*2+1] |
| 1065 |
pinsrd xm6, [topq +strideq*1-1], 1 |
| 1066 |
movhps xm5, [dstq +stride3q -1] |
| 1067 |
movhps xm9, [dst4q+strideq*1-1] |
| 1068 |
pinsrd xm10, [dst4q+stride3q +1], 1 |
| 1069 |
pinsrd xm6, [dstq +strideq*0-1], 2 |
| 1070 |
pinsrd xm10, [botq +strideq*0+1], 2 |
| 1071 |
pinsrd xm6, [dstq +strideq*1-1], 3 |
| 1072 |
pinsrd xm10, [botq +strideq*1+1], 3 |
| 1073 |
shufps xm11, xm5, xm9, q2020 |
| 1074 |
vinserti128 m6, xm11, 1 |
| 1075 |
pmovzxbw m11, [leftq-3] |
| 1076 |
psrldq xm5, 2 |
| 1077 |
psrldq xm9, 2 |
| 1078 |
shufps xm5, xm9, q2020 |
| 1079 |
movu m9, [blend_4x8_1] |
| 1080 |
vinserti128 m5, xm10, 1 |
| 1081 |
vpblendvb m6, m11, m9 |
| 1082 |
%endif |
| 1083 |
%else |
| 1084 |
lea r13, [blend_8x8_0] |
| 1085 |
movu m11, [r13+hq*2*2+16*2] |
| 1086 |
movq xm6, [top1q -1] |
| 1087 |
movq xm9, [top2q -1] |
| 1088 |
movq xm5, [dstq+strideq*2+1] |
| 1089 |
movq xm10, [dstq+stride3q +1] |
| 1090 |
vinserti128 m6, [dstq+strideq*0-1], 1 |
| 1091 |
vinserti128 m9, [dstq+strideq*1-1], 1 |
| 1092 |
vinserti128 m5, [botq+strideq*0+1], 1 |
| 1093 |
vinserti128 m10, [botq+strideq*1+1], 1 |
| 1094 |
punpcklqdq m6, m9 |
| 1095 |
punpcklqdq m5, m10 |
| 1096 |
vpblendvb m6, [rsp+gprsize+0x60+hq*8+64-8*2], m11 |
| 1097 |
%endif |
| 1098 |
ret |
| 1099 |
.d6k1: |
| 1100 |
%if %1 == 4 |
| 1101 |
%if %2 == 4 |
| 1102 |
movd xm6, [topq+strideq*0] |
| 1103 |
movd xm9, [topq+strideq*1] |
| 1104 |
movd xm5, [dstq+strideq*2] |
| 1105 |
movd xm10, [dstq+stride3q ] |
| 1106 |
vinserti128 m6, [dstq+strideq*0], 1 |
| 1107 |
vinserti128 m9, [dstq+strideq*1], 1 |
| 1108 |
vinserti128 m5, [botq+strideq*0], 1 |
| 1109 |
vinserti128 m10, [botq+strideq*1], 1 |
| 1110 |
punpckldq m6, m9 |
| 1111 |
punpckldq m5, m10 |
| 1112 |
%else |
| 1113 |
movd xm5, [dstq +strideq*2] |
| 1114 |
movd xm6, [topq +strideq*0] |
| 1115 |
movd xm9, [dst4q+strideq*2] |
| 1116 |
pinsrd xm5, [dstq +stride3q ], 1 |
| 1117 |
pinsrd xm6, [topq +strideq*1], 1 |
| 1118 |
pinsrd xm9, [dst4q+stride3q ], 1 |
| 1119 |
pinsrd xm5, [dst4q+strideq*0], 2 |
| 1120 |
pinsrd xm6, [dstq +strideq*0], 2 |
| 1121 |
pinsrd xm9, [botq +strideq*0], 2 |
| 1122 |
pinsrd xm5, [dst4q+strideq*1], 3 |
| 1123 |
pinsrd xm6, [dstq +strideq*1], 3 |
| 1124 |
pinsrd xm9, [botq +strideq*1], 3 |
| 1125 |
vinserti128 m6, xm5, 1 |
| 1126 |
vinserti128 m5, xm9, 1 |
| 1127 |
%endif |
| 1128 |
%else |
| 1129 |
movq xm5, [dstq+strideq*2] |
| 1130 |
movq xm9, [botq+strideq*0] |
| 1131 |
movq xm6, [top1q ] |
| 1132 |
movq xm10, [dstq+strideq*0] |
| 1133 |
movhps xm5, [dstq+stride3q ] |
| 1134 |
movhps xm9, [botq+strideq*1] |
| 1135 |
movhps xm6, [top2q ] |
| 1136 |
movhps xm10, [dstq+strideq*1] |
| 1137 |
vinserti128 m5, xm9, 1 |
| 1138 |
vinserti128 m6, xm10, 1 |
| 1139 |
%endif |
| 1140 |
ret |
| 1141 |
.d7k1: |
| 1142 |
%if %1 == 4 |
| 1143 |
%if %2 == 4 |
| 1144 |
movd xm5, [dstq+strideq*2-1] |
| 1145 |
movd xm9, [dstq+stride3q -1] |
| 1146 |
movd xm6, [topq+strideq*0+1] |
| 1147 |
movd xm10, [topq+strideq*1+1] |
| 1148 |
pinsrb xm5, [leftq+ 5], 0 |
| 1149 |
pinsrb xm9, [leftq+ 7], 0 |
| 1150 |
vinserti128 m6, [dstq+strideq*0+1], 1 |
| 1151 |
vinserti128 m10, [dstq+strideq*1+1], 1 |
| 1152 |
vinserti128 m5, [botq+strideq*0-1], 1 |
| 1153 |
vinserti128 m9, [botq+strideq*1-1], 1 |
| 1154 |
punpckldq m6, m10 |
| 1155 |
punpckldq m5, m9 |
| 1156 |
%else |
| 1157 |
movd xm6, [topq +strideq*0+1] |
| 1158 |
movq xm9, [dstq +strideq*2-1] |
| 1159 |
movq xm10, [dst4q+strideq*0-1] |
| 1160 |
movd xm11, [dst4q+strideq*2-1] |
| 1161 |
pinsrd xm6, [topq +strideq*1+1], 1 |
| 1162 |
movhps xm9, [dstq +stride3q -1] |
| 1163 |
movhps xm10, [dst4q+strideq*1-1] |
| 1164 |
pinsrd xm11, [dst4q+stride3q -1], 1 |
| 1165 |
pinsrd xm6, [dstq +strideq*0+1], 2 |
| 1166 |
pinsrd xm11, [botq +strideq*0-1], 2 |
| 1167 |
pinsrd xm6, [dstq +strideq*1+1], 3 |
| 1168 |
pinsrd xm11, [botq +strideq*1-1], 3 |
| 1169 |
shufps xm5, xm9, xm10, q2020 |
| 1170 |
vinserti128 m5, xm11, 1 |
| 1171 |
pmovzxbw m11, [leftq+5] |
| 1172 |
psrldq xm9, 2 |
| 1173 |
psrldq xm10, 2 |
| 1174 |
shufps xm9, xm10, q2020 |
| 1175 |
movu m10, [blend_4x8_1+8] |
| 1176 |
vinserti128 m6, xm9, 1 |
| 1177 |
vpblendvb m5, m11, m10 |
| 1178 |
%endif |
| 1179 |
%else |
| 1180 |
lea r13, [blend_8x8_0+16] |
| 1181 |
movq xm5, [dstq+strideq*2-1] |
| 1182 |
movq xm9, [botq+strideq*0-1] |
| 1183 |
movq xm6, [top1q +1] |
| 1184 |
movq xm10, [dstq+strideq*0+1] |
| 1185 |
movhps xm5, [dstq+stride3q -1] |
| 1186 |
movhps xm9, [botq+strideq*1-1] |
| 1187 |
movhps xm6, [top2q +1] |
| 1188 |
movhps xm10, [dstq+strideq*1+1] |
| 1189 |
movu m11, [r13+hq*2*2+16*2] |
| 1190 |
vinserti128 m5, xm9, 1 |
| 1191 |
vinserti128 m6, xm10, 1 |
| 1192 |
vpblendvb m5, [rsp+gprsize+0x60+hq*8+64+8*2], m11 |
| 1193 |
%endif |
| 1194 |
ret |
| 1195 |
|
| 1196 |
.border_block: |
| 1197 |
DEFINE_ARGS dst, stride, left, top, bot, pri, sec, stride3, dst4, edge |
| 1198 |
%define rstk rsp |
| 1199 |
%assign stack_offset stack_offset_entry |
| 1200 |
%assign regs_used 11 |
| 1201 |
ALLOC_STACK 2*16+(%2+4)*32, 16 |
| 1202 |
%define px rsp+2*16+2*32 |
| 1203 |
|
| 1204 |
pcmpeqw m14, m14 |
| 1205 |
psllw m14, 15 ; 0x8000 |
| 1206 |
|
| 1207 |
; prepare pixel buffers - body/right |
| 1208 |
%if %1 == 4 |
| 1209 |
INIT_XMM avx2 |
| 1210 |
%endif |
| 1211 |
%if %2 == 8 |
| 1212 |
lea dst4q, [dstq+strideq*4] |
| 1213 |
%endif |
| 1214 |
lea stride3q, [strideq*3] |
| 1215 |
test edgeb, 2 ; have_right |
| 1216 |
jz .no_right |
| 1217 |
pmovzxbw m1, [dstq+strideq*0] |
| 1218 |
pmovzxbw m2, [dstq+strideq*1] |
| 1219 |
pmovzxbw m3, [dstq+strideq*2] |
| 1220 |
pmovzxbw m4, [dstq+stride3q] |
| 1221 |
mova [px+0*32], m1 |
| 1222 |
mova [px+1*32], m2 |
| 1223 |
mova [px+2*32], m3 |
| 1224 |
mova [px+3*32], m4 |
| 1225 |
%if %2 == 8 |
| 1226 |
pmovzxbw m1, [dst4q+strideq*0] |
| 1227 |
pmovzxbw m2, [dst4q+strideq*1] |
| 1228 |
pmovzxbw m3, [dst4q+strideq*2] |
| 1229 |
pmovzxbw m4, [dst4q+stride3q] |
| 1230 |
mova [px+4*32], m1 |
| 1231 |
mova [px+5*32], m2 |
| 1232 |
mova [px+6*32], m3 |
| 1233 |
mova [px+7*32], m4 |
| 1234 |
%endif |
| 1235 |
jmp .body_done |
| 1236 |
.no_right: |
| 1237 |
%if %1 == 4 |
| 1238 |
movd xm1, [dstq+strideq*0] |
| 1239 |
movd xm2, [dstq+strideq*1] |
| 1240 |
movd xm3, [dstq+strideq*2] |
| 1241 |
movd xm4, [dstq+stride3q] |
| 1242 |
pmovzxbw xm1, xm1 |
| 1243 |
pmovzxbw xm2, xm2 |
| 1244 |
pmovzxbw xm3, xm3 |
| 1245 |
pmovzxbw xm4, xm4 |
| 1246 |
movq [px+0*32], xm1 |
| 1247 |
movq [px+1*32], xm2 |
| 1248 |
movq [px+2*32], xm3 |
| 1249 |
movq [px+3*32], xm4 |
| 1250 |
%else |
| 1251 |
pmovzxbw xm1, [dstq+strideq*0] |
| 1252 |
pmovzxbw xm2, [dstq+strideq*1] |
| 1253 |
pmovzxbw xm3, [dstq+strideq*2] |
| 1254 |
pmovzxbw xm4, [dstq+stride3q] |
| 1255 |
mova [px+0*32], xm1 |
| 1256 |
mova [px+1*32], xm2 |
| 1257 |
mova [px+2*32], xm3 |
| 1258 |
mova [px+3*32], xm4 |
| 1259 |
%endif |
| 1260 |
movd [px+0*32+%1*2], xm14 |
| 1261 |
movd [px+1*32+%1*2], xm14 |
| 1262 |
movd [px+2*32+%1*2], xm14 |
| 1263 |
movd [px+3*32+%1*2], xm14 |
| 1264 |
%if %2 == 8 |
| 1265 |
%if %1 == 4 |
| 1266 |
movd xm1, [dst4q+strideq*0] |
| 1267 |
movd xm2, [dst4q+strideq*1] |
| 1268 |
movd xm3, [dst4q+strideq*2] |
| 1269 |
movd xm4, [dst4q+stride3q] |
| 1270 |
pmovzxbw xm1, xm1 |
| 1271 |
pmovzxbw xm2, xm2 |
| 1272 |
pmovzxbw xm3, xm3 |
| 1273 |
pmovzxbw xm4, xm4 |
| 1274 |
movq [px+4*32], xm1 |
| 1275 |
movq [px+5*32], xm2 |
| 1276 |
movq [px+6*32], xm3 |
| 1277 |
movq [px+7*32], xm4 |
| 1278 |
%else |
| 1279 |
pmovzxbw xm1, [dst4q+strideq*0] |
| 1280 |
pmovzxbw xm2, [dst4q+strideq*1] |
| 1281 |
pmovzxbw xm3, [dst4q+strideq*2] |
| 1282 |
pmovzxbw xm4, [dst4q+stride3q] |
| 1283 |
mova [px+4*32], xm1 |
| 1284 |
mova [px+5*32], xm2 |
| 1285 |
mova [px+6*32], xm3 |
| 1286 |
mova [px+7*32], xm4 |
| 1287 |
%endif |
| 1288 |
movd [px+4*32+%1*2], xm14 |
| 1289 |
movd [px+5*32+%1*2], xm14 |
| 1290 |
movd [px+6*32+%1*2], xm14 |
| 1291 |
movd [px+7*32+%1*2], xm14 |
| 1292 |
%endif |
| 1293 |
.body_done: |
| 1294 |
|
| 1295 |
; top |
| 1296 |
test edgeb, 4 ; have_top |
| 1297 |
jz .no_top |
| 1298 |
test edgeb, 1 ; have_left |
| 1299 |
jz .top_no_left |
| 1300 |
test edgeb, 2 ; have_right |
| 1301 |
jz .top_no_right |
| 1302 |
pmovzxbw m1, [topq+strideq*0-(%1/2)] |
| 1303 |
pmovzxbw m2, [topq+strideq*1-(%1/2)] |
| 1304 |
movu [px-2*32-%1], m1 |
| 1305 |
movu [px-1*32-%1], m2 |
| 1306 |
jmp .top_done |
| 1307 |
.top_no_right: |
| 1308 |
pmovzxbw m1, [topq+strideq*0-%1] |
| 1309 |
pmovzxbw m2, [topq+strideq*1-%1] |
| 1310 |
movu [px-2*32-%1*2], m1 |
| 1311 |
movu [px-1*32-%1*2], m2 |
| 1312 |
movd [px-2*32+%1*2], xm14 |
| 1313 |
movd [px-1*32+%1*2], xm14 |
| 1314 |
jmp .top_done |
| 1315 |
.top_no_left: |
| 1316 |
test edgeb, 2 ; have_right |
| 1317 |
jz .top_no_left_right |
| 1318 |
pmovzxbw m1, [topq+strideq*0] |
| 1319 |
pmovzxbw m2, [topq+strideq*1] |
| 1320 |
mova [px-2*32+0], m1 |
| 1321 |
mova [px-1*32+0], m2 |
| 1322 |
movd [px-2*32-4], xm14 |
| 1323 |
movd [px-1*32-4], xm14 |
| 1324 |
jmp .top_done |
| 1325 |
.top_no_left_right: |
| 1326 |
%if %1 == 4 |
| 1327 |
movd xm1, [topq+strideq*0] |
| 1328 |
pinsrd xm1, [topq+strideq*1], 1 |
| 1329 |
pmovzxbw xm1, xm1 |
| 1330 |
movq [px-2*32+0], xm1 |
| 1331 |
movhps [px-1*32+0], xm1 |
| 1332 |
%else |
| 1333 |
pmovzxbw xm1, [topq+strideq*0] |
| 1334 |
pmovzxbw xm2, [topq+strideq*1] |
| 1335 |
mova [px-2*32+0], xm1 |
| 1336 |
mova [px-1*32+0], xm2 |
| 1337 |
%endif |
| 1338 |
movd [px-2*32-4], xm14 |
| 1339 |
movd [px-1*32-4], xm14 |
| 1340 |
movd [px-2*32+%1*2], xm14 |
| 1341 |
movd [px-1*32+%1*2], xm14 |
| 1342 |
jmp .top_done |
| 1343 |
.no_top: |
| 1344 |
movu [px-2*32-%1], m14 |
| 1345 |
movu [px-1*32-%1], m14 |
| 1346 |
.top_done: |
| 1347 |
|
| 1348 |
; left |
| 1349 |
test edgeb, 1 ; have_left |
| 1350 |
jz .no_left |
| 1351 |
pmovzxbw xm1, [leftq+ 0] |
| 1352 |
%if %2 == 8 |
| 1353 |
pmovzxbw xm2, [leftq+ 8] |
| 1354 |
%endif |
| 1355 |
movd [px+0*32-4], xm1 |
| 1356 |
pextrd [px+1*32-4], xm1, 1 |
| 1357 |
pextrd [px+2*32-4], xm1, 2 |
| 1358 |
pextrd [px+3*32-4], xm1, 3 |
| 1359 |
%if %2 == 8 |
| 1360 |
movd [px+4*32-4], xm2 |
| 1361 |
pextrd [px+5*32-4], xm2, 1 |
| 1362 |
pextrd [px+6*32-4], xm2, 2 |
| 1363 |
pextrd [px+7*32-4], xm2, 3 |
| 1364 |
%endif |
| 1365 |
jmp .left_done |
| 1366 |
.no_left: |
| 1367 |
movd [px+0*32-4], xm14 |
| 1368 |
movd [px+1*32-4], xm14 |
| 1369 |
movd [px+2*32-4], xm14 |
| 1370 |
movd [px+3*32-4], xm14 |
| 1371 |
%if %2 == 8 |
| 1372 |
movd [px+4*32-4], xm14 |
| 1373 |
movd [px+5*32-4], xm14 |
| 1374 |
movd [px+6*32-4], xm14 |
| 1375 |
movd [px+7*32-4], xm14 |
| 1376 |
%endif |
| 1377 |
.left_done: |
| 1378 |
|
| 1379 |
; bottom |
| 1380 |
DEFINE_ARGS dst, stride, _, _, bot, pri, sec, stride3, _, edge |
| 1381 |
test edgeb, 8 ; have_bottom |
| 1382 |
jz .no_bottom |
| 1383 |
test edgeb, 1 ; have_left |
| 1384 |
jz .bottom_no_left |
| 1385 |
test edgeb, 2 ; have_right |
| 1386 |
jz .bottom_no_right |
| 1387 |
pmovzxbw m1, [botq+strideq*0-(%1/2)] |
| 1388 |
pmovzxbw m2, [botq+strideq*1-(%1/2)] |
| 1389 |
movu [px+(%2+0)*32-%1], m1 |
| 1390 |
movu [px+(%2+1)*32-%1], m2 |
| 1391 |
jmp .bottom_done |
| 1392 |
.bottom_no_right: |
| 1393 |
pmovzxbw m1, [botq+strideq*0-%1] |
| 1394 |
pmovzxbw m2, [botq+strideq*1-%1] |
| 1395 |
movu [px+(%2+0)*32-%1*2], m1 |
| 1396 |
movu [px+(%2+1)*32-%1*2], m2 |
| 1397 |
%if %1 == 8 |
| 1398 |
movd [px+(%2-1)*32+%1*2], xm14 ; overwritten by previous movu |
| 1399 |
%endif |
| 1400 |
movd [px+(%2+0)*32+%1*2], xm14 |
| 1401 |
movd [px+(%2+1)*32+%1*2], xm14 |
| 1402 |
jmp .bottom_done |
| 1403 |
.bottom_no_left: |
| 1404 |
test edgeb, 2 ; have_right |
| 1405 |
jz .bottom_no_left_right |
| 1406 |
pmovzxbw m1, [botq+strideq*0] |
| 1407 |
pmovzxbw m2, [botq+strideq*1] |
| 1408 |
mova [px+(%2+0)*32+0], m1 |
| 1409 |
mova [px+(%2+1)*32+0], m2 |
| 1410 |
movd [px+(%2+0)*32-4], xm14 |
| 1411 |
movd [px+(%2+1)*32-4], xm14 |
| 1412 |
jmp .bottom_done |
| 1413 |
.bottom_no_left_right: |
| 1414 |
%if %1 == 4 |
| 1415 |
movd xm1, [botq+strideq*0] |
| 1416 |
pinsrd xm1, [botq+strideq*1], 1 |
| 1417 |
pmovzxbw xm1, xm1 |
| 1418 |
movq [px+(%2+0)*32+0], xm1 |
| 1419 |
movhps [px+(%2+1)*32+0], xm1 |
| 1420 |
%else |
| 1421 |
pmovzxbw xm1, [botq+strideq*0] |
| 1422 |
pmovzxbw xm2, [botq+strideq*1] |
| 1423 |
mova [px+(%2+0)*32+0], xm1 |
| 1424 |
mova [px+(%2+1)*32+0], xm2 |
| 1425 |
%endif |
| 1426 |
movd [px+(%2+0)*32-4], xm14 |
| 1427 |
movd [px+(%2+1)*32-4], xm14 |
| 1428 |
movd [px+(%2+0)*32+%1*2], xm14 |
| 1429 |
movd [px+(%2+1)*32+%1*2], xm14 |
| 1430 |
jmp .bottom_done |
| 1431 |
.no_bottom: |
| 1432 |
movu [px+(%2+0)*32-%1], m14 |
| 1433 |
movu [px+(%2+1)*32-%1], m14 |
| 1434 |
.bottom_done: |
| 1435 |
|
| 1436 |
; actual filter |
| 1437 |
INIT_YMM avx2 |
| 1438 |
DEFINE_ARGS dst, stride, _, pridmp, damping, pri, secdmp, stride3, zero |
| 1439 |
%undef edged |
| 1440 |
; register to shuffle values into after packing |
| 1441 |
vbroadcasti128 m12, [shufb_lohi] |
| 1442 |
|
| 1443 |
mov dampingd, r8m |
| 1444 |
xor zerod, zerod |
| 1445 |
movifnidn prid, prim |
| 1446 |
sub dampingd, 31 |
| 1447 |
movifnidn secdmpd, secdmpm |
| 1448 |
test prid, prid |
| 1449 |
jz .border_sec_only |
| 1450 |
movd xm0, prid |
| 1451 |
lzcnt pridmpd, prid |
| 1452 |
add pridmpd, dampingd |
| 1453 |
cmovs pridmpd, zerod |
| 1454 |
mov [rsp+0], pridmpq ; pri_shift |
| 1455 |
test secdmpd, secdmpd |
| 1456 |
jz .border_pri_only |
| 1457 |
movd xm1, secdmpd |
| 1458 |
lzcnt secdmpd, secdmpd |
| 1459 |
add secdmpd, dampingd |
| 1460 |
mov [rsp+8], secdmpq ; sec_shift |
| 1461 |
|
| 1462 |
DEFINE_ARGS dst, stride, _, pridmp, table, pri, secdmp, stride3 |
| 1463 |
lea tableq, [tap_table] |
| 1464 |
vpbroadcastb m13, [tableq+pridmpq] ; pri_shift_mask |
| 1465 |
vpbroadcastb m14, [tableq+secdmpq] ; sec_shift_mask |
| 1466 |
|
| 1467 |
; pri/sec_taps[k] [4 total] |
| 1468 |
DEFINE_ARGS dst, stride, _, dir, table, pri, sec, stride3 |
| 1469 |
vpbroadcastb m0, xm0 ; pri_strength |
| 1470 |
vpbroadcastb m1, xm1 ; sec_strength |
| 1471 |
and prid, 1 |
| 1472 |
lea priq, [tableq+priq*2+8] ; pri_taps |
| 1473 |
lea secq, [tableq+12] ; sec_taps |
| 1474 |
|
| 1475 |
BORDER_PREP_REGS %1, %2 |
| 1476 |
%if %1*%2*2/mmsize > 1 |
| 1477 |
.border_v_loop: |
| 1478 |
%endif |
| 1479 |
BORDER_LOAD_BLOCK %1, %2, 1 |
| 1480 |
.border_k_loop: |
| 1481 |
vpbroadcastb m2, [priq+kq] ; pri_taps |
| 1482 |
vpbroadcastb m3, [secq+kq] ; sec_taps |
| 1483 |
ACCUMULATE_TAP_WORD 0*2, [rsp+0], m13, m0, m2, %1, 1 |
| 1484 |
ACCUMULATE_TAP_WORD 2*2, [rsp+8], m14, m1, m3, %1, 1 |
| 1485 |
ACCUMULATE_TAP_WORD 6*2, [rsp+8], m14, m1, m3, %1, 1 |
| 1486 |
dec kq |
| 1487 |
jge .border_k_loop |
| 1488 |
|
| 1489 |
vpbroadcastd m10, [pw_2048] |
| 1490 |
BORDER_ADJUST_PIXEL %1, m10, 1 |
| 1491 |
%if %1*%2*2/mmsize > 1 |
| 1492 |
%define vloop_lines (mmsize/(%1*2)) |
| 1493 |
lea dstq, [dstq+strideq*vloop_lines] |
| 1494 |
add stkq, 32*vloop_lines |
| 1495 |
dec hd |
| 1496 |
jg .border_v_loop |
| 1497 |
%endif |
| 1498 |
RET |
| 1499 |
|
| 1500 |
.border_pri_only: |
| 1501 |
DEFINE_ARGS dst, stride, _, pridmp, table, pri, _, stride3 |
| 1502 |
lea tableq, [tap_table] |
| 1503 |
vpbroadcastb m13, [tableq+pridmpq] ; pri_shift_mask |
| 1504 |
DEFINE_ARGS dst, stride, _, dir, table, pri, _, stride3 |
| 1505 |
vpbroadcastb m0, xm0 ; pri_strength |
| 1506 |
and prid, 1 |
| 1507 |
lea priq, [tableq+priq*2+8] ; pri_taps |
| 1508 |
BORDER_PREP_REGS %1, %2 |
| 1509 |
vpbroadcastd m1, [pw_2048] |
| 1510 |
%if %1*%2*2/mmsize > 1 |
| 1511 |
.border_pri_v_loop: |
| 1512 |
%endif |
| 1513 |
BORDER_LOAD_BLOCK %1, %2 |
| 1514 |
.border_pri_k_loop: |
| 1515 |
vpbroadcastb m2, [priq+kq] ; pri_taps |
| 1516 |
ACCUMULATE_TAP_WORD 0*2, [rsp+0], m13, m0, m2, %1 |
| 1517 |
dec kq |
| 1518 |
jge .border_pri_k_loop |
| 1519 |
BORDER_ADJUST_PIXEL %1, m1 |
| 1520 |
%if %1*%2*2/mmsize > 1 |
| 1521 |
%define vloop_lines (mmsize/(%1*2)) |
| 1522 |
lea dstq, [dstq+strideq*vloop_lines] |
| 1523 |
add stkq, 32*vloop_lines |
| 1524 |
dec hd |
| 1525 |
jg .border_pri_v_loop |
| 1526 |
%endif |
| 1527 |
RET |
| 1528 |
|
| 1529 |
.border_sec_only: |
| 1530 |
DEFINE_ARGS dst, stride, _, _, damping, _, secdmp, stride3 |
| 1531 |
movd xm1, secdmpd |
| 1532 |
lzcnt secdmpd, secdmpd |
| 1533 |
add secdmpd, dampingd |
| 1534 |
mov [rsp+8], secdmpq ; sec_shift |
| 1535 |
DEFINE_ARGS dst, stride, _, _, table, _, secdmp, stride3 |
| 1536 |
lea tableq, [tap_table] |
| 1537 |
vpbroadcastb m14, [tableq+secdmpq] ; sec_shift_mask |
| 1538 |
DEFINE_ARGS dst, stride, _, dir, table, _, sec, stride3 |
| 1539 |
vpbroadcastb m1, xm1 ; sec_strength |
| 1540 |
lea secq, [tableq+12] ; sec_taps |
| 1541 |
BORDER_PREP_REGS %1, %2 |
| 1542 |
vpbroadcastd m0, [pw_2048] |
| 1543 |
%if %1*%2*2/mmsize > 1 |
| 1544 |
.border_sec_v_loop: |
| 1545 |
%endif |
| 1546 |
BORDER_LOAD_BLOCK %1, %2 |
| 1547 |
.border_sec_k_loop: |
| 1548 |
vpbroadcastb m3, [secq+kq] ; sec_taps |
| 1549 |
ACCUMULATE_TAP_WORD 2*2, [rsp+8], m14, m1, m3, %1 |
| 1550 |
ACCUMULATE_TAP_WORD 6*2, [rsp+8], m14, m1, m3, %1 |
| 1551 |
dec kq |
| 1552 |
jge .border_sec_k_loop |
| 1553 |
BORDER_ADJUST_PIXEL %1, m0 |
| 1554 |
%if %1*%2*2/mmsize > 1 |
| 1555 |
%define vloop_lines (mmsize/(%1*2)) |
| 1556 |
lea dstq, [dstq+strideq*vloop_lines] |
| 1557 |
add stkq, 32*vloop_lines |
| 1558 |
dec hd |
| 1559 |
jg .border_sec_v_loop |
| 1560 |
%endif |
| 1561 |
RET |
| 1562 |
%endmacro |
| 1563 |
|
| 1564 |
CDEF_FILTER 8, 8 |
| 1565 |
CDEF_FILTER 4, 8 |
| 1566 |
CDEF_FILTER 4, 4 |
| 1567 |
|
| 1568 |
INIT_YMM avx2 |
| 1569 |
cglobal cdef_dir_8bpc, 3, 4, 6, src, stride, var, stride3 |
| 1570 |
lea stride3q, [strideq*3] |
| 1571 |
movq xm0, [srcq+strideq*0] |
| 1572 |
movq xm1, [srcq+strideq*1] |
| 1573 |
movq xm2, [srcq+strideq*2] |
| 1574 |
movq xm3, [srcq+stride3q ] |
| 1575 |
lea srcq, [srcq+strideq*4] |
| 1576 |
vpbroadcastq m4, [srcq+stride3q ] |
| 1577 |
vpbroadcastq m5, [srcq+strideq*2] |
| 1578 |
vpblendd m0, m4, 0xf0 |
| 1579 |
vpblendd m1, m5, 0xf0 |
| 1580 |
vpbroadcastq m4, [srcq+strideq*1] |
| 1581 |
vpbroadcastq m5, [srcq+strideq*0] |
| 1582 |
vpblendd m2, m4, 0xf0 |
| 1583 |
vpblendd m3, m5, 0xf0 |
| 1584 |
pxor m4, m4 |
| 1585 |
punpcklbw m0, m4 |
| 1586 |
punpcklbw m1, m4 |
| 1587 |
punpcklbw m2, m4 |
| 1588 |
punpcklbw m3, m4 |
| 1589 |
cglobal_label .main |
| 1590 |
vpbroadcastd m4, [pw_128] |
| 1591 |
PROLOGUE 3, 4, 15 |
| 1592 |
psubw m0, m4 |
| 1593 |
psubw m1, m4 |
| 1594 |
psubw m2, m4 |
| 1595 |
psubw m3, m4 |
| 1596 |
|
| 1597 |
; shuffle registers to generate partial_sum_diag[0-1] together |
| 1598 |
vperm2i128 m7, m0, m0, 0x01 |
| 1599 |
vperm2i128 m6, m1, m1, 0x01 |
| 1600 |
vperm2i128 m5, m2, m2, 0x01 |
| 1601 |
vperm2i128 m4, m3, m3, 0x01 |
| 1602 |
|
| 1603 |
; start with partial_sum_hv[0-1] |
| 1604 |
paddw m8, m0, m1 |
| 1605 |
paddw m9, m2, m3 |
| 1606 |
phaddw m10, m0, m1 |
| 1607 |
phaddw m11, m2, m3 |
| 1608 |
paddw m8, m9 |
| 1609 |
phaddw m10, m11 |
| 1610 |
vextracti128 xm9, m8, 1 |
| 1611 |
vextracti128 xm11, m10, 1 |
| 1612 |
paddw xm8, xm9 ; partial_sum_hv[1] |
| 1613 |
phaddw xm10, xm11 ; partial_sum_hv[0] |
| 1614 |
vinserti128 m8, xm10, 1 |
| 1615 |
vpbroadcastd m9, [div_table+44] |
| 1616 |
pmaddwd m8, m8 |
| 1617 |
pmulld m8, m9 ; cost6[2a-d] | cost2[a-d] |
| 1618 |
|
| 1619 |
; create aggregates [lower half]: |
| 1620 |
; m9 = m0:01234567+m1:x0123456+m2:xx012345+m3:xxx01234+ |
| 1621 |
; m4:xxxx0123+m5:xxxxx012+m6:xxxxxx01+m7:xxxxxxx0 |
| 1622 |
; m10= m1:7xxxxxxx+m2:67xxxxxx+m3:567xxxxx+ |
| 1623 |
; m4:4567xxxx+m5:34567xxx+m6:234567xx+m7:1234567x |
| 1624 |
; and [upper half]: |
| 1625 |
; m9 = m0:xxxxxxx0+m1:xxxxxx01+m2:xxxxx012+m3:xxxx0123+ |
| 1626 |
; m4:xxx01234+m5:xx012345+m6:x0123456+m7:01234567 |
| 1627 |
; m10= m0:1234567x+m1:234567xx+m2:34567xxx+m3:4567xxxx+ |
| 1628 |
; m4:567xxxxx+m5:67xxxxxx+m6:7xxxxxxx |
| 1629 |
; and then shuffle m11 [shufw_6543210x], unpcklwd, pmaddwd, pmulld, paddd |
| 1630 |
|
| 1631 |
pslldq m9, m1, 2 |
| 1632 |
psrldq m10, m1, 14 |
| 1633 |
pslldq m11, m2, 4 |
| 1634 |
psrldq m12, m2, 12 |
| 1635 |
pslldq m13, m3, 6 |
| 1636 |
psrldq m14, m3, 10 |
| 1637 |
paddw m9, m11 |
| 1638 |
paddw m10, m12 |
| 1639 |
paddw m9, m13 |
| 1640 |
paddw m10, m14 |
| 1641 |
pslldq m11, m4, 8 |
| 1642 |
psrldq m12, m4, 8 |
| 1643 |
pslldq m13, m5, 10 |
| 1644 |
psrldq m14, m5, 6 |
| 1645 |
paddw m9, m11 |
| 1646 |
paddw m10, m12 |
| 1647 |
paddw m9, m13 |
| 1648 |
paddw m10, m14 |
| 1649 |
pslldq m11, m6, 12 |
| 1650 |
psrldq m12, m6, 4 |
| 1651 |
pslldq m13, m7, 14 |
| 1652 |
psrldq m14, m7, 2 |
| 1653 |
paddw m9, m11 |
| 1654 |
paddw m10, m12 |
| 1655 |
paddw m9, m13 |
| 1656 |
paddw m10, m14 ; partial_sum_diag[0/1][8-14,zero] |
| 1657 |
vbroadcasti128 m14, [shufw_6543210x] |
| 1658 |
vbroadcasti128 m13, [div_table+16] |
| 1659 |
vbroadcasti128 m12, [div_table+0] |
| 1660 |
paddw m9, m0 ; partial_sum_diag[0/1][0-7] |
| 1661 |
pshufb m10, m14 |
| 1662 |
punpckhwd m11, m9, m10 |
| 1663 |
punpcklwd m9, m10 |
| 1664 |
pmaddwd m11, m11 |
| 1665 |
pmaddwd m9, m9 |
| 1666 |
pmulld m11, m13 |
| 1667 |
pmulld m9, m12 |
| 1668 |
paddd m9, m11 ; cost0[a-d] | cost4[a-d] |
| 1669 |
|
| 1670 |
; merge horizontally and vertically for partial_sum_alt[0-3] |
| 1671 |
paddw m10, m0, m1 |
| 1672 |
paddw m11, m2, m3 |
| 1673 |
paddw m12, m4, m5 |
| 1674 |
paddw m13, m6, m7 |
| 1675 |
phaddw m0, m4 |
| 1676 |
phaddw m1, m5 |
| 1677 |
phaddw m2, m6 |
| 1678 |
phaddw m3, m7 |
| 1679 |
|
| 1680 |
; create aggregates [lower half]: |
| 1681 |
; m4 = m10:01234567+m11:x0123456+m12:xx012345+m13:xxx01234 |
| 1682 |
; m11= m11:7xxxxxxx+m12:67xxxxxx+m13:567xxxxx |
| 1683 |
; and [upper half]: |
| 1684 |
; m4 = m10:xxx01234+m11:xx012345+m12:x0123456+m13:01234567 |
| 1685 |
; m11= m10:567xxxxx+m11:67xxxxxx+m12:7xxxxxxx |
| 1686 |
; and then pshuflw m11 3012, unpcklwd, pmaddwd, pmulld, paddd |
| 1687 |
|
| 1688 |
pslldq m4, m11, 2 |
| 1689 |
psrldq m11, 14 |
| 1690 |
pslldq m5, m12, 4 |
| 1691 |
psrldq m12, 12 |
| 1692 |
pslldq m6, m13, 6 |
| 1693 |
psrldq m13, 10 |
| 1694 |
paddw m4, m10 |
| 1695 |
paddw m11, m12 |
| 1696 |
vpbroadcastd m12, [div_table+44] |
| 1697 |
paddw m5, m6 |
| 1698 |
paddw m11, m13 ; partial_sum_alt[3/2] right |
| 1699 |
vbroadcasti128 m13, [div_table+32] |
| 1700 |
paddw m4, m5 ; partial_sum_alt[3/2] left |
| 1701 |
pshuflw m5, m11, q3012 |
| 1702 |
punpckhwd m6, m11, m4 |
| 1703 |
punpcklwd m4, m5 |
| 1704 |
pmaddwd m6, m6 |
| 1705 |
pmaddwd m4, m4 |
| 1706 |
pmulld m6, m12 |
| 1707 |
pmulld m4, m13 |
| 1708 |
paddd m4, m6 ; cost7[a-d] | cost5[a-d] |
| 1709 |
|
| 1710 |
; create aggregates [lower half]: |
| 1711 |
; m5 = m0:01234567+m1:x0123456+m2:xx012345+m3:xxx01234 |
| 1712 |
; m1 = m1:7xxxxxxx+m2:67xxxxxx+m3:567xxxxx |
| 1713 |
; and [upper half]: |
| 1714 |
; m5 = m0:xxx01234+m1:xx012345+m2:x0123456+m3:01234567 |
| 1715 |
; m1 = m0:567xxxxx+m1:67xxxxxx+m2:7xxxxxxx |
| 1716 |
; and then pshuflw m1 3012, unpcklwd, pmaddwd, pmulld, paddd |
| 1717 |
|
| 1718 |
pslldq m5, m1, 2 |
| 1719 |
psrldq m1, 14 |
| 1720 |
pslldq m6, m2, 4 |
| 1721 |
psrldq m2, 12 |
| 1722 |
pslldq m7, m3, 6 |
| 1723 |
psrldq m3, 10 |
| 1724 |
paddw m5, m0 |
| 1725 |
paddw m1, m2 |
| 1726 |
paddw m6, m7 |
| 1727 |
paddw m1, m3 ; partial_sum_alt[0/1] right |
| 1728 |
paddw m5, m6 ; partial_sum_alt[0/1] left |
| 1729 |
pshuflw m0, m1, q3012 |
| 1730 |
punpckhwd m1, m5 |
| 1731 |
punpcklwd m5, m0 |
| 1732 |
pmaddwd m1, m1 |
| 1733 |
pmaddwd m5, m5 |
| 1734 |
pmulld m1, m12 |
| 1735 |
pmulld m5, m13 |
| 1736 |
paddd m5, m1 ; cost1[a-d] | cost3[a-d] |
| 1737 |
|
| 1738 |
mova xm0, [pd_47130256+ 16] |
| 1739 |
mova m1, [pd_47130256] |
| 1740 |
phaddd m9, m8 |
| 1741 |
phaddd m5, m4 |
| 1742 |
phaddd m9, m5 |
| 1743 |
vpermd m0, m9 ; cost[0-3] |
| 1744 |
vpermd m1, m9 ; cost[4-7] | cost[0-3] |
| 1745 |
|
| 1746 |
; now find the best cost |
| 1747 |
pmaxsd xm2, xm0, xm1 |
| 1748 |
pshufd xm3, xm2, q1032 |
| 1749 |
pmaxsd xm2, xm3 |
| 1750 |
pshufd xm3, xm2, q2301 |
| 1751 |
pmaxsd xm2, xm3 ; best cost |
| 1752 |
|
| 1753 |
; find the idx using minpos |
| 1754 |
; make everything other than the best cost negative via subtraction |
| 1755 |
; find the min of unsigned 16-bit ints to sort out the negative values |
| 1756 |
psubd xm4, xm1, xm2 |
| 1757 |
psubd xm3, xm0, xm2 |
| 1758 |
packssdw xm3, xm4 |
| 1759 |
phminposuw xm3, xm3 |
| 1760 |
|
| 1761 |
; convert idx to 32-bits |
| 1762 |
psrld xm3, 16 |
| 1763 |
movd eax, xm3 |
| 1764 |
|
| 1765 |
; get idx^4 complement |
| 1766 |
vpermd m3, m1 |
| 1767 |
psubd xm2, xm3 |
| 1768 |
psrld xm2, 10 |
| 1769 |
movd [varq], xm2 |
| 1770 |
RET |
| 1771 |
|
| 1772 |
%endif ; ARCH_X86_64 |