| 1 |
; Copyright © 2018, VideoLAN and dav1d authors |
| 2 |
; Copyright © 2018, Two Orioles, LLC |
| 3 |
; All rights reserved. |
| 4 |
; |
| 5 |
; Redistribution and use in source and binary forms, with or without |
| 6 |
; modification, are permitted provided that the following conditions are met: |
| 7 |
; |
| 8 |
; 1. Redistributions of source code must retain the above copyright notice, this |
| 9 |
; list of conditions and the following disclaimer. |
| 10 |
; |
| 11 |
; 2. Redistributions in binary form must reproduce the above copyright notice, |
| 12 |
; this list of conditions and the following disclaimer in the documentation |
| 13 |
; and/or other materials provided with the distribution. |
| 14 |
; |
| 15 |
; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND |
| 16 |
; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
| 17 |
; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE |
| 18 |
; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR |
| 19 |
; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES |
| 20 |
; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
| 21 |
; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND |
| 22 |
; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 23 |
; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS |
| 24 |
; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 25 |
|
| 26 |
%include "config.asm" |
| 27 |
%include "ext/x86/x86inc.asm" |
| 28 |
|
| 29 |
%if ARCH_X86_64 |
| 30 |
|
| 31 |
SECTION_RODATA 32 |
| 32 |
pd_47130256: dd 4, 7, 1, 3, 0, 2, 5, 6 |
| 33 |
div_table: dd 840, 420, 280, 210, 168, 140, 120, 105 |
| 34 |
dd 420, 210, 140, 105 |
| 35 |
shufw_6543210x: db 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1, 14, 15 |
| 36 |
shufb_lohi: db 0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15 |
| 37 |
tap_table: ; masks for 8 bit shifts |
| 38 |
db 0xFF, 0x7F, 0x3F, 0x1F, 0x0F, 0x07, 0x03, 0x01 |
| 39 |
; weights |
| 40 |
db 4, 2, 3, 3, 2, 1 |
| 41 |
db 0, 0 ; padding |
| 42 |
db -1, -2, 0, 0, 1, 2, 0, 0 |
| 43 |
db 0, -1, 0, 1, 1, 2, 0, 0 |
| 44 |
db 0, 0, 1, 2, -1, -2, 0, 0 |
| 45 |
db 0, 1, 1, 2, 0, -1, 0, 0 |
| 46 |
db 1, 2, 1, 2, 0, 0, 0, 0 |
| 47 |
db 1, 2, 1, 2, 0, 1, 0, 0 |
| 48 |
db 1, 2, -1, -2, 1, 2, 0, 0 |
| 49 |
db 1, 2, 0, -1, 1, 2, 0, 0 |
| 50 |
|
| 51 |
db 1, 2, 1, 2, 0, 0, 0, 0 |
| 52 |
db 1, 2, 1, 2, 0, -1, 0, 0 |
| 53 |
db 1, 2, 1, 2, 1, 2, 0, 0 |
| 54 |
db 1, 2, 0, 1, 1, 2, 0, 0 |
| 55 |
db 1, 2, 0, 0, 1, 2, 0, 0 |
| 56 |
db 0, 1, 0, -1, 1, 2, 0, 0 |
| 57 |
db 0, 0, 1, 2, 1, 2, 0, 0 |
| 58 |
db 0, -1, 1, 2, 0, 1, 0, 0 |
| 59 |
pw_128: times 2 dw 128 |
| 60 |
pw_2048: times 2 dw 2048 |
| 61 |
|
| 62 |
SECTION .text |
| 63 |
|
| 64 |
; stride unused |
| 65 |
%macro ACCUMULATE_TAP 7 ; tap_offset, shift, mask, strength, mul_tap, w, stride |
| 66 |
; load p0/p1 |
| 67 |
movsx offq, dword [offsets+kq*4+%1] ; off1 |
| 68 |
%if %6 == 4 |
| 69 |
movq xm5, [tmp0q+offq*2] ; p0 |
| 70 |
movq xm6, [tmp2q+offq*2] |
| 71 |
movhps xm5, [tmp1q+offq*2] |
| 72 |
movhps xm6, [tmp3q+offq*2] |
| 73 |
vinserti128 m5, xm6, 1 |
| 74 |
%else |
| 75 |
movu xm5, [tmp0q+offq*2] ; p0 |
| 76 |
vinserti128 m5, [tmp1q+offq*2], 1 |
| 77 |
%endif |
| 78 |
neg offq ; -off1 |
| 79 |
%if %6 == 4 |
| 80 |
movq xm6, [tmp0q+offq*2] ; p1 |
| 81 |
movq xm9, [tmp2q+offq*2] |
| 82 |
movhps xm6, [tmp1q+offq*2] |
| 83 |
movhps xm9, [tmp3q+offq*2] |
| 84 |
vinserti128 m6, xm9, 1 |
| 85 |
%else |
| 86 |
movu xm6, [tmp0q+offq*2] ; p1 |
| 87 |
vinserti128 m6, [tmp1q+offq*2], 1 |
| 88 |
%endif |
| 89 |
; out of bounds values are set to a value that is a both a large unsigned |
| 90 |
; value and a negative signed value. |
| 91 |
; use signed max and unsigned min to remove them |
| 92 |
pmaxsw m7, m5 ; max after p0 |
| 93 |
pminuw m8, m5 ; min after p0 |
| 94 |
pmaxsw m7, m6 ; max after p1 |
| 95 |
pminuw m8, m6 ; min after p1 |
| 96 |
|
| 97 |
; accumulate sum[m15] over p0/p1 |
| 98 |
; calculate difference before converting |
| 99 |
psubw m5, m4 ; diff_p0(p0 - px) |
| 100 |
psubw m6, m4 ; diff_p1(p1 - px) |
| 101 |
|
| 102 |
; convert to 8-bits with signed saturation |
| 103 |
; saturating to large diffs has no impact on the results |
| 104 |
packsswb m5, m6 |
| 105 |
|
| 106 |
; group into pairs so we can accumulate using maddubsw |
| 107 |
pshufb m5, m12 |
| 108 |
pabsb m9, m5 |
| 109 |
psignb m10, %5, m5 |
| 110 |
psrlw m5, m9, %2 ; emulate 8-bit shift |
| 111 |
pand m5, %3 |
| 112 |
psubusb m5, %4, m5 |
| 113 |
|
| 114 |
; use unsigned min since abs diff can equal 0x80 |
| 115 |
pminub m5, m9 |
| 116 |
pmaddubsw m5, m10 |
| 117 |
paddw m15, m5 |
| 118 |
%endmacro |
| 119 |
|
| 120 |
%macro CDEF_FILTER 2 ; w, h |
| 121 |
INIT_YMM avx2 |
| 122 |
%if %1*%2*2/mmsize > 1 |
| 123 |
%if %1 == 4 |
| 124 |
cglobal cdef_filter_%1x%2, 4, 13, 16, 64 |
| 125 |
%else |
| 126 |
cglobal cdef_filter_%1x%2, 4, 11, 16, 64 |
| 127 |
%endif |
| 128 |
%else |
| 129 |
cglobal cdef_filter_%1x%2, 4, 12, 16, 64 |
| 130 |
%endif |
| 131 |
%define offsets rsp+32 |
| 132 |
DEFINE_ARGS dst, dst_stride, tmp, tmp_stride, pri, sec, pridmp, table, \ |
| 133 |
secdmp, damping, dir |
| 134 |
lea tableq, [tap_table] |
| 135 |
|
| 136 |
; off1/2/3[k] [6 total] |
| 137 |
movd xm2, tmp_strided |
| 138 |
vpbroadcastd m2, xm2 |
| 139 |
mov dird, r6m |
| 140 |
pmovsxbd m3, [tableq+dirq*8+16] |
| 141 |
pmulld m2, m3 |
| 142 |
pmovsxbd m4, [tableq+dirq*8+16+64] |
| 143 |
paddd m2, m4 |
| 144 |
mova [offsets], m2 |
| 145 |
|
| 146 |
; register to shuffle values into after packing |
| 147 |
vbroadcasti128 m12, [shufb_lohi] |
| 148 |
|
| 149 |
movifnidn prid, prim |
| 150 |
mov dampingd, r7m |
| 151 |
lzcnt pridmpd, prid |
| 152 |
%if UNIX64 |
| 153 |
movd xm0, prid |
| 154 |
movd xm1, secd |
| 155 |
%endif |
| 156 |
lzcnt secdmpd, secm |
| 157 |
sub dampingd, 31 |
| 158 |
DEFINE_ARGS dst, dst_stride, tmp, tmp_stride, pri, sec, pridmp, table, \ |
| 159 |
secdmp, damping, zero |
| 160 |
xor zerod, zerod |
| 161 |
add pridmpd, dampingd |
| 162 |
cmovl pridmpd, zerod |
| 163 |
add secdmpd, dampingd |
| 164 |
cmovl secdmpd, zerod |
| 165 |
mov [rsp+0], pridmpq ; pri_shift |
| 166 |
mov [rsp+8], secdmpq ; sec_shift |
| 167 |
|
| 168 |
DEFINE_ARGS dst, dst_stride, tmp, tmp_stride, pri, sec, pridmp, table, \ |
| 169 |
secdmp |
| 170 |
vpbroadcastb m13, [tableq+pridmpq] ; pri_shift_mask |
| 171 |
vpbroadcastb m14, [tableq+secdmpq] ; sec_shift_mask |
| 172 |
|
| 173 |
; pri/sec_taps[k] [4 total] |
| 174 |
DEFINE_ARGS dst, dst_stride, tmp, tmp_stride, pri, sec, dummy, table, \ |
| 175 |
secdmp |
| 176 |
%if UNIX64 |
| 177 |
vpbroadcastb m0, xm0 ; pri_strength |
| 178 |
vpbroadcastb m1, xm1 ; sec_strength |
| 179 |
%else |
| 180 |
vpbroadcastb m0, prim |
| 181 |
vpbroadcastb m1, secm |
| 182 |
%endif |
| 183 |
and prid, 1 |
| 184 |
lea priq, [tableq+priq*2+8] ; pri_taps |
| 185 |
lea secq, [tableq+12] ; sec_taps |
| 186 |
%if %1*%2*2/mmsize > 1 |
| 187 |
%if %1 == 4 |
| 188 |
DEFINE_ARGS dst, dst_stride, tmp0, tmp_stride, pri, sec, dst_stride3, h, off, k, tmp1, tmp2, tmp3 |
| 189 |
lea dst_stride3q, [dst_strideq*3] |
| 190 |
%else |
| 191 |
DEFINE_ARGS dst, dst_stride, tmp0, tmp_stride, pri, sec, h, off, k, tmp1 |
| 192 |
%endif |
| 193 |
mov hd, %1*%2*2/mmsize |
| 194 |
%else |
| 195 |
DEFINE_ARGS dst, dst_stride, tmp0, tmp_stride, pri, sec, dst_stride3, off, k, tmp1, tmp2, tmp3 |
| 196 |
lea dst_stride3q, [dst_strideq*3] |
| 197 |
%endif |
| 198 |
pxor m11, m11 |
| 199 |
%if %1*%2*2/mmsize > 1 |
| 200 |
.v_loop: |
| 201 |
%endif |
| 202 |
lea tmp1q, [tmp0q+tmp_strideq*2] |
| 203 |
%if %1 == 4 |
| 204 |
lea tmp2q, [tmp0q+tmp_strideq*4] |
| 205 |
lea tmp3q, [tmp1q+tmp_strideq*4] |
| 206 |
%endif |
| 207 |
mov kd, 1 |
| 208 |
%if %1 == 4 |
| 209 |
movq xm4, [tmp0q] |
| 210 |
movhps xm4, [tmp1q] |
| 211 |
movq xm5, [tmp2q] |
| 212 |
movhps xm5, [tmp3q] |
| 213 |
vinserti128 m4, xm5, 1 |
| 214 |
%else |
| 215 |
mova xm4, [tmp0q] ; px |
| 216 |
vinserti128 m4, [tmp1q], 1 |
| 217 |
%endif |
| 218 |
pxor m15, m15 ; sum |
| 219 |
mova m7, m4 ; max |
| 220 |
mova m8, m4 ; min |
| 221 |
.k_loop: |
| 222 |
vpbroadcastb m2, [priq+kq] ; pri_taps |
| 223 |
vpbroadcastb m3, [secq+kq] ; sec_taps |
| 224 |
|
| 225 |
ACCUMULATE_TAP 0*8, [rsp+0], m13, m0, m2, %1, %3 |
| 226 |
ACCUMULATE_TAP 1*8, [rsp+8], m14, m1, m3, %1, %3 |
| 227 |
ACCUMULATE_TAP 2*8, [rsp+8], m14, m1, m3, %1, %3 |
| 228 |
|
| 229 |
dec kq |
| 230 |
jge .k_loop |
| 231 |
|
| 232 |
vpbroadcastd m10, [pw_2048] |
| 233 |
pcmpgtw m9, m11, m15 |
| 234 |
paddw m15, m9 |
| 235 |
pmulhrsw m15, m10 |
| 236 |
paddw m4, m15 |
| 237 |
pminsw m4, m7 |
| 238 |
pmaxsw m4, m8 |
| 239 |
packuswb m4, m4 |
| 240 |
vextracti128 xm5, m4, 1 |
| 241 |
%if %1 == 4 |
| 242 |
movd [dstq+dst_strideq*0], xm4 |
| 243 |
pextrd [dstq+dst_strideq*1], xm4, 1 |
| 244 |
movd [dstq+dst_strideq*2], xm5 |
| 245 |
pextrd [dstq+dst_stride3q], xm5, 1 |
| 246 |
%else |
| 247 |
movq [dstq+dst_strideq*0], xm4 |
| 248 |
movq [dstq+dst_strideq*1], xm5 |
| 249 |
%endif |
| 250 |
|
| 251 |
%if %1*%2*2/mmsize > 1 |
| 252 |
%define vloop_lines (mmsize/(%1*2)) |
| 253 |
lea dstq, [dstq+dst_strideq*vloop_lines] |
| 254 |
lea tmp0q, [tmp0q+tmp_strideq*2*vloop_lines] |
| 255 |
dec hd |
| 256 |
jg .v_loop |
| 257 |
%endif |
| 258 |
|
| 259 |
RET |
| 260 |
%endmacro |
| 261 |
|
| 262 |
CDEF_FILTER 8, 8 |
| 263 |
CDEF_FILTER 4, 8 |
| 264 |
CDEF_FILTER 4, 4 |
| 265 |
|
| 266 |
%endif ; ARCH_X86_64 |