| 1 |
; Copyright © 2020, VideoLAN and dav1d authors |
| 2 |
; Copyright © 2020, Two Orioles, LLC |
| 3 |
; All rights reserved. |
| 4 |
; |
| 5 |
; Redistribution and use in source and binary forms, with or without |
| 6 |
; modification, are permitted provided that the following conditions are met: |
| 7 |
; |
| 8 |
; 1. Redistributions of source code must retain the above copyright notice, this |
| 9 |
; list of conditions and the following disclaimer. |
| 10 |
; |
| 11 |
; 2. Redistributions in binary form must reproduce the above copyright notice, |
| 12 |
; this list of conditions and the following disclaimer in the documentation |
| 13 |
; and/or other materials provided with the distribution. |
| 14 |
; |
| 15 |
; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND |
| 16 |
; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
| 17 |
; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE |
| 18 |
; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR |
| 19 |
; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES |
| 20 |
; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
| 21 |
; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND |
| 22 |
; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 23 |
; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS |
| 24 |
; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 25 |
|
| 26 |
%include "config.asm" |
| 27 |
%include "src/asm/x86/x86inc.asm" |
| 28 |
|
| 29 |
%if ARCH_X86_64 |
| 30 |
|
| 31 |
SECTION_RODATA 64 |
| 32 |
|
| 33 |
%macro SMOOTH_WEIGHT_TABLE 1-* |
| 34 |
%rep %0 |
| 35 |
db %1-128, 127-%1 |
| 36 |
%rotate 1 |
| 37 |
%endrep |
| 38 |
%endmacro |
| 39 |
|
| 40 |
smooth_weights: SMOOTH_WEIGHT_TABLE \ |
| 41 |
0, 0, 255, 128, 255, 149, 85, 64, \ |
| 42 |
255, 197, 146, 105, 73, 50, 37, 32, \ |
| 43 |
255, 225, 196, 170, 145, 123, 102, 84, \ |
| 44 |
68, 54, 43, 33, 26, 20, 17, 16, \ |
| 45 |
255, 240, 225, 210, 196, 182, 169, 157, \ |
| 46 |
145, 133, 122, 111, 101, 92, 83, 74, \ |
| 47 |
66, 59, 52, 45, 39, 34, 29, 25, \ |
| 48 |
21, 17, 14, 12, 10, 9, 8, 8, \ |
| 49 |
255, 248, 240, 233, 225, 218, 210, 203, \ |
| 50 |
196, 189, 182, 176, 169, 163, 156, 150, \ |
| 51 |
144, 138, 133, 127, 121, 116, 111, 106, \ |
| 52 |
101, 96, 91, 86, 82, 77, 73, 69, \ |
| 53 |
65, 61, 57, 54, 50, 47, 44, 41, \ |
| 54 |
38, 35, 32, 29, 27, 25, 22, 20, \ |
| 55 |
18, 16, 15, 13, 12, 10, 9, 8, \ |
| 56 |
7, 6, 6, 5, 5, 4, 4, 4 |
| 57 |
|
| 58 |
; dav1d_filter_intra_taps[], reordered for VNNI: p1 p2 p3 p4, p6 p5 p0 __ |
| 59 |
filter_taps: db 10, 0, 0, 0, 2, 10, 0, 0, 1, 1, 10, 0, 1, 1, 2, 10 |
| 60 |
db 6, 0, 0, 0, 2, 6, 0, 0, 2, 2, 6, 0, 1, 2, 2, 6 |
| 61 |
db 0, 12, -6, 0, 0, 9, -5, 0, 0, 7, -3, 0, 0, 5, -3, 0 |
| 62 |
db 12, 2, -4, 0, 9, 2, -3, 0, 7, 2, -3, 0, 5, 3, -3, 0 |
| 63 |
db 16, 0, 0, 0, 0, 16, 0, 0, 0, 0, 16, 0, 0, 0, 0, 16 |
| 64 |
db 16, 0, 0, 0, 0, 16, 0, 0, 0, 0, 16, 0, 0, 0, 0, 16 |
| 65 |
db 0, 10,-10, 0, 0, 6, -6, 0, 0, 4, -4, 0, 0, 2, -2, 0 |
| 66 |
db 10, 0,-10, 0, 6, 0, -6, 0, 4, 0, -4, 0, 2, 0, -2, 0 |
| 67 |
db 8, 0, 0, 0, 0, 8, 0, 0, 0, 0, 8, 0, 0, 0, 0, 8 |
| 68 |
db 4, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4 |
| 69 |
db 0, 16, -8, 0, 0, 16, -8, 0, 0, 16, -8, 0, 0, 16, -8, 0 |
| 70 |
db 16, 0, -4, 0, 16, 0, -4, 0, 16, 0, -4, 0, 16, 0, -4, 0 |
| 71 |
db 8, 0, 0, 0, 3, 8, 0, 0, 2, 3, 8, 0, 1, 2, 3, 8 |
| 72 |
db 4, 0, 0, 0, 3, 4, 0, 0, 2, 3, 4, 0, 2, 2, 3, 4 |
| 73 |
db 0, 10, -2, 0, 0, 6, -1, 0, 0, 4, -1, 0, 0, 2, 0, 0 |
| 74 |
db 10, 3, -1, 0, 6, 4, -1, 0, 4, 4, -1, 0, 3, 3, -1, 0 |
| 75 |
db 14, 0, 0, 0, 0, 14, 0, 0, 0, 0, 14, 0, 0, 0, 0, 14 |
| 76 |
db 12, 0, 0, 0, 1, 12, 0, 0, 0, 0, 12, 0, 0, 0, 1, 12 |
| 77 |
db 0, 14,-12, 0, 0, 12,-10, 0, 0, 11, -9, 0, 0, 10, -8, 0 |
| 78 |
db 14, 0,-10, 0, 12, 0, -9, 0, 11, 1, -8, 0, 9, 1, -7, 0 |
| 79 |
filter_perm: db 0, 1, 2, 3, 24, 25, 26, 27, 4, 5, 6, 7, 28, 29, 30, 31 |
| 80 |
db 15, 11, 7, 3, 15, 11, 7, 3, 15, 11, 7, 3, 15, 11, 7,131 |
| 81 |
db 31, 27, 23, 19, 31, 27, 23, 19, 31, 27, 23, 19, 31, 27, 23,147 |
| 82 |
db 47, 43, 39, 35, 47, 43, 39, 35, 47, 43, 39, 35, 47, 43, 39,163 |
| 83 |
filter_end: dd 2, 3, 16, 17, -1, -1, 20, 21, 0, 6, 24, 30, 1, 7, 25, 31 |
| 84 |
smooth_shuf: db 7, 7, 7, 7, 0, 1, 0, 1, 3, 3, 3, 3, 8, 9, 8, 9 |
| 85 |
db 5, 5, 5, 5, 4, 5, 4, 5, 1, 1, 1, 1, 12, 13, 12, 13 |
| 86 |
db 6, 6, 6, 6, 2, 3, 2, 3, 2, 2, 2, 2, 10, 11, 10, 11 |
| 87 |
db 4, 4, 4, 4, 6, 7, 6, 7, 0, 0, 0, 0, 14, 15, 14, 15 |
| 88 |
smooth_endA: db 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31 |
| 89 |
db 33, 35, 37, 39, 41, 43, 45, 47, 49, 51, 53, 55, 57, 59, 61, 63 |
| 90 |
db 65, 67, 69, 71, 73, 75, 77, 79, 81, 83, 85, 87, 89, 91, 93, 95 |
| 91 |
db 97, 99,101,103,105,107,109,111,113,115,117,119,121,123,125,127 |
| 92 |
smooth_endB: db 1, 3, 5, 7, 9, 11, 13, 15, 65, 67, 69, 71, 73, 75, 77, 79 |
| 93 |
db 17, 19, 21, 23, 25, 27, 29, 31, 81, 83, 85, 87, 89, 91, 93, 95 |
| 94 |
db 33, 35, 37, 39, 41, 43, 45, 47, 97, 99,101,103,105,107,109,111 |
| 95 |
db 49, 51, 53, 55, 57, 59, 61, 63,113,115,117,119,121,123,125,127 |
| 96 |
ipred_h_shuf: db 7, 7, 7, 7, 6, 6, 6, 6, 5, 5, 5, 5, 4, 4, 4, 4 |
| 97 |
db 3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0 |
| 98 |
|
| 99 |
pb_127_m127: times 2 db 127, -127 |
| 100 |
pb_128: times 4 db 128 |
| 101 |
pw_128: times 2 dw 128 |
| 102 |
pw_255: times 2 dw 255 |
| 103 |
|
| 104 |
%define pb_1 (ipred_h_shuf+24) |
| 105 |
%define pb_2 (ipred_h_shuf+20) |
| 106 |
%define pb_3 (ipred_h_shuf+16) |
| 107 |
%define pd_8 (filter_taps+128) |
| 108 |
|
| 109 |
%macro JMP_TABLE 3-* |
| 110 |
%xdefine %1_%2_table (%%table - 2*4) |
| 111 |
%xdefine %%base mangle(private_prefix %+ _%1_%2) |
| 112 |
%%table: |
| 113 |
%rep %0 - 2 |
| 114 |
dd %%base %+ .%3 - (%%table - 2*4) |
| 115 |
%rotate 1 |
| 116 |
%endrep |
| 117 |
%endmacro |
| 118 |
|
| 119 |
%define ipred_dc_splat_8bpc_avx512icl_table (ipred_dc_8bpc_avx512icl_table + 10*4) |
| 120 |
|
| 121 |
JMP_TABLE ipred_h_8bpc, avx512icl, w4, w8, w16, w32, w64 |
| 122 |
JMP_TABLE ipred_paeth_8bpc, avx512icl, w4, w8, w16, w32, w64 |
| 123 |
JMP_TABLE ipred_smooth_8bpc, avx512icl, w4, w8, w16, w32, w64 |
| 124 |
JMP_TABLE ipred_smooth_v_8bpc, avx512icl, w4, w8, w16, w32, w64 |
| 125 |
JMP_TABLE ipred_smooth_h_8bpc, avx512icl, w4, w8, w16, w32, w64 |
| 126 |
JMP_TABLE ipred_dc_8bpc, avx512icl, h4, h8, h16, h32, h64, w4, w8, w16, w32, w64, \ |
| 127 |
s4-10*4, s8-10*4, s16-10*4, s32-10*4, s64-10*4 |
| 128 |
JMP_TABLE ipred_dc_left_8bpc, avx512icl, h4, h8, h16, h32, h64 |
| 129 |
JMP_TABLE pal_pred_8bpc, avx512icl, w4, w8, w16, w32, w64 |
| 130 |
|
| 131 |
SECTION .text |
| 132 |
|
| 133 |
INIT_ZMM avx512icl |
| 134 |
cglobal ipred_dc_top_8bpc, 3, 7, 5, dst, stride, tl, w, h |
| 135 |
lea r5, [ipred_dc_left_8bpc_avx512icl_table] |
| 136 |
movd xm0, wm |
| 137 |
tzcnt wd, wm |
| 138 |
inc tlq |
| 139 |
movifnidn hd, hm |
| 140 |
movu ym1, [tlq] |
| 141 |
movd xmm3, wd |
| 142 |
movsxd r6, [r5+wq*4] |
| 143 |
vpbroadcastd ym2, [r5-ipred_dc_left_8bpc_avx512icl_table+pb_1] |
| 144 |
psrld xm0, 1 |
| 145 |
vpdpbusd ym0, ym1, ym2 |
| 146 |
add r6, r5 |
| 147 |
add r5, ipred_dc_splat_8bpc_avx512icl_table-ipred_dc_left_8bpc_avx512icl_table |
| 148 |
movsxd wq, [r5+wq*4] |
| 149 |
add wq, r5 |
| 150 |
jmp r6 |
| 151 |
|
| 152 |
cglobal ipred_dc_left_8bpc, 3, 7, 5, dst, stride, tl, w, h, stride3 |
| 153 |
lea r5, [ipred_dc_left_8bpc_avx512icl_table] |
| 154 |
mov hd, hm |
| 155 |
tzcnt r6d, hd |
| 156 |
sub tlq, hq |
| 157 |
tzcnt wd, wm |
| 158 |
movd xm0, hm |
| 159 |
movu ym1, [tlq] |
| 160 |
movd xmm3, r6d |
| 161 |
movsxd r6, [r5+r6*4] |
| 162 |
vpbroadcastd ym2, [r5-ipred_dc_left_8bpc_avx512icl_table+pb_1] |
| 163 |
psrld xm0, 1 |
| 164 |
vpdpbusd ym0, ym1, ym2 |
| 165 |
add r6, r5 |
| 166 |
add r5, ipred_dc_splat_8bpc_avx512icl_table-ipred_dc_left_8bpc_avx512icl_table |
| 167 |
movsxd wq, [r5+wq*4] |
| 168 |
add wq, r5 |
| 169 |
jmp r6 |
| 170 |
.h64: |
| 171 |
movu ym1, [tlq+32] ; unaligned when jumping here from dc_top |
| 172 |
vpdpbusd ym0, ym1, ym2 |
| 173 |
.h32: |
| 174 |
vextracti32x4 xm1, ym0, 1 |
| 175 |
paddd xm0, xm1 |
| 176 |
.h16: |
| 177 |
punpckhqdq xm1, xm0, xm0 |
| 178 |
paddd xm0, xm1 |
| 179 |
.h8: |
| 180 |
psrlq xm1, xm0, 32 |
| 181 |
paddd xm0, xm1 |
| 182 |
.h4: |
| 183 |
vpsrlvd xm0, xmm3 |
| 184 |
lea stride3q, [strideq*3] |
| 185 |
vpbroadcastb m0, xm0 |
| 186 |
jmp wq |
| 187 |
|
| 188 |
cglobal ipred_dc_8bpc, 3, 7, 5, dst, stride, tl, w, h, stride3 |
| 189 |
movifnidn hd, hm |
| 190 |
movifnidn wd, wm |
| 191 |
tzcnt r6d, hd |
| 192 |
lea r5d, [wq+hq] |
| 193 |
movd xm0, r5d |
| 194 |
tzcnt r5d, r5d |
| 195 |
movd xmm4, r5d |
| 196 |
lea r5, [ipred_dc_8bpc_avx512icl_table] |
| 197 |
tzcnt wd, wd |
| 198 |
movsxd r6, [r5+r6*4] |
| 199 |
movsxd wq, [r5+wq*4+5*4] |
| 200 |
vpbroadcastd ym3, [r5-ipred_dc_8bpc_avx512icl_table+pb_1] |
| 201 |
psrld xm0, 1 |
| 202 |
add r6, r5 |
| 203 |
add wq, r5 |
| 204 |
lea stride3q, [strideq*3] |
| 205 |
jmp r6 |
| 206 |
.h4: |
| 207 |
movd xmm1, [tlq-4] |
| 208 |
vpdpbusd xm0, xmm1, xm3 |
| 209 |
jmp wq |
| 210 |
.w4: |
| 211 |
movd xmm1, [tlq+1] |
| 212 |
vpdpbusd xm0, xmm1, xm3 |
| 213 |
cmp hd, 4 |
| 214 |
jg .w4_mul |
| 215 |
psrlw xmm0, xm0, 3 |
| 216 |
jmp .w4_end |
| 217 |
.w4_mul: |
| 218 |
punpckhqdq xmm1, xm0, xm0 |
| 219 |
lea r2d, [hq*2] |
| 220 |
mov r6d, 0x55563334 |
| 221 |
paddd xmm1, xm0 |
| 222 |
shrx r6d, r6d, r2d |
| 223 |
psrlq xmm0, xmm1, 32 |
| 224 |
paddd xmm0, xmm1 |
| 225 |
movd xmm1, r6d |
| 226 |
psrld xmm0, 2 |
| 227 |
pmulhuw xmm0, xmm1 |
| 228 |
.w4_end: |
| 229 |
vpbroadcastb xm0, xmm0 |
| 230 |
.s4: |
| 231 |
movd [dstq+strideq*0], xm0 |
| 232 |
movd [dstq+strideq*1], xm0 |
| 233 |
movd [dstq+strideq*2], xm0 |
| 234 |
movd [dstq+stride3q ], xm0 |
| 235 |
lea dstq, [dstq+strideq*4] |
| 236 |
sub hd, 4 |
| 237 |
jg .s4 |
| 238 |
RET |
| 239 |
.h8: |
| 240 |
movq xmm1, [tlq-8] |
| 241 |
vpdpbusd xm0, xmm1, xm3 |
| 242 |
jmp wq |
| 243 |
.w8: |
| 244 |
movq xmm1, [tlq+1] |
| 245 |
vextracti32x4 xm2, ym0, 1 |
| 246 |
vpdpbusd xm0, xmm1, xm3 |
| 247 |
paddd xmm2, xm2, xm0 |
| 248 |
punpckhqdq xmm0, xmm2, xmm2 |
| 249 |
paddd xmm0, xmm2 |
| 250 |
psrlq xmm1, xmm0, 32 |
| 251 |
paddd xmm0, xmm1 |
| 252 |
vpsrlvd xmm0, xmm4 |
| 253 |
cmp hd, 8 |
| 254 |
je .w8_end |
| 255 |
mov r6d, 0x5556 |
| 256 |
mov r2d, 0x3334 |
| 257 |
cmp hd, 32 |
| 258 |
cmove r6d, r2d |
| 259 |
movd xmm1, r6d |
| 260 |
pmulhuw xmm0, xmm1 |
| 261 |
.w8_end: |
| 262 |
vpbroadcastb xm0, xmm0 |
| 263 |
.s8: |
| 264 |
movq [dstq+strideq*0], xm0 |
| 265 |
movq [dstq+strideq*1], xm0 |
| 266 |
movq [dstq+strideq*2], xm0 |
| 267 |
movq [dstq+stride3q ], xm0 |
| 268 |
lea dstq, [dstq+strideq*4] |
| 269 |
sub hd, 4 |
| 270 |
jg .s8 |
| 271 |
RET |
| 272 |
.h16: |
| 273 |
mova xmm1, [tlq-16] |
| 274 |
vpdpbusd xm0, xmm1, xm3 |
| 275 |
jmp wq |
| 276 |
.w16: |
| 277 |
movu xmm1, [tlq+1] |
| 278 |
vextracti32x4 xm2, ym0, 1 |
| 279 |
vpdpbusd xm0, xmm1, xm3 |
| 280 |
paddd xmm2, xm2, xm0 |
| 281 |
punpckhqdq xmm0, xmm2, xmm2 |
| 282 |
paddd xmm0, xmm2 |
| 283 |
psrlq xmm1, xmm0, 32 |
| 284 |
paddd xmm0, xmm1 |
| 285 |
vpsrlvd xmm0, xmm4 |
| 286 |
cmp hd, 16 |
| 287 |
je .w16_end |
| 288 |
mov r6d, 0x5556 |
| 289 |
mov r2d, 0x3334 |
| 290 |
test hb, 8|32 |
| 291 |
cmovz r6d, r2d |
| 292 |
movd xmm1, r6d |
| 293 |
pmulhuw xmm0, xmm1 |
| 294 |
.w16_end: |
| 295 |
vpbroadcastb xm0, xmm0 |
| 296 |
.s16: |
| 297 |
mova [dstq+strideq*0], xm0 |
| 298 |
mova [dstq+strideq*1], xm0 |
| 299 |
mova [dstq+strideq*2], xm0 |
| 300 |
mova [dstq+stride3q ], xm0 |
| 301 |
lea dstq, [dstq+strideq*4] |
| 302 |
sub hd, 4 |
| 303 |
jg .s16 |
| 304 |
RET |
| 305 |
.h32: |
| 306 |
mova ym1, [tlq-32] |
| 307 |
vpdpbusd ym0, ym1, ym3 |
| 308 |
jmp wq |
| 309 |
.w32: |
| 310 |
movu ym1, [tlq+1] |
| 311 |
vpdpbusd ym0, ym1, ym3 |
| 312 |
vextracti32x4 xm1, ym0, 1 |
| 313 |
paddd xmm1, xm1, xm0 |
| 314 |
punpckhqdq xmm0, xmm1, xmm1 |
| 315 |
paddd xmm0, xmm1 |
| 316 |
psrlq xmm1, xmm0, 32 |
| 317 |
paddd xmm0, xmm1 |
| 318 |
vpsrlvd xmm0, xmm4 |
| 319 |
cmp hd, 32 |
| 320 |
je .w32_end |
| 321 |
lea r2d, [hq*2] |
| 322 |
mov r6d, 0x33345556 |
| 323 |
shrx r6d, r6d, r2d |
| 324 |
movd xmm1, r6d |
| 325 |
pmulhuw xmm0, xmm1 |
| 326 |
.w32_end: |
| 327 |
vpbroadcastb ym0, xmm0 |
| 328 |
.s32: |
| 329 |
mova [dstq+strideq*0], ym0 |
| 330 |
mova [dstq+strideq*1], ym0 |
| 331 |
mova [dstq+strideq*2], ym0 |
| 332 |
mova [dstq+stride3q ], ym0 |
| 333 |
lea dstq, [dstq+strideq*4] |
| 334 |
sub hd, 4 |
| 335 |
jg .s32 |
| 336 |
RET |
| 337 |
.h64: |
| 338 |
mova ym1, [tlq-64] |
| 339 |
mova ym2, [tlq-32] |
| 340 |
vpdpbusd ym0, ym1, ym3 |
| 341 |
vpdpbusd ym0, ym2, ym3 |
| 342 |
jmp wq |
| 343 |
.w64: |
| 344 |
movu ym1, [tlq+ 1] |
| 345 |
movu ym2, [tlq+33] |
| 346 |
vpdpbusd ym0, ym1, ym3 |
| 347 |
vpdpbusd ym0, ym2, ym3 |
| 348 |
vextracti32x4 xm1, ym0, 1 |
| 349 |
paddd xmm1, xm1, xm0 |
| 350 |
punpckhqdq xmm0, xmm1, xmm1 |
| 351 |
paddd xmm0, xmm1 |
| 352 |
psrlq xmm1, xmm0, 32 |
| 353 |
paddd xmm0, xmm1 |
| 354 |
vpsrlvd xmm0, xmm4 |
| 355 |
cmp hd, 64 |
| 356 |
je .w64_end |
| 357 |
mov r6d, 0x33345556 |
| 358 |
shrx r6d, r6d, hd |
| 359 |
movd xmm1, r6d |
| 360 |
pmulhuw xmm0, xmm1 |
| 361 |
.w64_end: |
| 362 |
vpbroadcastb m0, xmm0 |
| 363 |
.s64: |
| 364 |
mova [dstq+strideq*0], m0 |
| 365 |
mova [dstq+strideq*1], m0 |
| 366 |
mova [dstq+strideq*2], m0 |
| 367 |
mova [dstq+stride3q ], m0 |
| 368 |
lea dstq, [dstq+strideq*4] |
| 369 |
sub hd, 4 |
| 370 |
jg .s64 |
| 371 |
RET |
| 372 |
|
| 373 |
cglobal ipred_dc_128_8bpc, 2, 7, 5, dst, stride, tl, w, h, stride3 |
| 374 |
lea r5, [ipred_dc_splat_8bpc_avx512icl_table] |
| 375 |
tzcnt wd, wm |
| 376 |
movifnidn hd, hm |
| 377 |
movsxd wq, [r5+wq*4] |
| 378 |
vpbroadcastd m0, [r5-ipred_dc_splat_8bpc_avx512icl_table+pb_128] |
| 379 |
add wq, r5 |
| 380 |
lea stride3q, [strideq*3] |
| 381 |
jmp wq |
| 382 |
|
| 383 |
cglobal ipred_v_8bpc, 3, 7, 5, dst, stride, tl, w, h, stride3 |
| 384 |
lea r5, [ipred_dc_splat_8bpc_avx512icl_table] |
| 385 |
tzcnt wd, wm |
| 386 |
movu m0, [tlq+1] |
| 387 |
movifnidn hd, hm |
| 388 |
movsxd wq, [r5+wq*4] |
| 389 |
add wq, r5 |
| 390 |
lea stride3q, [strideq*3] |
| 391 |
jmp wq |
| 392 |
|
| 393 |
cglobal ipred_h_8bpc, 3, 7, 8, dst, stride, tl, w, h, stride3 |
| 394 |
%define base r6-ipred_h_8bpc_avx512icl_table |
| 395 |
lea r6, [ipred_h_8bpc_avx512icl_table] |
| 396 |
tzcnt wd, wm |
| 397 |
mov hd, hm |
| 398 |
movsxd wq, [r6+wq*4] |
| 399 |
lea stride3q, [strideq*3] |
| 400 |
sub tlq, hq |
| 401 |
add wq, r6 |
| 402 |
jmp wq |
| 403 |
.w4: |
| 404 |
mova xmm1, [base+ipred_h_shuf+16] |
| 405 |
.w4_loop: |
| 406 |
movd xmm0, [tlq+hq-4] |
| 407 |
pshufb xmm0, xmm1 |
| 408 |
movd [dstq+strideq*0], xmm0 |
| 409 |
pextrd [dstq+strideq*1], xmm0, 1 |
| 410 |
pextrd [dstq+strideq*2], xmm0, 2 |
| 411 |
pextrd [dstq+stride3q ], xmm0, 3 |
| 412 |
lea dstq, [dstq+strideq*4] |
| 413 |
sub hd, 4 |
| 414 |
jg .w4_loop |
| 415 |
RET |
| 416 |
.w8: |
| 417 |
movsldup xmm2, [base+ipred_h_shuf+16] |
| 418 |
movshdup xmm3, [base+ipred_h_shuf+16] |
| 419 |
.w8_loop: |
| 420 |
movd xmm1, [tlq+hq-4] |
| 421 |
pshufb xmm0, xmm1, xmm2 |
| 422 |
pshufb xmm1, xmm3 |
| 423 |
movq [dstq+strideq*0], xmm0 |
| 424 |
movq [dstq+strideq*1], xmm1 |
| 425 |
movhps [dstq+strideq*2], xmm0 |
| 426 |
movhps [dstq+stride3q ], xmm1 |
| 427 |
lea dstq, [dstq+strideq*4] |
| 428 |
sub hd, 4 |
| 429 |
jg .w8_loop |
| 430 |
RET |
| 431 |
.w16: |
| 432 |
movsldup m1, [base+smooth_shuf] |
| 433 |
.w16_loop: |
| 434 |
vpbroadcastd m0, [tlq+hq-4] |
| 435 |
pshufb m0, m1 |
| 436 |
mova [dstq+strideq*0], xm0 |
| 437 |
vextracti32x4 [dstq+strideq*1], m0, 2 |
| 438 |
vextracti32x4 [dstq+strideq*2], ym0, 1 |
| 439 |
vextracti32x4 [dstq+stride3q ], m0, 3 |
| 440 |
lea dstq, [dstq+strideq*4] |
| 441 |
sub hd, 4 |
| 442 |
jg .w16 |
| 443 |
RET |
| 444 |
.w32: |
| 445 |
vpbroadcastd ym3, [base+pb_1] |
| 446 |
vpord m2, m3, [base+pb_2] {1to16} |
| 447 |
.w32_loop: |
| 448 |
vpbroadcastd m1, [tlq+hq-4] |
| 449 |
pshufb m0, m1, m2 |
| 450 |
pshufb m1, m3 |
| 451 |
mova [dstq+strideq*0], ym0 |
| 452 |
vextracti32x8 [dstq+strideq*1], m0, 1 |
| 453 |
mova [dstq+strideq*2], ym1 |
| 454 |
vextracti32x8 [dstq+stride3q ], m1, 1 |
| 455 |
lea dstq, [dstq+strideq*4] |
| 456 |
sub hd, 4 |
| 457 |
jg .w32_loop |
| 458 |
RET |
| 459 |
.w64: |
| 460 |
vpbroadcastd m4, [base+pb_3] |
| 461 |
vpbroadcastd m5, [base+pb_2] |
| 462 |
vpbroadcastd m6, [base+pb_1] |
| 463 |
pxor m7, m7 |
| 464 |
.w64_loop: |
| 465 |
vpbroadcastd m3, [tlq+hq-4] |
| 466 |
pshufb m0, m3, m4 |
| 467 |
pshufb m1, m3, m5 |
| 468 |
pshufb m2, m3, m6 |
| 469 |
pshufb m3, m7 |
| 470 |
mova [dstq+strideq*0], m0 |
| 471 |
mova [dstq+strideq*1], m1 |
| 472 |
mova [dstq+strideq*2], m2 |
| 473 |
mova [dstq+stride3q ], m3 |
| 474 |
lea dstq, [dstq+strideq*4] |
| 475 |
sub hd, 4 |
| 476 |
jg .w64_loop |
| 477 |
RET |
| 478 |
|
| 479 |
%macro PAETH 0 |
| 480 |
psubusb m1, m5, m4 |
| 481 |
psubusb m0, m4, m5 |
| 482 |
por m1, m0 ; tdiff |
| 483 |
pavgb m2, m6, m4 |
| 484 |
vpcmpub k1, m1, m7, 1 ; tdiff < ldiff |
| 485 |
vpblendmb m0{k1}, m4, m6 |
| 486 |
vpternlogd m4, m6, m8, 0x28 ; (m4 ^ m6) & m8 |
| 487 |
psubusb m3, m5, m2 |
| 488 |
psubb m2, m4 |
| 489 |
psubusb m2, m5 |
| 490 |
por m2, m3 |
| 491 |
pminub m1, m7 |
| 492 |
paddusb m2, m2 |
| 493 |
por m2, m4 ; min(tldiff, 255) |
| 494 |
vpcmpub k1, m2, m1, 1 ; tldiff < ldiff && tldiff < tdiff |
| 495 |
vmovdqu8 m0{k1}, m5 |
| 496 |
%endmacro |
| 497 |
|
| 498 |
cglobal ipred_paeth_8bpc, 3, 7, 10, dst, stride, tl, w, h, top, stride3 |
| 499 |
lea r6, [ipred_paeth_8bpc_avx512icl_table] |
| 500 |
tzcnt wd, wm |
| 501 |
vpbroadcastb m5, [tlq] ; topleft |
| 502 |
mov hd, hm |
| 503 |
movsxd wq, [r6+wq*4] |
| 504 |
vpbroadcastd m8, [r6-ipred_paeth_8bpc_avx512icl_table+pb_1] |
| 505 |
lea topq, [tlq+1] |
| 506 |
sub tlq, hq |
| 507 |
add wq, r6 |
| 508 |
lea stride3q, [strideq*3] |
| 509 |
jmp wq |
| 510 |
INIT_YMM avx512icl |
| 511 |
.w4: |
| 512 |
vpbroadcastd m6, [topq] |
| 513 |
mova m9, [ipred_h_shuf] |
| 514 |
psubusb m7, m5, m6 |
| 515 |
psubusb m0, m6, m5 |
| 516 |
por m7, m0 ; ldiff |
| 517 |
.w4_loop: |
| 518 |
vpbroadcastq m4, [tlq+hq-8] |
| 519 |
pshufb m4, m9 ; left |
| 520 |
PAETH |
| 521 |
movd [dstq+strideq*0], xm0 |
| 522 |
pextrd [dstq+strideq*1], xm0, 1 |
| 523 |
pextrd [dstq+strideq*2], xm0, 2 |
| 524 |
pextrd [dstq+stride3q ], xm0, 3 |
| 525 |
sub hd, 8 |
| 526 |
jl .w4_ret |
| 527 |
vextracti32x4 xm0, m0, 1 |
| 528 |
lea dstq, [dstq+strideq*4] |
| 529 |
movd [dstq+strideq*0], xm0 |
| 530 |
pextrd [dstq+strideq*1], xm0, 1 |
| 531 |
pextrd [dstq+strideq*2], xm0, 2 |
| 532 |
pextrd [dstq+stride3q ], xm0, 3 |
| 533 |
lea dstq, [dstq+strideq*4] |
| 534 |
jg .w4_loop |
| 535 |
.w4_ret: |
| 536 |
RET |
| 537 |
INIT_ZMM avx512icl |
| 538 |
.w8: |
| 539 |
vpbroadcastq m6, [topq] |
| 540 |
movsldup m9, [smooth_shuf] |
| 541 |
psubusb m7, m5, m6 |
| 542 |
psubusb m0, m6, m5 |
| 543 |
por m7, m0 |
| 544 |
.w8_loop: |
| 545 |
vpbroadcastq m4, [tlq+hq-8] |
| 546 |
pshufb m4, m9 |
| 547 |
PAETH |
| 548 |
vextracti32x4 xm1, m0, 2 |
| 549 |
vextracti32x4 xm2, ym0, 1 |
| 550 |
vextracti32x4 xm3, m0, 3 |
| 551 |
movq [dstq+strideq*0], xm0 |
| 552 |
movq [dstq+strideq*1], xm1 |
| 553 |
movq [dstq+strideq*2], xm2 |
| 554 |
movq [dstq+stride3q ], xm3 |
| 555 |
sub hd, 8 |
| 556 |
jl .w8_ret |
| 557 |
lea dstq, [dstq+strideq*4] |
| 558 |
movhps [dstq+strideq*0], xm0 |
| 559 |
movhps [dstq+strideq*1], xm1 |
| 560 |
movhps [dstq+strideq*2], xm2 |
| 561 |
movhps [dstq+stride3q ], xm3 |
| 562 |
lea dstq, [dstq+strideq*4] |
| 563 |
jg .w8_loop |
| 564 |
.w8_ret: |
| 565 |
RET |
| 566 |
.w16: |
| 567 |
vbroadcasti32x4 m6, [topq] |
| 568 |
movsldup m9, [smooth_shuf] |
| 569 |
psubusb m7, m5, m6 |
| 570 |
psubusb m0, m6, m5 |
| 571 |
por m7, m0 |
| 572 |
.w16_loop: |
| 573 |
vpbroadcastd m4, [tlq+hq-4] |
| 574 |
pshufb m4, m9 |
| 575 |
PAETH |
| 576 |
mova [dstq+strideq*0], xm0 |
| 577 |
vextracti32x4 [dstq+strideq*1], m0, 2 |
| 578 |
vextracti32x4 [dstq+strideq*2], ym0, 1 |
| 579 |
vextracti32x4 [dstq+stride3q ], m0, 3 |
| 580 |
lea dstq, [dstq+strideq*4] |
| 581 |
sub hd, 4 |
| 582 |
jg .w16_loop |
| 583 |
RET |
| 584 |
.w32: |
| 585 |
vbroadcasti32x8 m6, [topq] |
| 586 |
mova ym9, ym8 |
| 587 |
psubusb m7, m5, m6 |
| 588 |
psubusb m0, m6, m5 |
| 589 |
por m7, m0 |
| 590 |
.w32_loop: |
| 591 |
vpbroadcastd m4, [tlq+hq-2] |
| 592 |
pshufb m4, m9 |
| 593 |
PAETH |
| 594 |
mova [dstq+strideq*0], ym0 |
| 595 |
vextracti32x8 [dstq+strideq*1], m0, 1 |
| 596 |
lea dstq, [dstq+strideq*2] |
| 597 |
sub hd, 2 |
| 598 |
jg .w32_loop |
| 599 |
RET |
| 600 |
.w64: |
| 601 |
movu m6, [topq] |
| 602 |
psubusb m7, m5, m6 |
| 603 |
psubusb m0, m6, m5 |
| 604 |
por m7, m0 |
| 605 |
.w64_loop: |
| 606 |
vpbroadcastb m4, [tlq+hq-1] |
| 607 |
PAETH |
| 608 |
mova [dstq], m0 |
| 609 |
add dstq, strideq |
| 610 |
dec hd |
| 611 |
jg .w64_loop |
| 612 |
RET |
| 613 |
|
| 614 |
cglobal ipred_smooth_v_8bpc, 3, 7, 7, dst, stride, tl, w, h, weights, stride3 |
| 615 |
%define base r6-ipred_smooth_v_8bpc_avx512icl_table |
| 616 |
lea r6, [ipred_smooth_v_8bpc_avx512icl_table] |
| 617 |
tzcnt wd, wm |
| 618 |
mov hd, hm |
| 619 |
movsxd wq, [r6+wq*4] |
| 620 |
vpbroadcastd m0, [base+pb_127_m127] |
| 621 |
vpbroadcastd m1, [base+pw_128] |
| 622 |
lea weightsq, [base+smooth_weights+hq*4] |
| 623 |
neg hq |
| 624 |
vpbroadcastb m4, [tlq+hq] ; bottom |
| 625 |
add wq, r6 |
| 626 |
lea stride3q, [strideq*3] |
| 627 |
jmp wq |
| 628 |
.w4: |
| 629 |
vpbroadcastd m2, [tlq+1] |
| 630 |
movshdup m5, [smooth_shuf] |
| 631 |
mova ym6, [smooth_endA] |
| 632 |
punpcklbw m2, m4 ; top, bottom |
| 633 |
pmaddubsw m3, m2, m0 |
| 634 |
paddw m1, m2 ; 1 * top + 256 * bottom + 128, overflow is ok |
| 635 |
paddw m3, m1 ; 128 * top + 129 * bottom + 128 |
| 636 |
.w4_loop: |
| 637 |
vbroadcasti32x4 m0, [weightsq+hq*2] |
| 638 |
pshufb m0, m5 |
| 639 |
pmaddubsw m0, m2, m0 |
| 640 |
paddw m0, m3 |
| 641 |
vpermb m0, m6, m0 |
| 642 |
vextracti32x4 xm1, ym0, 1 |
| 643 |
movd [dstq+strideq*0], xm0 |
| 644 |
movd [dstq+strideq*1], xm1 |
| 645 |
pextrd [dstq+strideq*2], xm0, 2 |
| 646 |
pextrd [dstq+stride3q ], xm1, 2 |
| 647 |
add hq, 8 |
| 648 |
jg .ret |
| 649 |
lea dstq, [dstq+strideq*4] |
| 650 |
pextrd [dstq+strideq*0], xm0, 1 |
| 651 |
pextrd [dstq+strideq*1], xm1, 1 |
| 652 |
pextrd [dstq+strideq*2], xm0, 3 |
| 653 |
pextrd [dstq+stride3q ], xm1, 3 |
| 654 |
lea dstq, [dstq+strideq*4] |
| 655 |
jl .w4_loop |
| 656 |
.ret: |
| 657 |
RET |
| 658 |
.w8: |
| 659 |
vpbroadcastq m2, [tlq+1] |
| 660 |
movshdup m5, [smooth_shuf] |
| 661 |
mova ym6, [smooth_endA] |
| 662 |
punpcklbw m2, m4 |
| 663 |
pmaddubsw m3, m2, m0 |
| 664 |
paddw m1, m2 |
| 665 |
paddw m3, m1 |
| 666 |
.w8_loop: |
| 667 |
vpbroadcastq m0, [weightsq+hq*2] |
| 668 |
pshufb m0, m5 |
| 669 |
pmaddubsw m0, m2, m0 |
| 670 |
paddw m0, m3 |
| 671 |
vpermb m0, m6, m0 |
| 672 |
vextracti32x4 xm1, ym0, 1 |
| 673 |
movq [dstq+strideq*0], xm0 |
| 674 |
movq [dstq+strideq*1], xm1 |
| 675 |
movhps [dstq+strideq*2], xm0 |
| 676 |
movhps [dstq+stride3q ], xm1 |
| 677 |
lea dstq, [dstq+strideq*4] |
| 678 |
add hq, 4 |
| 679 |
jl .w8_loop |
| 680 |
RET |
| 681 |
.w16: |
| 682 |
vbroadcasti32x4 m3, [tlq+1] |
| 683 |
movshdup m6, [smooth_shuf] |
| 684 |
mova m7, [smooth_endB] |
| 685 |
punpcklbw m2, m3, m4 |
| 686 |
punpckhbw m3, m4 |
| 687 |
pmaddubsw m4, m2, m0 |
| 688 |
pmaddubsw m5, m3, m0 |
| 689 |
paddw m0, m1, m2 |
| 690 |
paddw m1, m3 |
| 691 |
paddw m4, m0 |
| 692 |
paddw m5, m1 |
| 693 |
.w16_loop: |
| 694 |
vpbroadcastq m1, [weightsq+hq*2] |
| 695 |
pshufb m1, m6 |
| 696 |
pmaddubsw m0, m2, m1 |
| 697 |
pmaddubsw m1, m3, m1 |
| 698 |
paddw m0, m4 |
| 699 |
paddw m1, m5 |
| 700 |
vpermt2b m0, m7, m1 |
| 701 |
mova [dstq+strideq*0], xm0 |
| 702 |
vextracti32x4 [dstq+strideq*1], m0, 2 |
| 703 |
vextracti32x4 [dstq+strideq*2], ym0, 1 |
| 704 |
vextracti32x4 [dstq+stride3q ], m0, 3 |
| 705 |
lea dstq, [dstq+strideq*4] |
| 706 |
add hq, 4 |
| 707 |
jl .w16_loop |
| 708 |
RET |
| 709 |
.w32: |
| 710 |
vbroadcasti32x8 m3, [tlq+1] |
| 711 |
movshdup m6, [smooth_shuf] |
| 712 |
mova m7, [smooth_endB] |
| 713 |
punpcklbw m2, m3, m4 |
| 714 |
punpckhbw m3, m4 |
| 715 |
pmaddubsw m4, m2, m0 |
| 716 |
pmaddubsw m5, m3, m0 |
| 717 |
paddw m0, m1, m2 |
| 718 |
paddw m1, m3 |
| 719 |
paddw m4, m0 |
| 720 |
paddw m5, m1 |
| 721 |
.w32_loop: |
| 722 |
vpbroadcastd m1, [weightsq+hq*2] |
| 723 |
pshufb m1, m6 |
| 724 |
pmaddubsw m0, m2, m1 |
| 725 |
pmaddubsw m1, m3, m1 |
| 726 |
paddw m0, m4 |
| 727 |
paddw m1, m5 |
| 728 |
vpermt2b m0, m7, m1 |
| 729 |
mova [dstq+strideq*0], ym0 |
| 730 |
vextracti32x8 [dstq+strideq*1], m0, 1 |
| 731 |
lea dstq, [dstq+strideq*2] |
| 732 |
add hq, 2 |
| 733 |
jl .w32_loop |
| 734 |
RET |
| 735 |
.w64: |
| 736 |
movu m3, [tlq+1] |
| 737 |
mova m6, [smooth_endB] |
| 738 |
punpcklbw m2, m3, m4 |
| 739 |
punpckhbw m3, m4 |
| 740 |
pmaddubsw m4, m2, m0 |
| 741 |
pmaddubsw m5, m3, m0 |
| 742 |
paddw m0, m1, m2 |
| 743 |
paddw m1, m3 |
| 744 |
paddw m4, m0 |
| 745 |
paddw m5, m1 |
| 746 |
.w64_loop: |
| 747 |
vpbroadcastw m1, [weightsq+hq*2] |
| 748 |
pmaddubsw m0, m2, m1 |
| 749 |
pmaddubsw m1, m3, m1 |
| 750 |
paddw m0, m4 |
| 751 |
paddw m1, m5 |
| 752 |
vpermt2b m0, m6, m1 |
| 753 |
mova [dstq], m0 |
| 754 |
add dstq, strideq |
| 755 |
inc hq |
| 756 |
jl .w64_loop |
| 757 |
RET |
| 758 |
|
| 759 |
cglobal ipred_smooth_h_8bpc, 4, 7, 11, dst, stride, tl, w, h, stride3 |
| 760 |
%define base r5-ipred_smooth_h_8bpc_avx512icl_table |
| 761 |
lea r5, [ipred_smooth_h_8bpc_avx512icl_table] |
| 762 |
mov r6d, wd |
| 763 |
tzcnt wd, wd |
| 764 |
vpbroadcastb m4, [tlq+r6] ; right |
| 765 |
mov hd, hm |
| 766 |
movsxd wq, [r5+wq*4] |
| 767 |
vpbroadcastd m5, [base+pb_127_m127] |
| 768 |
vpbroadcastd m6, [base+pw_128] |
| 769 |
sub tlq, hq |
| 770 |
add wq, r5 |
| 771 |
vpmovb2m k1, m6 |
| 772 |
lea stride3q, [strideq*3] |
| 773 |
jmp wq |
| 774 |
.w4: |
| 775 |
movsldup m3, [smooth_shuf] |
| 776 |
vpbroadcastq m7, [smooth_weights+4*2] |
| 777 |
mova ym8, [smooth_endA] |
| 778 |
.w4_loop: |
| 779 |
vpbroadcastq m0, [tlq+hq-8] |
| 780 |
mova m2, m4 |
| 781 |
vpshufb m2{k1}, m0, m3 ; left, right |
| 782 |
pmaddubsw m0, m2, m5 |
| 783 |
pmaddubsw m1, m2, m7 |
| 784 |
paddw m2, m6 |
| 785 |
paddw m0, m2 |
| 786 |
paddw m0, m1 |
| 787 |
vpermb m0, m8, m0 |
| 788 |
vextracti32x4 xm1, ym0, 1 |
| 789 |
movd [dstq+strideq*0], xm0 |
| 790 |
movd [dstq+strideq*1], xm1 |
| 791 |
pextrd [dstq+strideq*2], xm0, 2 |
| 792 |
pextrd [dstq+stride3q ], xm1, 2 |
| 793 |
sub hd, 8 |
| 794 |
jl .ret |
| 795 |
lea dstq, [dstq+strideq*4] |
| 796 |
pextrd [dstq+strideq*0], xm0, 1 |
| 797 |
pextrd [dstq+strideq*1], xm1, 1 |
| 798 |
pextrd [dstq+strideq*2], xm0, 3 |
| 799 |
pextrd [dstq+stride3q ], xm1, 3 |
| 800 |
lea dstq, [dstq+strideq*4] |
| 801 |
jg .w4_loop |
| 802 |
.ret: |
| 803 |
RET |
| 804 |
.w8: |
| 805 |
movsldup m3, [smooth_shuf] |
| 806 |
vbroadcasti32x4 m7, [smooth_weights+8*2] |
| 807 |
mova ym8, [smooth_endA] |
| 808 |
.w8_loop: |
| 809 |
vpbroadcastd m0, [tlq+hq-4] |
| 810 |
mova m2, m4 |
| 811 |
vpshufb m2{k1}, m0, m3 |
| 812 |
pmaddubsw m0, m2, m5 |
| 813 |
pmaddubsw m1, m2, m7 |
| 814 |
paddw m2, m6 |
| 815 |
paddw m0, m2 |
| 816 |
paddw m0, m1 |
| 817 |
vpermb m0, m8, m0 |
| 818 |
vextracti32x4 xm1, ym0, 1 |
| 819 |
movq [dstq+strideq*0], xm0 |
| 820 |
movq [dstq+strideq*1], xm1 |
| 821 |
movhps [dstq+strideq*2], xm0 |
| 822 |
movhps [dstq+stride3q ], xm1 |
| 823 |
lea dstq, [dstq+strideq*4] |
| 824 |
sub hd, 4 |
| 825 |
jg .w8_loop |
| 826 |
RET |
| 827 |
.w16: |
| 828 |
movsldup m7, [smooth_shuf] |
| 829 |
vbroadcasti32x4 m8, [smooth_weights+16*2] |
| 830 |
vbroadcasti32x4 m9, [smooth_weights+16*3] |
| 831 |
mova m10, [smooth_endB] |
| 832 |
.w16_loop: |
| 833 |
vpbroadcastd m0, [tlq+hq-4] |
| 834 |
mova m3, m4 |
| 835 |
vpshufb m3{k1}, m0, m7 |
| 836 |
pmaddubsw m2, m3, m5 |
| 837 |
pmaddubsw m0, m3, m8 |
| 838 |
pmaddubsw m1, m3, m9 |
| 839 |
paddw m3, m6 |
| 840 |
paddw m2, m3 |
| 841 |
paddw m0, m2 |
| 842 |
paddw m1, m2 |
| 843 |
vpermt2b m0, m10, m1 |
| 844 |
mova [dstq+strideq*0], xm0 |
| 845 |
vextracti32x4 [dstq+strideq*1], m0, 2 |
| 846 |
vextracti32x4 [dstq+strideq*2], ym0, 1 |
| 847 |
vextracti32x4 [dstq+stride3q ], m0, 3 |
| 848 |
lea dstq, [dstq+strideq*4] |
| 849 |
sub hd, 4 |
| 850 |
jg .w16_loop |
| 851 |
RET |
| 852 |
.w32: |
| 853 |
mova m10, [smooth_endA] |
| 854 |
vpbroadcastd ym7, [pb_1] |
| 855 |
vbroadcasti32x8 m8, [smooth_weights+32*2] |
| 856 |
vbroadcasti32x8 m9, [smooth_weights+32*3] |
| 857 |
vshufi32x4 m10, m10, q3120 |
| 858 |
.w32_loop: |
| 859 |
vpbroadcastd m0, [tlq+hq-2] |
| 860 |
mova m3, m4 |
| 861 |
vpshufb m3{k1}, m0, m7 |
| 862 |
pmaddubsw m2, m3, m5 |
| 863 |
pmaddubsw m0, m3, m8 |
| 864 |
pmaddubsw m1, m3, m9 |
| 865 |
paddw m3, m6 |
| 866 |
paddw m2, m3 |
| 867 |
paddw m0, m2 |
| 868 |
paddw m1, m2 |
| 869 |
vpermt2b m0, m10, m1 |
| 870 |
mova [dstq+strideq*0], ym0 |
| 871 |
vextracti32x8 [dstq+strideq*1], m0, 1 |
| 872 |
lea dstq, [dstq+strideq*2] |
| 873 |
sub hd, 2 |
| 874 |
jg .w32_loop |
| 875 |
RET |
| 876 |
.w64: |
| 877 |
mova m7, [smooth_weights+64*2] |
| 878 |
mova m8, [smooth_weights+64*3] |
| 879 |
mova m9, [smooth_endA] |
| 880 |
.w64_loop: |
| 881 |
mova m3, m4 |
| 882 |
vpbroadcastb m3{k1}, [tlq+hq-1] |
| 883 |
pmaddubsw m2, m3, m5 |
| 884 |
pmaddubsw m0, m3, m7 |
| 885 |
pmaddubsw m1, m3, m8 |
| 886 |
paddw m3, m6 |
| 887 |
paddw m2, m3 |
| 888 |
paddw m0, m2 |
| 889 |
paddw m1, m2 |
| 890 |
vpermt2b m0, m9, m1 |
| 891 |
mova [dstq], m0 |
| 892 |
add dstq, strideq |
| 893 |
dec hd |
| 894 |
jg .w64_loop |
| 895 |
RET |
| 896 |
|
| 897 |
cglobal ipred_smooth_8bpc, 4, 7, 16, dst, stride, tl, w, h, v_weights, stride3 |
| 898 |
%define base r5-ipred_smooth_8bpc_avx512icl_table |
| 899 |
lea r5, [ipred_smooth_8bpc_avx512icl_table] |
| 900 |
mov r6d, wd |
| 901 |
tzcnt wd, wd |
| 902 |
mov hd, hm |
| 903 |
vpbroadcastb m6, [tlq+r6] ; right |
| 904 |
sub tlq, hq |
| 905 |
movsxd wq, [r5+wq*4] |
| 906 |
vpbroadcastd m7, [base+pb_127_m127] |
| 907 |
vpbroadcastb m0, [tlq] ; bottom |
| 908 |
vpbroadcastd m1, [base+pw_255] |
| 909 |
add wq, r5 |
| 910 |
lea v_weightsq, [base+smooth_weights+hq*2] |
| 911 |
vpmovb2m k1, m1 |
| 912 |
lea stride3q, [strideq*3] |
| 913 |
jmp wq |
| 914 |
.w4: |
| 915 |
vpbroadcastd m8, [tlq+hq+1] |
| 916 |
movsldup m4, [smooth_shuf] |
| 917 |
movshdup m5, [smooth_shuf] |
| 918 |
vpbroadcastq m9, [smooth_weights+4*2] |
| 919 |
mova ym11, [smooth_endA] |
| 920 |
|
| 921 |
punpcklbw m8, m0 ; top, bottom |
| 922 |
pmaddubsw m10, m8, m7 |
| 923 |
paddw m1, m8 ; 1 * top + 256 * bottom + 255 |
| 924 |
paddw m10, m1 ; 128 * top + 129 * bottom + 255 |
| 925 |
.w4_loop: |
| 926 |
vpbroadcastq m1, [tlq+hq-8] |
| 927 |
vbroadcasti32x4 m0, [v_weightsq] |
| 928 |
add v_weightsq, 16 |
| 929 |
mova m2, m6 |
| 930 |
vpshufb m2{k1}, m1, m4 ; left, right |
| 931 |
pmaddubsw m1, m2, m7 ; 127 * left - 127 * right |
| 932 |
pshufb m0, m5 |
| 933 |
pmaddubsw m0, m8, m0 |
| 934 |
paddw m1, m2 ; 128 * left + 129 * right |
| 935 |
pmaddubsw m2, m9 |
| 936 |
paddw m0, m10 |
| 937 |
paddw m1, m2 |
| 938 |
pavgw m0, m1 |
| 939 |
vpermb m0, m11, m0 |
| 940 |
vextracti32x4 xm1, ym0, 1 |
| 941 |
movd [dstq+strideq*0], xm0 |
| 942 |
movd [dstq+strideq*1], xm1 |
| 943 |
pextrd [dstq+strideq*2], xm0, 2 |
| 944 |
pextrd [dstq+stride3q ], xm1, 2 |
| 945 |
sub hd, 8 |
| 946 |
jl .ret |
| 947 |
lea dstq, [dstq+strideq*4] |
| 948 |
pextrd [dstq+strideq*0], xm0, 1 |
| 949 |
pextrd [dstq+strideq*1], xm1, 1 |
| 950 |
pextrd [dstq+strideq*2], xm0, 3 |
| 951 |
pextrd [dstq+stride3q ], xm1, 3 |
| 952 |
lea dstq, [dstq+strideq*4] |
| 953 |
jg .w4_loop |
| 954 |
.ret: |
| 955 |
RET |
| 956 |
.w8: |
| 957 |
vpbroadcastq m8, [tlq+hq+1] |
| 958 |
movsldup m4, [smooth_shuf] |
| 959 |
movshdup m5, [smooth_shuf] |
| 960 |
vbroadcasti32x4 m9, [smooth_weights+8*2] |
| 961 |
mova ym11, [smooth_endA] |
| 962 |
punpcklbw m8, m0 |
| 963 |
pmaddubsw m10, m8, m7 |
| 964 |
paddw m1, m8 |
| 965 |
paddw m10, m1 |
| 966 |
.w8_loop: |
| 967 |
vpbroadcastd m1, [tlq+hq-4] |
| 968 |
vpbroadcastq m0, [v_weightsq] |
| 969 |
add v_weightsq, 8 |
| 970 |
mova m2, m6 |
| 971 |
vpshufb m2{k1}, m1, m4 |
| 972 |
pmaddubsw m1, m2, m7 |
| 973 |
pshufb m0, m5 |
| 974 |
pmaddubsw m0, m8, m0 |
| 975 |
paddw m1, m2 |
| 976 |
pmaddubsw m2, m9 |
| 977 |
paddw m0, m10 |
| 978 |
paddw m1, m2 |
| 979 |
pavgw m0, m1 |
| 980 |
vpermb m0, m11, m0 |
| 981 |
vextracti32x4 xm1, ym0, 1 |
| 982 |
movq [dstq+strideq*0], xm0 |
| 983 |
movq [dstq+strideq*1], xm1 |
| 984 |
movhps [dstq+strideq*2], xm0 |
| 985 |
movhps [dstq+stride3q ], xm1 |
| 986 |
lea dstq, [dstq+strideq*4] |
| 987 |
sub hd, 4 |
| 988 |
jg .w8_loop |
| 989 |
RET |
| 990 |
.w16: |
| 991 |
vbroadcasti32x4 m9, [tlq+hq+1] |
| 992 |
movsldup m5, [smooth_shuf] |
| 993 |
movshdup m10, [smooth_shuf] |
| 994 |
vbroadcasti32x4 m11, [smooth_weights+16*2] |
| 995 |
vbroadcasti32x4 m12, [smooth_weights+16*3] |
| 996 |
mova m15, [smooth_endB] |
| 997 |
punpcklbw m8, m9, m0 |
| 998 |
punpckhbw m9, m0 |
| 999 |
pmaddubsw m13, m8, m7 |
| 1000 |
pmaddubsw m14, m9, m7 |
| 1001 |
paddw m0, m1, m8 |
| 1002 |
paddw m1, m9 |
| 1003 |
paddw m13, m0 |
| 1004 |
paddw m14, m1 |
| 1005 |
.w16_loop: |
| 1006 |
vpbroadcastd m0, [tlq+hq-4] |
| 1007 |
vpbroadcastq m1, [v_weightsq] |
| 1008 |
add v_weightsq, 8 |
| 1009 |
mova m4, m6 |
| 1010 |
vpshufb m4{k1}, m0, m5 |
| 1011 |
pmaddubsw m2, m4, m7 |
| 1012 |
pshufb m1, m10 |
| 1013 |
pmaddubsw m0, m8, m1 |
| 1014 |
pmaddubsw m1, m9, m1 |
| 1015 |
paddw m2, m4 |
| 1016 |
pmaddubsw m3, m4, m11 |
| 1017 |
pmaddubsw m4, m12 |
| 1018 |
paddw m0, m13 |
| 1019 |
paddw m1, m14 |
| 1020 |
paddw m3, m2 |
| 1021 |
paddw m4, m2 |
| 1022 |
pavgw m0, m3 |
| 1023 |
pavgw m1, m4 |
| 1024 |
vpermt2b m0, m15, m1 |
| 1025 |
mova [dstq+strideq*0], xm0 |
| 1026 |
vextracti32x4 [dstq+strideq*1], m0, 2 |
| 1027 |
vextracti32x4 [dstq+strideq*2], ym0, 1 |
| 1028 |
vextracti32x4 [dstq+stride3q ], m0, 3 |
| 1029 |
lea dstq, [dstq+strideq*4] |
| 1030 |
sub hd, 4 |
| 1031 |
jg .w16_loop |
| 1032 |
RET |
| 1033 |
.w32: |
| 1034 |
vbroadcasti32x8 m9, [tlq+hq+1] |
| 1035 |
movshdup m10, [smooth_shuf] |
| 1036 |
mova m12, [smooth_weights+32*2] |
| 1037 |
vpbroadcastd ym5, [pb_1] |
| 1038 |
mova m15, [smooth_endB] |
| 1039 |
punpcklbw m8, m9, m0 |
| 1040 |
punpckhbw m9, m0 |
| 1041 |
pmaddubsw m13, m8, m7 |
| 1042 |
pmaddubsw m14, m9, m7 |
| 1043 |
vshufi32x4 m11, m12, m12, q2020 |
| 1044 |
vshufi32x4 m12, m12, q3131 |
| 1045 |
paddw m0, m1, m8 |
| 1046 |
paddw m1, m9 |
| 1047 |
paddw m13, m0 |
| 1048 |
paddw m14, m1 |
| 1049 |
.w32_loop: |
| 1050 |
vpbroadcastd m0, [tlq+hq-2] |
| 1051 |
vpbroadcastd m1, [v_weightsq] |
| 1052 |
add v_weightsq, 4 |
| 1053 |
mova m4, m6 |
| 1054 |
vpshufb m4{k1}, m0, m5 |
| 1055 |
pmaddubsw m2, m4, m7 |
| 1056 |
pshufb m1, m10 |
| 1057 |
pmaddubsw m0, m8, m1 |
| 1058 |
pmaddubsw m1, m9, m1 |
| 1059 |
paddw m2, m4 |
| 1060 |
pmaddubsw m3, m4, m11 |
| 1061 |
pmaddubsw m4, m12 |
| 1062 |
paddw m0, m13 |
| 1063 |
paddw m1, m14 |
| 1064 |
paddw m3, m2 |
| 1065 |
paddw m4, m2 |
| 1066 |
pavgw m0, m3 |
| 1067 |
pavgw m1, m4 |
| 1068 |
vpermt2b m0, m15, m1 |
| 1069 |
mova [dstq+strideq*0], ym0 |
| 1070 |
vextracti32x8 [dstq+strideq*1], m0, 1 |
| 1071 |
lea dstq, [dstq+strideq*2] |
| 1072 |
sub hd, 2 |
| 1073 |
jg .w32_loop |
| 1074 |
RET |
| 1075 |
.w64: |
| 1076 |
movu m9, [tlq+hq+1] |
| 1077 |
mova m11, [smooth_weights+64*2] |
| 1078 |
mova m2, [smooth_weights+64*3] |
| 1079 |
mova m14, [smooth_endB] |
| 1080 |
punpcklbw m8, m9, m0 |
| 1081 |
punpckhbw m9, m0 |
| 1082 |
pmaddubsw m12, m8, m7 |
| 1083 |
pmaddubsw m13, m9, m7 |
| 1084 |
vshufi32x4 m10, m11, m2, q2020 |
| 1085 |
vshufi32x4 m11, m2, q3131 |
| 1086 |
paddw m0, m1, m8 |
| 1087 |
paddw m1, m9 |
| 1088 |
paddw m12, m0 |
| 1089 |
paddw m13, m1 |
| 1090 |
.w64_loop: |
| 1091 |
mova m4, m6 |
| 1092 |
vpbroadcastb m4{k1}, [tlq+hq-1] |
| 1093 |
vpbroadcastw m1, [v_weightsq] |
| 1094 |
add v_weightsq, 2 |
| 1095 |
pmaddubsw m2, m4, m7 |
| 1096 |
pmaddubsw m0, m8, m1 |
| 1097 |
pmaddubsw m1, m9, m1 |
| 1098 |
paddw m2, m4 |
| 1099 |
pmaddubsw m3, m4, m10 |
| 1100 |
pmaddubsw m4, m11 |
| 1101 |
paddw m0, m12 |
| 1102 |
paddw m1, m13 |
| 1103 |
paddw m3, m2 |
| 1104 |
paddw m4, m2 |
| 1105 |
pavgw m0, m3 |
| 1106 |
pavgw m1, m4 |
| 1107 |
vpermt2b m0, m14, m1 |
| 1108 |
mova [dstq], m0 |
| 1109 |
add dstq, strideq |
| 1110 |
dec hd |
| 1111 |
jg .w64_loop |
| 1112 |
RET |
| 1113 |
|
| 1114 |
cglobal pal_pred_8bpc, 4, 7, 5, dst, stride, pal, idx, w, h, stride3 |
| 1115 |
lea r6, [pal_pred_8bpc_avx512icl_table] |
| 1116 |
tzcnt wd, wm |
| 1117 |
vbroadcasti32x4 m4, [palq] |
| 1118 |
movifnidn hd, hm |
| 1119 |
movsxd wq, [r6+wq*4] |
| 1120 |
packuswb m4, m4 |
| 1121 |
add wq, r6 |
| 1122 |
lea stride3q, [strideq*3] |
| 1123 |
jmp wq |
| 1124 |
.w4: |
| 1125 |
pshufb xmm0, xm4, [idxq] |
| 1126 |
add idxq, 16 |
| 1127 |
movd [dstq+strideq*0], xmm0 |
| 1128 |
pextrd [dstq+strideq*1], xmm0, 1 |
| 1129 |
pextrd [dstq+strideq*2], xmm0, 2 |
| 1130 |
pextrd [dstq+stride3q ], xmm0, 3 |
| 1131 |
lea dstq, [dstq+strideq*4] |
| 1132 |
sub hd, 4 |
| 1133 |
jg .w4 |
| 1134 |
RET |
| 1135 |
.w8: |
| 1136 |
pshufb xmm0, xm4, [idxq+16*0] |
| 1137 |
pshufb xmm1, xm4, [idxq+16*1] |
| 1138 |
add idxq, 16*2 |
| 1139 |
movq [dstq+strideq*0], xmm0 |
| 1140 |
movhps [dstq+strideq*1], xmm0 |
| 1141 |
movq [dstq+strideq*2], xmm1 |
| 1142 |
movhps [dstq+stride3q ], xmm1 |
| 1143 |
lea dstq, [dstq+strideq*4] |
| 1144 |
sub hd, 4 |
| 1145 |
jg .w8 |
| 1146 |
RET |
| 1147 |
.w16: |
| 1148 |
pshufb m0, m4, [idxq] |
| 1149 |
add idxq, 64 |
| 1150 |
mova [dstq+strideq*0], xm0 |
| 1151 |
vextracti32x4 [dstq+strideq*1], ym0, 1 |
| 1152 |
vextracti32x4 [dstq+strideq*2], m0, 2 |
| 1153 |
vextracti32x4 [dstq+stride3q ], m0, 3 |
| 1154 |
lea dstq, [dstq+strideq*4] |
| 1155 |
sub hd, 4 |
| 1156 |
jg .w16 |
| 1157 |
RET |
| 1158 |
.w32: |
| 1159 |
pshufb m0, m4, [idxq+64*0] |
| 1160 |
pshufb m1, m4, [idxq+64*1] |
| 1161 |
add idxq, 64*2 |
| 1162 |
mova [dstq+strideq*0], ym0 |
| 1163 |
vextracti32x8 [dstq+strideq*1], m0, 1 |
| 1164 |
mova [dstq+strideq*2], ym1 |
| 1165 |
vextracti32x8 [dstq+stride3q ], m1, 1 |
| 1166 |
lea dstq, [dstq+strideq*4] |
| 1167 |
sub hd, 4 |
| 1168 |
jg .w32 |
| 1169 |
RET |
| 1170 |
.w64: |
| 1171 |
pshufb m0, m4, [idxq+64*0] |
| 1172 |
pshufb m1, m4, [idxq+64*1] |
| 1173 |
pshufb m2, m4, [idxq+64*2] |
| 1174 |
pshufb m3, m4, [idxq+64*3] |
| 1175 |
add idxq, 64*4 |
| 1176 |
mova [dstq+strideq*0], m0 |
| 1177 |
mova [dstq+strideq*1], m1 |
| 1178 |
mova [dstq+strideq*2], m2 |
| 1179 |
mova [dstq+stride3q ], m3 |
| 1180 |
lea dstq, [dstq+strideq*4] |
| 1181 |
sub hd, 4 |
| 1182 |
jg .w64 |
| 1183 |
RET |
| 1184 |
|
| 1185 |
; The ipred_filter code processes 4x2 blocks in the following order |
| 1186 |
; which increases parallelism compared to doing things row by row. |
| 1187 |
; Some redundant blocks are calculated for w > 4. |
| 1188 |
; w4 w8 w16 w32 |
| 1189 |
; 1 1 2 1 2 3 4 1 2 3 4 9 a b c |
| 1190 |
; 2 2 3 2 3 4 5 2 3 4 5 a b c d |
| 1191 |
; 3 3 4 3 4 5 6 3 4 5 6 b c d e |
| 1192 |
; 4 4 5 4 5 6 7 4 5 6 7 c d e f |
| 1193 |
; 5 5 6 5 6 7 8 5 6 7 8 d e f g |
| 1194 |
; 6 6 7 6 7 8 9 6 7 8 9 e f g h |
| 1195 |
; 7 7 8 7 8 9 a 7 8 9 a f g h i |
| 1196 |
; ___ 8 ___ 8 9 ___ 8 9 a b ___ 8 9 a b g h i j ___ |
| 1197 |
; 9 9 a b h i j |
| 1198 |
; a b i j |
| 1199 |
; b j |
| 1200 |
|
| 1201 |
cglobal ipred_filter_8bpc, 4, 7, 14, dst, stride, tl, w, h, flt |
| 1202 |
%define base r6-filter_taps |
| 1203 |
lea r6, [filter_taps] |
| 1204 |
%ifidn fltd, fltm |
| 1205 |
movzx fltd, fltb |
| 1206 |
%else |
| 1207 |
movzx fltd, byte fltm |
| 1208 |
%endif |
| 1209 |
vpbroadcastd xmm2, [tlq+1] ; t0 t0 t0 t0 |
| 1210 |
movifnidn hd, hm |
| 1211 |
shl fltd, 6 |
| 1212 |
vpbroadcastd m6, [base+pd_8] |
| 1213 |
vpbroadcastd xmm3, [tlq-2] ; l1 l0 tl __ |
| 1214 |
vbroadcasti32x4 m7, [r6+fltq+16*0] ; p1 p2 p3 p4 |
| 1215 |
vbroadcasti32x4 m8, [r6+fltq+16*1] |
| 1216 |
vbroadcasti32x4 m9, [r6+fltq+16*2] ; p6 p5 p0 __ |
| 1217 |
vbroadcasti32x4 m10, [r6+fltq+16*3] |
| 1218 |
mova xmm0, xm6 |
| 1219 |
vpdpbusd xmm0, xmm2, xm7 |
| 1220 |
mova xmm1, xm6 |
| 1221 |
vpdpbusd xmm1, xmm2, xm8 |
| 1222 |
vpdpbusd xmm0, xmm3, xm9 |
| 1223 |
vpdpbusd xmm1, xmm3, xm10 |
| 1224 |
packssdw xmm0, xmm1 |
| 1225 |
cmp wd, 8 |
| 1226 |
jb .w4 |
| 1227 |
vpbroadcastd ym2, [tlq+5] |
| 1228 |
mova m11, [base+filter_perm] |
| 1229 |
mov r5, 0xffffffffffff000f |
| 1230 |
psrldq xmm2, 1 ; __ t0 |
| 1231 |
kmovq k1, r5 ; 0x000f |
| 1232 |
psraw xm5, xmm0, 4 |
| 1233 |
packuswb xmm2, xm5 ; __ t0 a0 b0 |
| 1234 |
pshufd ym2{k1}, ymm2, q3333 ; b0 b0 b0 b0 t1 t1 t1 t1 |
| 1235 |
je .w8 |
| 1236 |
kxnorb k3, k3, k3 ; 0x00ff |
| 1237 |
vpbroadcastd xm3, [tlq-4] |
| 1238 |
kandnq k2, k3, k1 ; 0xffffffffffff0000 |
| 1239 |
vpermb ym3{k2}, ym11, ymm2 ; l3 l2 l1 __ b3 a3 t3 __ |
| 1240 |
mova ym0, ym6 |
| 1241 |
vpdpbusd ym0, ym2, ym7 |
| 1242 |
mova ym1, ym6 |
| 1243 |
vpdpbusd ym1, ym2, ym8 |
| 1244 |
pshufb ym5{k2}, ym2, ym11 ; a0 b0 __ t0 |
| 1245 |
vpbroadcastd m2, [tlq+9] |
| 1246 |
vpdpbusd ym0, ym3, ym9 |
| 1247 |
vpdpbusd ym1, ym3, ym10 |
| 1248 |
vpbroadcastd xm3, [tlq-6] ; l5 l4 l3 __ |
| 1249 |
kunpckbw k4, k1, k3 ; 0x0fff |
| 1250 |
packssdw ym0, ym1 |
| 1251 |
psraw ym0, 4 ; a0 d0 a1 b1 |
| 1252 |
packuswb ym5, ym0 ; a0 b0 c0 d0 __ t1 a1 b1 |
| 1253 |
pshufd m2{k3}, m5, q3333 ; d0 d0 d0 d0 b1 b1 b1 b1 t2 t2 t2 t2 |
| 1254 |
vpermb m3{k2}, m11, m5 ; l5 l4 l3 __ d3 c3 b3 __ b7 a7 t7 __ |
| 1255 |
mova m4, m6 |
| 1256 |
vpdpbusd m4, m2, m7 |
| 1257 |
mova m1, m6 |
| 1258 |
vpdpbusd m1, m2, m8 |
| 1259 |
psrldq m0, m2, 1 ; __ d0 __ b0 __ t0 |
| 1260 |
vpbroadcastd m2, [tlq+13] |
| 1261 |
vpdpbusd m4, m3, m9 |
| 1262 |
vpdpbusd m1, m3, m10 |
| 1263 |
mova m12, [base+filter_end] |
| 1264 |
lea r5d, [hq-6] |
| 1265 |
mov r6, dstq |
| 1266 |
cmovp hd, r5d ; w == 16 ? h : h - 6 |
| 1267 |
packssdw m4, m1 |
| 1268 |
psraw m4, 4 ; e0 f0 c1 d1 a2 b2 |
| 1269 |
packuswb m0, m4 ; __ d0 e0 f0 __ b1 c1 d1 __ t2 a2 b2 |
| 1270 |
pshufd m2{k4}, m0, q3333 ; f0 f0 f0 f0 d1 d1 d1 d1 b2 b2 b2 b2 t3 t3 t3 t3 |
| 1271 |
.w16_loop: |
| 1272 |
vpbroadcastd xm3, [tlq-8] |
| 1273 |
vpermb m3{k2}, m11, m0 ; l7 l6 l5 __ f3 e3 d3 __ d7 c7 b7 __ bb ab tb __ |
| 1274 |
mova m1, m6 |
| 1275 |
vpdpbusd m1, m2, m7 |
| 1276 |
mova m0, m6 |
| 1277 |
vpdpbusd m0, m2, m8 |
| 1278 |
sub tlq, 2 |
| 1279 |
vpdpbusd m1, m3, m9 |
| 1280 |
vpdpbusd m0, m3, m10 |
| 1281 |
packssdw m1, m0 |
| 1282 |
mova m0, m4 |
| 1283 |
psraw m4, m1, 4 ; g0 h0 e1 f1 c2 d2 a3 b3 |
| 1284 |
packuswb m0, m4 ; e0 f0 g0 h0 c1 d1 e1 f1 a2 b2 c2 d2 __ __ a3 b3 |
| 1285 |
pshufd m2, m0, q3333 ; h0 h0 h0 h0 f1 f1 f1 f1 d2 d2 d2 d2 b3 b3 b3 b3 |
| 1286 |
vpermt2d m5, m12, m0 ; c0 d0 e0 f0 __ __ c1 d1 a0 a1 a2 a3 b0 b1 b2 b3 |
| 1287 |
vextracti32x4 [dstq+strideq*0], m5, 2 |
| 1288 |
vextracti32x4 [dstq+strideq*1], m5, 3 |
| 1289 |
lea dstq, [dstq+strideq*2] |
| 1290 |
sub hd, 2 |
| 1291 |
jg .w16_loop |
| 1292 |
cmp wd, 16 |
| 1293 |
je .ret |
| 1294 |
mova xm13, [filter_perm+16] |
| 1295 |
mova xmm3, [r6+strideq*0] |
| 1296 |
punpckhdq xmm3, [r6+strideq*1] |
| 1297 |
vpbroadcastd m2{k1}, [tlq+r5+17] ; t4 t4 t4 t4 f1 f1 f1 f1 d2 d2 d2 d2 b3 b3 b3 b3 |
| 1298 |
pinsrb xm3, xmm3, [tlq+r5+16], 7 |
| 1299 |
pshufb xm3, xm13 |
| 1300 |
vpermb m3{k2}, m11, m0 ; bf af tf __ h3 g3 f3 __ f7 e7 d7 __ db cb bb __ |
| 1301 |
mova m0, m6 |
| 1302 |
vpdpbusd m0, m2, m7 |
| 1303 |
mova m1, m6 |
| 1304 |
vpdpbusd m1, m2, m8 |
| 1305 |
kunpckbw k5, k3, k1 ; 0xff0f |
| 1306 |
lea r3, [strideq*3] |
| 1307 |
vpdpbusd m0, m3, m9 |
| 1308 |
vpdpbusd m1, m3, m10 |
| 1309 |
packssdw m0, m1 |
| 1310 |
psraw m0, 4 ; a4 b4 g1 h1 e2 f2 c3 d3 |
| 1311 |
packuswb m4, m0 ; g0 h0 a4 b4 e1 f1 g1 h1 c2 d2 e2 f2 __ __ c3 d3 |
| 1312 |
vpblendmb m1{k3}, m4, m2 ; __ t4 a4 b4 e1 f1 g1 h1 c2 d2 e2 f2 __ __ c3 d3 |
| 1313 |
vpbroadcastd ym2, [tlq+r5+21] |
| 1314 |
pshufd m2{k5}, m4, q3333 ; b4 b4 b4 b4 t5 t5 t5 t5 f2 f2 f2 f2 d3 d3 d3 d3 |
| 1315 |
vpermt2d m5, m12, m4 ; e0 f0 g0 h0 __ __ e1 f1 c0 c1 c2 c3 d0 d1 d2 d3 |
| 1316 |
vextracti32x4 [dstq+strideq*0], m5, 2 |
| 1317 |
vextracti32x4 [dstq+strideq*1], m5, 3 |
| 1318 |
punpckhqdq xmm3, [r6+r3] |
| 1319 |
pinsrb xmm3, [r6+strideq*2+15], 11 |
| 1320 |
pshufb xm3, xmm3, xm13 |
| 1321 |
vpermb m3{k2}, m11, m1 ; df cf bf __ bj aj tj __ h7 g7 f7 __ fb eb db __ |
| 1322 |
mova m4, m6 |
| 1323 |
vpdpbusd m4, m2, m7 |
| 1324 |
mova m1, m6 |
| 1325 |
vpdpbusd m1, m2, m8 |
| 1326 |
kxnord k3, k3, k4 ; 0xfffff0ff |
| 1327 |
lea r4, [strideq*5] |
| 1328 |
vpdpbusd m4, m3, m9 |
| 1329 |
vpdpbusd m1, m3, m10 |
| 1330 |
packssdw m4, m1 |
| 1331 |
psraw m4, 4 ; c4 d4 a5 b5 g2 h2 e3 f3 |
| 1332 |
packuswb m0, m4 ; a4 b4 c4 d4 g1 h1 a5 b5 e2 f2 g2 h2 __ __ e3 f3 |
| 1333 |
vpblendmw m1{k3}, m2, m0 ; a4 b4 c4 d4 __ t5 a5 b5 e2 f2 g2 h2 __ __ e3 f3 |
| 1334 |
vpbroadcastd m2, [tlq+r5+25] |
| 1335 |
pshufd m2{k3}, m0, q3333 ; d4 d4 d4 d4 b5 b5 b5 b5 t6 t6 t6 t6 f3 f3 f3 f3 |
| 1336 |
vpermt2d m5, m12, m0 ; g0 h0 a4 b4 __ __ g1 h1 e0 e1 e2 e3 f0 f1 f2 f3 |
| 1337 |
vextracti32x4 [dstq+strideq*2], m5, 2 |
| 1338 |
vextracti32x4 [dstq+r3 ], m5, 3 |
| 1339 |
punpckhqdq xmm3, [r6+r4] |
| 1340 |
pinsrb xmm3, [r6+strideq*4+15], 11 |
| 1341 |
pshufb xm3, xmm3, xm13 |
| 1342 |
vpermb m3{k2}, m11, m1 ; ff ef df __ dj cj bj __ bn an tn __ hb hb fb __ |
| 1343 |
mova m0, m6 |
| 1344 |
vpdpbusd m0, m2, m7 |
| 1345 |
mova m1, m6 |
| 1346 |
vpdpbusd m1, m2, m8 |
| 1347 |
kunpckwd k1, k1, k2 ; 0x000f0000 |
| 1348 |
vpdpbusd m0, m3, m9 |
| 1349 |
vpdpbusd m1, m3, m10 |
| 1350 |
packssdw m0, m1 |
| 1351 |
psraw m0, 4 ; e4 f4 c5 d5 a6 b6 g3 h3 |
| 1352 |
packuswb m4, m0 ; c4 d4 e4 f4 a5 b5 c5 d5 g2 h2 a6 b6 __ __ g3 h3 |
| 1353 |
vpblendmw m1{k1}, m4, m2 ; c4 d4 e4 f4 a5 b5 c5 d5 __ t6 a6 b6 __ __ g3 h3 |
| 1354 |
vpbroadcastd m2, [tlq+r5+29] |
| 1355 |
pshufd m2{k4}, m4, q3333 ; f4 f4 f4 f4 d5 d5 d5 d5 b6 b6 b6 b6 t7 t7 t7 t7 |
| 1356 |
vpermt2d m5, m12, m4 ; a4 b4 c4 d4 __ __ a5 b5 g0 g1 g2 g3 h0 h1 h2 h3 |
| 1357 |
vextracti32x4 [dstq+strideq*4], m5, 2 |
| 1358 |
vextracti32x4 [dstq+r4 ], m5, 3 |
| 1359 |
lea r0, [strideq+r3*2] |
| 1360 |
.w32_loop: |
| 1361 |
punpckhqdq xmm3, [r6+r0] |
| 1362 |
pinsrb xmm3, [r6+r3*2+15], 11 |
| 1363 |
pshufb xm3, xmm3, xm13 |
| 1364 |
vpermb m3{k2}, m11, m1 ; hf gf ff __ fj ej dj __ dn cn bn __ br ar tr __ |
| 1365 |
.w32_loop_tail: |
| 1366 |
mova m4, m6 |
| 1367 |
vpdpbusd m4, m2, m7 |
| 1368 |
mova m1, m6 |
| 1369 |
vpdpbusd m1, m2, m8 |
| 1370 |
vpdpbusd m4, m3, m9 |
| 1371 |
vpdpbusd m1, m3, m10 |
| 1372 |
packssdw m4, m1 |
| 1373 |
mova m1, m0 |
| 1374 |
psraw m0, m4, 4 ; g4 h4 e5 f5 c6 d6 a7 b7 |
| 1375 |
packuswb m1, m0 ; e4 f4 g4 h4 c5 d5 e5 f5 a6 b6 c6 d6 __ __ a7 b7 |
| 1376 |
pshufd m2, m1, q3333 ; h4 h4 h4 h4 f5 f5 f5 f5 d6 d6 d6 d6 b7 b7 b7 b7 |
| 1377 |
vpermt2d m5, m12, m1 ; c4 d4 e4 f4 __ __ c5 d5 a4 a5 a6 a7 b4 b5 b6 b7 |
| 1378 |
vextracti32x4 [r6+strideq*0+16], m5, 2 |
| 1379 |
vextracti32x4 [r6+strideq*1+16], m5, 3 |
| 1380 |
lea r6, [r6+strideq*2] |
| 1381 |
sub r5d, 2 |
| 1382 |
jg .w32_loop |
| 1383 |
vpermb m3, m11, m1 |
| 1384 |
cmp r5d, -6 |
| 1385 |
jg .w32_loop_tail |
| 1386 |
.ret: |
| 1387 |
RET |
| 1388 |
.w8: |
| 1389 |
vpermb ym3, ym11, ymm2 |
| 1390 |
.w8_loop: |
| 1391 |
vpbroadcastd ym3{k1}, [tlq-4] ; l3 l2 l1 __ b3 a3 t3 __ |
| 1392 |
mova ym0, ym6 |
| 1393 |
vpdpbusd ym0, ym2, ym7 |
| 1394 |
mova ym1, ym6 |
| 1395 |
vpdpbusd ym1, ym2, ym8 |
| 1396 |
sub tlq, 2 |
| 1397 |
vpdpbusd ym0, ym3, ym9 |
| 1398 |
vpdpbusd ym1, ym3, ym10 |
| 1399 |
mova ym3, ym5 |
| 1400 |
packssdw ym0, ym1 |
| 1401 |
psraw ym5, ym0, 4 ; c0 d0 a1 b1 |
| 1402 |
packuswb ym3, ym5 ; a0 b0 c0 d0 __ __ a1 b1 |
| 1403 |
pshufd ym2, ym3, q3333 ; d0 d0 d0 d0 b1 b1 b1 b1 |
| 1404 |
vpermb ym3, ym11, ym3 ; a0 a1 b0 b1 |
| 1405 |
movq [dstq+strideq*0], xm3 |
| 1406 |
movhps [dstq+strideq*1], xm3 |
| 1407 |
lea dstq, [dstq+strideq*2] |
| 1408 |
sub hd, 2 |
| 1409 |
jg .w8_loop |
| 1410 |
RET |
| 1411 |
.w4_loop: |
| 1412 |
vpbroadcastd xmm3, [tlq-4] ; l3 l2 l1 __ |
| 1413 |
mova xmm0, xm6 |
| 1414 |
vpdpbusd xmm0, xmm2, xm7 |
| 1415 |
mova xmm1, xm6 |
| 1416 |
vpdpbusd xmm1, xmm2, xm8 |
| 1417 |
sub tlq, 2 |
| 1418 |
vpdpbusd xmm0, xmm3, xm9 |
| 1419 |
vpdpbusd xmm1, xmm3, xm10 |
| 1420 |
packssdw xmm0, xmm1 |
| 1421 |
.w4: |
| 1422 |
psraw xmm0, 4 ; a0 b0 |
| 1423 |
packuswb xmm0, xmm0 |
| 1424 |
movd [dstq+strideq*0], xmm0 |
| 1425 |
pshufd xmm2, xmm0, q1111 ; b0 b0 b0 b0 |
| 1426 |
movd [dstq+strideq*1], xmm2 |
| 1427 |
lea dstq, [dstq+strideq*2] |
| 1428 |
sub hd, 2 |
| 1429 |
jg .w4_loop |
| 1430 |
RET |
| 1431 |
|
| 1432 |
%endif ; ARCH_X86_64 |