| 1 |
; Copyright © 2022, VideoLAN and dav1d authors |
| 2 |
; Copyright © 2022, Two Orioles, LLC |
| 3 |
; All rights reserved. |
| 4 |
; |
| 5 |
; Redistribution and use in source and binary forms, with or without |
| 6 |
; modification, are permitted provided that the following conditions are met: |
| 7 |
; |
| 8 |
; 1. Redistributions of source code must retain the above copyright notice, this |
| 9 |
; list of conditions and the following disclaimer. |
| 10 |
; |
| 11 |
; 2. Redistributions in binary form must reproduce the above copyright notice, |
| 12 |
; this list of conditions and the following disclaimer in the documentation |
| 13 |
; and/or other materials provided with the distribution. |
| 14 |
; |
| 15 |
; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND |
| 16 |
; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
| 17 |
; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE |
| 18 |
; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR |
| 19 |
; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES |
| 20 |
; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
| 21 |
; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND |
| 22 |
; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 23 |
; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS |
| 24 |
; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 25 |
|
| 26 |
%include "config.asm" |
| 27 |
%include "ext/x86/x86inc.asm" |
| 28 |
%include "x86/filmgrain_common.asm" |
| 29 |
|
| 30 |
%if ARCH_X86_64 |
| 31 |
|
| 32 |
SECTION_RODATA 64 |
| 33 |
|
| 34 |
pb_even: db 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30 |
| 35 |
db 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62 |
| 36 |
db 64, 66, 68, 70, 72, 74, 76, 78, 80, 82, 84, 86, 88, 90, 92, 94 |
| 37 |
db 96, 98,100,102,104,106,108,110,112,114,116,118,120,122,124,126 |
| 38 |
pb_odd: db 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31 |
| 39 |
db 33, 35, 37, 39, 41, 43, 45, 47, 49, 51, 53, 55, 57, 59, 61, 63 |
| 40 |
db 65, 67, 69, 71, 73, 75, 77, 79, 81, 83, 85, 87, 89, 91, 93, 95 |
| 41 |
db 97, 99,101,103,105,107,109,111,113,115,117,119,121,123,125,127 |
| 42 |
interleave_hl: db 8, 0, 9, 1, 10, 2, 11, 3, 12, 4, 13, 5, 14, 6, 15, 7 |
| 43 |
pb_27_17_17_27: db 27, 17, 17, 27, 0, 32, 0, 32 |
| 44 |
pb_23_22_0_32: db 23, 22, 0, 32, 0, 32, 0, 32 |
| 45 |
pb_27_17: times 2 db 27, 17 |
| 46 |
pb_23_22: times 2 db 23, 22 |
| 47 |
pw_8: times 2 dw 8 |
| 48 |
pw_1024: times 2 dw 1024 |
| 49 |
pb_17_27: times 2 db 17, 27 |
| 50 |
fg_max: times 4 db 255 |
| 51 |
times 4 db 240 |
| 52 |
times 4 db 235 |
| 53 |
fg_min: times 4 db 0 |
| 54 |
times 4 db 16 |
| 55 |
noise_rnd: times 2 dw 128 |
| 56 |
times 2 dw 64 |
| 57 |
times 2 dw 32 |
| 58 |
times 2 dw 16 |
| 59 |
|
| 60 |
SECTION .text |
| 61 |
|
| 62 |
INIT_ZMM avx512icl |
| 63 |
cglobal fgy_32x32xn_8bpc, 6, 13, 22, dst, src, stride, fg_data, w, scaling, \ |
| 64 |
grain_lut, h, sby, see, overlap |
| 65 |
%define base r11-fg_min |
| 66 |
lea r11, [fg_min] |
| 67 |
mov r6d, [fg_dataq+FGData.scaling_shift] |
| 68 |
mov r7d, [fg_dataq+FGData.clip_to_restricted_range] |
| 69 |
mov sbyd, sbym |
| 70 |
mov overlapd, [fg_dataq+FGData.overlap_flag] |
| 71 |
mov r12, 0x0000000f0000000f ; h_overlap mask |
| 72 |
mova m0, [scalingq+64*0] |
| 73 |
mova m1, [scalingq+64*1] |
| 74 |
mova m2, [scalingq+64*2] |
| 75 |
mova m3, [scalingq+64*3] |
| 76 |
kmovq k1, r12 |
| 77 |
vbroadcasti32x4 m4, [base+interleave_hl] |
| 78 |
vpbroadcastd ym16, [base+pb_27_17] |
| 79 |
vpbroadcastd m12, [base+pb_17_27] |
| 80 |
vpbroadcastd m6, [base+noise_rnd+r6*4-32] |
| 81 |
test sbyd, sbyd |
| 82 |
setnz r6b |
| 83 |
vpbroadcastd m7, [base+fg_min+r7*4] |
| 84 |
vpbroadcastd m8, [base+fg_max+r7*8] |
| 85 |
pxor m5, m5 |
| 86 |
vpbroadcastd m9, [base+pw_1024] |
| 87 |
vpbroadcastq m10, [base+pb_27_17_17_27] |
| 88 |
vmovdqa64 m12{k1}, m16 |
| 89 |
test r6b, overlapb |
| 90 |
jnz .v_overlap |
| 91 |
|
| 92 |
imul seed, sbyd, (173 << 24) | 37 |
| 93 |
add seed, (105 << 24) | 178 |
| 94 |
rorx seed, seed, 24 |
| 95 |
movzx seed, seew |
| 96 |
xor seed, [fg_dataq+FGData.seed] |
| 97 |
|
| 98 |
DEFINE_ARGS dst, src, stride, src_bak, w, offx, offy, \ |
| 99 |
h, sby, see, overlap |
| 100 |
|
| 101 |
lea src_bakq, [srcq+wq] |
| 102 |
neg wq |
| 103 |
sub dstq, srcq |
| 104 |
.loop_x: |
| 105 |
rorx r6, seeq, 1 |
| 106 |
or seed, 0xeff4 |
| 107 |
test seeb, seeh |
| 108 |
lea seed, [r6+0x8000] |
| 109 |
cmovp seed, r6d ; updated seed |
| 110 |
rorx offyd, seed, 8 |
| 111 |
rorx offxq, seeq, 12 |
| 112 |
and offyd, 0xf |
| 113 |
imul offyd, 164 |
| 114 |
lea offxd, [offyq+offxq*2+829] ; offy*stride+offx |
| 115 |
|
| 116 |
DEFINE_ARGS dst, src, stride, src_bak, w, offxy, grain_lut, \ |
| 117 |
h, sby, see, overlap |
| 118 |
|
| 119 |
mov grain_lutq, grain_lutmp |
| 120 |
mov hd, hm |
| 121 |
.loop_y: |
| 122 |
movu ym21, [grain_lutq+offxyq-82] |
| 123 |
vinserti32x8 m21, [grain_lutq+offxyq+ 0], 1 |
| 124 |
call .add_noise |
| 125 |
sub hb, 2 |
| 126 |
jg .loop_y |
| 127 |
add wq, 32 |
| 128 |
jge .end |
| 129 |
lea srcq, [src_bakq+wq] |
| 130 |
test overlapd, overlapd |
| 131 |
jz .loop_x |
| 132 |
test sbyd, sbyd |
| 133 |
jnz .hv_overlap |
| 134 |
|
| 135 |
.loop_x_h_overlap: |
| 136 |
rorx r6, seeq, 1 |
| 137 |
or seed, 0xeff4 |
| 138 |
test seeb, seeh |
| 139 |
lea seed, [r6+0x8000] |
| 140 |
cmovp seed, r6d ; updated seed |
| 141 |
|
| 142 |
DEFINE_ARGS dst, src, stride, src_bak, w, offx, offy, \ |
| 143 |
h, sby, see, left_offxy |
| 144 |
|
| 145 |
rorx offyd, seed, 8 |
| 146 |
mov left_offxyd, offxd ; previous column's offy*stride |
| 147 |
rorx offxq, seeq, 12 |
| 148 |
and offyd, 0xf |
| 149 |
imul offyd, 164 |
| 150 |
lea offxd, [offyq+offxq*2+829] ; offy*stride+offx |
| 151 |
|
| 152 |
DEFINE_ARGS dst, src, stride, src_bak, w, offxy, grain_lut, \ |
| 153 |
h, sby, see, left_offxy |
| 154 |
|
| 155 |
mov grain_lutq, grain_lutmp |
| 156 |
mov hd, hm |
| 157 |
.loop_y_h_overlap: |
| 158 |
movu ym20, [grain_lutq+offxyq-82] |
| 159 |
vinserti32x8 m20, [grain_lutq+offxyq+ 0], 1 |
| 160 |
movd xm19, [grain_lutq+left_offxyq-50] |
| 161 |
vinserti32x4 m19, [grain_lutq+left_offxyq+32], 2 |
| 162 |
punpcklbw m19, m20 |
| 163 |
pmaddubsw m19, m10, m19 |
| 164 |
pmulhrsw m19, m9 |
| 165 |
punpckhbw m21, m20, m5 |
| 166 |
packsswb m20{k1}, m19, m19 |
| 167 |
punpcklbw m20, m5, m20 |
| 168 |
call .add_noise_h |
| 169 |
sub hb, 2 |
| 170 |
jg .loop_y_h_overlap |
| 171 |
add wq, 32 |
| 172 |
jge .end |
| 173 |
lea srcq, [src_bakq+wq] |
| 174 |
test sbyd, sbyd |
| 175 |
jnz .hv_overlap |
| 176 |
jmp .loop_x_h_overlap |
| 177 |
|
| 178 |
.v_overlap: |
| 179 |
DEFINE_ARGS dst, src, stride, fg_data, w, offy, offx, \ |
| 180 |
h, sby, see, overlap |
| 181 |
|
| 182 |
movzx r6d, sbyb |
| 183 |
imul seed, [fg_dataq+FGData.seed], 0x00010001 |
| 184 |
imul r7d, r6d, 173 * 0x00010001 |
| 185 |
imul r6d, 37 * 0x01000100 |
| 186 |
add r7d, (105 << 16) | 188 |
| 187 |
add r6d, (178 << 24) | (141 << 8) |
| 188 |
and r7d, 0x00ff00ff |
| 189 |
and r6d, 0xff00ff00 |
| 190 |
xor seed, r7d |
| 191 |
xor seed, r6d ; (cur_seed << 16) | top_seed |
| 192 |
|
| 193 |
DEFINE_ARGS dst, src, stride, src_bak, w, offx, offy, \ |
| 194 |
h, sby, see, overlap |
| 195 |
|
| 196 |
lea src_bakq, [srcq+wq] |
| 197 |
neg wq |
| 198 |
sub dstq, srcq |
| 199 |
|
| 200 |
; we assume from the block above that bits 8-15 of r7d are zero'ed |
| 201 |
mov r6d, seed |
| 202 |
or seed, 0xeff4eff4 |
| 203 |
test seeb, seeh |
| 204 |
setp r7b ; parity of top_seed |
| 205 |
shr seed, 16 |
| 206 |
shl r7d, 16 |
| 207 |
test seeb, seeh |
| 208 |
setp r7b ; parity of cur_seed |
| 209 |
or r6d, 0x00010001 |
| 210 |
xor r7d, r6d |
| 211 |
rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed |
| 212 |
rorx offyd, seed, 8 |
| 213 |
rorx offxd, seed, 12 |
| 214 |
and offyd, 0xf000f |
| 215 |
and offxd, 0xf000f |
| 216 |
imul offyd, 164 |
| 217 |
; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy |
| 218 |
lea offxd, [offyq+offxq*2+0x10001*829+32*82] |
| 219 |
|
| 220 |
DEFINE_ARGS dst, src, stride, src_bak, w, offxy, grain_lut, \ |
| 221 |
h, sby, see, overlap, top_offxy |
| 222 |
|
| 223 |
mov grain_lutq, grain_lutmp |
| 224 |
mov hd, hm |
| 225 |
movzx top_offxyd, offxyw |
| 226 |
shr offxyd, 16 |
| 227 |
movu ym19, [grain_lutq+offxyq-82] |
| 228 |
vinserti32x8 m19, [grain_lutq+offxyq+ 0], 1 |
| 229 |
movu ym21, [grain_lutq+top_offxyq-82] |
| 230 |
vinserti32x8 m21, [grain_lutq+top_offxyq+ 0], 1 |
| 231 |
punpckhbw m20, m21, m19 |
| 232 |
punpcklbw m21, m19 |
| 233 |
call .add_noise_v |
| 234 |
sub hb, 2 |
| 235 |
jg .loop_y |
| 236 |
add wq, 32 |
| 237 |
jge .end |
| 238 |
lea srcq, [src_bakq+wq] |
| 239 |
|
| 240 |
; since fg_dataq.overlap is guaranteed to be set, we never jump back |
| 241 |
; to .v_overlap, and instead always fall-through to h+v overlap |
| 242 |
.hv_overlap: |
| 243 |
; we assume from the block above that bits 8-15 of r7d are zero'ed |
| 244 |
mov r6d, seed |
| 245 |
or seed, 0xeff4eff4 |
| 246 |
test seeb, seeh |
| 247 |
setp r7b ; parity of top_seed |
| 248 |
shr seed, 16 |
| 249 |
shl r7d, 16 |
| 250 |
test seeb, seeh |
| 251 |
setp r7b ; parity of cur_seed |
| 252 |
or r6d, 0x00010001 |
| 253 |
xor r7d, r6d |
| 254 |
rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed |
| 255 |
|
| 256 |
DEFINE_ARGS dst, src, stride, src_bak, w, offx, offy, \ |
| 257 |
h, sby, see, left_offxy, top_offxy, topleft_offxy |
| 258 |
|
| 259 |
mov topleft_offxyd, top_offxyd |
| 260 |
rorx offyd, seed, 8 |
| 261 |
mov left_offxyd, offxd |
| 262 |
rorx offxd, seed, 12 |
| 263 |
and offyd, 0xf000f |
| 264 |
and offxd, 0xf000f |
| 265 |
imul offyd, 164 |
| 266 |
; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy |
| 267 |
lea offxd, [offyq+offxq*2+0x10001*829+32*82] |
| 268 |
|
| 269 |
DEFINE_ARGS dst, src, stride, src_bak, w, offxy, grain_lut, \ |
| 270 |
h, sby, see, left_offxy, top_offxy, topleft_offxy |
| 271 |
|
| 272 |
mov grain_lutq, grain_lutmp |
| 273 |
mov hd, hm |
| 274 |
movzx top_offxyd, offxyw |
| 275 |
shr offxyd, 16 |
| 276 |
movu ym19, [grain_lutq+offxyq-82] |
| 277 |
vinserti32x8 m19, [grain_lutq+offxyq+ 0], 1 |
| 278 |
movd xm16, [grain_lutq+left_offxyq-50] |
| 279 |
vinserti32x4 m16, [grain_lutq+left_offxyq+32], 2 |
| 280 |
movu ym21, [grain_lutq+top_offxyq-82] |
| 281 |
vinserti32x8 m21, [grain_lutq+top_offxyq+ 0], 1 |
| 282 |
movd xm17, [grain_lutq+topleft_offxyq-50] |
| 283 |
vinserti32x4 m17, [grain_lutq+topleft_offxyq+32], 2 |
| 284 |
; do h interpolation first (so top | top/left -> top, left | cur -> cur) |
| 285 |
punpcklbw m16, m19 |
| 286 |
pmaddubsw m16, m10, m16 |
| 287 |
punpcklbw m17, m21 |
| 288 |
pmaddubsw m17, m10, m17 |
| 289 |
punpckhbw m20, m21, m19 |
| 290 |
pmulhrsw m16, m9 |
| 291 |
pmulhrsw m17, m9 |
| 292 |
packsswb m19{k1}, m16, m16 |
| 293 |
packsswb m21{k1}, m17, m17 |
| 294 |
; followed by v interpolation (top | cur -> cur) |
| 295 |
punpcklbw m21, m19 |
| 296 |
call .add_noise_v |
| 297 |
sub hb, 2 |
| 298 |
jg .loop_y_h_overlap |
| 299 |
add wq, 32 |
| 300 |
lea srcq, [src_bakq+wq] |
| 301 |
jl .hv_overlap |
| 302 |
.end: |
| 303 |
RET |
| 304 |
ALIGN function_align |
| 305 |
.add_noise_v: |
| 306 |
pmaddubsw m20, m12, m20 |
| 307 |
pmaddubsw m21, m12, m21 |
| 308 |
pmulhrsw m20, m9 |
| 309 |
pmulhrsw m21, m9 |
| 310 |
packsswb m21, m20 |
| 311 |
.add_noise: |
| 312 |
punpcklbw m20, m5, m21 |
| 313 |
punpckhbw m21, m5 |
| 314 |
.add_noise_h: |
| 315 |
mova ym18, [srcq+strideq*0] |
| 316 |
vinserti32x8 m18, [srcq+strideq*1], 1 |
| 317 |
mova m19, m0 |
| 318 |
punpcklbw m16, m18, m5 |
| 319 |
vpermt2b m19, m18, m1 ; scaling[ 0..127] |
| 320 |
vpmovb2m k2, m18 |
| 321 |
punpckhbw m17, m18, m5 |
| 322 |
vpermi2b m18, m2, m3 ; scaling[128..255] |
| 323 |
vmovdqu8 m19{k2}, m18 ; scaling[src] |
| 324 |
pshufb m19, m4 |
| 325 |
pmaddubsw m18, m19, m20 |
| 326 |
pmaddubsw m19, m21 |
| 327 |
add grain_lutq, 82*2 |
| 328 |
pmulhrsw m18, m6 ; noise |
| 329 |
pmulhrsw m19, m6 |
| 330 |
paddw m16, m18 |
| 331 |
paddw m17, m19 |
| 332 |
packuswb m16, m17 |
| 333 |
pmaxub m16, m7 |
| 334 |
pminub m16, m8 |
| 335 |
mova [dstq+srcq], ym16 |
| 336 |
add srcq, strideq |
| 337 |
vextracti32x8 [dstq+srcq], m16, 1 |
| 338 |
add srcq, strideq |
| 339 |
ret |
| 340 |
|
| 341 |
%macro FGUV_FN 3 ; name, ss_hor, ss_ver |
| 342 |
cglobal fguv_32x32xn_i%1_8bpc, 6, 14+%2, 22, dst, src, stride, fg_data, w, \ |
| 343 |
scaling, grain_lut, h, sby, luma, \ |
| 344 |
overlap, uv_pl, is_id, _, stride3 |
| 345 |
lea r11, [fg_min] |
| 346 |
mov r6d, [fg_dataq+FGData.scaling_shift] |
| 347 |
mov r7d, [fg_dataq+FGData.clip_to_restricted_range] |
| 348 |
mov r9d, is_idm |
| 349 |
mov sbyd, sbym |
| 350 |
mov overlapd, [fg_dataq+FGData.overlap_flag] |
| 351 |
%if %2 |
| 352 |
mov r12, 0x000f000f000f000f ; h_overlap mask |
| 353 |
vpbroadcastq m10, [base+pb_23_22_0_32] |
| 354 |
lea stride3q, [strideq*3] |
| 355 |
%else |
| 356 |
mov r12, 0x0000000f0000000f |
| 357 |
vpbroadcastq m10, [base+pb_27_17_17_27] |
| 358 |
%endif |
| 359 |
mova m0, [scalingq+64*0] |
| 360 |
mova m1, [scalingq+64*1] |
| 361 |
mova m2, [scalingq+64*2] |
| 362 |
mova m3, [scalingq+64*3] |
| 363 |
kmovq k1, r12 |
| 364 |
vbroadcasti32x4 m4, [base+interleave_hl] |
| 365 |
vpbroadcastd m6, [base+noise_rnd+r6*4-32] |
| 366 |
vpbroadcastd m7, [base+fg_min+r7*4] |
| 367 |
shlx r7d, r7d, r9d |
| 368 |
vpbroadcastd m8, [base+fg_max+r7*4] |
| 369 |
test sbyd, sbyd |
| 370 |
setnz r7b |
| 371 |
vpbroadcastd m9, [base+pw_1024] |
| 372 |
mova m11, [base+pb_even] |
| 373 |
mova m12, [base+pb_odd] |
| 374 |
pxor m5, m5 |
| 375 |
mov r5, r10mp ; lstride |
| 376 |
cmp byte [fg_dataq+FGData.chroma_scaling_from_luma], 0 |
| 377 |
jne .csfl |
| 378 |
|
| 379 |
%macro %%FGUV_32x32xN_LOOP 3 ; not-csfl, ss_hor, ss_ver |
| 380 |
DEFINE_ARGS dst, src, stride, fg_data, w, lstride, grain_lut, \ |
| 381 |
h, sby, see, overlap, uv_pl, _, _, stride3 |
| 382 |
%if %1 |
| 383 |
mov r6d, uv_plm |
| 384 |
vpbroadcastd m16, [base+pw_8] |
| 385 |
vbroadcasti32x4 m14, [fg_dataq+FGData.uv_mult+r6*4] |
| 386 |
vpbroadcastw m15, [fg_dataq+FGData.uv_offset+r6*4] |
| 387 |
pshufb m14, m16 ; uv_luma_mult, uv_mult |
| 388 |
%endif |
| 389 |
test r7b, overlapb |
| 390 |
jnz %%v_overlap |
| 391 |
|
| 392 |
imul seed, sbyd, (173 << 24) | 37 |
| 393 |
add seed, (105 << 24) | 178 |
| 394 |
rorx seed, seed, 24 |
| 395 |
movzx seed, seew |
| 396 |
xor seed, [fg_dataq+FGData.seed] |
| 397 |
|
| 398 |
DEFINE_ARGS dst, src, stride, luma, w, lstride, grain_lut, \ |
| 399 |
offx, offy, see, overlap, _, _, _, stride3 |
| 400 |
|
| 401 |
mov lumaq, r9mp |
| 402 |
lea r11, [srcq+wq] |
| 403 |
lea r12, [dstq+wq] |
| 404 |
lea r13, [lumaq+wq*(1+%2)] |
| 405 |
mov r11mp, r11 |
| 406 |
mov r12mp, r12 |
| 407 |
neg wq |
| 408 |
|
| 409 |
%%loop_x: |
| 410 |
rorx r6, seeq, 1 |
| 411 |
or seed, 0xeff4 |
| 412 |
test seeb, seeh |
| 413 |
lea seed, [r6+0x8000] |
| 414 |
cmovp seed, r6d ; updated seed |
| 415 |
rorx offyd, seed, 8 |
| 416 |
rorx offxq, seeq, 12 |
| 417 |
and offyd, 0xf |
| 418 |
imul offyd, 164>>%3 |
| 419 |
lea offyd, [offyq+offxq*(2-%2)+(3+(6>>%3))*82+3+(6>>%2)] ; offy*stride+offx |
| 420 |
|
| 421 |
DEFINE_ARGS dst, src, stride, luma, w, lstride, grain_lut, \ |
| 422 |
h, offxy, see, overlap, _, _, _, stride3 |
| 423 |
|
| 424 |
mov grain_lutq, grain_lutmp |
| 425 |
mov hd, hm |
| 426 |
%%loop_y: |
| 427 |
%if %2 |
| 428 |
movu xm21, [grain_lutq+offxyq+82*0] |
| 429 |
vinserti128 ym21, [grain_lutq+offxyq+82*1], 1 |
| 430 |
vinserti32x4 m21, [grain_lutq+offxyq+82*2], 2 |
| 431 |
vinserti32x4 m21, [grain_lutq+offxyq+82*3], 3 |
| 432 |
%else |
| 433 |
movu ym21, [grain_lutq+offxyq+82*0] |
| 434 |
vinserti32x8 m21, [grain_lutq+offxyq+82*1], 1 |
| 435 |
%endif |
| 436 |
call %%add_noise |
| 437 |
sub hb, 2<<%2 |
| 438 |
jg %%loop_y |
| 439 |
add wq, 32>>%2 |
| 440 |
jge .end |
| 441 |
mov srcq, r11mp |
| 442 |
mov dstq, r12mp |
| 443 |
lea lumaq, [r13+wq*(1<<%2)] |
| 444 |
add srcq, wq |
| 445 |
add dstq, wq |
| 446 |
test overlapd, overlapd |
| 447 |
jz %%loop_x |
| 448 |
cmp dword r8m, 0 ; sby |
| 449 |
jne %%hv_overlap |
| 450 |
|
| 451 |
; horizontal overlap (without vertical overlap) |
| 452 |
%%loop_x_h_overlap: |
| 453 |
rorx r6, seeq, 1 |
| 454 |
or seed, 0xeff4 |
| 455 |
test seeb, seeh |
| 456 |
lea seed, [r6+0x8000] |
| 457 |
cmovp seed, r6d ; updated seed |
| 458 |
|
| 459 |
DEFINE_ARGS dst, src, stride, luma, w, lstride, grain_lut, \ |
| 460 |
offx, offy, see, left_offxy, _, _, _, stride3 |
| 461 |
|
| 462 |
lea left_offxyd, [offyq+(32>>%2)] ; previous column's offy*stride+offx |
| 463 |
rorx offyd, seed, 8 |
| 464 |
rorx offxq, seeq, 12 |
| 465 |
and offyd, 0xf |
| 466 |
imul offyd, 164>>%3 |
| 467 |
lea offyd, [offyq+offxq*(2-%2)+(3+(6>>%3))*82+3+(6>>%2)] ; offy*stride+offx |
| 468 |
|
| 469 |
DEFINE_ARGS dst, src, stride, luma, w, lstride, grain_lut, \ |
| 470 |
h, offxy, see, left_offxy, _, _, _, stride3 |
| 471 |
|
| 472 |
mov grain_lutq, grain_lutmp |
| 473 |
mov hd, hm |
| 474 |
%%loop_y_h_overlap: |
| 475 |
%if %2 |
| 476 |
movu xm20, [grain_lutq+offxyq +82*0] |
| 477 |
movd xm19, [grain_lutq+left_offxyq+82*0] |
| 478 |
vinserti32x4 ym20, [grain_lutq+offxyq +82*1], 1 |
| 479 |
vinserti32x4 ym19, [grain_lutq+left_offxyq+82*1], 1 |
| 480 |
vinserti32x4 m20, [grain_lutq+offxyq +82*2], 2 |
| 481 |
vinserti32x4 m19, [grain_lutq+left_offxyq+82*2], 2 |
| 482 |
vinserti32x4 m20, [grain_lutq+offxyq +82*3], 3 |
| 483 |
vinserti32x4 m19, [grain_lutq+left_offxyq+82*3], 3 |
| 484 |
%else |
| 485 |
movu ym20, [grain_lutq+offxyq + 0] |
| 486 |
movd xm19, [grain_lutq+left_offxyq+ 0] |
| 487 |
vinserti32x8 m20, [grain_lutq+offxyq +82], 1 |
| 488 |
vinserti32x4 m19, [grain_lutq+left_offxyq+82], 2 |
| 489 |
%endif |
| 490 |
punpcklbw m19, m20 |
| 491 |
pmaddubsw m19, m10, m19 |
| 492 |
punpckhbw m21, m20, m5 |
| 493 |
pmulhrsw m19, m9 |
| 494 |
vpacksswb m20{k1}, m19, m19 |
| 495 |
punpcklbw m20, m5, m20 |
| 496 |
call %%add_noise_h |
| 497 |
sub hb, 2<<%2 |
| 498 |
jg %%loop_y_h_overlap |
| 499 |
add wq, 32>>%2 |
| 500 |
jge .end |
| 501 |
mov srcq, r11mp |
| 502 |
mov dstq, r12mp |
| 503 |
lea lumaq, [r13+wq*(1<<%2)] |
| 504 |
add srcq, wq |
| 505 |
add dstq, wq |
| 506 |
cmp dword r8m, 0 ; sby |
| 507 |
jne %%hv_overlap |
| 508 |
jmp %%loop_x_h_overlap |
| 509 |
|
| 510 |
%%v_overlap: |
| 511 |
DEFINE_ARGS dst, src, stride, fg_data, w, lstride, grain_lut, \ |
| 512 |
_, sby, see, overlap, _, _, _, stride3 |
| 513 |
|
| 514 |
movzx sbyd, sbyb |
| 515 |
imul seed, [fg_dataq+FGData.seed], 0x00010001 |
| 516 |
imul r7d, sbyd, 173 * 0x00010001 |
| 517 |
imul sbyd, 37 * 0x01000100 |
| 518 |
add r7d, (105 << 16) | 188 |
| 519 |
add sbyd, (178 << 24) | (141 << 8) |
| 520 |
and r7d, 0x00ff00ff |
| 521 |
and sbyd, 0xff00ff00 |
| 522 |
xor seed, r7d |
| 523 |
xor seed, sbyd ; (cur_seed << 16) | top_seed |
| 524 |
|
| 525 |
%if %3 |
| 526 |
vpbroadcastd m13, [base+pb_23_22] |
| 527 |
kxnorw k3, k3, k3 ; v_overlap mask |
| 528 |
%elif %2 |
| 529 |
vbroadcasti32x8 m13, [base+pb_27_17] |
| 530 |
kxnord k3, k3, k3 |
| 531 |
pshufd m13, m13, q0000 ; 8x27_17, 8x17_27 |
| 532 |
%else |
| 533 |
vpbroadcastd ym16, [base+pb_27_17] |
| 534 |
vpbroadcastd m13, [base+pb_17_27] |
| 535 |
vmovdqa64 m13{k1}, m16 |
| 536 |
%endif |
| 537 |
|
| 538 |
DEFINE_ARGS dst, src, stride, luma, w, lstride, grain_lut, \ |
| 539 |
offx, offy, see, overlap, top_offxy, _, _, stride3 |
| 540 |
|
| 541 |
mov lumaq, r9mp |
| 542 |
lea r11, [srcq+wq] |
| 543 |
lea r12, [dstq+wq] |
| 544 |
lea r13, [lumaq+wq*(1<<%2)] |
| 545 |
mov r11mp, r11 |
| 546 |
mov r12mp, r12 |
| 547 |
neg wq |
| 548 |
|
| 549 |
; we assume from the block above that bits 8-15 of r7d are zero'ed |
| 550 |
mov r6d, seed |
| 551 |
or seed, 0xeff4eff4 |
| 552 |
test seeb, seeh |
| 553 |
setp r7b ; parity of top_seed |
| 554 |
shr seed, 16 |
| 555 |
shl r7d, 16 |
| 556 |
test seeb, seeh |
| 557 |
setp r7b ; parity of cur_seed |
| 558 |
or r6d, 0x00010001 |
| 559 |
xor r7d, r6d |
| 560 |
rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed |
| 561 |
rorx offyd, seed, 8 |
| 562 |
rorx offxd, seed, 12 |
| 563 |
and offyd, 0x000f000f |
| 564 |
and offxd, 0x000f000f |
| 565 |
imul offyd, 164>>%3 |
| 566 |
; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy |
| 567 |
lea offyd, [offyq+offxq*(2-%2)+0x10001*((3+(6>>%3))*82+3+(6>>%2))+(32>>%3)*82] |
| 568 |
|
| 569 |
DEFINE_ARGS dst, src, stride, luma, w, lstride, grain_lut, \ |
| 570 |
h, offxy, see, overlap, top_offxy, _, _, stride3 |
| 571 |
|
| 572 |
mov grain_lutq, grain_lutmp |
| 573 |
mov hd, hm |
| 574 |
movzx top_offxyd, offxyw |
| 575 |
shr offxyd, 16 |
| 576 |
|
| 577 |
%if %3 |
| 578 |
movu xm18, [grain_lutq+offxyq+82*0] |
| 579 |
movu xm20, [grain_lutq+top_offxyq+82*0] |
| 580 |
; only interpolate first line, insert remaining line unmodified |
| 581 |
vbroadcasti128 ym21, [grain_lutq+offxyq+82*1] |
| 582 |
vinserti32x4 m21, [grain_lutq+offxyq+82*2], 2 |
| 583 |
vinserti32x4 m21, [grain_lutq+offxyq+82*3], 3 |
| 584 |
punpcklbw xm19, xm20, xm18 |
| 585 |
punpckhbw xm20, xm18 |
| 586 |
%elif %2 |
| 587 |
movu xm18, [grain_lutq+offxyq+82*0] |
| 588 |
vinserti128 ym18, [grain_lutq+offxyq+82*1], 1 |
| 589 |
movu xm20, [grain_lutq+top_offxyq+82*0] |
| 590 |
vinserti32x4 ym20, [grain_lutq+top_offxyq+82*1], 1 |
| 591 |
vbroadcasti32x4 m21, [grain_lutq+offxyq+82*2] |
| 592 |
vinserti32x4 m21, [grain_lutq+offxyq+82*3], 3 |
| 593 |
punpcklbw ym19, ym20, ym18 |
| 594 |
punpckhbw ym20, ym18 |
| 595 |
%else |
| 596 |
movu ym21, [grain_lutq+offxyq+82*0] |
| 597 |
vinserti32x8 m21, [grain_lutq+offxyq+82*1], 1 |
| 598 |
movu ym20, [grain_lutq+top_offxyq+82*0] |
| 599 |
vinserti32x8 m20, [grain_lutq+top_offxyq+82*1], 1 |
| 600 |
%endif |
| 601 |
call %%add_noise_v |
| 602 |
sub hb, 2<<%2 |
| 603 |
jg %%loop_y |
| 604 |
add wq, 32>>%2 |
| 605 |
jge .end |
| 606 |
mov srcq, r11mp |
| 607 |
mov dstq, r12mp |
| 608 |
lea lumaq, [r13+wq*(1<<%2)] |
| 609 |
add srcq, wq |
| 610 |
add dstq, wq |
| 611 |
|
| 612 |
%%hv_overlap: |
| 613 |
; we assume from the block above that bits 8-15 of r7d are zero'ed |
| 614 |
mov r6d, seed |
| 615 |
or seed, 0xeff4eff4 |
| 616 |
test seeb, seeh |
| 617 |
setp r7b ; parity of top_seed |
| 618 |
shr seed, 16 |
| 619 |
shl r7d, 16 |
| 620 |
test seeb, seeh |
| 621 |
setp r7b ; parity of cur_seed |
| 622 |
or r6d, 0x00010001 |
| 623 |
xor r7d, r6d |
| 624 |
rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed |
| 625 |
|
| 626 |
DEFINE_ARGS dst, src, stride, luma, w, lstride, grain_lut, \ |
| 627 |
offx, offy, see, left_offxy, top_offxy, topleft_offxy, _, stride3 |
| 628 |
|
| 629 |
lea topleft_offxyd, [top_offxyq+(32>>%2)] |
| 630 |
lea left_offxyd, [offyq+(32>>%2)] |
| 631 |
rorx offyd, seed, 8 |
| 632 |
rorx offxd, seed, 12 |
| 633 |
and offyd, 0x000f000f |
| 634 |
and offxd, 0x000f000f |
| 635 |
imul offyd, 164>>%3 |
| 636 |
; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy |
| 637 |
lea offyd, [offyq+offxq*(2-%2)+0x10001*((3+(6>>%3))*82+3+(6>>%2))+(32>>%3)*82] |
| 638 |
|
| 639 |
DEFINE_ARGS dst, src, stride, luma, w, lstride, grain_lut, \ |
| 640 |
h, offxy, see, left_offxy, top_offxy, topleft_offxy, _, stride3 |
| 641 |
|
| 642 |
mov grain_lutq, grain_lutmp |
| 643 |
mov hd, hm |
| 644 |
movzx top_offxyd, offxyw |
| 645 |
shr offxyd, 16 |
| 646 |
|
| 647 |
%if %2 |
| 648 |
movu xm21, [grain_lutq+offxyq+82*0] |
| 649 |
movd xm16, [grain_lutq+left_offxyq+82*0] |
| 650 |
vinserti128 ym21, [grain_lutq+offxyq+82*1], 1 |
| 651 |
vinserti128 ym16, [grain_lutq+left_offxyq+82*1], 1 |
| 652 |
vinserti32x4 m21, [grain_lutq+offxyq+82*2], 2 |
| 653 |
vinserti32x4 m16, [grain_lutq+left_offxyq+82*2], 2 |
| 654 |
vinserti32x4 m21, [grain_lutq+offxyq+82*3], 3 |
| 655 |
vinserti32x4 m16, [grain_lutq+left_offxyq+82*3], 3 |
| 656 |
movd xm18, [grain_lutq+topleft_offxyq+82*0] |
| 657 |
movu xm20, [grain_lutq+top_offxyq] |
| 658 |
; do h interpolation first (so top | top/left -> top, left | cur -> cur) |
| 659 |
punpcklbw m16, m21 |
| 660 |
%if %3 |
| 661 |
punpcklbw xm18, xm20 |
| 662 |
%else |
| 663 |
vinserti128 ym18, [grain_lutq+topleft_offxyq+82*1], 1 |
| 664 |
vinserti128 ym20, [grain_lutq+top_offxyq+82*1], 1 |
| 665 |
punpcklbw ym18, ym20 |
| 666 |
%endif |
| 667 |
punpcklqdq m16, m18 |
| 668 |
pmaddubsw m16, m10, m16 |
| 669 |
pmulhrsw m16, m9 |
| 670 |
packsswb m16, m16 |
| 671 |
vmovdqu8 m21{k1}, m16 |
| 672 |
%if %3 |
| 673 |
vpalignr xm20{k1}, xm16, xm16, 4 |
| 674 |
punpcklbw xm19, xm20, xm21 |
| 675 |
punpckhbw xm20, xm21 |
| 676 |
%else |
| 677 |
vpalignr ym20{k1}, ym16, ym16, 4 |
| 678 |
punpcklbw ym19, ym20, ym21 |
| 679 |
punpckhbw ym20, ym21 |
| 680 |
%endif |
| 681 |
%else |
| 682 |
movu ym21, [grain_lutq+offxyq+82*0] |
| 683 |
vinserti32x8 m21, [grain_lutq+offxyq+82*1], 1 |
| 684 |
movd xm16, [grain_lutq+left_offxyq+82*0] |
| 685 |
vinserti32x4 m16, [grain_lutq+left_offxyq+82*1], 2 |
| 686 |
movu ym20, [grain_lutq+top_offxyq+82*0] |
| 687 |
vinserti32x8 m20, [grain_lutq+top_offxyq+82*1], 1 |
| 688 |
movd xm18, [grain_lutq+topleft_offxyq+82*0] |
| 689 |
vinserti32x4 m18, [grain_lutq+topleft_offxyq+82*1], 2 |
| 690 |
punpcklbw m16, m21 |
| 691 |
punpcklbw m18, m20 |
| 692 |
punpcklqdq m16, m18 |
| 693 |
pmaddubsw m16, m10, m16 |
| 694 |
pmulhrsw m16, m9 |
| 695 |
packsswb m16, m16 |
| 696 |
vpalignr m20{k1}, m16, m16, 4 |
| 697 |
vmovdqu8 m21{k1}, m16 |
| 698 |
%endif |
| 699 |
call %%add_noise_v |
| 700 |
sub hb, 2<<%2 |
| 701 |
jg %%loop_y_h_overlap |
| 702 |
add wq, 32>>%2 |
| 703 |
jge .end |
| 704 |
mov srcq, r11mp |
| 705 |
mov dstq, r12mp |
| 706 |
lea lumaq, [r13+wq*(1<<%2)] |
| 707 |
add srcq, wq |
| 708 |
add dstq, wq |
| 709 |
jmp %%hv_overlap |
| 710 |
ALIGN function_align |
| 711 |
%%add_noise_v: |
| 712 |
%if %3 |
| 713 |
pmaddubsw xm19, xm13, xm19 |
| 714 |
pmaddubsw xm20, xm13, xm20 |
| 715 |
pmulhrsw xm19, xm9 |
| 716 |
pmulhrsw xm20, xm9 |
| 717 |
vpacksswb m21{k3}, m19, m20 |
| 718 |
%elif %2 |
| 719 |
pmaddubsw ym19, ym13, ym19 |
| 720 |
pmaddubsw ym20, ym13, ym20 |
| 721 |
pmulhrsw ym19, ym9 |
| 722 |
pmulhrsw ym20, ym9 |
| 723 |
vpacksswb m21{k3}, m19, m20 |
| 724 |
%else |
| 725 |
punpcklbw m19, m20, m21 |
| 726 |
punpckhbw m20, m21 |
| 727 |
pmaddubsw m19, m13, m19 |
| 728 |
pmaddubsw m20, m13, m20 |
| 729 |
pmulhrsw m19, m9 |
| 730 |
pmulhrsw m20, m9 |
| 731 |
packsswb m21, m19, m20 |
| 732 |
%endif |
| 733 |
%%add_noise: |
| 734 |
punpcklbw m20, m5, m21 |
| 735 |
punpckhbw m21, m5 |
| 736 |
%%add_noise_h: |
| 737 |
mova ym18, [lumaq+lstrideq*(0<<%3)] |
| 738 |
vinserti32x8 m18, [lumaq+lstrideq*(1<<%3)], 1 |
| 739 |
%if %2 |
| 740 |
lea lumaq, [lumaq+lstrideq*(2<<%3)] |
| 741 |
mova ym16, [lumaq+lstrideq*(0<<%3)] |
| 742 |
vinserti32x8 m16, [lumaq+lstrideq*(1<<%3)], 1 |
| 743 |
mova xm17, [srcq+strideq*0] |
| 744 |
mova m19, m11 |
| 745 |
vpermi2b m19, m18, m16 |
| 746 |
vinserti128 ym17, [srcq+strideq*1], 1 |
| 747 |
vpermt2b m18, m12, m16 |
| 748 |
vinserti32x4 m17, [srcq+strideq*2], 2 |
| 749 |
pavgb m18, m19 |
| 750 |
vinserti32x4 m17, [srcq+stride3q ], 3 |
| 751 |
%else |
| 752 |
mova ym17, [srcq+strideq*0] |
| 753 |
vinserti32x8 m17, [srcq+strideq*1], 1 |
| 754 |
%endif |
| 755 |
%if %1 |
| 756 |
punpckhbw m19, m18, m17 |
| 757 |
punpcklbw m18, m17 ; { luma, chroma } |
| 758 |
pmaddubsw m19, m14 |
| 759 |
pmaddubsw m18, m14 |
| 760 |
psraw m19, 6 |
| 761 |
psraw m18, 6 |
| 762 |
paddw m19, m15 |
| 763 |
paddw m18, m15 |
| 764 |
packuswb m18, m19 |
| 765 |
.add_noise_main: |
| 766 |
mova m19, m0 |
| 767 |
vpermt2b m19, m18, m1 ; scaling[ 0..127] |
| 768 |
vpmovb2m k2, m18 |
| 769 |
vpermi2b m18, m2, m3 ; scaling[128..255] |
| 770 |
vmovdqu8 m19{k2}, m18 ; scaling[src] |
| 771 |
pshufb m19, m4 |
| 772 |
pmaddubsw m18, m19, m20 |
| 773 |
pmaddubsw m19, m21 |
| 774 |
add grain_lutq, 82*2<<%2 |
| 775 |
lea lumaq, [lumaq+lstrideq*(2<<%3)] |
| 776 |
lea srcq, [srcq+strideq*(2<<%2)] |
| 777 |
pmulhrsw m18, m6 ; noise |
| 778 |
pmulhrsw m19, m6 |
| 779 |
punpcklbw m16, m17, m5 ; chroma |
| 780 |
punpckhbw m17, m5 |
| 781 |
paddw m16, m18 |
| 782 |
paddw m17, m19 |
| 783 |
packuswb m16, m17 |
| 784 |
pmaxub m16, m7 |
| 785 |
pminub m16, m8 |
| 786 |
%if %2 |
| 787 |
mova [dstq+strideq*0], xm16 |
| 788 |
vextracti128 [dstq+strideq*1], ym16, 1 |
| 789 |
vextracti32x4 [dstq+strideq*2], m16, 2 |
| 790 |
vextracti32x4 [dstq+stride3q ], m16, 3 |
| 791 |
%else |
| 792 |
mova [dstq+strideq*0], ym16 |
| 793 |
vextracti32x8 [dstq+strideq*1], m16, 1 |
| 794 |
%endif |
| 795 |
lea dstq, [dstq+strideq*(2<<%2)] |
| 796 |
ret |
| 797 |
%else |
| 798 |
jmp .add_noise_main |
| 799 |
%endif |
| 800 |
%endmacro |
| 801 |
|
| 802 |
%%FGUV_32x32xN_LOOP 1, %2, %3 |
| 803 |
.csfl: |
| 804 |
%%FGUV_32x32xN_LOOP 0, %2, %3 |
| 805 |
.end: |
| 806 |
RET |
| 807 |
%endmacro |
| 808 |
|
| 809 |
FGUV_FN 420, 1, 1 |
| 810 |
FGUV_FN 422, 1, 0 |
| 811 |
FGUV_FN 444, 0, 0 |
| 812 |
|
| 813 |
%endif ; ARCH_X86_64 |