Debian Codegraph @

.cargo/vendor/rav1e/src/x86/filmgrain16_avx512.asm

raw
1 ; Copyright © 2022, VideoLAN and dav1d authors
2 ; Copyright © 2022, Two Orioles, LLC
3 ; All rights reserved.
4 ;
5 ; Redistribution and use in source and binary forms, with or without
6 ; modification, are permitted provided that the following conditions are met:
7 ;
8 ; 1. Redistributions of source code must retain the above copyright notice, this
9 ; list of conditions and the following disclaimer.
10 ;
11 ; 2. Redistributions in binary form must reproduce the above copyright notice,
12 ; this list of conditions and the following disclaimer in the documentation
13 ; and/or other materials provided with the distribution.
14 ;
15 ; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
16 ; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
17 ; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
18 ; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
19 ; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
20 ; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
21 ; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
22 ; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 ; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
24 ; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25
26 %include "config.asm"
27 %include "ext/x86/x86inc.asm"
28 %include "x86/filmgrain_common.asm"
29
30 %if ARCH_X86_64
31
32 SECTION_RODATA 64
33 pb_0to63: db 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
34 db 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31
35 db 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47
36 db 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63
37 scale_mask: db -1, -1, 0, -1, -1, -1, 4, -1, -1, -1, 8, -1, -1, -1, 12, -1
38 scale_shift: dw 7, 7, 6, 6, 5, 5, 4, 4
39 pw_27_17_17_27: dw 108, 68, 68, 108, 27, 17, 17, 27
40 pw_23_22: dw 92, 88, 0, 128, 23, 22, 0, 32
41 fg_min: times 2 dw 0
42 times 2 dw 64
43 times 2 dw 256
44 fg_max: times 2 dw 1023
45 times 2 dw 4095
46 times 2 dw 960
47 times 2 dw 3840
48 times 2 dw 940
49 times 2 dw 3760
50 scale_rnd: dd 64
51 dd 16
52 uv_offset_mul: dd 256
53 dd 1024
54 pb_8_9_0_1: db 8, 9, 0, 1
55
56 SECTION .text
57
58 INIT_ZMM avx512icl
59 cglobal fgy_32x32xn_16bpc, 6, 15, 21, dst, src, stride, fg_data, w, scaling, \
60 grain_lut, offx, sby, see, offy, src_bak
61 %define base r11-fg_min
62 lea r11, [fg_min]
63 mov r6d, r9m ; bdmax
64 mov r9d, [fg_dataq+FGData.clip_to_restricted_range]
65 mov r7d, [fg_dataq+FGData.scaling_shift]
66 mov sbyd, sbym
67 vpbroadcastd m6, r9m
68 shr r6d, 11 ; is_12bpc
69 vbroadcasti32x4 m7, [base+scale_mask]
70 shlx r10d, r9d, r6d
71 vpbroadcastd m10, [base+scale_shift+r7*4-32]
72 lea r9d, [r6+r9*4]
73 vpbroadcastd m8, [base+fg_min+r10*4]
74 kxnorw k1, k1, k1 ; 0xffff
75 vpbroadcastd m9, [base+fg_max+r9*4]
76 mov r12, 0xeeeeeeeeeeeeeeee
77 vpbroadcastd m19, [base+scale_rnd+r6*4]
78 kshiftrb k2, k1, 4 ; 0xf
79 vpbroadcastq xm20, [base+pw_27_17_17_27+r6*8]
80 kmovq k3, r12
81 vpbroadcastd m11, [base+scale_shift+r6*8+4]
82 test sbyd, sbyd
83 setnz r7b
84 vpbroadcastd m12, [base+pw_27_17_17_27+r6*8+0]
85 vpbroadcastd m13, [base+pw_27_17_17_27+r6*8+4]
86 test r7b, [fg_dataq+FGData.overlap_flag]
87 jnz .v_overlap
88
89 imul seed, sbyd, (173 << 24) | 37
90 add seed, (105 << 24) | 178
91 rorx seed, seed, 24
92 movzx seed, seew
93 xor seed, [fg_dataq+FGData.seed]
94 lea src_bakq, [srcq+wq*2]
95 neg wq
96 sub dstq, srcq
97
98 .loop_x:
99 rorx r6, seeq, 1
100 or seed, 0xeff4
101 test seeb, seeh
102 lea seed, [r6+0x8000]
103 cmovp seed, r6d ; updated seed
104 rorx offyd, seed, 8
105 rorx offxq, seeq, 12
106 and offyd, 0xf
107 imul offyd, 164
108 lea offyd, [offyq+offxq*2+747] ; offy*stride+offx
109
110 DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, h, \
111 sby, see, offxy, src_bak
112
113 mov grain_lutq, grain_lutmp
114 mov hd, hm
115 .loop_y:
116 movu m4, [grain_lutq+offxyq*2+82*0]
117 movu m5, [grain_lutq+offxyq*2+82*2]
118 call .add_noise
119 sub hb, 2
120 jg .loop_y
121 add wq, 32
122 jge .end
123 lea srcq, [src_bakq+wq*2]
124 cmp byte [fg_dataq+FGData.overlap_flag], 0
125 je .loop_x
126 test sbyd, sbyd
127 jnz .hv_overlap
128
129 ; horizontal overlap (without vertical overlap)
130 .loop_x_h_overlap:
131 rorx r6, seeq, 1
132 or seed, 0xeff4
133 test seeb, seeh
134 lea seed, [r6+0x8000]
135 cmovp seed, r6d ; updated seed
136
137 DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, offx, \
138 sby, see, offy, src_bak, left_offxy
139
140 lea left_offxyd, [offyq+73] ; previous column's offy*stride+offx
141 rorx offyd, seed, 8
142 rorx offxq, seeq, 12
143 and offyd, 0xf
144 imul offyd, 164
145 lea offyd, [offyq+offxq*2+747] ; offy*stride+offx
146
147 DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, h, \
148 sby, see, offxy, src_bak, left_offxy
149
150 mov grain_lutq, grain_lutmp
151 mov hd, hm
152 .loop_y_h_overlap:
153 movu m4, [grain_lutq+offxyq*2+82*0]
154 movu m5, [grain_lutq+offxyq*2+82*2]
155 movd xm17, [grain_lutq+left_offxyq*2-82*1]
156 pinsrd xm17, [grain_lutq+left_offxyq*2+82*1], 1
157 punpckldq xm16, xm4, xm5
158 punpcklwd xm17, xm16
159 mova xm16, xm19
160 vpdpwssd xm16, xm20, xm17
161 psrad xm16, 1
162 packssdw xm16, xm16
163 vpsravw xm16, xm11
164 vmovdqu8 m4{k2}, m16
165 vpalignr m5{k2}, m16, m16, 4
166 call .add_noise
167 sub hb, 2
168 jg .loop_y_h_overlap
169 add wq, 32
170 jge .end
171 lea srcq, [src_bakq+wq*2]
172 test sbyd, sbyd
173 jnz .hv_overlap
174 jmp .loop_x_h_overlap
175
176 .v_overlap:
177 movzx sbyd, sbyb
178 imul seed, [fg_dataq+FGData.seed], 0x00010001
179 imul r7d, sbyd, 173 * 0x00010001
180 imul sbyd, 37 * 0x01000100
181 add r7d, (105 << 16) | 188
182 add sbyd, (178 << 24) | (141 << 8)
183 and r7d, 0x00ff00ff
184 and sbyd, 0xff00ff00
185 xor seed, r7d
186 xor seed, sbyd ; (cur_seed << 16) | top_seed
187 lea src_bakq, [srcq+wq*2]
188 neg wq
189 sub dstq, srcq
190
191 ; we assume from the block above that bits 8-15 of r7d are zero'ed
192 mov r6d, seed
193 or seed, 0xeff4eff4
194 test seeb, seeh
195 setp r7b ; parity of top_seed
196 shr seed, 16
197 shl r7d, 16
198 test seeb, seeh
199 setp r7b ; parity of cur_seed
200 or r6d, 0x00010001
201 xor r7d, r6d
202 rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed
203
204 DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, offx, \
205 sby, see, offy, src_bak, _, top_offxy
206
207 rorx offyd, seed, 8
208 rorx offxd, seed, 12
209 and offyd, 0xf000f
210 and offxd, 0xf000f
211 imul offyd, 164
212 ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy
213 lea offyd, [offyq+offxq*2+0x10001*747+32*82]
214
215 DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, h, \
216 sby, see, offxy, src_bak, _, top_offxy
217
218 mov grain_lutq, grain_lutmp
219 mov hd, hm
220 movzx top_offxyd, offxyw
221 shr offxyd, 16
222
223 movu m16, [grain_lutq+offxyq*2+82*0]
224 movu m0, [grain_lutq+top_offxyq*2+82*0]
225 movu m17, [grain_lutq+offxyq*2+82*2]
226 movu m1, [grain_lutq+top_offxyq*2+82*2]
227 punpckhwd m4, m0, m16
228 punpcklwd m0, m16
229 punpckhwd m5, m1, m17
230 punpcklwd m1, m17
231 call .add_noise_v
232 sub hb, 2
233 jg .loop_y
234 add wq, 32
235 jge .end
236 lea srcq, [src_bakq+wq*2]
237
238 ; since fg_dataq.overlap is guaranteed to be set, we never jump back
239 ; to .v_overlap, and instead always fall-through to .hv_overlap
240 .hv_overlap:
241 ; we assume from the block above that bits 8-15 of r7d are zero'ed
242 mov r6d, seed
243 or seed, 0xeff4eff4
244 test seeb, seeh
245 setp r7b ; parity of top_seed
246 shr seed, 16
247 shl r7d, 16
248 test seeb, seeh
249 setp r7b ; parity of cur_seed
250 or r6d, 0x00010001
251 xor r7d, r6d
252 rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed
253
254 DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, offx, \
255 sby, see, offy, src_bak, left_offxy, top_offxy, topleft_offxy
256
257 lea topleft_offxyd, [top_offxyq+73]
258 lea left_offxyd, [offyq+73]
259 rorx offyd, seed, 8
260 rorx offxd, seed, 12
261 and offyd, 0xf000f
262 and offxd, 0xf000f
263 imul offyd, 164
264 ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy
265 lea offyd, [offyq+offxq*2+0x10001*747+32*82]
266
267 DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, h, \
268 sby, see, offxy, src_bak, left_offxy, top_offxy, topleft_offxy
269
270 mov grain_lutq, grain_lutmp
271 mov hd, hm
272 movzx top_offxyd, offxyw
273 shr offxyd, 16
274
275 movu m5, [grain_lutq+offxyq*2+82*0]
276 movu m0, [grain_lutq+top_offxyq*2+82*0]
277 movd xm17, [grain_lutq+left_offxyq*2-82*1]
278 pinsrd xm17, [grain_lutq+topleft_offxyq*2-82*1], 1
279 movu m2, [grain_lutq+offxyq*2+82*2]
280 movu m1, [grain_lutq+top_offxyq*2+82*2]
281 movd xm18, [grain_lutq+left_offxyq*2+82*1]
282 pinsrd xm18, [grain_lutq+topleft_offxyq*2+82*1], 1
283 punpckldq xm16, xm5, xm0
284 punpcklwd xm17, xm16
285 mova xm16, xm19
286 vpdpwssd xm16, xm20, xm17
287 punpckldq xm17, xm2, xm1
288 punpcklwd xm18, xm17
289 mova xm17, xm19
290 vpdpwssd xm17, xm20, xm18
291 punpckhwd m4, m0, m5
292 punpcklwd m0, m5
293 punpckhwd m5, m1, m2
294 punpcklwd m1, m2
295 psrad xm16, 1
296 psrad xm17, 1
297 packssdw xm16, xm17
298 vpsravw xm16, xm11
299 vpshuflw m0{k2}, m16, q1302
300 punpckhqdq xm16, xm16
301 vpshuflw m1{k2}, m16, q1302
302 call .add_noise_v
303 sub hb, 2
304 jg .loop_y_h_overlap
305 add wq, 32
306 lea srcq, [src_bakq+wq*2]
307 jl .hv_overlap
308 .end:
309 RET
310 ALIGN function_align
311 .add_noise_v:
312 mova m2, m19
313 vpdpwssd m2, m12, m4
314 mova m3, m19
315 vpdpwssd m3, m13, m5
316 mova m4, m19
317 vpdpwssd m4, m12, m0
318 mova m5, m19
319 vpdpwssd m5, m13, m1
320 REPX {psrad x, 1}, m2, m3, m4, m5
321 packssdw m4, m2
322 packssdw m5, m3
323 vpsravw m4, m11
324 vpsravw m5, m11
325 .add_noise:
326 mova m0, [srcq+strideq*0]
327 mova m1, [srcq+strideq*1]
328 kmovw k4, k1
329 pand m16, m6, m0
330 psrld m3, m0, 16
331 vpgatherdd m2{k4}, [scalingq+m16]
332 vpcmpud k4, m3, m6, 2 ; px <= bdmax
333 vpgatherdd m16{k4}, [scalingq+m3]
334 kmovw k4, k1
335 pand m17, m6, m1
336 vpgatherdd m3{k4}, [scalingq+m17]
337 vpshufb m2{k3}, m16, m7
338 psrld m16, m1, 16
339 vpcmpud k4, m16, m6, 2
340 vpgatherdd m17{k4}, [scalingq+m16]
341 vpshufb m3{k3}, m17, m7
342 vpsllvw m2, m10
343 vpsllvw m3, m10
344 pmulhrsw m4, m2
345 pmulhrsw m5, m3
346 add grain_lutq, 82*4
347 paddw m0, m4
348 paddw m1, m5
349 pmaxsw m0, m8
350 pmaxsw m1, m8
351 pminsw m0, m9
352 pminsw m1, m9
353 mova [dstq+srcq], m0
354 add srcq, strideq
355 mova [dstq+srcq], m1
356 add srcq, strideq
357 ret
358
359 %macro FGUV_FN 3 ; name, ss_hor, ss_ver
360 cglobal fguv_32x32xn_i%1_16bpc, 6, 15, 22, dst, src, stride, fg_data, w, scaling, \
361 grain_lut, h, sby, luma, lstride, uv_pl, is_id
362 %define base r12-fg_min
363 lea r12, [fg_min]
364 mov r9d, r13m ; bdmax
365 mov r7d, [fg_dataq+FGData.scaling_shift]
366 mov r6d, [fg_dataq+FGData.clip_to_restricted_range]
367 mov r11d, is_idm
368 kxnorw k1, k1, k1 ; 0xffff
369 vpbroadcastd m5, r13m
370 mov r13, 0xeeeeeeeeeeeeeeee
371 vbroadcasti32x4 m6, [base+scale_mask]
372 shr r9d, 11 ; is_12bpc
373 vpbroadcastd m7, [base+scale_shift+r7*4-32]
374 shlx r10d, r6d, r9d
375 mov sbyd, sbym
376 shlx r6d, r6d, r11d
377 vpbroadcastd m8, [base+fg_min+r10*4]
378 lea r6d, [r9+r6*2]
379 vpbroadcastd m9, [base+fg_max+r6*4]
380 kmovq k2, r13
381 vpbroadcastd m20, [base+scale_rnd+r9*4]
382 packssdw m4, m5, m5
383 vpbroadcastd m21, [base+scale_shift+r9*8+4]
384 %if %2
385 mova m12, [base+pb_0to63] ; pw_even
386 mov r13d, 0x0101
387 vpbroadcastq m10, [base+pw_23_22+r9*8]
388 kmovw k3, r13d
389 %if %3
390 pshufd m11, m10, q0000
391 %else
392 vpbroadcastd ym16, [base+pw_27_17_17_27+r9*8+0]
393 vpbroadcastd m11, [base+pw_27_17_17_27+r9*8+4]
394 vmovdqu16 m11{k1}, m16
395 %endif
396 psrlw m13, m12, 8 ; pw_odd
397 %else
398 vpbroadcastq m10, [base+pw_27_17_17_27+r9*8]
399 kshiftrb k3, k1, 7 ; 0x01
400 kshiftrb k4, k1, 4 ; 0x0f
401 pshufd m11, m10, q0000
402 %endif
403 mov lstrideq, r10mp
404 test sbyd, sbyd
405 setnz r7b
406 cmp byte [fg_dataq+FGData.chroma_scaling_from_luma], 0
407 jne .csfl
408
409 %macro %%FGUV_32x32xN_LOOP 3 ; not-csfl, ss_hor, ss_ver
410 DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \
411 _, sby, see, lstride
412
413 %if %1
414 mov r6d, r11m
415 vpbroadcastd m0, [base+uv_offset_mul+r9*4]
416 vpbroadcastd m1, [base+pb_8_9_0_1]
417 vpbroadcastd m14, [fg_dataq+FGData.uv_offset+r6*4]
418 vbroadcasti32x4 m15, [fg_dataq+FGData.uv_mult+r6*4]
419 pmaddwd m14, m0
420 pshufb m15, m1 ; { uv_luma_mult, uv_mult }
421 %endif
422 test r7b, [fg_dataq+FGData.overlap_flag]
423 jnz %%v_overlap
424
425 imul seed, sbyd, (173 << 24) | 37
426 add seed, (105 << 24) | 178
427 rorx seed, seed, 24
428 movzx seed, seew
429 xor seed, [fg_dataq+FGData.seed]
430
431 DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \
432 offx, offy, see, lstride, luma
433
434 mov lumaq, r9mp
435 lea r12, [srcq+wq*2]
436 lea r13, [dstq+wq*2]
437 lea r14, [lumaq+wq*(2<<%2)]
438 mov r9mp, r12
439 mov r10mp, r13
440 mov r11mp, r14
441 neg wq
442
443 %%loop_x:
444 rorx r6, seeq, 1
445 or seed, 0xeff4
446 test seeb, seeh
447 lea seed, [r6+0x8000]
448 cmovp seed, r6d ; updated seed
449 rorx offyd, seed, 8
450 rorx offxq, seeq, 12
451 and offyd, 0xf
452 imul offyd, 164>>%3
453 lea offyd, [offyq+offxq*(2-%2)+(3+(6>>%3))*82+(3+(6>>%2))] ; offy*stride+offx
454
455 DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \
456 h, offxy, see, lstride, luma
457
458 mov grain_lutq, grain_lutmp
459 mov hd, hm
460 %%loop_y:
461 %if %2
462 movu ym18, [grain_lutq+offxyq*2+82*0]
463 vinserti32x8 m18, [grain_lutq+offxyq*2+82*2], 1
464 movu ym19, [grain_lutq+offxyq*2+82*4]
465 vinserti32x8 m19, [grain_lutq+offxyq*2+82*6], 1
466 %else
467 movu m18, [grain_lutq+offxyq*2+82*0]
468 movu m19, [grain_lutq+offxyq*2+82*2]
469 %endif
470 call %%add_noise
471 sub hb, 2<<%2
472 jg %%loop_y
473 add wq, 32>>%2
474 jge .end
475 mov srcq, r9mp
476 mov dstq, r10mp
477 mov lumaq, r11mp
478 lea srcq, [srcq+wq*2]
479 lea dstq, [dstq+wq*2]
480 lea lumaq, [lumaq+wq*(2<<%2)]
481 cmp byte [fg_dataq+FGData.overlap_flag], 0
482 je %%loop_x
483 cmp dword r8m, 0 ; sby
484 jne %%hv_overlap
485
486 ; horizontal overlap (without vertical overlap)
487 %%loop_x_h_overlap:
488 rorx r6, seeq, 1
489 or seed, 0xEFF4
490 test seeb, seeh
491 lea seed, [r6+0x8000]
492 cmovp seed, r6d ; updated seed
493
494 DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \
495 offx, offy, see, lstride, luma, left_offxy
496
497 lea left_offxyd, [offyq+(32>>%2)] ; previous column's offy*stride+offx
498 rorx offyd, seed, 8
499 rorx offxq, seeq, 12
500 and offyd, 0xf
501 imul offyd, 164>>%3
502 lea offyd, [offyq+offxq*(2-%2)+(3+(6>>%3))*82+3+(6>>%2)] ; offy*stride+offx
503
504 DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \
505 h, offxy, see, lstride, luma, left_offxy
506
507 mov grain_lutq, grain_lutmp
508 mov hd, hm
509 %%loop_y_h_overlap:
510 %if %2
511 movu ym18, [grain_lutq+offxyq*2+82*0]
512 vinserti32x8 m18, [grain_lutq+offxyq*2+82*2], 1
513 movu ym19, [grain_lutq+offxyq*2+82*4]
514 vinserti32x8 m19, [grain_lutq+offxyq*2+82*6], 1
515 movd xm16, [grain_lutq+left_offxyq*2+82*0]
516 vinserti32x4 m16, [grain_lutq+left_offxyq*2+82*2], 2
517 movd xm17, [grain_lutq+left_offxyq*2+82*4]
518 vinserti32x4 m17, [grain_lutq+left_offxyq*2+82*6], 2
519 punpckldq m16, m17
520 punpckldq m17, m18, m19
521 punpcklwd m16, m17
522 mova m17, m20
523 vpdpwssd m17, m16, m10
524 psrad m17, 1
525 packssdw m17, m17
526 vpsravw m17, m21
527 %else
528 movu m18, [grain_lutq+offxyq*2+82*0]
529 movu m19, [grain_lutq+offxyq*2+82*2]
530 movd xm16, [grain_lutq+left_offxyq*2+82*0]
531 pinsrd xm16, [grain_lutq+left_offxyq*2+82*2], 1
532 punpckldq xm17, xm18, xm19
533 punpcklwd xm16, xm17
534 mova xm17, xm20
535 vpdpwssd xm17, xm16, xm10
536 psrad xm17, 1
537 packssdw xm17, xm17
538 vpsravw xm17, xm21
539 %endif
540 vmovdqa32 m18{k3}, m17
541 vpshufd m19{k3}, m17, q0321
542 call %%add_noise
543 sub hb, 2<<%2
544 jg %%loop_y_h_overlap
545 add wq, 32>>%2
546 jge .end
547 mov srcq, r9mp
548 mov dstq, r10mp
549 mov lumaq, r11mp
550 lea srcq, [srcq+wq*2]
551 lea dstq, [dstq+wq*2]
552 lea lumaq, [lumaq+wq*(2<<%2)]
553 cmp dword r8m, 0 ; sby
554 jne %%hv_overlap
555 jmp %%loop_x_h_overlap
556
557 %%v_overlap:
558 DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \
559 _, sby, see, lstride
560
561 movzx sbyd, sbyb
562 imul seed, [fg_dataq+FGData.seed], 0x00010001
563 imul r7d, sbyd, 173 * 0x00010001
564 imul sbyd, 37 * 0x01000100
565 add r7d, (105 << 16) | 188
566 add sbyd, (178 << 24) | (141 << 8)
567 and r7d, 0x00ff00ff
568 and sbyd, 0xff00ff00
569 xor seed, r7d
570 xor seed, sbyd ; (cur_seed << 16) | top_seed
571
572 DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \
573 offx, offy, see, lstride, luma, _, top_offxy
574
575 mov lumaq, r9mp
576 lea r12, [srcq+wq*2]
577 lea r13, [dstq+wq*2]
578 lea r14, [lumaq+wq*(2<<%2)]
579 mov r9mp, r12
580 mov r10mp, r13
581 mov r11mp, r14
582 neg wq
583
584 ; we assume from the block above that bits 8-15 of r7d are zero'ed
585 mov r6d, seed
586 or seed, 0xeff4eff4
587 test seeb, seeh
588 setp r7b ; parity of top_seed
589 shr seed, 16
590 shl r7d, 16
591 test seeb, seeh
592 setp r7b ; parity of cur_seed
593 or r6d, 0x00010001
594 xor r7d, r6d
595 rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed
596
597 rorx offyd, seed, 8
598 rorx offxd, seed, 12
599 and offyd, 0xf000f
600 and offxd, 0xf000f
601 imul offyd, 164>>%3
602 ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy
603 lea offyd, [offyq+offxq*(2-%2)+0x10001*((3+(6>>%3))*82+3+(6>>%2))+(32>>%3)*82]
604
605 DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \
606 h, offxy, see, lstride, luma, _, top_offxy
607
608 mov grain_lutq, grain_lutmp
609 mov hd, hm
610 movzx top_offxyd, offxyw
611 shr offxyd, 16
612
613 %if %3
614 movu ym16, [grain_lutq+offxyq*2+82*0]
615 movu ym1, [grain_lutq+top_offxyq*2+82*0]
616 vbroadcasti32x8 m18, [grain_lutq+offxyq*2+82*2]
617 movu ym19, [grain_lutq+offxyq*2+82*4]
618 vinserti32x8 m19, [grain_lutq+offxyq*2+82*6], 1
619 punpcklwd ym17, ym1, ym16
620 punpckhwd ym1, ym16
621 %elif %2
622 movu ym18, [grain_lutq+offxyq*2+82*0]
623 vinserti32x8 m18, [grain_lutq+offxyq*2+82*2], 1
624 movu ym17, [grain_lutq+top_offxyq*2+82*0]
625 vinserti32x8 m17, [grain_lutq+top_offxyq*2+82*2], 1
626 movu ym19, [grain_lutq+offxyq*2+82*4]
627 vinserti32x8 m19, [grain_lutq+offxyq*2+82*6], 1
628 punpcklwd m16, m17, m18
629 punpckhwd m17, m18
630 %else
631 movu m18, [grain_lutq+offxyq*2+82*0]
632 movu m19, [grain_lutq+top_offxyq*2+82*0]
633 movu m2, [grain_lutq+offxyq*2+82*2]
634 movu m16, [grain_lutq+top_offxyq*2+82*2]
635 punpckhwd m1, m19, m18
636 punpcklwd m19, m18
637 punpckhwd m18, m2, m16
638 punpcklwd m2, m16
639 %endif
640 call %%add_noise_v
641 sub hb, 2<<%2
642 jg %%loop_y
643 add wq, 32>>%2
644 jge .end
645 mov srcq, r9mp
646 mov dstq, r10mp
647 mov lumaq, r11mp
648 lea srcq, [srcq+wq*2]
649 lea dstq, [dstq+wq*2]
650 lea lumaq, [lumaq+wq*(2<<%2)]
651
652 ; since fg_dataq.overlap is guaranteed to be set, we never jump back
653 ; to %%v_overlap, and instead always fall-through to %%hv_overlap
654 %%hv_overlap:
655 ; we assume from the block above that bits 8-15 of r7d are zero'ed
656 mov r6d, seed
657 or seed, 0xeff4eff4
658 test seeb, seeh
659 setp r7b ; parity of top_seed
660 shr seed, 16
661 shl r7d, 16
662 test seeb, seeh
663 setp r7b ; parity of cur_seed
664 or r6d, 0x00010001
665 xor r7d, r6d
666 rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed
667
668 DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \
669 offx, offy, see, lstride, luma, left_offxy, top_offxy, topleft_offxy
670
671 lea topleft_offxyq, [top_offxyq+(32>>%2)]
672 lea left_offxyq, [offyq+(32>>%2)]
673 rorx offyd, seed, 8
674 rorx offxd, seed, 12
675 and offyd, 0xf000f
676 and offxd, 0xf000f
677 imul offyd, 164>>%3
678 ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy
679 lea offyd, [offyq+offxq*(2-%2)+0x10001*((3+(6>>%3))*82+3+(6>>%2))+(32>>%3)*82]
680
681 DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \
682 h, offxy, see, lstride, luma, left_offxy, top_offxy, topleft_offxy
683
684 mov grain_lutq, grain_lutmp
685 mov hd, hm
686 movzx top_offxyd, offxyw
687 shr offxyd, 16
688
689 ; grain = grain_lut[offy+y][offx+x]
690 %if %2
691 movd xm16, [grain_lutq+left_offxyq*2+82*0]
692 vinserti32x4 m16, [grain_lutq+left_offxyq*2+82*2], 2
693 movd xm17, [grain_lutq+left_offxyq*2+82*4]
694 vinserti32x4 m17, [grain_lutq+left_offxyq*2+82*6], 2
695 movu ym18, [grain_lutq+offxyq*2+82*0]
696 vinserti32x8 m18, [grain_lutq+offxyq*2+82*2], 1
697 movu ym19, [grain_lutq+offxyq*2+82*4]
698 vinserti32x8 m19, [grain_lutq+offxyq*2+82*6], 1
699 punpckldq m16, m17
700 punpckldq m17, m18, m19
701 punpcklwd m16, m17
702 movu ym1, [grain_lutq+top_offxyq*2+82*0]
703 movd xm17, [grain_lutq+topleft_offxyq*2+82*0]
704 mova m0, m20
705 vpdpwssd m0, m16, m10
706 %if %3
707 punpcklwd xm17, xm1
708 mova xm16, xm20
709 vpdpwssd xm16, xm17, xm10
710 psrad xm16, 1
711 %else
712 vinserti32x8 m1, [grain_lutq+top_offxyq*2+82*2], 1
713 vinserti32x4 m17, [grain_lutq+topleft_offxyq*2+82*2], 2
714 punpcklwd m17, m1
715 mova m16, m20
716 vpdpwssd m16, m17, m10
717 psrad m16, 1
718 %endif
719 psrad m0, 1
720 packssdw m0, m16
721 vpsravw m0, m21
722 vmovdqa32 m18{k3}, m0
723 vpshufd m19{k3}, m0, q0321
724 %if %3
725 vpunpckhdq ym1{k3}, ym0, ym0
726 punpcklwd ym17, ym1, ym18
727 punpckhwd ym1, ym18
728 %else
729 vpunpckhdq m1{k3}, m0, m0
730 punpcklwd m16, m1, m18
731 punpckhwd m17, m1, m18
732 %endif
733 %else
734 movu m18, [grain_lutq+offxyq*2+82*0]
735 movu m19, [grain_lutq+top_offxyq*2+82*0]
736 movd xm17, [grain_lutq+left_offxyq*2+82*0]
737 pinsrd xm17, [grain_lutq+topleft_offxyq*2+82*0], 1
738 punpckldq xm16, xm18, xm19
739 punpcklwd xm17, xm16
740 movu m2, [grain_lutq+offxyq*2+82*2]
741 movu m0, [grain_lutq+top_offxyq*2+82*2]
742 movd xm16, [grain_lutq+left_offxyq*2+82*2]
743 pinsrd xm16, [grain_lutq+topleft_offxyq*2+82*2], 1
744 punpckldq xm1, xm2, xm0
745 punpcklwd xm1, xm16, xm1
746 mova xm16, xm20
747 vpdpwssd xm16, xm17, xm10
748 mova xm17, xm20
749 vpdpwssd xm17, xm1, xm10
750 punpckhwd m1, m19, m18
751 punpcklwd m19, m18
752 punpckhwd m18, m2, m0
753 punpcklwd m2, m0
754 psrad xm16, 1
755 psrad xm17, 1
756 packssdw xm16, xm17
757 vpsravw xm16, xm21
758 vpshuflw m19{k4}, m16, q1302
759 punpckhqdq xm16, xm16
760 vpshuflw m2{k4}, m16, q3120
761 %endif
762 call %%add_noise_v
763 sub hb, 2<<%2
764 jg %%loop_y_h_overlap
765 add wq, 32>>%2
766 jge .end
767 mov srcq, r9mp
768 mov dstq, r10mp
769 mov lumaq, r11mp
770 lea srcq, [srcq+wq*2]
771 lea dstq, [dstq+wq*2]
772 lea lumaq, [lumaq+wq*(2<<%2)]
773 jmp %%hv_overlap
774
775 ALIGN function_align
776 %%add_noise_v:
777 %if %3
778 mova ym16, ym20
779 vpdpwssd ym16, ym17, ym11
780 mova ym17, ym20
781 vpdpwssd ym17, ym1, ym11
782 psrad ym16, 1
783 psrad ym17, 1
784 packssdw ym16, ym17
785 vpsravw m18{k1}, m16, m21
786 %elif %2
787 mova m18, m20
788 vpdpwssd m18, m16, m11
789 mova m16, m20
790 vpdpwssd m16, m17, m11
791 psrad m18, 1
792 psrad m16, 1
793 packssdw m18, m16
794 vpsravw m18, m21
795 %else
796 mova m16, m20
797 vpdpwssd m16, m1, m11
798 mova m17, m20
799 vpdpwssd m17, m18, m11
800 mova m18, m20
801 vpdpwssd m18, m19, m11
802 mova m19, m20
803 vpdpwssd m19, m2, m11
804 REPX {psrad x, 1}, m16, m17, m18, m19
805 packssdw m18, m16
806 packssdw m19, m17
807 vpsravw m18, m21
808 vpsravw m19, m21
809 %endif
810 %%add_noise:
811 %if %2
812 mova m2, [lumaq+lstrideq*(0<<%3)]
813 mova m0, [lumaq+lstrideq*(1<<%3)]
814 lea lumaq, [lumaq+lstrideq*(2<<%3)]
815 mova m3, [lumaq+lstrideq*(0<<%3)]
816 mova m1, [lumaq+lstrideq*(1<<%3)]
817 mova m16, m12
818 vpermi2w m16, m2, m0
819 vpermt2w m2, m13, m0
820 mova m17, m12
821 vpermi2w m17, m3, m1
822 vpermt2w m3, m13, m1
823 pavgw m2, m16
824 pavgw m3, m17
825 %elif %1
826 mova m2, [lumaq+lstrideq*0]
827 mova m3, [lumaq+lstrideq*1]
828 %endif
829 %if %2
830 mova ym16, [srcq+strideq*0]
831 vinserti32x8 m16, [srcq+strideq*1], 1
832 lea srcq, [srcq+strideq*2]
833 %else
834 mova m16, [srcq+strideq*0]
835 %endif
836 %if %1
837 punpckhwd m17, m2, m16
838 mova m0, m14
839 vpdpwssd m0, m17, m15
840 punpcklwd m17, m2, m16
841 mova m2, m14
842 vpdpwssd m2, m17, m15
843 %endif
844 %if %2
845 mova ym17, [srcq+strideq*0]
846 vinserti32x8 m17, [srcq+strideq*1], 1
847 %else
848 mova m17, [srcq+strideq*1]
849 %endif
850 %if %1
851 psrad m0, 6
852 psrad m2, 6
853 packusdw m2, m0
854 punpckhwd m0, m3, m17
855 mova m1, m14
856 vpdpwssd m1, m15, m0
857 punpcklwd m0, m3, m17
858 mova m3, m14
859 vpdpwssd m3, m15, m0
860 psrad m1, 6
861 psrad m3, 6
862 packusdw m3, m1
863 pminuw m2, m4
864 pminuw m3, m4
865
866 .add_noise_main:
867 ; scaling[luma_src]
868 kmovw k5, k1
869 pand m1, m5, m2
870 vpgatherdd m0{k5}, [scalingq+m1]
871 kmovw k5, k1
872 psrld m2, 16
873 vpgatherdd m1{k5}, [scalingq+m2]
874 vpshufb m0{k2}, m1, m6
875 kmovw k5, k1
876 psrld m1, m3, 16
877 vpgatherdd m2{k5}, [scalingq+m1]
878 kmovw k5, k1
879 pand m3, m5
880 vpgatherdd m1{k5}, [scalingq+m3]
881 vpshufb m1{k2}, m2, m6
882
883 ; noise = round2(scaling[luma_src] * grain, scaling_shift)
884 vpsllvw m0, m7
885 vpsllvw m1, m7
886 pmulhrsw m18, m0
887 pmulhrsw m19, m1
888 add grain_lutq, 82*(4<<%2)
889 lea lumaq, [lumaq+lstrideq*(2<<%3)]
890 lea srcq, [srcq+strideq*2]
891 paddw m16, m18
892 paddw m17, m19
893 pmaxsw m16, m8
894 pmaxsw m17, m8
895 pminsw m16, m9
896 pminsw m17, m9
897 %if %2
898 mova [dstq+strideq*0], ym16
899 vextracti32x8 [dstq+strideq*1], m16, 1
900 lea dstq, [dstq+strideq*2]
901 mova [dstq+strideq*0], ym17
902 vextracti32x8 [dstq+strideq*1], m17, 1
903 %else
904 mova [dstq+strideq*0], m16
905 mova [dstq+strideq*1], m17
906 %endif
907 lea dstq, [dstq+strideq*2]
908 ret
909 %else
910 %if %2
911 pand m2, m4
912 pand m3, m4
913 %else
914 pand m2, m4, [lumaq+lstrideq*0]
915 pand m3, m4, [lumaq+lstrideq*1]
916 %endif
917 jmp .add_noise_main
918 %endif
919 %endmacro
920
921 %%FGUV_32x32xN_LOOP 1, %2, %3
922 .csfl:
923 %%FGUV_32x32xN_LOOP 0, %2, %3
924 .end:
925 RET
926 %endmacro
927
928 FGUV_FN 420, 1, 1
929 FGUV_FN 422, 1, 0
930 FGUV_FN 444, 0, 0
931
932 %endif