Debian Codegraph @

.cargo/vendor/rav1e/src/x86/filmgrain_avx512.asm

raw
1 ; Copyright © 2022, VideoLAN and dav1d authors
2 ; Copyright © 2022, Two Orioles, LLC
3 ; All rights reserved.
4 ;
5 ; Redistribution and use in source and binary forms, with or without
6 ; modification, are permitted provided that the following conditions are met:
7 ;
8 ; 1. Redistributions of source code must retain the above copyright notice, this
9 ; list of conditions and the following disclaimer.
10 ;
11 ; 2. Redistributions in binary form must reproduce the above copyright notice,
12 ; this list of conditions and the following disclaimer in the documentation
13 ; and/or other materials provided with the distribution.
14 ;
15 ; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
16 ; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
17 ; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
18 ; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
19 ; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
20 ; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
21 ; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
22 ; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 ; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
24 ; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25
26 %include "config.asm"
27 %include "ext/x86/x86inc.asm"
28 %include "x86/filmgrain_common.asm"
29
30 %if ARCH_X86_64
31
32 SECTION_RODATA 64
33
34 pb_even: db 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30
35 db 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62
36 db 64, 66, 68, 70, 72, 74, 76, 78, 80, 82, 84, 86, 88, 90, 92, 94
37 db 96, 98,100,102,104,106,108,110,112,114,116,118,120,122,124,126
38 pb_odd: db 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31
39 db 33, 35, 37, 39, 41, 43, 45, 47, 49, 51, 53, 55, 57, 59, 61, 63
40 db 65, 67, 69, 71, 73, 75, 77, 79, 81, 83, 85, 87, 89, 91, 93, 95
41 db 97, 99,101,103,105,107,109,111,113,115,117,119,121,123,125,127
42 interleave_hl: db 8, 0, 9, 1, 10, 2, 11, 3, 12, 4, 13, 5, 14, 6, 15, 7
43 pb_27_17_17_27: db 27, 17, 17, 27, 0, 32, 0, 32
44 pb_23_22_0_32: db 23, 22, 0, 32, 0, 32, 0, 32
45 pb_27_17: times 2 db 27, 17
46 pb_23_22: times 2 db 23, 22
47 pw_8: times 2 dw 8
48 pw_1024: times 2 dw 1024
49 pb_17_27: times 2 db 17, 27
50 fg_max: times 4 db 255
51 times 4 db 240
52 times 4 db 235
53 fg_min: times 4 db 0
54 times 4 db 16
55 noise_rnd: times 2 dw 128
56 times 2 dw 64
57 times 2 dw 32
58 times 2 dw 16
59
60 SECTION .text
61
62 INIT_ZMM avx512icl
63 cglobal fgy_32x32xn_8bpc, 6, 13, 22, dst, src, stride, fg_data, w, scaling, \
64 grain_lut, h, sby, see, overlap
65 %define base r11-fg_min
66 lea r11, [fg_min]
67 mov r6d, [fg_dataq+FGData.scaling_shift]
68 mov r7d, [fg_dataq+FGData.clip_to_restricted_range]
69 mov sbyd, sbym
70 mov overlapd, [fg_dataq+FGData.overlap_flag]
71 mov r12, 0x0000000f0000000f ; h_overlap mask
72 mova m0, [scalingq+64*0]
73 mova m1, [scalingq+64*1]
74 mova m2, [scalingq+64*2]
75 mova m3, [scalingq+64*3]
76 kmovq k1, r12
77 vbroadcasti32x4 m4, [base+interleave_hl]
78 vpbroadcastd ym16, [base+pb_27_17]
79 vpbroadcastd m12, [base+pb_17_27]
80 vpbroadcastd m6, [base+noise_rnd+r6*4-32]
81 test sbyd, sbyd
82 setnz r6b
83 vpbroadcastd m7, [base+fg_min+r7*4]
84 vpbroadcastd m8, [base+fg_max+r7*8]
85 pxor m5, m5
86 vpbroadcastd m9, [base+pw_1024]
87 vpbroadcastq m10, [base+pb_27_17_17_27]
88 vmovdqa64 m12{k1}, m16
89 test r6b, overlapb
90 jnz .v_overlap
91
92 imul seed, sbyd, (173 << 24) | 37
93 add seed, (105 << 24) | 178
94 rorx seed, seed, 24
95 movzx seed, seew
96 xor seed, [fg_dataq+FGData.seed]
97
98 DEFINE_ARGS dst, src, stride, src_bak, w, offx, offy, \
99 h, sby, see, overlap
100
101 lea src_bakq, [srcq+wq]
102 neg wq
103 sub dstq, srcq
104 .loop_x:
105 rorx r6, seeq, 1
106 or seed, 0xeff4
107 test seeb, seeh
108 lea seed, [r6+0x8000]
109 cmovp seed, r6d ; updated seed
110 rorx offyd, seed, 8
111 rorx offxq, seeq, 12
112 and offyd, 0xf
113 imul offyd, 164
114 lea offxd, [offyq+offxq*2+829] ; offy*stride+offx
115
116 DEFINE_ARGS dst, src, stride, src_bak, w, offxy, grain_lut, \
117 h, sby, see, overlap
118
119 mov grain_lutq, grain_lutmp
120 mov hd, hm
121 .loop_y:
122 movu ym21, [grain_lutq+offxyq-82]
123 vinserti32x8 m21, [grain_lutq+offxyq+ 0], 1
124 call .add_noise
125 sub hb, 2
126 jg .loop_y
127 add wq, 32
128 jge .end
129 lea srcq, [src_bakq+wq]
130 test overlapd, overlapd
131 jz .loop_x
132 test sbyd, sbyd
133 jnz .hv_overlap
134
135 .loop_x_h_overlap:
136 rorx r6, seeq, 1
137 or seed, 0xeff4
138 test seeb, seeh
139 lea seed, [r6+0x8000]
140 cmovp seed, r6d ; updated seed
141
142 DEFINE_ARGS dst, src, stride, src_bak, w, offx, offy, \
143 h, sby, see, left_offxy
144
145 rorx offyd, seed, 8
146 mov left_offxyd, offxd ; previous column's offy*stride
147 rorx offxq, seeq, 12
148 and offyd, 0xf
149 imul offyd, 164
150 lea offxd, [offyq+offxq*2+829] ; offy*stride+offx
151
152 DEFINE_ARGS dst, src, stride, src_bak, w, offxy, grain_lut, \
153 h, sby, see, left_offxy
154
155 mov grain_lutq, grain_lutmp
156 mov hd, hm
157 .loop_y_h_overlap:
158 movu ym20, [grain_lutq+offxyq-82]
159 vinserti32x8 m20, [grain_lutq+offxyq+ 0], 1
160 movd xm19, [grain_lutq+left_offxyq-50]
161 vinserti32x4 m19, [grain_lutq+left_offxyq+32], 2
162 punpcklbw m19, m20
163 pmaddubsw m19, m10, m19
164 pmulhrsw m19, m9
165 punpckhbw m21, m20, m5
166 packsswb m20{k1}, m19, m19
167 punpcklbw m20, m5, m20
168 call .add_noise_h
169 sub hb, 2
170 jg .loop_y_h_overlap
171 add wq, 32
172 jge .end
173 lea srcq, [src_bakq+wq]
174 test sbyd, sbyd
175 jnz .hv_overlap
176 jmp .loop_x_h_overlap
177
178 .v_overlap:
179 DEFINE_ARGS dst, src, stride, fg_data, w, offy, offx, \
180 h, sby, see, overlap
181
182 movzx r6d, sbyb
183 imul seed, [fg_dataq+FGData.seed], 0x00010001
184 imul r7d, r6d, 173 * 0x00010001
185 imul r6d, 37 * 0x01000100
186 add r7d, (105 << 16) | 188
187 add r6d, (178 << 24) | (141 << 8)
188 and r7d, 0x00ff00ff
189 and r6d, 0xff00ff00
190 xor seed, r7d
191 xor seed, r6d ; (cur_seed << 16) | top_seed
192
193 DEFINE_ARGS dst, src, stride, src_bak, w, offx, offy, \
194 h, sby, see, overlap
195
196 lea src_bakq, [srcq+wq]
197 neg wq
198 sub dstq, srcq
199
200 ; we assume from the block above that bits 8-15 of r7d are zero'ed
201 mov r6d, seed
202 or seed, 0xeff4eff4
203 test seeb, seeh
204 setp r7b ; parity of top_seed
205 shr seed, 16
206 shl r7d, 16
207 test seeb, seeh
208 setp r7b ; parity of cur_seed
209 or r6d, 0x00010001
210 xor r7d, r6d
211 rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed
212 rorx offyd, seed, 8
213 rorx offxd, seed, 12
214 and offyd, 0xf000f
215 and offxd, 0xf000f
216 imul offyd, 164
217 ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy
218 lea offxd, [offyq+offxq*2+0x10001*829+32*82]
219
220 DEFINE_ARGS dst, src, stride, src_bak, w, offxy, grain_lut, \
221 h, sby, see, overlap, top_offxy
222
223 mov grain_lutq, grain_lutmp
224 mov hd, hm
225 movzx top_offxyd, offxyw
226 shr offxyd, 16
227 movu ym19, [grain_lutq+offxyq-82]
228 vinserti32x8 m19, [grain_lutq+offxyq+ 0], 1
229 movu ym21, [grain_lutq+top_offxyq-82]
230 vinserti32x8 m21, [grain_lutq+top_offxyq+ 0], 1
231 punpckhbw m20, m21, m19
232 punpcklbw m21, m19
233 call .add_noise_v
234 sub hb, 2
235 jg .loop_y
236 add wq, 32
237 jge .end
238 lea srcq, [src_bakq+wq]
239
240 ; since fg_dataq.overlap is guaranteed to be set, we never jump back
241 ; to .v_overlap, and instead always fall-through to h+v overlap
242 .hv_overlap:
243 ; we assume from the block above that bits 8-15 of r7d are zero'ed
244 mov r6d, seed
245 or seed, 0xeff4eff4
246 test seeb, seeh
247 setp r7b ; parity of top_seed
248 shr seed, 16
249 shl r7d, 16
250 test seeb, seeh
251 setp r7b ; parity of cur_seed
252 or r6d, 0x00010001
253 xor r7d, r6d
254 rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed
255
256 DEFINE_ARGS dst, src, stride, src_bak, w, offx, offy, \
257 h, sby, see, left_offxy, top_offxy, topleft_offxy
258
259 mov topleft_offxyd, top_offxyd
260 rorx offyd, seed, 8
261 mov left_offxyd, offxd
262 rorx offxd, seed, 12
263 and offyd, 0xf000f
264 and offxd, 0xf000f
265 imul offyd, 164
266 ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy
267 lea offxd, [offyq+offxq*2+0x10001*829+32*82]
268
269 DEFINE_ARGS dst, src, stride, src_bak, w, offxy, grain_lut, \
270 h, sby, see, left_offxy, top_offxy, topleft_offxy
271
272 mov grain_lutq, grain_lutmp
273 mov hd, hm
274 movzx top_offxyd, offxyw
275 shr offxyd, 16
276 movu ym19, [grain_lutq+offxyq-82]
277 vinserti32x8 m19, [grain_lutq+offxyq+ 0], 1
278 movd xm16, [grain_lutq+left_offxyq-50]
279 vinserti32x4 m16, [grain_lutq+left_offxyq+32], 2
280 movu ym21, [grain_lutq+top_offxyq-82]
281 vinserti32x8 m21, [grain_lutq+top_offxyq+ 0], 1
282 movd xm17, [grain_lutq+topleft_offxyq-50]
283 vinserti32x4 m17, [grain_lutq+topleft_offxyq+32], 2
284 ; do h interpolation first (so top | top/left -> top, left | cur -> cur)
285 punpcklbw m16, m19
286 pmaddubsw m16, m10, m16
287 punpcklbw m17, m21
288 pmaddubsw m17, m10, m17
289 punpckhbw m20, m21, m19
290 pmulhrsw m16, m9
291 pmulhrsw m17, m9
292 packsswb m19{k1}, m16, m16
293 packsswb m21{k1}, m17, m17
294 ; followed by v interpolation (top | cur -> cur)
295 punpcklbw m21, m19
296 call .add_noise_v
297 sub hb, 2
298 jg .loop_y_h_overlap
299 add wq, 32
300 lea srcq, [src_bakq+wq]
301 jl .hv_overlap
302 .end:
303 RET
304 ALIGN function_align
305 .add_noise_v:
306 pmaddubsw m20, m12, m20
307 pmaddubsw m21, m12, m21
308 pmulhrsw m20, m9
309 pmulhrsw m21, m9
310 packsswb m21, m20
311 .add_noise:
312 punpcklbw m20, m5, m21
313 punpckhbw m21, m5
314 .add_noise_h:
315 mova ym18, [srcq+strideq*0]
316 vinserti32x8 m18, [srcq+strideq*1], 1
317 mova m19, m0
318 punpcklbw m16, m18, m5
319 vpermt2b m19, m18, m1 ; scaling[ 0..127]
320 vpmovb2m k2, m18
321 punpckhbw m17, m18, m5
322 vpermi2b m18, m2, m3 ; scaling[128..255]
323 vmovdqu8 m19{k2}, m18 ; scaling[src]
324 pshufb m19, m4
325 pmaddubsw m18, m19, m20
326 pmaddubsw m19, m21
327 add grain_lutq, 82*2
328 pmulhrsw m18, m6 ; noise
329 pmulhrsw m19, m6
330 paddw m16, m18
331 paddw m17, m19
332 packuswb m16, m17
333 pmaxub m16, m7
334 pminub m16, m8
335 mova [dstq+srcq], ym16
336 add srcq, strideq
337 vextracti32x8 [dstq+srcq], m16, 1
338 add srcq, strideq
339 ret
340
341 %macro FGUV_FN 3 ; name, ss_hor, ss_ver
342 cglobal fguv_32x32xn_i%1_8bpc, 6, 14+%2, 22, dst, src, stride, fg_data, w, \
343 scaling, grain_lut, h, sby, luma, \
344 overlap, uv_pl, is_id, _, stride3
345 lea r11, [fg_min]
346 mov r6d, [fg_dataq+FGData.scaling_shift]
347 mov r7d, [fg_dataq+FGData.clip_to_restricted_range]
348 mov r9d, is_idm
349 mov sbyd, sbym
350 mov overlapd, [fg_dataq+FGData.overlap_flag]
351 %if %2
352 mov r12, 0x000f000f000f000f ; h_overlap mask
353 vpbroadcastq m10, [base+pb_23_22_0_32]
354 lea stride3q, [strideq*3]
355 %else
356 mov r12, 0x0000000f0000000f
357 vpbroadcastq m10, [base+pb_27_17_17_27]
358 %endif
359 mova m0, [scalingq+64*0]
360 mova m1, [scalingq+64*1]
361 mova m2, [scalingq+64*2]
362 mova m3, [scalingq+64*3]
363 kmovq k1, r12
364 vbroadcasti32x4 m4, [base+interleave_hl]
365 vpbroadcastd m6, [base+noise_rnd+r6*4-32]
366 vpbroadcastd m7, [base+fg_min+r7*4]
367 shlx r7d, r7d, r9d
368 vpbroadcastd m8, [base+fg_max+r7*4]
369 test sbyd, sbyd
370 setnz r7b
371 vpbroadcastd m9, [base+pw_1024]
372 mova m11, [base+pb_even]
373 mova m12, [base+pb_odd]
374 pxor m5, m5
375 mov r5, r10mp ; lstride
376 cmp byte [fg_dataq+FGData.chroma_scaling_from_luma], 0
377 jne .csfl
378
379 %macro %%FGUV_32x32xN_LOOP 3 ; not-csfl, ss_hor, ss_ver
380 DEFINE_ARGS dst, src, stride, fg_data, w, lstride, grain_lut, \
381 h, sby, see, overlap, uv_pl, _, _, stride3
382 %if %1
383 mov r6d, uv_plm
384 vpbroadcastd m16, [base+pw_8]
385 vbroadcasti32x4 m14, [fg_dataq+FGData.uv_mult+r6*4]
386 vpbroadcastw m15, [fg_dataq+FGData.uv_offset+r6*4]
387 pshufb m14, m16 ; uv_luma_mult, uv_mult
388 %endif
389 test r7b, overlapb
390 jnz %%v_overlap
391
392 imul seed, sbyd, (173 << 24) | 37
393 add seed, (105 << 24) | 178
394 rorx seed, seed, 24
395 movzx seed, seew
396 xor seed, [fg_dataq+FGData.seed]
397
398 DEFINE_ARGS dst, src, stride, luma, w, lstride, grain_lut, \
399 offx, offy, see, overlap, _, _, _, stride3
400
401 mov lumaq, r9mp
402 lea r11, [srcq+wq]
403 lea r12, [dstq+wq]
404 lea r13, [lumaq+wq*(1+%2)]
405 mov r11mp, r11
406 mov r12mp, r12
407 neg wq
408
409 %%loop_x:
410 rorx r6, seeq, 1
411 or seed, 0xeff4
412 test seeb, seeh
413 lea seed, [r6+0x8000]
414 cmovp seed, r6d ; updated seed
415 rorx offyd, seed, 8
416 rorx offxq, seeq, 12
417 and offyd, 0xf
418 imul offyd, 164>>%3
419 lea offyd, [offyq+offxq*(2-%2)+(3+(6>>%3))*82+3+(6>>%2)] ; offy*stride+offx
420
421 DEFINE_ARGS dst, src, stride, luma, w, lstride, grain_lut, \
422 h, offxy, see, overlap, _, _, _, stride3
423
424 mov grain_lutq, grain_lutmp
425 mov hd, hm
426 %%loop_y:
427 %if %2
428 movu xm21, [grain_lutq+offxyq+82*0]
429 vinserti128 ym21, [grain_lutq+offxyq+82*1], 1
430 vinserti32x4 m21, [grain_lutq+offxyq+82*2], 2
431 vinserti32x4 m21, [grain_lutq+offxyq+82*3], 3
432 %else
433 movu ym21, [grain_lutq+offxyq+82*0]
434 vinserti32x8 m21, [grain_lutq+offxyq+82*1], 1
435 %endif
436 call %%add_noise
437 sub hb, 2<<%2
438 jg %%loop_y
439 add wq, 32>>%2
440 jge .end
441 mov srcq, r11mp
442 mov dstq, r12mp
443 lea lumaq, [r13+wq*(1<<%2)]
444 add srcq, wq
445 add dstq, wq
446 test overlapd, overlapd
447 jz %%loop_x
448 cmp dword r8m, 0 ; sby
449 jne %%hv_overlap
450
451 ; horizontal overlap (without vertical overlap)
452 %%loop_x_h_overlap:
453 rorx r6, seeq, 1
454 or seed, 0xeff4
455 test seeb, seeh
456 lea seed, [r6+0x8000]
457 cmovp seed, r6d ; updated seed
458
459 DEFINE_ARGS dst, src, stride, luma, w, lstride, grain_lut, \
460 offx, offy, see, left_offxy, _, _, _, stride3
461
462 lea left_offxyd, [offyq+(32>>%2)] ; previous column's offy*stride+offx
463 rorx offyd, seed, 8
464 rorx offxq, seeq, 12
465 and offyd, 0xf
466 imul offyd, 164>>%3
467 lea offyd, [offyq+offxq*(2-%2)+(3+(6>>%3))*82+3+(6>>%2)] ; offy*stride+offx
468
469 DEFINE_ARGS dst, src, stride, luma, w, lstride, grain_lut, \
470 h, offxy, see, left_offxy, _, _, _, stride3
471
472 mov grain_lutq, grain_lutmp
473 mov hd, hm
474 %%loop_y_h_overlap:
475 %if %2
476 movu xm20, [grain_lutq+offxyq +82*0]
477 movd xm19, [grain_lutq+left_offxyq+82*0]
478 vinserti32x4 ym20, [grain_lutq+offxyq +82*1], 1
479 vinserti32x4 ym19, [grain_lutq+left_offxyq+82*1], 1
480 vinserti32x4 m20, [grain_lutq+offxyq +82*2], 2
481 vinserti32x4 m19, [grain_lutq+left_offxyq+82*2], 2
482 vinserti32x4 m20, [grain_lutq+offxyq +82*3], 3
483 vinserti32x4 m19, [grain_lutq+left_offxyq+82*3], 3
484 %else
485 movu ym20, [grain_lutq+offxyq + 0]
486 movd xm19, [grain_lutq+left_offxyq+ 0]
487 vinserti32x8 m20, [grain_lutq+offxyq +82], 1
488 vinserti32x4 m19, [grain_lutq+left_offxyq+82], 2
489 %endif
490 punpcklbw m19, m20
491 pmaddubsw m19, m10, m19
492 punpckhbw m21, m20, m5
493 pmulhrsw m19, m9
494 vpacksswb m20{k1}, m19, m19
495 punpcklbw m20, m5, m20
496 call %%add_noise_h
497 sub hb, 2<<%2
498 jg %%loop_y_h_overlap
499 add wq, 32>>%2
500 jge .end
501 mov srcq, r11mp
502 mov dstq, r12mp
503 lea lumaq, [r13+wq*(1<<%2)]
504 add srcq, wq
505 add dstq, wq
506 cmp dword r8m, 0 ; sby
507 jne %%hv_overlap
508 jmp %%loop_x_h_overlap
509
510 %%v_overlap:
511 DEFINE_ARGS dst, src, stride, fg_data, w, lstride, grain_lut, \
512 _, sby, see, overlap, _, _, _, stride3
513
514 movzx sbyd, sbyb
515 imul seed, [fg_dataq+FGData.seed], 0x00010001
516 imul r7d, sbyd, 173 * 0x00010001
517 imul sbyd, 37 * 0x01000100
518 add r7d, (105 << 16) | 188
519 add sbyd, (178 << 24) | (141 << 8)
520 and r7d, 0x00ff00ff
521 and sbyd, 0xff00ff00
522 xor seed, r7d
523 xor seed, sbyd ; (cur_seed << 16) | top_seed
524
525 %if %3
526 vpbroadcastd m13, [base+pb_23_22]
527 kxnorw k3, k3, k3 ; v_overlap mask
528 %elif %2
529 vbroadcasti32x8 m13, [base+pb_27_17]
530 kxnord k3, k3, k3
531 pshufd m13, m13, q0000 ; 8x27_17, 8x17_27
532 %else
533 vpbroadcastd ym16, [base+pb_27_17]
534 vpbroadcastd m13, [base+pb_17_27]
535 vmovdqa64 m13{k1}, m16
536 %endif
537
538 DEFINE_ARGS dst, src, stride, luma, w, lstride, grain_lut, \
539 offx, offy, see, overlap, top_offxy, _, _, stride3
540
541 mov lumaq, r9mp
542 lea r11, [srcq+wq]
543 lea r12, [dstq+wq]
544 lea r13, [lumaq+wq*(1<<%2)]
545 mov r11mp, r11
546 mov r12mp, r12
547 neg wq
548
549 ; we assume from the block above that bits 8-15 of r7d are zero'ed
550 mov r6d, seed
551 or seed, 0xeff4eff4
552 test seeb, seeh
553 setp r7b ; parity of top_seed
554 shr seed, 16
555 shl r7d, 16
556 test seeb, seeh
557 setp r7b ; parity of cur_seed
558 or r6d, 0x00010001
559 xor r7d, r6d
560 rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed
561 rorx offyd, seed, 8
562 rorx offxd, seed, 12
563 and offyd, 0x000f000f
564 and offxd, 0x000f000f
565 imul offyd, 164>>%3
566 ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy
567 lea offyd, [offyq+offxq*(2-%2)+0x10001*((3+(6>>%3))*82+3+(6>>%2))+(32>>%3)*82]
568
569 DEFINE_ARGS dst, src, stride, luma, w, lstride, grain_lut, \
570 h, offxy, see, overlap, top_offxy, _, _, stride3
571
572 mov grain_lutq, grain_lutmp
573 mov hd, hm
574 movzx top_offxyd, offxyw
575 shr offxyd, 16
576
577 %if %3
578 movu xm18, [grain_lutq+offxyq+82*0]
579 movu xm20, [grain_lutq+top_offxyq+82*0]
580 ; only interpolate first line, insert remaining line unmodified
581 vbroadcasti128 ym21, [grain_lutq+offxyq+82*1]
582 vinserti32x4 m21, [grain_lutq+offxyq+82*2], 2
583 vinserti32x4 m21, [grain_lutq+offxyq+82*3], 3
584 punpcklbw xm19, xm20, xm18
585 punpckhbw xm20, xm18
586 %elif %2
587 movu xm18, [grain_lutq+offxyq+82*0]
588 vinserti128 ym18, [grain_lutq+offxyq+82*1], 1
589 movu xm20, [grain_lutq+top_offxyq+82*0]
590 vinserti32x4 ym20, [grain_lutq+top_offxyq+82*1], 1
591 vbroadcasti32x4 m21, [grain_lutq+offxyq+82*2]
592 vinserti32x4 m21, [grain_lutq+offxyq+82*3], 3
593 punpcklbw ym19, ym20, ym18
594 punpckhbw ym20, ym18
595 %else
596 movu ym21, [grain_lutq+offxyq+82*0]
597 vinserti32x8 m21, [grain_lutq+offxyq+82*1], 1
598 movu ym20, [grain_lutq+top_offxyq+82*0]
599 vinserti32x8 m20, [grain_lutq+top_offxyq+82*1], 1
600 %endif
601 call %%add_noise_v
602 sub hb, 2<<%2
603 jg %%loop_y
604 add wq, 32>>%2
605 jge .end
606 mov srcq, r11mp
607 mov dstq, r12mp
608 lea lumaq, [r13+wq*(1<<%2)]
609 add srcq, wq
610 add dstq, wq
611
612 %%hv_overlap:
613 ; we assume from the block above that bits 8-15 of r7d are zero'ed
614 mov r6d, seed
615 or seed, 0xeff4eff4
616 test seeb, seeh
617 setp r7b ; parity of top_seed
618 shr seed, 16
619 shl r7d, 16
620 test seeb, seeh
621 setp r7b ; parity of cur_seed
622 or r6d, 0x00010001
623 xor r7d, r6d
624 rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed
625
626 DEFINE_ARGS dst, src, stride, luma, w, lstride, grain_lut, \
627 offx, offy, see, left_offxy, top_offxy, topleft_offxy, _, stride3
628
629 lea topleft_offxyd, [top_offxyq+(32>>%2)]
630 lea left_offxyd, [offyq+(32>>%2)]
631 rorx offyd, seed, 8
632 rorx offxd, seed, 12
633 and offyd, 0x000f000f
634 and offxd, 0x000f000f
635 imul offyd, 164>>%3
636 ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy
637 lea offyd, [offyq+offxq*(2-%2)+0x10001*((3+(6>>%3))*82+3+(6>>%2))+(32>>%3)*82]
638
639 DEFINE_ARGS dst, src, stride, luma, w, lstride, grain_lut, \
640 h, offxy, see, left_offxy, top_offxy, topleft_offxy, _, stride3
641
642 mov grain_lutq, grain_lutmp
643 mov hd, hm
644 movzx top_offxyd, offxyw
645 shr offxyd, 16
646
647 %if %2
648 movu xm21, [grain_lutq+offxyq+82*0]
649 movd xm16, [grain_lutq+left_offxyq+82*0]
650 vinserti128 ym21, [grain_lutq+offxyq+82*1], 1
651 vinserti128 ym16, [grain_lutq+left_offxyq+82*1], 1
652 vinserti32x4 m21, [grain_lutq+offxyq+82*2], 2
653 vinserti32x4 m16, [grain_lutq+left_offxyq+82*2], 2
654 vinserti32x4 m21, [grain_lutq+offxyq+82*3], 3
655 vinserti32x4 m16, [grain_lutq+left_offxyq+82*3], 3
656 movd xm18, [grain_lutq+topleft_offxyq+82*0]
657 movu xm20, [grain_lutq+top_offxyq]
658 ; do h interpolation first (so top | top/left -> top, left | cur -> cur)
659 punpcklbw m16, m21
660 %if %3
661 punpcklbw xm18, xm20
662 %else
663 vinserti128 ym18, [grain_lutq+topleft_offxyq+82*1], 1
664 vinserti128 ym20, [grain_lutq+top_offxyq+82*1], 1
665 punpcklbw ym18, ym20
666 %endif
667 punpcklqdq m16, m18
668 pmaddubsw m16, m10, m16
669 pmulhrsw m16, m9
670 packsswb m16, m16
671 vmovdqu8 m21{k1}, m16
672 %if %3
673 vpalignr xm20{k1}, xm16, xm16, 4
674 punpcklbw xm19, xm20, xm21
675 punpckhbw xm20, xm21
676 %else
677 vpalignr ym20{k1}, ym16, ym16, 4
678 punpcklbw ym19, ym20, ym21
679 punpckhbw ym20, ym21
680 %endif
681 %else
682 movu ym21, [grain_lutq+offxyq+82*0]
683 vinserti32x8 m21, [grain_lutq+offxyq+82*1], 1
684 movd xm16, [grain_lutq+left_offxyq+82*0]
685 vinserti32x4 m16, [grain_lutq+left_offxyq+82*1], 2
686 movu ym20, [grain_lutq+top_offxyq+82*0]
687 vinserti32x8 m20, [grain_lutq+top_offxyq+82*1], 1
688 movd xm18, [grain_lutq+topleft_offxyq+82*0]
689 vinserti32x4 m18, [grain_lutq+topleft_offxyq+82*1], 2
690 punpcklbw m16, m21
691 punpcklbw m18, m20
692 punpcklqdq m16, m18
693 pmaddubsw m16, m10, m16
694 pmulhrsw m16, m9
695 packsswb m16, m16
696 vpalignr m20{k1}, m16, m16, 4
697 vmovdqu8 m21{k1}, m16
698 %endif
699 call %%add_noise_v
700 sub hb, 2<<%2
701 jg %%loop_y_h_overlap
702 add wq, 32>>%2
703 jge .end
704 mov srcq, r11mp
705 mov dstq, r12mp
706 lea lumaq, [r13+wq*(1<<%2)]
707 add srcq, wq
708 add dstq, wq
709 jmp %%hv_overlap
710 ALIGN function_align
711 %%add_noise_v:
712 %if %3
713 pmaddubsw xm19, xm13, xm19
714 pmaddubsw xm20, xm13, xm20
715 pmulhrsw xm19, xm9
716 pmulhrsw xm20, xm9
717 vpacksswb m21{k3}, m19, m20
718 %elif %2
719 pmaddubsw ym19, ym13, ym19
720 pmaddubsw ym20, ym13, ym20
721 pmulhrsw ym19, ym9
722 pmulhrsw ym20, ym9
723 vpacksswb m21{k3}, m19, m20
724 %else
725 punpcklbw m19, m20, m21
726 punpckhbw m20, m21
727 pmaddubsw m19, m13, m19
728 pmaddubsw m20, m13, m20
729 pmulhrsw m19, m9
730 pmulhrsw m20, m9
731 packsswb m21, m19, m20
732 %endif
733 %%add_noise:
734 punpcklbw m20, m5, m21
735 punpckhbw m21, m5
736 %%add_noise_h:
737 mova ym18, [lumaq+lstrideq*(0<<%3)]
738 vinserti32x8 m18, [lumaq+lstrideq*(1<<%3)], 1
739 %if %2
740 lea lumaq, [lumaq+lstrideq*(2<<%3)]
741 mova ym16, [lumaq+lstrideq*(0<<%3)]
742 vinserti32x8 m16, [lumaq+lstrideq*(1<<%3)], 1
743 mova xm17, [srcq+strideq*0]
744 mova m19, m11
745 vpermi2b m19, m18, m16
746 vinserti128 ym17, [srcq+strideq*1], 1
747 vpermt2b m18, m12, m16
748 vinserti32x4 m17, [srcq+strideq*2], 2
749 pavgb m18, m19
750 vinserti32x4 m17, [srcq+stride3q ], 3
751 %else
752 mova ym17, [srcq+strideq*0]
753 vinserti32x8 m17, [srcq+strideq*1], 1
754 %endif
755 %if %1
756 punpckhbw m19, m18, m17
757 punpcklbw m18, m17 ; { luma, chroma }
758 pmaddubsw m19, m14
759 pmaddubsw m18, m14
760 psraw m19, 6
761 psraw m18, 6
762 paddw m19, m15
763 paddw m18, m15
764 packuswb m18, m19
765 .add_noise_main:
766 mova m19, m0
767 vpermt2b m19, m18, m1 ; scaling[ 0..127]
768 vpmovb2m k2, m18
769 vpermi2b m18, m2, m3 ; scaling[128..255]
770 vmovdqu8 m19{k2}, m18 ; scaling[src]
771 pshufb m19, m4
772 pmaddubsw m18, m19, m20
773 pmaddubsw m19, m21
774 add grain_lutq, 82*2<<%2
775 lea lumaq, [lumaq+lstrideq*(2<<%3)]
776 lea srcq, [srcq+strideq*(2<<%2)]
777 pmulhrsw m18, m6 ; noise
778 pmulhrsw m19, m6
779 punpcklbw m16, m17, m5 ; chroma
780 punpckhbw m17, m5
781 paddw m16, m18
782 paddw m17, m19
783 packuswb m16, m17
784 pmaxub m16, m7
785 pminub m16, m8
786 %if %2
787 mova [dstq+strideq*0], xm16
788 vextracti128 [dstq+strideq*1], ym16, 1
789 vextracti32x4 [dstq+strideq*2], m16, 2
790 vextracti32x4 [dstq+stride3q ], m16, 3
791 %else
792 mova [dstq+strideq*0], ym16
793 vextracti32x8 [dstq+strideq*1], m16, 1
794 %endif
795 lea dstq, [dstq+strideq*(2<<%2)]
796 ret
797 %else
798 jmp .add_noise_main
799 %endif
800 %endmacro
801
802 %%FGUV_32x32xN_LOOP 1, %2, %3
803 .csfl:
804 %%FGUV_32x32xN_LOOP 0, %2, %3
805 .end:
806 RET
807 %endmacro
808
809 FGUV_FN 420, 1, 1
810 FGUV_FN 422, 1, 0
811 FGUV_FN 444, 0, 0
812
813 %endif ; ARCH_X86_64