Debian Codegraph @

.cargo/vendor/rav1e/src/x86/looprestoration_avx2.asm

raw
1 ; Copyright © 2018, VideoLAN and dav1d authors
2 ; Copyright © 2018, Two Orioles, LLC
3 ; All rights reserved.
4 ;
5 ; Redistribution and use in source and binary forms, with or without
6 ; modification, are permitted provided that the following conditions are met:
7 ;
8 ; 1. Redistributions of source code must retain the above copyright notice, this
9 ; list of conditions and the following disclaimer.
10 ;
11 ; 2. Redistributions in binary form must reproduce the above copyright notice,
12 ; this list of conditions and the following disclaimer in the documentation
13 ; and/or other materials provided with the distribution.
14 ;
15 ; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
16 ; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
17 ; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
18 ; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
19 ; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
20 ; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
21 ; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
22 ; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 ; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
24 ; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25
26 %include "config.asm"
27 %include "ext/x86/x86inc.asm"
28
29 %if ARCH_X86_64
30
31 SECTION_RODATA 32
32
33 wiener_l_shuf: db 4, 4, 4, 4, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
34 pb_0to31: db 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
35 db 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31
36 wiener_shufA: db 1, 7, 2, 8, 3, 9, 4, 10, 5, 11, 6, 12, 7, 13, 8, 14
37 wiener_shufB: db 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10
38 wiener_shufC: db 6, 5, 7, 6, 8, 7, 9, 8, 10, 9, 11, 10, 12, 11, 13, 12
39 sgr_r_ext: times 16 db 1
40 times 16 db 9
41
42 ; dword version of dav1d_sgr_x_by_x[] for use with gathers, wastes a bit of
43 ; cache but eliminates some shifts in the inner sgr loop which is overall a win
44 const sgr_x_by_x_avx2
45 dd 255,128, 85, 64, 51, 43, 37, 32, 28, 26, 23, 21, 20, 18, 17, 16
46 dd 15, 14, 13, 13, 12, 12, 11, 11, 10, 10, 9, 9, 9, 9, 8, 8
47 dd 8, 8, 7, 7, 7, 7, 7, 6, 6, 6, 6, 6, 6, 6, 5, 5
48 dd 5, 5, 5, 5, 5, 5, 5, 5, 4, 4, 4, 4, 4, 4, 4, 4
49 dd 4, 4, 4, 4, 4, 4, 4, 4, 4, 3, 3, 3, 3, 3, 3, 3
50 dd 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3
51 dd 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2
52 dd 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2
53 dd 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2
54 dd 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2
55 dd 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1
56 dd 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1
57 dd 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1
58 dd 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1
59 dd 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1
60 dd 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0
61
62 times 4 db -1 ; needed for 16-bit sgr
63 pb_m5: times 4 db -5
64 pb_3: times 4 db 3
65 pw_5_6: dw 5, 6
66
67 sgr_l_shuf: db 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11
68 sgr_shuf: db 1, -1, 2, -1, 3, -1, 4, -1, 5, -1, 6, -1, 7, -1, 8, -1
69 db 9, -1, 10, -1, 11, -1, 12, -1
70
71 pw_256: times 2 dw 256
72 pw_2056: times 2 dw 2056
73 pw_m16380: times 2 dw -16380
74 pd_25: dd 25
75 pd_34816: dd 34816
76 pd_m4096: dd -4096
77 pd_0xf00801c7: dd 0xf00801c7
78 pd_0xf00800a4: dd 0xf00800a4
79
80 SECTION .text
81
82 DECLARE_REG_TMP 8, 7, 9, 11, 12, 13, 14 ; ring buffer pointers
83
84 INIT_YMM avx2
85 cglobal wiener_filter7_8bpc, 4, 15, 16, -384*12-16, dst, stride, left, lpf, \
86 w, h, edge, flt
87 mov fltq, r6mp
88 movifnidn hd, hm
89 mov edged, r7m
90 mov wd, wm
91 vbroadcasti128 m6, [wiener_shufA]
92 vpbroadcastb m11, [fltq+ 0] ; x0 x0
93 vbroadcasti128 m7, [wiener_shufB]
94 vpbroadcastd m12, [fltq+ 2]
95 vbroadcasti128 m8, [wiener_shufC]
96 packsswb m12, m12 ; x1 x2
97 vpbroadcastw m13, [fltq+ 6] ; x3
98 vbroadcasti128 m9, [sgr_shuf+6]
99 add lpfq, wq
100 vpbroadcastd m10, [pw_m16380]
101 vpbroadcastd m14, [fltq+16] ; y0 y1
102 add dstq, wq
103 vpbroadcastd m15, [fltq+20] ; y2 y3
104 lea t1, [rsp+wq*2+16]
105 psllw m14, 5
106 neg wq
107 psllw m15, 5
108 test edgeb, 4 ; LR_HAVE_TOP
109 jz .no_top
110 call .h_top
111 add lpfq, strideq
112 mov t6, t1
113 mov t5, t1
114 add t1, 384*2
115 call .h_top
116 lea r10, [lpfq+strideq*4]
117 mov lpfq, dstq
118 mov t4, t1
119 add t1, 384*2
120 add r10, strideq
121 mov [rsp], r10 ; below
122 call .h
123 mov t3, t1
124 mov t2, t1
125 dec hd
126 jz .v1
127 add lpfq, strideq
128 add t1, 384*2
129 call .h
130 mov t2, t1
131 dec hd
132 jz .v2
133 add lpfq, strideq
134 add t1, 384*2
135 call .h
136 dec hd
137 jz .v3
138 .main:
139 lea t0, [t1+384*2]
140 .main_loop:
141 call .hv
142 dec hd
143 jnz .main_loop
144 test edgeb, 8 ; LR_HAVE_BOTTOM
145 jz .v3
146 mov lpfq, [rsp]
147 call .hv_bottom
148 add lpfq, strideq
149 call .hv_bottom
150 .v1:
151 call .v
152 RET
153 .no_top:
154 lea r10, [lpfq+strideq*4]
155 mov lpfq, dstq
156 lea r10, [r10+strideq*2]
157 mov [rsp], r10
158 call .h
159 mov t6, t1
160 mov t5, t1
161 mov t4, t1
162 mov t3, t1
163 mov t2, t1
164 dec hd
165 jz .v1
166 add lpfq, strideq
167 add t1, 384*2
168 call .h
169 mov t2, t1
170 dec hd
171 jz .v2
172 add lpfq, strideq
173 add t1, 384*2
174 call .h
175 dec hd
176 jz .v3
177 lea t0, [t1+384*2]
178 call .hv
179 dec hd
180 jz .v3
181 add t0, 384*8
182 call .hv
183 dec hd
184 jnz .main
185 .v3:
186 call .v
187 .v2:
188 call .v
189 jmp .v1
190 .extend_right:
191 movd xm2, r10d
192 vpbroadcastd m0, [pb_3]
193 vpbroadcastd m1, [pb_m5]
194 vpbroadcastb m2, xm2
195 movu m3, [pb_0to31]
196 psubb m0, m2
197 psubb m1, m2
198 pminub m0, m3
199 pminub m1, m3
200 pshufb m4, m0
201 pshufb m5, m1
202 ret
203 .h:
204 mov r10, wq
205 test edgeb, 1 ; LR_HAVE_LEFT
206 jz .h_extend_left
207 movd xm4, [leftq]
208 vpblendd m4, [lpfq+r10-4], 0xfe
209 add leftq, 4
210 jmp .h_main
211 .h_extend_left:
212 vbroadcasti128 m5, [lpfq+r10] ; avoid accessing memory located
213 mova m4, [lpfq+r10] ; before the start of the buffer
214 palignr m4, m5, 12
215 pshufb m4, [wiener_l_shuf]
216 jmp .h_main
217 .h_top:
218 mov r10, wq
219 test edgeb, 1 ; LR_HAVE_LEFT
220 jz .h_extend_left
221 .h_loop:
222 movu m4, [lpfq+r10-4]
223 .h_main:
224 movu m5, [lpfq+r10+4]
225 test edgeb, 2 ; LR_HAVE_RIGHT
226 jnz .h_have_right
227 cmp r10d, -34
228 jl .h_have_right
229 call .extend_right
230 .h_have_right:
231 pshufb m0, m4, m6
232 pmaddubsw m0, m11
233 pshufb m1, m5, m6
234 pmaddubsw m1, m11
235 pshufb m2, m4, m7
236 pmaddubsw m2, m12
237 pshufb m3, m5, m7
238 pmaddubsw m3, m12
239 paddw m0, m2
240 pshufb m2, m4, m8
241 pmaddubsw m2, m12
242 paddw m1, m3
243 pshufb m3, m5, m8
244 pmaddubsw m3, m12
245 pshufb m4, m9
246 paddw m0, m2
247 pmullw m2, m4, m13
248 pshufb m5, m9
249 paddw m1, m3
250 pmullw m3, m5, m13
251 psllw m4, 7
252 psllw m5, 7
253 paddw m4, m10
254 paddw m5, m10
255 paddw m0, m2
256 vpbroadcastd m2, [pw_2056]
257 paddw m1, m3
258 paddsw m0, m4
259 paddsw m1, m5
260 psraw m0, 3
261 psraw m1, 3
262 paddw m0, m2
263 paddw m1, m2
264 mova [t1+r10*2+ 0], m0
265 mova [t1+r10*2+32], m1
266 add r10, 32
267 jl .h_loop
268 ret
269 ALIGN function_align
270 .hv:
271 add lpfq, strideq
272 mov r10, wq
273 test edgeb, 1 ; LR_HAVE_LEFT
274 jz .hv_extend_left
275 movd xm4, [leftq]
276 vpblendd m4, [lpfq+r10-4], 0xfe
277 add leftq, 4
278 jmp .hv_main
279 .hv_extend_left:
280 movu m4, [lpfq+r10-4]
281 pshufb m4, [wiener_l_shuf]
282 jmp .hv_main
283 .hv_bottom:
284 mov r10, wq
285 test edgeb, 1 ; LR_HAVE_LEFT
286 jz .hv_extend_left
287 .hv_loop:
288 movu m4, [lpfq+r10-4]
289 .hv_main:
290 movu m5, [lpfq+r10+4]
291 test edgeb, 2 ; LR_HAVE_RIGHT
292 jnz .hv_have_right
293 cmp r10d, -34
294 jl .hv_have_right
295 call .extend_right
296 .hv_have_right:
297 pshufb m0, m4, m6
298 pmaddubsw m0, m11
299 pshufb m1, m5, m6
300 pmaddubsw m1, m11
301 pshufb m2, m4, m7
302 pmaddubsw m2, m12
303 pshufb m3, m5, m7
304 pmaddubsw m3, m12
305 paddw m0, m2
306 pshufb m2, m4, m8
307 pmaddubsw m2, m12
308 paddw m1, m3
309 pshufb m3, m5, m8
310 pmaddubsw m3, m12
311 pshufb m4, m9
312 paddw m0, m2
313 pmullw m2, m4, m13
314 pshufb m5, m9
315 paddw m1, m3
316 pmullw m3, m5, m13
317 psllw m4, 7
318 psllw m5, 7
319 paddw m4, m10
320 paddw m5, m10
321 paddw m0, m2
322 paddw m1, m3
323 mova m2, [t4+r10*2]
324 paddw m2, [t2+r10*2]
325 mova m3, [t3+r10*2]
326 paddsw m0, m4
327 vpbroadcastd m4, [pw_2056]
328 paddsw m1, m5
329 mova m5, [t5+r10*2]
330 paddw m5, [t1+r10*2]
331 psraw m0, 3
332 psraw m1, 3
333 paddw m0, m4
334 paddw m1, m4
335 paddw m4, m0, [t6+r10*2]
336 mova [t0+r10*2], m0
337 punpcklwd m0, m2, m3
338 pmaddwd m0, m15
339 punpckhwd m2, m3
340 pmaddwd m2, m15
341 punpcklwd m3, m4, m5
342 pmaddwd m3, m14
343 punpckhwd m4, m5
344 pmaddwd m4, m14
345 paddd m0, m3
346 paddd m4, m2
347 mova m2, [t4+r10*2+32]
348 paddw m2, [t2+r10*2+32]
349 mova m3, [t3+r10*2+32]
350 mova m5, [t5+r10*2+32]
351 paddw m5, [t1+r10*2+32]
352 packuswb m0, m4
353 paddw m4, m1, [t6+r10*2+32]
354 mova [t0+r10*2+32], m1
355 punpcklwd m1, m2, m3
356 pmaddwd m1, m15
357 punpckhwd m2, m3
358 pmaddwd m2, m15
359 punpcklwd m3, m4, m5
360 pmaddwd m3, m14
361 punpckhwd m4, m5
362 pmaddwd m4, m14
363 paddd m1, m3
364 paddd m2, m4
365 packuswb m1, m2
366 psrlw m0, 8
367 psrlw m1, 8
368 packuswb m0, m1
369 mova [dstq+r10], m0
370 add r10, 32
371 jl .hv_loop
372 mov t6, t5
373 mov t5, t4
374 mov t4, t3
375 mov t3, t2
376 mov t2, t1
377 mov t1, t0
378 mov t0, t6
379 add dstq, strideq
380 ret
381 .v:
382 mov r10, wq
383 .v_loop:
384 mova m2, [t4+r10*2+ 0]
385 paddw m2, [t2+r10*2+ 0]
386 mova m4, [t3+r10*2+ 0]
387 mova m6, [t1+r10*2+ 0]
388 paddw m8, m6, [t6+r10*2+ 0]
389 paddw m6, [t5+r10*2+ 0]
390 mova m3, [t4+r10*2+32]
391 paddw m3, [t2+r10*2+32]
392 mova m5, [t3+r10*2+32]
393 mova m7, [t1+r10*2+32]
394 paddw m9, m7, [t6+r10*2+32]
395 paddw m7, [t5+r10*2+32]
396 punpcklwd m0, m2, m4
397 pmaddwd m0, m15
398 punpckhwd m2, m4
399 pmaddwd m2, m15
400 punpcklwd m4, m8, m6
401 pmaddwd m4, m14
402 punpckhwd m6, m8, m6
403 pmaddwd m6, m14
404 punpcklwd m1, m3, m5
405 pmaddwd m1, m15
406 punpckhwd m3, m5
407 pmaddwd m3, m15
408 punpcklwd m5, m9, m7
409 pmaddwd m5, m14
410 punpckhwd m7, m9, m7
411 pmaddwd m7, m14
412 paddd m0, m4
413 paddd m2, m6
414 paddd m1, m5
415 paddd m3, m7
416 packuswb m0, m2
417 packuswb m1, m3
418 psrlw m0, 8
419 psrlw m1, 8
420 packuswb m0, m1
421 mova [dstq+r10], m0
422 add r10, 32
423 jl .v_loop
424 mov t6, t5
425 mov t5, t4
426 mov t4, t3
427 mov t3, t2
428 mov t2, t1
429 add dstq, strideq
430 ret
431
432 cglobal wiener_filter5_8bpc, 4, 13, 16, 384*8+16, dst, stride, left, lpf, \
433 w, h, edge, flt
434 mov fltq, r6mp
435 movifnidn hd, hm
436 mov edged, r7m
437 mov wd, wm
438 vbroadcasti128 m6, [wiener_shufB]
439 vpbroadcastd m12, [fltq+ 2]
440 vbroadcasti128 m7, [wiener_shufC]
441 packsswb m12, m12 ; x1 x2
442 vpbroadcastw m13, [fltq+ 6] ; x3
443 vbroadcasti128 m8, [sgr_shuf+6]
444 add lpfq, wq
445 vpbroadcastd m9, [pw_m16380]
446 vpbroadcastd m10, [pw_2056]
447 mova m11, [wiener_l_shuf]
448 vpbroadcastd m14, [fltq+16] ; __ y1
449 add dstq, wq
450 vpbroadcastd m15, [fltq+20] ; y2 y3
451 lea t1, [rsp+wq*2+16]
452 psllw m14, 5
453 neg wq
454 psllw m15, 5
455 test edgeb, 4 ; LR_HAVE_TOP
456 jz .no_top
457 call .h_top
458 add lpfq, strideq
459 mov t4, t1
460 add t1, 384*2
461 call .h_top
462 lea r10, [lpfq+strideq*4]
463 mov lpfq, dstq
464 mov t3, t1
465 add t1, 384*2
466 add r10, strideq
467 mov [rsp], r10 ; below
468 call .h
469 mov t2, t1
470 dec hd
471 jz .v1
472 add lpfq, strideq
473 add t1, 384*2
474 call .h
475 dec hd
476 jz .v2
477 .main:
478 mov t0, t4
479 .main_loop:
480 call .hv
481 dec hd
482 jnz .main_loop
483 test edgeb, 8 ; LR_HAVE_BOTTOM
484 jz .v2
485 mov lpfq, [rsp]
486 call .hv_bottom
487 add lpfq, strideq
488 call .hv_bottom
489 .end:
490 RET
491 .no_top:
492 lea r10, [lpfq+strideq*4]
493 mov lpfq, dstq
494 lea r10, [r10+strideq*2]
495 mov [rsp], r10
496 call .h
497 mov t4, t1
498 mov t3, t1
499 mov t2, t1
500 dec hd
501 jz .v1
502 add lpfq, strideq
503 add t1, 384*2
504 call .h
505 dec hd
506 jz .v2
507 lea t0, [t1+384*2]
508 call .hv
509 dec hd
510 jz .v2
511 add t0, 384*6
512 call .hv
513 dec hd
514 jnz .main
515 .v2:
516 call .v
517 mov t4, t3
518 mov t3, t2
519 mov t2, t1
520 add dstq, strideq
521 .v1:
522 call .v
523 jmp .end
524 .h:
525 mov r10, wq
526 test edgeb, 1 ; LR_HAVE_LEFT
527 jz .h_extend_left
528 movd xm4, [leftq]
529 vpblendd m4, [lpfq+r10-4], 0xfe
530 add leftq, 4
531 jmp .h_main
532 .h_extend_left:
533 vbroadcasti128 m5, [lpfq+r10] ; avoid accessing memory located
534 mova m4, [lpfq+r10] ; before the start of the buffer
535 palignr m4, m5, 12
536 pshufb m4, m11
537 jmp .h_main
538 .h_top:
539 mov r10, wq
540 test edgeb, 1 ; LR_HAVE_LEFT
541 jz .h_extend_left
542 .h_loop:
543 movu m4, [lpfq+r10-4]
544 .h_main:
545 movu m5, [lpfq+r10+4]
546 test edgeb, 2 ; LR_HAVE_RIGHT
547 jnz .h_have_right
548 cmp r10d, -33
549 jl .h_have_right
550 call mangle(private_prefix %+ _wiener_filter7_8bpc_avx2).extend_right
551 .h_have_right:
552 pshufb m0, m4, m6
553 pmaddubsw m0, m12
554 pshufb m1, m5, m6
555 pmaddubsw m1, m12
556 pshufb m2, m4, m7
557 pmaddubsw m2, m12
558 pshufb m3, m5, m7
559 pmaddubsw m3, m12
560 pshufb m4, m8
561 paddw m0, m2
562 pmullw m2, m4, m13
563 pshufb m5, m8
564 paddw m1, m3
565 pmullw m3, m5, m13
566 psllw m4, 7
567 psllw m5, 7
568 paddw m4, m9
569 paddw m5, m9
570 paddw m0, m2
571 paddw m1, m3
572 paddsw m0, m4
573 paddsw m1, m5
574 psraw m0, 3
575 psraw m1, 3
576 paddw m0, m10
577 paddw m1, m10
578 mova [t1+r10*2+ 0], m0
579 mova [t1+r10*2+32], m1
580 add r10, 32
581 jl .h_loop
582 ret
583 ALIGN function_align
584 .hv:
585 add lpfq, strideq
586 mov r10, wq
587 test edgeb, 1 ; LR_HAVE_LEFT
588 jz .hv_extend_left
589 movd xm4, [leftq]
590 vpblendd m4, [lpfq+r10-4], 0xfe
591 add leftq, 4
592 jmp .hv_main
593 .hv_extend_left:
594 movu m4, [lpfq+r10-4]
595 pshufb m4, m11
596 jmp .hv_main
597 .hv_bottom:
598 mov r10, wq
599 test edgeb, 1 ; LR_HAVE_LEFT
600 jz .hv_extend_left
601 .hv_loop:
602 movu m4, [lpfq+r10-4]
603 .hv_main:
604 movu m5, [lpfq+r10+4]
605 test edgeb, 2 ; LR_HAVE_RIGHT
606 jnz .hv_have_right
607 cmp r10d, -33
608 jl .hv_have_right
609 call mangle(private_prefix %+ _wiener_filter7_8bpc_avx2).extend_right
610 .hv_have_right:
611 pshufb m0, m4, m6
612 pmaddubsw m0, m12
613 pshufb m1, m5, m6
614 pmaddubsw m1, m12
615 pshufb m2, m4, m7
616 pmaddubsw m2, m12
617 pshufb m3, m5, m7
618 pmaddubsw m3, m12
619 pshufb m4, m8
620 paddw m0, m2
621 pmullw m2, m4, m13
622 pshufb m5, m8
623 paddw m1, m3
624 pmullw m3, m5, m13
625 psllw m4, 7
626 psllw m5, 7
627 paddw m4, m9
628 paddw m5, m9
629 paddw m0, m2
630 paddw m1, m3
631 mova m2, [t3+r10*2]
632 paddw m2, [t1+r10*2]
633 mova m3, [t2+r10*2]
634 paddsw m0, m4
635 paddsw m1, m5
636 psraw m0, 3
637 psraw m1, 3
638 paddw m0, m10
639 paddw m1, m10
640 paddw m4, m0, [t4+r10*2]
641 mova [t0+r10*2], m0
642 punpcklwd m0, m2, m3
643 pmaddwd m0, m15
644 punpckhwd m2, m3
645 pmaddwd m2, m15
646 punpcklwd m3, m4, m4
647 pmaddwd m3, m14
648 punpckhwd m4, m4
649 pmaddwd m4, m14
650 paddd m0, m3
651 paddd m4, m2
652 mova m2, [t3+r10*2+32]
653 paddw m2, [t1+r10*2+32]
654 mova m3, [t2+r10*2+32]
655 packuswb m0, m4
656 paddw m4, m1, [t4+r10*2+32]
657 mova [t0+r10*2+32], m1
658 punpcklwd m1, m2, m3
659 pmaddwd m1, m15
660 punpckhwd m2, m3
661 pmaddwd m2, m15
662 punpcklwd m3, m4, m4
663 pmaddwd m3, m14
664 punpckhwd m4, m4
665 pmaddwd m4, m14
666 paddd m1, m3
667 paddd m2, m4
668 packuswb m1, m2
669 psrlw m0, 8
670 psrlw m1, 8
671 packuswb m0, m1
672 mova [dstq+r10], m0
673 add r10, 32
674 jl .hv_loop
675 mov t4, t3
676 mov t3, t2
677 mov t2, t1
678 mov t1, t0
679 mov t0, t4
680 add dstq, strideq
681 ret
682 .v:
683 mov r10, wq
684 psrld m13, m14, 16 ; y1 __
685 .v_loop:
686 mova m6, [t1+r10*2+ 0]
687 paddw m2, m6, [t3+r10*2+ 0]
688 mova m4, [t2+r10*2+ 0]
689 mova m7, [t1+r10*2+32]
690 paddw m3, m7, [t3+r10*2+32]
691 mova m5, [t2+r10*2+32]
692 paddw m6, [t4+r10*2+ 0]
693 paddw m7, [t4+r10*2+32]
694 punpcklwd m0, m2, m4
695 pmaddwd m0, m15
696 punpckhwd m2, m4
697 pmaddwd m2, m15
698 punpcklwd m1, m3, m5
699 pmaddwd m1, m15
700 punpckhwd m3, m5
701 pmaddwd m3, m15
702 punpcklwd m5, m7, m6
703 pmaddwd m4, m5, m14
704 punpckhwd m7, m6
705 pmaddwd m6, m7, m14
706 pmaddwd m5, m13
707 pmaddwd m7, m13
708 paddd m0, m4
709 paddd m2, m6
710 paddd m1, m5
711 paddd m3, m7
712 packuswb m0, m2
713 packuswb m1, m3
714 psrlw m0, 8
715 psrlw m1, 8
716 packuswb m0, m1
717 mova [dstq+r10], m0
718 add r10, 32
719 jl .v_loop
720 ret
721
722 cglobal sgr_filter_5x5_8bpc, 4, 13, 16, 400*24+16, dst, stride, left, lpf, \
723 w, h, edge, params
724 %define base r12-sgr_x_by_x_avx2-256*4
725 lea r12, [sgr_x_by_x_avx2+256*4]
726 mov paramsq, r6mp
727 mov wd, wm
728 movifnidn hd, hm
729 mov edged, r7m
730 vbroadcasti128 m8, [base+sgr_shuf+0]
731 vbroadcasti128 m9, [base+sgr_shuf+8]
732 add lpfq, wq
733 vbroadcasti128 m10, [base+sgr_shuf+2]
734 add dstq, wq
735 vbroadcasti128 m11, [base+sgr_shuf+6]
736 lea t3, [rsp+wq*4+16+400*12]
737 vpbroadcastd m12, [paramsq+0] ; s0
738 pxor m6, m6
739 vpbroadcastw m7, [paramsq+8] ; w0
740 lea t1, [rsp+wq*2+20]
741 vpbroadcastd m13, [base+pd_0xf00800a4]
742 neg wq
743 vpbroadcastd m14, [base+pd_34816] ; (1 << 11) + (1 << 15)
744 psllw m7, 4
745 vpbroadcastd m15, [base+pd_m4096]
746 test edgeb, 4 ; LR_HAVE_TOP
747 jz .no_top
748 call .h_top
749 add lpfq, strideq
750 mov t2, t1
751 call .top_fixup
752 add t1, 400*6
753 call .h_top
754 lea r10, [lpfq+strideq*4]
755 mov lpfq, dstq
756 add r10, strideq
757 mov [rsp], r10 ; below
758 mov t0, t2
759 dec hd
760 jz .height1
761 or edged, 16
762 call .h
763 .main:
764 add lpfq, strideq
765 call .hv
766 call .prep_n
767 sub hd, 2
768 jl .extend_bottom
769 .main_loop:
770 add lpfq, strideq
771 test hd, hd
772 jz .odd_height
773 call .h
774 add lpfq, strideq
775 call .hv
776 call .n0
777 call .n1
778 sub hd, 2
779 jge .main_loop
780 test edgeb, 8 ; LR_HAVE_BOTTOM
781 jz .extend_bottom
782 mov lpfq, [rsp]
783 call .h_top
784 add lpfq, strideq
785 call .hv_bottom
786 .end:
787 call .n0
788 call .n1
789 .end2:
790 RET
791 .height1:
792 call .hv
793 call .prep_n
794 jmp .odd_height_end
795 .odd_height:
796 call .hv
797 call .n0
798 call .n1
799 .odd_height_end:
800 call .v
801 call .n0
802 jmp .end2
803 .extend_bottom:
804 call .v
805 jmp .end
806 .no_top:
807 lea r10, [lpfq+strideq*4]
808 mov lpfq, dstq
809 lea r10, [r10+strideq*2]
810 mov [rsp], r10
811 call .h
812 lea t2, [t1+400*6]
813 call .top_fixup
814 dec hd
815 jz .no_top_height1
816 or edged, 16
817 mov t0, t1
818 mov t1, t2
819 jmp .main
820 .no_top_height1:
821 call .v
822 call .prep_n
823 jmp .odd_height_end
824 .extend_right:
825 movd xm2, r10d
826 mova m0, [sgr_r_ext]
827 vpbroadcastb m2, xm2
828 psubb m0, m2
829 pminub m0, [pb_0to31]
830 pshufb m5, m0
831 ret
832 .h: ; horizontal boxsum
833 lea r10, [wq-2]
834 test edgeb, 1 ; LR_HAVE_LEFT
835 jz .h_extend_left
836 vpbroadcastd xm0, [leftq]
837 mova xm5, [lpfq+wq]
838 palignr xm5, xm0, 12
839 add leftq, 4
840 jmp .h_main
841 .h_extend_left:
842 mova xm5, [lpfq+wq]
843 pshufb xm5, [base+sgr_l_shuf]
844 jmp .h_main
845 .h_top:
846 lea r10, [wq-2]
847 test edgeb, 1 ; LR_HAVE_LEFT
848 jz .h_extend_left
849 .h_loop:
850 movu xm5, [lpfq+r10-2]
851 .h_main:
852 vinserti128 m5, [lpfq+r10+6], 1
853 test edgeb, 2 ; LR_HAVE_RIGHT
854 jnz .h_have_right
855 cmp r10d, -18
856 jl .h_have_right
857 call .extend_right
858 .h_have_right:
859 pshufb m3, m5, m8
860 pmullw m4, m3, m3
861 pshufb m2, m5, m9
862 paddw m0, m3, m2
863 shufps m3, m2, q2121
864 paddw m0, m3
865 punpcklwd m1, m2, m3
866 pmaddwd m1, m1
867 punpckhwd m2, m3
868 pmaddwd m2, m2
869 punpcklwd m3, m4, m6
870 paddd m1, m3
871 punpckhwd m4, m6
872 paddd m2, m4
873 pshufb m4, m5, m10
874 paddw m0, m4
875 pshufb m5, m11
876 paddw m0, m5 ; sum
877 punpcklwd m3, m4, m5
878 pmaddwd m3, m3
879 punpckhwd m4, m5
880 pmaddwd m4, m4
881 test edgeb, 16 ; y > 0
882 jz .h_loop_end
883 paddw m0, [t1+r10*2+400*0]
884 paddd m1, [t1+r10*2+400*2]
885 paddd m2, [t1+r10*2+400*4]
886 .h_loop_end:
887 paddd m1, m3 ; sumsq
888 paddd m2, m4
889 mova [t1+r10*2+400*0], m0
890 mova [t1+r10*2+400*2], m1
891 mova [t1+r10*2+400*4], m2
892 add r10, 16
893 jl .h_loop
894 ret
895 .top_fixup:
896 lea r10, [wq-2]
897 .top_fixup_loop: ; the sums of the first row needs to be doubled
898 mova m0, [t1+r10*2+400*0]
899 mova m1, [t1+r10*2+400*2]
900 mova m2, [t1+r10*2+400*4]
901 paddw m0, m0
902 paddd m1, m1
903 paddd m2, m2
904 mova [t2+r10*2+400*0], m0
905 mova [t2+r10*2+400*2], m1
906 mova [t2+r10*2+400*4], m2
907 add r10, 16
908 jl .top_fixup_loop
909 ret
910 ALIGN function_align
911 .hv: ; horizontal boxsum + vertical boxsum + ab
912 lea r10, [wq-2]
913 test edgeb, 1 ; LR_HAVE_LEFT
914 jz .hv_extend_left
915 vpbroadcastd xm0, [leftq]
916 mova xm5, [lpfq+wq]
917 palignr xm5, xm0, 12
918 add leftq, 4
919 jmp .hv_main
920 .hv_extend_left:
921 mova xm5, [lpfq+wq]
922 pshufb xm5, [base+sgr_l_shuf]
923 jmp .hv_main
924 .hv_bottom:
925 lea r10, [wq-2]
926 test edgeb, 1 ; LR_HAVE_LEFT
927 jz .hv_extend_left
928 .hv_loop:
929 movu xm5, [lpfq+r10-2]
930 .hv_main:
931 vinserti128 m5, [lpfq+r10+6], 1
932 test edgeb, 2 ; LR_HAVE_RIGHT
933 jnz .hv_have_right
934 cmp r10d, -18
935 jl .hv_have_right
936 call .extend_right
937 .hv_have_right:
938 pshufb m1, m5, m8
939 pmullw m4, m1, m1
940 pshufb m3, m5, m9
941 paddw m0, m1, m3
942 shufps m1, m3, q2121
943 paddw m0, m1
944 punpcklwd m2, m3, m1
945 pmaddwd m2, m2
946 punpckhwd m3, m1
947 pmaddwd m3, m3
948 punpcklwd m1, m4, m6
949 paddd m2, m1
950 punpckhwd m4, m6
951 paddd m3, m4
952 pshufb m1, m5, m10
953 paddw m0, m1
954 pshufb m5, m11
955 paddw m0, m5 ; h sum
956 punpcklwd m4, m5, m1
957 pmaddwd m4, m4
958 punpckhwd m5, m1
959 pmaddwd m5, m5
960 paddw m1, m0, [t1+r10*2+400*0]
961 paddd m2, m4 ; h sumsq
962 paddd m3, m5
963 paddd m4, m2, [t1+r10*2+400*2]
964 paddd m5, m3, [t1+r10*2+400*4]
965 test hd, hd
966 jz .hv_last_row
967 .hv_main2:
968 paddw m1, [t2+r10*2+400*0] ; hv sum
969 paddd m4, [t2+r10*2+400*2] ; hv sumsq
970 paddd m5, [t2+r10*2+400*4]
971 mova [t0+r10*2+400*0], m0
972 mova [t0+r10*2+400*2], m2
973 mova [t0+r10*2+400*4], m3
974 vpbroadcastd m2, [pd_25]
975 punpcklwd m0, m1, m6 ; b
976 punpckhwd m1, m6
977 pmulld m4, m2 ; a * 25
978 pmulld m5, m2
979 pmaddwd m2, m0, m0 ; b * b
980 pmaddwd m3, m1, m1
981 psubd m4, m2 ; p
982 psubd m5, m3
983 pmulld m4, m12 ; p * s
984 pmulld m5, m12
985 pmaddwd m0, m13 ; b * 164
986 pmaddwd m1, m13
987 paddusw m4, m13
988 paddusw m5, m13
989 psrad m3, m4, 20 ; min(z, 255) - 256
990 vpgatherdd m2, [r12+m3*4], m4 ; x
991 psrad m4, m5, 20
992 vpgatherdd m3, [r12+m4*4], m5
993 pmulld m0, m2
994 pmulld m1, m3
995 paddd m0, m14 ; x * b * 164 + (1 << 11) + (1 << 15)
996 paddd m1, m14
997 pand m0, m15
998 pand m1, m15
999 por m0, m2 ; a | (b << 12)
1000 por m1, m3
1001 mova [t3+r10*4+ 8], xm0 ; The neighbor calculations requires
1002 vextracti128 [t3+r10*4+40], m0, 1 ; 13 bits for a and 21 bits for b.
1003 mova [t3+r10*4+24], xm1 ; Packing them allows for 12+20, but
1004 vextracti128 [t3+r10*4+56], m1, 1 ; that gets us most of the way.
1005 add r10, 16
1006 jl .hv_loop
1007 mov t2, t1
1008 mov t1, t0
1009 mov t0, t2
1010 ret
1011 .hv_last_row: ; esoteric edge case for odd heights
1012 mova [t1+r10*2+400*0], m1
1013 paddw m1, m0
1014 mova [t1+r10*2+400*2], m4
1015 paddd m4, m2
1016 mova [t1+r10*2+400*4], m5
1017 paddd m5, m3
1018 jmp .hv_main2
1019 .v: ; vertical boxsum + ab
1020 lea r10, [wq-2]
1021 .v_loop:
1022 mova m0, [t1+r10*2+400*0]
1023 mova m2, [t1+r10*2+400*2]
1024 mova m3, [t1+r10*2+400*4]
1025 paddw m1, m0, [t2+r10*2+400*0]
1026 paddd m4, m2, [t2+r10*2+400*2]
1027 paddd m5, m3, [t2+r10*2+400*4]
1028 paddw m0, m0
1029 paddd m2, m2
1030 paddd m3, m3
1031 paddw m1, m0 ; hv sum
1032 paddd m4, m2 ; hv sumsq
1033 paddd m5, m3
1034 vpbroadcastd m2, [pd_25]
1035 punpcklwd m0, m1, m6 ; b
1036 punpckhwd m1, m6
1037 pmulld m4, m2 ; a * 25
1038 pmulld m5, m2
1039 pmaddwd m2, m0, m0 ; b * b
1040 pmaddwd m3, m1, m1
1041 psubd m4, m2 ; p
1042 psubd m5, m3
1043 pmulld m4, m12 ; p * s
1044 pmulld m5, m12
1045 pmaddwd m0, m13 ; b * 164
1046 pmaddwd m1, m13
1047 paddusw m4, m13
1048 paddusw m5, m13
1049 psrad m3, m4, 20 ; min(z, 255) - 256
1050 vpgatherdd m2, [r12+m3*4], m4 ; x
1051 psrad m4, m5, 20
1052 vpgatherdd m3, [r12+m4*4], m5
1053 pmulld m0, m2
1054 pmulld m1, m3
1055 paddd m0, m14 ; x * b * 164 + (1 << 11) + (1 << 15)
1056 paddd m1, m14
1057 pand m0, m15
1058 pand m1, m15
1059 por m0, m2 ; a | (b << 12)
1060 por m1, m3
1061 mova [t3+r10*4+ 8], xm0
1062 vextracti128 [t3+r10*4+40], m0, 1
1063 mova [t3+r10*4+24], xm1
1064 vextracti128 [t3+r10*4+56], m1, 1
1065 add r10, 16
1066 jl .v_loop
1067 ret
1068 .prep_n: ; initial neighbor setup
1069 mov r10, wq
1070 .prep_n_loop:
1071 movu m0, [t3+r10*4+ 4]
1072 movu m1, [t3+r10*4+36]
1073 paddd m2, m0, [t3+r10*4+ 0]
1074 paddd m3, m1, [t3+r10*4+32]
1075 paddd m2, [t3+r10*4+ 8]
1076 paddd m3, [t3+r10*4+40]
1077 paddd m0, m2
1078 pslld m2, 2
1079 paddd m1, m3
1080 pslld m3, 2
1081 paddd m2, m0 ; ab 565
1082 paddd m3, m1
1083 pandn m0, m15, m2 ; a
1084 psrld m2, 12 ; b
1085 pandn m1, m15, m3
1086 psrld m3, 12
1087 mova [t3+r10*4+400*4+ 0], m0
1088 mova [t3+r10*4+400*8+ 0], m2
1089 mova [t3+r10*4+400*4+32], m1
1090 mova [t3+r10*4+400*8+32], m3
1091 add r10, 16
1092 jl .prep_n_loop
1093 ret
1094 ALIGN function_align
1095 .n0: ; neighbor + output (even rows)
1096 mov r10, wq
1097 .n0_loop:
1098 movu m0, [t3+r10*4+ 4]
1099 movu m1, [t3+r10*4+36]
1100 paddd m2, m0, [t3+r10*4+ 0]
1101 paddd m3, m1, [t3+r10*4+32]
1102 paddd m2, [t3+r10*4+ 8]
1103 paddd m3, [t3+r10*4+40]
1104 paddd m0, m2
1105 pslld m2, 2
1106 paddd m1, m3
1107 pslld m3, 2
1108 paddd m2, m0
1109 paddd m3, m1
1110 pandn m0, m15, m2
1111 psrld m2, 12
1112 pandn m1, m15, m3
1113 psrld m3, 12
1114 paddd m4, m0, [t3+r10*4+400*4+ 0] ; a
1115 paddd m5, m1, [t3+r10*4+400*4+32]
1116 mova [t3+r10*4+400*4+ 0], m0
1117 mova [t3+r10*4+400*4+32], m1
1118 paddd m0, m2, [t3+r10*4+400*8+ 0] ; b
1119 paddd m1, m3, [t3+r10*4+400*8+32]
1120 mova [t3+r10*4+400*8+ 0], m2
1121 mova [t3+r10*4+400*8+32], m3
1122 pmovzxbd m2, [dstq+r10+0]
1123 pmovzxbd m3, [dstq+r10+8]
1124 pmaddwd m4, m2 ; a * src
1125 pmaddwd m5, m3
1126 packssdw m2, m3
1127 psubd m0, m4 ; b - a * src + (1 << 8)
1128 psubd m1, m5
1129 psrad m0, 9
1130 psrad m1, 9
1131 packssdw m0, m1
1132 pmulhrsw m0, m7
1133 paddw m0, m2
1134 vextracti128 xm1, m0, 1
1135 packuswb xm0, xm1
1136 pshufd xm0, xm0, q3120
1137 mova [dstq+r10], xm0
1138 add r10, 16
1139 jl .n0_loop
1140 add dstq, strideq
1141 ret
1142 ALIGN function_align
1143 .n1: ; neighbor + output (odd rows)
1144 mov r10, wq
1145 .n1_loop:
1146 pmovzxbd m2, [dstq+r10+0]
1147 pmovzxbd m3, [dstq+r10+8]
1148 pmaddwd m4, m2, [t3+r10*4+400*4+ 0] ; a * src
1149 pmaddwd m5, m3, [t3+r10*4+400*4+32]
1150 mova m0, [t3+r10*4+400*8+ 0] ; b
1151 mova m1, [t3+r10*4+400*8+32]
1152 packssdw m2, m3
1153 psubd m0, m4 ; b - a * src + (1 << 7)
1154 psubd m1, m5
1155 psrad m0, 8
1156 psrad m1, 8
1157 packssdw m0, m1
1158 pmulhrsw m0, m7
1159 paddw m0, m2
1160 vextracti128 xm1, m0, 1
1161 packuswb xm0, xm1
1162 pshufd xm0, xm0, q3120
1163 mova [dstq+r10], xm0
1164 add r10, 16
1165 jl .n1_loop
1166 add dstq, strideq
1167 ret
1168
1169 cglobal sgr_filter_3x3_8bpc, 4, 15, 15, -400*28-16, dst, stride, left, lpf, \
1170 w, h, edge, params
1171 %define base r14-sgr_x_by_x_avx2-256*4
1172 mov paramsq, r6mp
1173 mov wd, wm
1174 movifnidn hd, hm
1175 mov edged, r7m
1176 lea r14, [sgr_x_by_x_avx2+256*4]
1177 vbroadcasti128 m8, [base+sgr_shuf+2]
1178 add lpfq, wq
1179 vbroadcasti128 m9, [base+sgr_shuf+4]
1180 add dstq, wq
1181 vbroadcasti128 m10, [base+sgr_shuf+6]
1182 lea t3, [rsp+wq*4+16+400*12]
1183 vpbroadcastd m11, [paramsq+ 4] ; s1
1184 pxor m6, m6
1185 vpbroadcastw m7, [paramsq+10] ; w1
1186 lea t1, [rsp+wq*2+20]
1187 vpbroadcastd m12, [base+pd_0xf00801c7]
1188 neg wq
1189 vpbroadcastd m13, [base+pd_34816] ; (1 << 11) + (1 << 15)
1190 psllw m7, 4
1191 vpbroadcastd m14, [base+pd_m4096]
1192 test edgeb, 4 ; LR_HAVE_TOP
1193 jz .no_top
1194 call .h_top
1195 add lpfq, strideq
1196 mov t2, t1
1197 add t1, 400*6
1198 call .h_top
1199 lea t4, [lpfq+strideq*4]
1200 mov lpfq, dstq
1201 add t4, strideq
1202 mov [rsp], t4 ; below
1203 mov t0, t2
1204 call .hv
1205 .main:
1206 mov t5, t3
1207 add t3, 400*4
1208 dec hd
1209 jz .height1
1210 add lpfq, strideq
1211 call .hv
1212 call .prep_n
1213 dec hd
1214 jz .extend_bottom
1215 .main_loop:
1216 add lpfq, strideq
1217 call .hv
1218 call .n
1219 dec hd
1220 jnz .main_loop
1221 test edgeb, 8 ; LR_HAVE_BOTTOM
1222 jz .extend_bottom
1223 mov lpfq, [rsp]
1224 call .hv_bottom
1225 call .n
1226 add lpfq, strideq
1227 call .hv_bottom
1228 .end:
1229 call .n
1230 RET
1231 .height1:
1232 call .v
1233 call .prep_n
1234 mov t2, t1
1235 call .v
1236 jmp .end
1237 .extend_bottom:
1238 call .v
1239 call .n
1240 mov t2, t1
1241 call .v
1242 jmp .end
1243 .no_top:
1244 lea t4, [lpfq+strideq*4]
1245 mov lpfq, dstq
1246 lea t4, [t4+strideq*2]
1247 mov [rsp], t4
1248 call .h
1249 lea t0, [t1+400*6]
1250 mov t2, t1
1251 call .v
1252 jmp .main
1253 .h: ; horizontal boxsum
1254 lea r10, [wq-2]
1255 test edgeb, 1 ; LR_HAVE_LEFT
1256 jz .h_extend_left
1257 vpbroadcastd xm0, [leftq]
1258 mova xm5, [lpfq+wq]
1259 palignr xm5, xm0, 12
1260 add leftq, 4
1261 jmp .h_main
1262 .h_extend_left:
1263 mova xm5, [lpfq+wq]
1264 pshufb xm5, [base+sgr_l_shuf]
1265 jmp .h_main
1266 .h_top:
1267 lea r10, [wq-2]
1268 test edgeb, 1 ; LR_HAVE_LEFT
1269 jz .h_extend_left
1270 .h_loop:
1271 movu xm5, [lpfq+r10-2]
1272 .h_main:
1273 vinserti128 m5, [lpfq+r10+6], 1
1274 test edgeb, 2 ; LR_HAVE_RIGHT
1275 jnz .h_have_right
1276 cmp r10d, -17
1277 jl .h_have_right
1278 call mangle(private_prefix %+ _sgr_filter_5x5_8bpc_avx2).extend_right
1279 .h_have_right:
1280 pshufb m0, m5, m8
1281 pmullw m2, m0, m0
1282 pshufb m4, m5, m9
1283 paddw m0, m4
1284 pshufb m5, m10
1285 paddw m0, m5 ; sum
1286 punpcklwd m3, m4, m5
1287 pmaddwd m3, m3
1288 punpckhwd m4, m5
1289 pmaddwd m4, m4
1290 punpcklwd m1, m2, m6
1291 punpckhwd m2, m6
1292 mova [t1+r10*2+400*0], m0
1293 paddd m1, m3 ; sumsq
1294 paddd m2, m4
1295 mova [t1+r10*2+400*2], m1
1296 mova [t1+r10*2+400*4], m2
1297 add r10, 16
1298 jl .h_loop
1299 ret
1300 ALIGN function_align
1301 .hv: ; horizontal boxsum + vertical boxsum + ab
1302 lea r10, [wq-2]
1303 test edgeb, 1 ; LR_HAVE_LEFT
1304 jz .hv_extend_left
1305 vpbroadcastd xm0, [leftq]
1306 mova xm5, [lpfq+wq]
1307 palignr xm5, xm0, 12
1308 add leftq, 4
1309 jmp .hv_main
1310 .hv_extend_left:
1311 mova xm5, [lpfq+wq]
1312 pshufb xm5, [base+sgr_l_shuf]
1313 jmp .hv_main
1314 .hv_bottom:
1315 lea r10, [wq-2]
1316 test edgeb, 1 ; LR_HAVE_LEFT
1317 jz .hv_extend_left
1318 .hv_loop:
1319 movu xm5, [lpfq+r10-2]
1320 .hv_main:
1321 vinserti128 m5, [lpfq+r10+6], 1
1322 test edgeb, 2 ; LR_HAVE_RIGHT
1323 jnz .hv_have_right
1324 cmp r10d, -17
1325 jl .hv_have_right
1326 call mangle(private_prefix %+ _sgr_filter_5x5_8bpc_avx2).extend_right
1327 .hv_have_right:
1328 pshufb m0, m5, m8
1329 pmullw m3, m0, m0
1330 pshufb m1, m5, m9
1331 paddw m0, m1
1332 pshufb m5, m10
1333 paddw m0, m5 ; h sum
1334 punpcklwd m4, m5, m1
1335 pmaddwd m4, m4
1336 punpckhwd m5, m1
1337 pmaddwd m5, m5
1338 paddw m1, m0, [t2+r10*2+400*0]
1339 paddw m1, [t1+r10*2+400*0] ; hv sum
1340 punpcklwd m2, m3, m6
1341 punpckhwd m3, m6
1342 paddd m4, m2 ; h sumsq
1343 paddd m5, m3
1344 paddd m2, m4, [t2+r10*2+400*2]
1345 paddd m3, m5, [t2+r10*2+400*4]
1346 paddd m2, [t1+r10*2+400*2] ; hv sumsq
1347 paddd m3, [t1+r10*2+400*4]
1348 mova [t0+r10*2+400*0], m0
1349 punpcklwd m0, m1, m6 ; b
1350 punpckhwd m1, m6
1351 mova [t0+r10*2+400*2], m4
1352 pslld m4, m2, 3
1353 mova [t0+r10*2+400*4], m5
1354 pslld m5, m3, 3
1355 paddd m4, m2 ; a * 9
1356 pmaddwd m2, m0, m0 ; b * b
1357 paddd m5, m3
1358 pmaddwd m3, m1, m1
1359 psubd m4, m2 ; p
1360 psubd m5, m3
1361 pmulld m4, m11 ; p * s
1362 pmulld m5, m11
1363 pmaddwd m0, m12 ; b * 455
1364 pmaddwd m1, m12
1365 paddusw m4, m12
1366 paddusw m5, m12
1367 psrad m3, m4, 20 ; min(z, 255) - 256
1368 vpgatherdd m2, [r14+m3*4], m4
1369 psrad m4, m5, 20
1370 vpgatherdd m3, [r14+m4*4], m5
1371 pmulld m0, m2
1372 pmulld m1, m3
1373 paddd m0, m13 ; x * b * 455 + (1 << 11) + (1 << 15)
1374 paddd m1, m13
1375 pand m0, m14
1376 pand m1, m14
1377 por m0, m2 ; a | (b << 12)
1378 por m1, m3
1379 mova [t3+r10*4+ 8], xm0
1380 vextracti128 [t3+r10*4+40], m0, 1
1381 mova [t3+r10*4+24], xm1
1382 vextracti128 [t3+r10*4+56], m1, 1
1383 add r10, 16
1384 jl .hv_loop
1385 mov t2, t1
1386 mov t1, t0
1387 mov t0, t2
1388 ret
1389 .v: ; vertical boxsum + ab
1390 lea r10, [wq-2]
1391 .v_loop:
1392 mova m1, [t1+r10*2+400*0]
1393 paddw m1, m1
1394 paddw m1, [t2+r10*2+400*0] ; hv sum
1395 mova m2, [t1+r10*2+400*2]
1396 mova m3, [t1+r10*2+400*4]
1397 paddd m2, m2
1398 paddd m3, m3
1399 paddd m2, [t2+r10*2+400*2] ; hv sumsq
1400 paddd m3, [t2+r10*2+400*4]
1401 punpcklwd m0, m1, m6 ; b
1402 punpckhwd m1, m6
1403 pslld m4, m2, 3
1404 pslld m5, m3, 3
1405 paddd m4, m2 ; a * 9
1406 pmaddwd m2, m0, m0 ; b * b
1407 paddd m5, m3
1408 pmaddwd m3, m1, m1
1409 psubd m4, m2 ; p
1410 psubd m5, m3
1411 pmulld m4, m11 ; p * s
1412 pmulld m5, m11
1413 pmaddwd m0, m12 ; b * 455
1414 pmaddwd m1, m12
1415 paddusw m4, m12
1416 paddusw m5, m12
1417 psrad m3, m4, 20 ; min(z, 255) - 256
1418 vpgatherdd m2, [r14+m3*4], m4
1419 psrad m4, m5, 20
1420 vpgatherdd m3, [r14+m4*4], m5
1421 pmulld m0, m2
1422 pmulld m1, m3
1423 paddd m0, m13 ; x * b * 455 + (1 << 11) + (1 << 15)
1424 paddd m1, m13
1425 pand m0, m14
1426 pand m1, m14
1427 por m0, m2 ; a | (b << 12)
1428 por m1, m3
1429 mova [t3+r10*4+ 8], xm0
1430 vextracti128 [t3+r10*4+40], m0, 1
1431 mova [t3+r10*4+24], xm1
1432 vextracti128 [t3+r10*4+56], m1, 1
1433 add r10, 16
1434 jl .v_loop
1435 ret
1436 .prep_n: ; initial neighbor setup
1437 mov r10, wq
1438 mov t4, t3
1439 add t3, 400*4
1440 .prep_n_loop:
1441 mova m2, [t5+r10*4+0]
1442 mova m3, [t4+r10*4+0]
1443 paddd m2, [t5+r10*4+8]
1444 paddd m3, [t4+r10*4+8]
1445 paddd m0, m2, [t5+r10*4+4]
1446 paddd m1, m3, [t4+r10*4+4]
1447 pslld m0, 2
1448 paddd m1, m1 ; ab[ 0] 222
1449 psubd m0, m2 ; ab[-1] 343
1450 mova [t3+r10*4+400*4], m1
1451 paddd m1, m1
1452 mova [t5+r10*4], m0
1453 psubd m1, m3 ; ab[ 0] 343
1454 mova [t4+r10*4], m1
1455 add r10, 8
1456 jl .prep_n_loop
1457 ret
1458 ; a+b are packed together in a single dword, but we can't do the
1459 ; full neighbor calculations before splitting them since we don't
1460 ; have sufficient precision. The solution is to do the calculations
1461 ; in two equal halves and split a and b before doing the final sum.
1462 ALIGN function_align
1463 .n: ; neighbor + output
1464 mov r10, wq
1465 .n_loop:
1466 mova m4, [t3+r10*4+ 0]
1467 paddd m4, [t3+r10*4+ 8]
1468 paddd m5, m4, [t3+r10*4+ 4]
1469 paddd m5, m5 ; ab[+1] 222
1470 mova m2, [t3+r10*4+400*4+ 0]
1471 paddd m0, m2, [t5+r10*4+ 0] ; ab[ 0] 222 + ab[-1] 343
1472 mova m3, [t3+r10*4+400*4+32]
1473 paddd m1, m3, [t5+r10*4+32]
1474 mova [t3+r10*4+400*4+ 0], m5
1475 paddd m5, m5
1476 psubd m5, m4 ; ab[+1] 343
1477 mova [t5+r10*4+ 0], m5
1478 paddd m2, m5 ; ab[ 0] 222 + ab[+1] 343
1479 mova m4, [t3+r10*4+32]
1480 paddd m4, [t3+r10*4+40]
1481 paddd m5, m4, [t3+r10*4+36]
1482 paddd m5, m5
1483 mova [t3+r10*4+400*4+32], m5
1484 paddd m5, m5
1485 psubd m5, m4
1486 mova [t5+r10*4+32], m5
1487 pandn m4, m14, m0
1488 psrld m0, 12
1489 paddd m3, m5
1490 pandn m5, m14, m2
1491 psrld m2, 12
1492 paddd m4, m5 ; a
1493 pandn m5, m14, m1
1494 psrld m1, 12
1495 paddd m0, m2 ; b + (1 << 8)
1496 pandn m2, m14, m3
1497 psrld m3, 12
1498 paddd m5, m2
1499 pmovzxbd m2, [dstq+r10+0]
1500 paddd m1, m3
1501 pmovzxbd m3, [dstq+r10+8]
1502 pmaddwd m4, m2 ; a * src
1503 pmaddwd m5, m3
1504 packssdw m2, m3
1505 psubd m0, m4 ; b - a * src + (1 << 8)
1506 psubd m1, m5
1507 psrad m0, 9
1508 psrad m1, 9
1509 packssdw m0, m1
1510 pmulhrsw m0, m7
1511 paddw m0, m2
1512 vextracti128 xm1, m0, 1
1513 packuswb xm0, xm1
1514 pshufd xm0, xm0, q3120
1515 mova [dstq+r10], xm0
1516 add r10, 16
1517 jl .n_loop
1518 mov r10, t5
1519 mov t5, t4
1520 mov t4, r10
1521 add dstq, strideq
1522 ret
1523
1524 cglobal sgr_filter_mix_8bpc, 4, 13, 16, 400*56+8, dst, stride, left, lpf, \
1525 w, h, edge, params
1526 %define base r12-sgr_x_by_x_avx2-256*4
1527 lea r12, [sgr_x_by_x_avx2+256*4]
1528 mov paramsq, r6mp
1529 mov wd, wm
1530 movifnidn hd, hm
1531 mov edged, r7m
1532 vbroadcasti128 m9, [base+sgr_shuf+0]
1533 vbroadcasti128 m10, [base+sgr_shuf+8]
1534 add lpfq, wq
1535 vbroadcasti128 m11, [base+sgr_shuf+2]
1536 vbroadcasti128 m12, [base+sgr_shuf+6]
1537 add dstq, wq
1538 vpbroadcastd m15, [paramsq+8] ; w0 w1
1539 lea t3, [rsp+wq*4+400*24+8]
1540 vpbroadcastd m13, [paramsq+0] ; s0
1541 pxor m7, m7
1542 vpbroadcastd m14, [paramsq+4] ; s1
1543 lea t1, [rsp+wq*2+12]
1544 neg wq
1545 psllw m15, 2 ; to reuse existing pd_m4096 register for rounding
1546 test edgeb, 4 ; LR_HAVE_TOP
1547 jz .no_top
1548 call .h_top
1549 add lpfq, strideq
1550 mov t2, t1
1551 call mangle(private_prefix %+ _sgr_filter_5x5_8bpc_avx2).top_fixup
1552 add t1, 400*12
1553 call .h_top
1554 lea r10, [lpfq+strideq*4]
1555 mov lpfq, dstq
1556 add r10, strideq
1557 mov [rsp], r10 ; below
1558 call .hv0
1559 .main:
1560 dec hd
1561 jz .height1
1562 add lpfq, strideq
1563 call .hv1
1564 call .prep_n
1565 sub hd, 2
1566 jl .extend_bottom
1567 .main_loop:
1568 add lpfq, strideq
1569 call .hv0
1570 test hd, hd
1571 jz .odd_height
1572 add lpfq, strideq
1573 call .hv1
1574 call .n0
1575 call .n1
1576 sub hd, 2
1577 jge .main_loop
1578 test edgeb, 8 ; LR_HAVE_BOTTOM
1579 jz .extend_bottom
1580 mov lpfq, [rsp]
1581 call .hv0_bottom
1582 add lpfq, strideq
1583 call .hv1_bottom
1584 .end:
1585 call .n0
1586 call .n1
1587 .end2:
1588 RET
1589 .height1:
1590 call .v1
1591 call .prep_n
1592 jmp .odd_height_end
1593 .odd_height:
1594 call .v1
1595 call .n0
1596 call .n1
1597 .odd_height_end:
1598 call .v0
1599 call .v1
1600 call .n0
1601 jmp .end2
1602 .extend_bottom:
1603 call .v0
1604 call .v1
1605 jmp .end
1606 .no_top:
1607 lea r10, [lpfq+strideq*4]
1608 mov lpfq, dstq
1609 lea r10, [r10+strideq*2]
1610 mov [rsp], r10
1611 call .h
1612 lea t2, [t1+400*12]
1613 lea r10, [wq-2]
1614 .top_fixup_loop:
1615 mova m0, [t1+r10*2+400* 0]
1616 mova m1, [t1+r10*2+400* 2]
1617 mova m2, [t1+r10*2+400* 4]
1618 paddw m0, m0
1619 mova m3, [t1+r10*2+400* 6]
1620 paddd m1, m1
1621 mova m4, [t1+r10*2+400* 8]
1622 paddd m2, m2
1623 mova m5, [t1+r10*2+400*10]
1624 mova [t2+r10*2+400* 0], m0
1625 mova [t2+r10*2+400* 2], m1
1626 mova [t2+r10*2+400* 4], m2
1627 mova [t2+r10*2+400* 6], m3
1628 mova [t2+r10*2+400* 8], m4
1629 mova [t2+r10*2+400*10], m5
1630 add r10, 16
1631 jl .top_fixup_loop
1632 call .v0
1633 jmp .main
1634 .h: ; horizontal boxsums
1635 lea r10, [wq-2]
1636 test edgeb, 1 ; LR_HAVE_LEFT
1637 jz .h_extend_left
1638 vpbroadcastd xm0, [leftq]
1639 mova xm5, [lpfq+wq]
1640 palignr xm5, xm0, 12
1641 add leftq, 4
1642 jmp .h_main
1643 .h_extend_left:
1644 mova xm5, [lpfq+wq]
1645 pshufb xm5, [base+sgr_l_shuf]
1646 jmp .h_main
1647 .h_top:
1648 lea r10, [wq-2]
1649 test edgeb, 1 ; LR_HAVE_LEFT
1650 jz .h_extend_left
1651 .h_loop:
1652 movu xm5, [lpfq+r10-2]
1653 .h_main:
1654 vinserti128 m5, [lpfq+r10+6], 1
1655 test edgeb, 2 ; LR_HAVE_RIGHT
1656 jnz .h_have_right
1657 cmp r10d, -18
1658 jl .h_have_right
1659 call mangle(private_prefix %+ _sgr_filter_5x5_8bpc_avx2).extend_right
1660 .h_have_right:
1661 pshufb m6, m5, m9
1662 pshufb m4, m5, m10
1663 paddw m8, m6, m4
1664 shufps m0, m6, m4, q2121
1665 pmullw m3, m0, m0
1666 pshufb m2, m5, m11
1667 paddw m0, m2
1668 pshufb m5, m12
1669 paddw m0, m5 ; sum3
1670 punpcklwd m1, m2, m5
1671 pmaddwd m1, m1
1672 punpckhwd m2, m5
1673 pmaddwd m2, m2
1674 punpcklwd m5, m6, m4
1675 pmaddwd m5, m5
1676 punpckhwd m6, m4
1677 pmaddwd m6, m6
1678 punpcklwd m4, m3, m7
1679 paddd m1, m4 ; sumsq3
1680 punpckhwd m3, m7
1681 paddd m2, m3
1682 mova [t1+r10*2+400* 6], m0
1683 mova [t1+r10*2+400* 8], m1
1684 mova [t1+r10*2+400*10], m2
1685 paddw m8, m0 ; sum5
1686 paddd m5, m1 ; sumsq5
1687 paddd m6, m2
1688 mova [t1+r10*2+400* 0], m8
1689 mova [t1+r10*2+400* 2], m5
1690 mova [t1+r10*2+400* 4], m6
1691 add r10, 16
1692 jl .h_loop
1693 ret
1694 ALIGN function_align
1695 .hv0: ; horizontal boxsums + vertical boxsum3 + ab3 (even rows)
1696 lea r10, [wq-2]
1697 test edgeb, 1 ; LR_HAVE_LEFT
1698 jz .hv0_extend_left
1699 vpbroadcastd xm0, [leftq]
1700 mova xm5, [lpfq+wq]
1701 palignr xm5, xm0, 12
1702 add leftq, 4
1703 jmp .hv0_main
1704 .hv0_extend_left:
1705 mova xm5, [lpfq+wq]
1706 pshufb xm5, [base+sgr_l_shuf]
1707 jmp .hv0_main
1708 .hv0_bottom:
1709 lea r10, [wq-2]
1710 test edgeb, 1 ; LR_HAVE_LEFT
1711 jz .hv0_extend_left
1712 .hv0_loop:
1713 movu xm5, [lpfq+r10-2]
1714 .hv0_main:
1715 vinserti128 m5, [lpfq+r10+6], 1
1716 test edgeb, 2 ; LR_HAVE_RIGHT
1717 jnz .hv0_have_right
1718 cmp r10d, -18
1719 jl .hv0_have_right
1720 call mangle(private_prefix %+ _sgr_filter_5x5_8bpc_avx2).extend_right
1721 .hv0_have_right:
1722 pshufb m6, m5, m9
1723 pshufb m4, m5, m10
1724 paddw m8, m6, m4
1725 shufps m1, m6, m4, q2121
1726 pmullw m0, m1, m1
1727 pshufb m3, m5, m11
1728 paddw m1, m3
1729 pshufb m5, m12
1730 paddw m1, m5 ; sum3
1731 punpcklwd m2, m3, m5
1732 pmaddwd m2, m2
1733 punpckhwd m3, m5
1734 pmaddwd m3, m3
1735 punpcklwd m5, m6, m4
1736 pmaddwd m5, m5
1737 punpckhwd m6, m4
1738 pmaddwd m6, m6
1739 punpcklwd m4, m0, m7
1740 paddd m2, m4 ; sumsq3
1741 punpckhwd m0, m7
1742 paddd m3, m0
1743 paddw m8, m1 ; sum5
1744 paddd m5, m2 ; sumsq5
1745 paddd m6, m3
1746 mova [t3+r10*4+400*8+ 8], m8 ; we need a clean copy of the last row
1747 mova [t3+r10*4+400*0+ 8], m5 ; in case height is odd
1748 mova [t3+r10*4+400*0+40], m6
1749 paddw m8, [t1+r10*2+400* 0]
1750 paddd m5, [t1+r10*2+400* 2]
1751 paddd m6, [t1+r10*2+400* 4]
1752 mova [t1+r10*2+400* 0], m8
1753 mova [t1+r10*2+400* 2], m5
1754 mova [t1+r10*2+400* 4], m6
1755 paddw m0, m1, [t1+r10*2+400* 6]
1756 paddd m4, m2, [t1+r10*2+400* 8]
1757 paddd m5, m3, [t1+r10*2+400*10]
1758 mova [t1+r10*2+400* 6], m1
1759 mova [t1+r10*2+400* 8], m2
1760 mova [t1+r10*2+400*10], m3
1761 paddw m1, m0, [t2+r10*2+400* 6]
1762 paddd m2, m4, [t2+r10*2+400* 8]
1763 paddd m3, m5, [t2+r10*2+400*10]
1764 mova [t2+r10*2+400* 6], m0
1765 mova [t2+r10*2+400* 8], m4
1766 mova [t2+r10*2+400*10], m5
1767 punpcklwd m0, m1, m7 ; b3
1768 punpckhwd m1, m7
1769 pslld m4, m2, 3
1770 pslld m5, m3, 3
1771 paddd m4, m2 ; a3 * 9
1772 pmaddwd m2, m0, m0 ; b3 * b
1773 paddd m5, m3
1774 pmaddwd m3, m1, m1
1775 psubd m4, m2 ; p3
1776 vpbroadcastd m2, [base+pd_0xf00801c7]
1777 psubd m5, m3
1778 pmulld m4, m14 ; p3 * s1
1779 pmulld m5, m14
1780 pmaddwd m0, m2 ; b3 * 455
1781 pmaddwd m1, m2
1782 paddusw m4, m2
1783 paddusw m5, m2
1784 psrad m3, m4, 20 ; min(z3, 255) - 256
1785 vpgatherdd m2, [r12+m3*4], m4
1786 psrad m4, m5, 20
1787 vpgatherdd m3, [r12+m4*4], m5
1788 vpbroadcastd m4, [base+pd_34816]
1789 pmulld m0, m2
1790 vpbroadcastd m5, [base+pd_m4096]
1791 pmulld m1, m3
1792 paddd m0, m4 ; x3 * b3 * 455 + (1 << 11) + (1 << 15)
1793 paddd m1, m4
1794 pand m0, m5
1795 pand m1, m5
1796 por m0, m2 ; a3 | (b3 << 12)
1797 por m1, m3
1798 mova [t3+r10*4+400*4+ 8], xm0
1799 vextracti128 [t3+r10*4+400*4+40], m0, 1
1800 mova [t3+r10*4+400*4+24], xm1
1801 vextracti128 [t3+r10*4+400*4+56], m1, 1
1802 add r10, 16
1803 jl .hv0_loop
1804 ret
1805 ALIGN function_align
1806 .hv1: ; horizontal boxsums + vertical boxsums + ab (odd rows)
1807 lea r10, [wq-2]
1808 test edgeb, 1 ; LR_HAVE_LEFT
1809 jz .hv1_extend_left
1810 vpbroadcastd xm0, [leftq]
1811 mova xm5, [lpfq+wq]
1812 palignr xm5, xm0, 12
1813 add leftq, 4
1814 jmp .hv1_main
1815 .hv1_extend_left:
1816 mova xm5, [lpfq+wq]
1817 pshufb xm5, [base+sgr_l_shuf]
1818 jmp .hv1_main
1819 .hv1_bottom:
1820 lea r10, [wq-2]
1821 test edgeb, 1 ; LR_HAVE_LEFT
1822 jz .hv1_extend_left
1823 .hv1_loop:
1824 movu xm5, [lpfq+r10-2]
1825 .hv1_main:
1826 vinserti128 m5, [lpfq+r10+6], 1
1827 test edgeb, 2 ; LR_HAVE_RIGHT
1828 jnz .hv1_have_right
1829 cmp r10d, -18
1830 jl .hv1_have_right
1831 call mangle(private_prefix %+ _sgr_filter_5x5_8bpc_avx2).extend_right
1832 .hv1_have_right:
1833 pshufb m6, m5, m9
1834 pshufb m3, m5, m10
1835 paddw m8, m6, m3
1836 shufps m2, m6, m3, q2121
1837 pmullw m1, m2, m2
1838 pshufb m0, m5, m11
1839 paddw m2, m0
1840 pshufb m5, m12
1841 paddw m2, m5 ; sum3
1842 punpcklwd m4, m5, m0
1843 pmaddwd m4, m4
1844 punpckhwd m5, m0
1845 pmaddwd m5, m5
1846 punpcklwd m0, m6, m3
1847 pmaddwd m0, m0
1848 punpckhwd m6, m3
1849 pmaddwd m6, m6
1850 punpcklwd m3, m1, m7
1851 paddd m4, m3 ; sumsq3
1852 punpckhwd m1, m7
1853 paddd m5, m1
1854 paddw m1, m2, [t2+r10*2+400* 6]
1855 mova [t2+r10*2+400* 6], m2
1856 paddw m8, m2 ; sum5
1857 paddd m2, m4, [t2+r10*2+400* 8]
1858 paddd m3, m5, [t2+r10*2+400*10]
1859 mova [t2+r10*2+400* 8], m4
1860 mova [t2+r10*2+400*10], m5
1861 paddd m4, m0 ; sumsq5
1862 paddd m5, m6
1863 punpcklwd m0, m1, m7 ; b3
1864 punpckhwd m1, m7
1865 pslld m6, m2, 3
1866 pslld m7, m3, 3
1867 paddd m6, m2 ; a3 * 9
1868 pmaddwd m2, m0, m0 ; b3 * b3
1869 paddd m7, m3
1870 pmaddwd m3, m1, m1
1871 psubd m6, m2 ; p3
1872 vpbroadcastd m2, [base+pd_0xf00801c7]
1873 psubd m7, m3
1874 pmulld m6, m14 ; p3 * s1
1875 pmulld m7, m14
1876 pmaddwd m0, m2 ; b3 * 455
1877 pmaddwd m1, m2
1878 paddusw m6, m2
1879 paddusw m7, m2
1880 psrad m3, m6, 20 ; min(z3, 255) - 256
1881 vpgatherdd m2, [r12+m3*4], m6
1882 psrad m6, m7, 20
1883 vpgatherdd m3, [r12+m6*4], m7
1884 vpbroadcastd m6, [base+pd_34816] ; x3
1885 pmulld m0, m2
1886 vpbroadcastd m7, [base+pd_m4096]
1887 pmulld m1, m3
1888 paddd m0, m6 ; x3 * b3 * 455 + (1 << 11) + (1 << 15)
1889 paddd m1, m6
1890 pand m0, m7
1891 pand m7, m1
1892 por m0, m2 ; a3 | (b3 << 12)
1893 por m7, m3
1894 paddw m1, m8, [t2+r10*2+400*0]
1895 paddd m2, m4, [t2+r10*2+400*2]
1896 paddd m3, m5, [t2+r10*2+400*4]
1897 paddw m1, [t1+r10*2+400*0]
1898 paddd m2, [t1+r10*2+400*2]
1899 paddd m3, [t1+r10*2+400*4]
1900 mova [t2+r10*2+400*0], m8
1901 mova [t2+r10*2+400*2], m4
1902 mova [t2+r10*2+400*4], m5
1903 mova [t3+r10*4+400*8+ 8], xm0
1904 vextracti128 [t3+r10*4+400*8+40], m0, 1
1905 mova [t3+r10*4+400*8+24], xm7
1906 vextracti128 [t3+r10*4+400*8+56], m7, 1
1907 vpbroadcastd m4, [base+pd_25]
1908 pxor m7, m7
1909 punpcklwd m0, m1, m7 ; b5
1910 punpckhwd m1, m7
1911 pmulld m2, m4 ; a5 * 25
1912 pmulld m3, m4
1913 pmaddwd m4, m0, m0 ; b5 * b5
1914 pmaddwd m5, m1, m1
1915 psubd m2, m4 ; p5
1916 vpbroadcastd m4, [base+pd_0xf00800a4]
1917 psubd m3, m5
1918 pmulld m2, m13 ; p5 * s0
1919 pmulld m3, m13
1920 pmaddwd m0, m4 ; b5 * 164
1921 pmaddwd m1, m4
1922 paddusw m2, m4
1923 paddusw m3, m4
1924 psrad m5, m2, 20 ; min(z5, 255) - 256
1925 vpgatherdd m4, [r12+m5*4], m2 ; x5
1926 psrad m2, m3, 20
1927 vpgatherdd m5, [r12+m2*4], m3
1928 pmulld m0, m4
1929 pmulld m1, m5
1930 paddd m0, m6 ; x5 * b5 * 164 + (1 << 11) + (1 << 15)
1931 paddd m1, m6
1932 vpbroadcastd m6, [base+pd_m4096]
1933 pand m0, m6
1934 pand m1, m6
1935 por m0, m4 ; a5 | (b5 << 12)
1936 por m1, m5
1937 mova [t3+r10*4+400*0+ 8], xm0
1938 vextracti128 [t3+r10*4+400*0+40], m0, 1
1939 mova [t3+r10*4+400*0+24], xm1
1940 vextracti128 [t3+r10*4+400*0+56], m1, 1
1941 add r10, 16
1942 jl .hv1_loop
1943 mov r10, t2
1944 mov t2, t1
1945 mov t1, r10
1946 ret
1947 .v0: ; vertical boxsums + ab3 (even rows)
1948 lea r10, [wq-2]
1949 vpbroadcastd m6, [base+pd_34816]
1950 vpbroadcastd m8, [base+pd_m4096]
1951 .v0_loop:
1952 mova m0, [t1+r10*2+400* 6]
1953 mova m4, [t1+r10*2+400* 8]
1954 mova m5, [t1+r10*2+400*10]
1955 paddw m0, m0
1956 paddd m4, m4
1957 paddd m5, m5
1958 paddw m1, m0, [t2+r10*2+400* 6]
1959 paddd m2, m4, [t2+r10*2+400* 8]
1960 paddd m3, m5, [t2+r10*2+400*10]
1961 mova [t2+r10*2+400* 6], m0
1962 mova [t2+r10*2+400* 8], m4
1963 mova [t2+r10*2+400*10], m5
1964 punpcklwd m0, m1, m7 ; b3
1965 punpckhwd m1, m7
1966 pslld m4, m2, 3
1967 pslld m5, m3, 3
1968 paddd m4, m2 ; a3 * 9
1969 pmaddwd m2, m0, m0 ; b3 * b3
1970 paddd m5, m3
1971 pmaddwd m3, m1, m1
1972 psubd m4, m2 ; p3
1973 vpbroadcastd m2, [base+pd_0xf00801c7]
1974 psubd m5, m3
1975 pmulld m4, m14 ; p3 * s1
1976 pmulld m5, m14
1977 pmaddwd m0, m2 ; b3 * 455
1978 pmaddwd m1, m2
1979 paddusw m4, m2
1980 paddusw m5, m2
1981 psrad m3, m4, 20 ; min(z3, 255) - 256
1982 vpgatherdd m2, [r12+m3*4], m4 ; x3
1983 psrad m4, m5, 20
1984 vpgatherdd m3, [r12+m4*4], m5
1985 pmulld m0, m2
1986 pmulld m1, m3
1987 paddd m0, m6 ; x3 * b3 * 455 + (1 << 11) + (1 << 15)
1988 paddd m1, m6
1989 pand m0, m8
1990 pand m1, m8
1991 por m0, m2 ; a3 | (b3 << 12)
1992 por m1, m3
1993 mova m2, [t1+r10*2+400*0]
1994 mova m3, [t1+r10*2+400*2]
1995 mova m4, [t1+r10*2+400*4]
1996 mova [t3+r10*4+400*8+ 8], m2
1997 mova [t3+r10*4+400*0+ 8], m3
1998 mova [t3+r10*4+400*0+40], m4
1999 paddw m2, m2 ; cc5
2000 paddd m3, m3
2001 paddd m4, m4
2002 mova [t1+r10*2+400*0], m2
2003 mova [t1+r10*2+400*2], m3
2004 mova [t1+r10*2+400*4], m4
2005 mova [t3+r10*4+400*4+ 8], xm0
2006 vextracti128 [t3+r10*4+400*4+40], m0, 1
2007 mova [t3+r10*4+400*4+24], xm1
2008 vextracti128 [t3+r10*4+400*4+56], m1, 1
2009 add r10, 16
2010 jl .v0_loop
2011 ret
2012 .v1: ; vertical boxsums + ab (odd rows)
2013 lea r10, [wq-2]
2014 .v1_loop:
2015 mova m4, [t1+r10*2+400* 6]
2016 mova m5, [t1+r10*2+400* 8]
2017 mova m6, [t1+r10*2+400*10]
2018 paddw m1, m4, [t2+r10*2+400* 6]
2019 paddd m2, m5, [t2+r10*2+400* 8]
2020 paddd m3, m6, [t2+r10*2+400*10]
2021 mova [t2+r10*2+400* 6], m4
2022 mova [t2+r10*2+400* 8], m5
2023 mova [t2+r10*2+400*10], m6
2024 punpcklwd m0, m1, m7 ; b3
2025 punpckhwd m1, m7
2026 pslld m4, m2, 3
2027 pslld m5, m3, 3
2028 paddd m4, m2 ; a3 * 9
2029 pmaddwd m2, m0, m0 ; b3 * b3
2030 paddd m5, m3
2031 pmaddwd m3, m1, m1
2032 psubd m4, m2 ; p3
2033 vpbroadcastd m2, [base+pd_0xf00801c7]
2034 psubd m5, m3
2035 pmulld m4, m14 ; p3 * s1
2036 pmulld m5, m14
2037 pmaddwd m0, m2 ; b3 * 455
2038 pmaddwd m1, m2
2039 paddusw m4, m2
2040 paddusw m5, m2
2041 psrad m3, m4, 20 ; min(z3, 255) - 256
2042 vpgatherdd m2, [r12+m3*4], m4 ; x3
2043 psrad m4, m5, 20
2044 vpgatherdd m3, [r12+m4*4], m5
2045 vpbroadcastd m4, [base+pd_34816]
2046 pmulld m0, m2
2047 vpbroadcastd m8, [base+pd_m4096]
2048 pmulld m1, m3
2049 paddd m0, m4 ; x3 * b3 * 455 + (1 << 11) + (1 << 15)
2050 paddd m1, m4
2051 pand m0, m8
2052 pand m8, m1
2053 por m0, m2 ; a3 | (b3 << 12)
2054 por m8, m3
2055 mova m4, [t3+r10*4+400*8+ 8]
2056 mova m5, [t3+r10*4+400*0+ 8]
2057 mova m6, [t3+r10*4+400*0+40]
2058 paddw m1, m4, [t2+r10*2+400*0]
2059 paddd m2, m5, [t2+r10*2+400*2]
2060 paddd m3, m6, [t2+r10*2+400*4]
2061 paddw m1, [t1+r10*2+400*0]
2062 paddd m2, [t1+r10*2+400*2]
2063 paddd m3, [t1+r10*2+400*4]
2064 mova [t2+r10*2+400*0], m4
2065 mova [t2+r10*2+400*2], m5
2066 mova [t2+r10*2+400*4], m6
2067 vpbroadcastd m4, [base+pd_25]
2068 mova [t3+r10*4+400*8+ 8], xm0
2069 vextracti128 [t3+r10*4+400*8+40], m0, 1
2070 mova [t3+r10*4+400*8+24], xm8
2071 vextracti128 [t3+r10*4+400*8+56], m8, 1
2072 punpcklwd m0, m1, m7 ; b5
2073 punpckhwd m1, m7
2074 pmulld m2, m4 ; a5 * 25
2075 pmulld m3, m4
2076 pmaddwd m4, m0, m0 ; b5 * b5
2077 pmaddwd m5, m1, m1
2078 psubd m2, m4 ; p5
2079 vpbroadcastd m4, [base+pd_0xf00800a4]
2080 psubd m3, m5
2081 pmulld m2, m13 ; p5 * s0
2082 pmulld m3, m13
2083 pmaddwd m0, m4 ; b5 * 164
2084 pmaddwd m1, m4
2085 paddusw m2, m4
2086 paddusw m3, m4
2087 psrad m5, m2, 20 ; min(z5, 255) - 256
2088 vpgatherdd m4, [r12+m5*4], m2 ; x5
2089 psrad m2, m3, 20
2090 vpgatherdd m5, [r12+m2*4], m3
2091 pmulld m0, m4
2092 vpbroadcastd m6, [base+pd_34816]
2093 pmulld m1, m5
2094 paddd m0, m6 ; x5 * b5 * 164 + (1 << 11) + (1 << 15)
2095 paddd m1, m6
2096 vpbroadcastd m6, [base+pd_m4096]
2097 pand m0, m6
2098 pand m1, m6
2099 por m0, m4 ; a5 | (b5 << 12)
2100 por m1, m5
2101 mova [t3+r10*4+400*0+ 8], xm0
2102 vextracti128 [t3+r10*4+400*0+40], m0, 1
2103 mova [t3+r10*4+400*0+24], xm1
2104 vextracti128 [t3+r10*4+400*0+56], m1, 1
2105 add r10, 16
2106 jl .v1_loop
2107 mov r10, t2
2108 mov t2, t1
2109 mov t1, r10
2110 ret
2111 .prep_n: ; initial neighbor setup
2112 mov r10, wq
2113 .prep_n_loop:
2114 movu m0, [t3+r10*4+400*0+4]
2115 paddd m1, m0, [t3+r10*4+400*0+0]
2116 mova m4, [t3+r10*4+400*4+0]
2117 paddd m1, [t3+r10*4+400*0+8]
2118 mova m5, [t3+r10*4+400*8+0]
2119 paddd m4, [t3+r10*4+400*4+8]
2120 paddd m5, [t3+r10*4+400*8+8]
2121 paddd m2, m4, [t3+r10*4+400*4+4]
2122 paddd m3, m5, [t3+r10*4+400*8+4]
2123 paddd m0, m1
2124 pslld m1, 2
2125 pslld m2, 2
2126 paddd m1, m0 ; ab5 565
2127 paddd m3, m3 ; ab3[ 0] 222
2128 psubd m2, m4 ; ab3[-1] 343
2129 mova [t3+r10*4+400*20], m3
2130 pandn m0, m6, m1 ; a5 565
2131 mova [t3+r10*4+400*24], m2
2132 psrld m1, 12 ; b5 565
2133 mova [t3+r10*4+400*12], m0
2134 paddd m3, m3
2135 mova [t3+r10*4+400*16], m1
2136 psubd m3, m5 ; ab3[ 0] 343
2137 mova [t3+r10*4+400*28], m3
2138 add r10, 8
2139 jl .prep_n_loop
2140 ret
2141 ALIGN function_align
2142 .n0: ; neighbor + output (even rows)
2143 mov r10, wq
2144 .n0_loop:
2145 movu m0, [t3+r10*4+4]
2146 paddd m4, m0, [t3+r10*4+0]
2147 paddd m4, [t3+r10*4+8]
2148 paddd m0, m4
2149 pslld m4, 2
2150 paddd m4, m0
2151 pandn m0, m6, m4
2152 psrld m4, 12
2153 paddd m2, m0, [t3+r10*4+400*12] ; a5
2154 mova [t3+r10*4+400*12], m0
2155 paddd m0, m4, [t3+r10*4+400*16] ; b5 + (1 << 8)
2156 mova [t3+r10*4+400*16], m4
2157 mova m3, [t3+r10*4+400*4+0]
2158 paddd m3, [t3+r10*4+400*4+8]
2159 paddd m5, m3, [t3+r10*4+400*4+4]
2160 paddd m5, m5 ; ab3[ 1] 222
2161 mova m4, [t3+r10*4+400*20]
2162 paddd m1, m4, [t3+r10*4+400*24] ; ab3[ 0] 222 + ab3[-1] 343
2163 mova [t3+r10*4+400*20], m5
2164 paddd m5, m5
2165 psubd m5, m3 ; ab3[ 1] 343
2166 mova [t3+r10*4+400*24], m5
2167 paddd m4, m5 ; ab3[ 0] 222 + ab3[ 1] 343
2168 pandn m3, m6, m1
2169 psrld m1, 12
2170 pandn m5, m6, m4
2171 psrld m4, 12
2172 paddd m3, m5 ; a3
2173 paddd m1, m4 ; b3 + (1 << 8)
2174 pmovzxbd m4, [dstq+r10]
2175 pmaddwd m2, m4 ; a5 * src
2176 pmaddwd m3, m4 ; a3 * src
2177 psubd m0, m2 ; b5 - a5 * src + (1 << 8)
2178 psubd m1, m3 ; b3 - a3 * src + (1 << 8)
2179 psrld m0, 9
2180 pslld m1, 7
2181 pblendw m0, m1, 0xaa
2182 pmaddwd m0, m15
2183 psubd m0, m6
2184 psrad m0, 13
2185 paddd m0, m4
2186 vextracti128 xm1, m0, 1
2187 packssdw xm0, xm1
2188 packuswb xm0, xm0
2189 movq [dstq+r10], xm0
2190 add r10, 8
2191 jl .n0_loop
2192 add dstq, strideq
2193 ret
2194 ALIGN function_align
2195 .n1: ; neighbor + output (odd rows)
2196 mov r10, wq
2197 .n1_loop:
2198 mova m3, [t3+r10*4+400*8+0]
2199 paddd m3, [t3+r10*4+400*8+8]
2200 paddd m5, m3, [t3+r10*4+400*8+4]
2201 paddd m5, m5 ; ab3[ 1] 222
2202 mova m4, [t3+r10*4+400*20]
2203 paddd m1, m4, [t3+r10*4+400*28] ; ab3[ 0] 222 + ab3[-1] 343
2204 mova [t3+r10*4+400*20], m5
2205 paddd m5, m5
2206 psubd m5, m3 ; ab3[ 1] 343
2207 mova [t3+r10*4+400*28], m5
2208 paddd m4, m5 ; ab3[ 0] 222 + ab3[ 1] 343
2209 pandn m3, m6, m1
2210 psrld m1, 12
2211 pandn m5, m6, m4
2212 psrld m4, 12
2213 paddd m3, m5 ; -a3
2214 paddd m1, m4 ; b3 + (1 << 8)
2215 pmovzxbd m4, [dstq+r10]
2216 pmaddwd m2, m4, [t3+r10*4+400*12] ; -a5 * src
2217 mova m0, [t3+r10*4+400*16] ; b5 + (1 << 7)
2218 pmaddwd m3, m4 ; -a3 * src
2219 psubd m0, m2 ; a5 * src + b5 + (1 << 7)
2220 psubd m1, m3 ; a3 * src + b3 + (1 << 8)
2221 psrld m0, 8
2222 pslld m1, 7
2223 pblendw m0, m1, 0xaa
2224 pmaddwd m0, m15
2225 psubd m0, m6
2226 psrad m0, 13
2227 paddd m0, m4
2228 vextracti128 xm1, m0, 1
2229 packssdw xm0, xm1
2230 packuswb xm0, xm0
2231 movq [dstq+r10], xm0
2232 add r10, 8
2233 jl .n1_loop
2234 add dstq, strideq
2235 ret
2236
2237 %endif ; ARCH_X86_64