Debian Codegraph @

.cargo/vendor/rav1e/src/x86/msac.asm

raw
1 ; Copyright © 2019, VideoLAN and dav1d authors
2 ; Copyright © 2019, Two Orioles, LLC
3 ; All rights reserved.
4 ;
5 ; Redistribution and use in source and binary forms, with or without
6 ; modification, are permitted provided that the following conditions are met:
7 ;
8 ; 1. Redistributions of source code must retain the above copyright notice, this
9 ; list of conditions and the following disclaimer.
10 ;
11 ; 2. Redistributions in binary form must reproduce the above copyright notice,
12 ; this list of conditions and the following disclaimer in the documentation
13 ; and/or other materials provided with the distribution.
14 ;
15 ; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
16 ; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
17 ; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
18 ; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
19 ; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
20 ; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
21 ; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
22 ; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 ; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
24 ; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25
26 %include "config.asm"
27 %include "ext/x86/x86inc.asm"
28
29 SECTION_RODATA 64 ; avoids cacheline splits
30
31 min_prob: dw 60, 56, 52, 48, 44, 40, 36, 32, 28, 24, 20, 16, 12, 8, 4, 0
32 pw_0xff00: times 8 dw 0xff00
33 pw_32: times 8 dw 32
34
35 %if ARCH_X86_64
36 %define resp resq
37 %define movp movq
38 %define c_shuf q3333
39 %macro DECODE_SYMBOL_ADAPT_INIT 0-1
40 %endmacro
41 %else
42 %define resp resd
43 %define movp movd
44 %define c_shuf q1111
45 %macro DECODE_SYMBOL_ADAPT_INIT 0-1 0 ; hi_tok
46 mov t0, r0m
47 mov t1, r1m
48 %if %1 == 0
49 mov t2, r2m
50 %endif
51 %if STACK_ALIGNMENT >= 16
52 sub esp, 40-%1*4
53 %else
54 mov eax, esp
55 and esp, ~15
56 sub esp, 40-%1*4
57 mov [esp], eax
58 %endif
59 %endmacro
60 %endif
61
62 struc msac
63 .buf: resp 1
64 .end: resp 1
65 .dif: resp 1
66 .rng: resd 1
67 .cnt: resd 1
68 .update_cdf: resd 1
69 endstruc
70
71 %define m(x, y) mangle(private_prefix %+ _ %+ x %+ y)
72
73 SECTION .text
74
75 %if WIN64
76 DECLARE_REG_TMP 0, 1, 2, 3, 4, 5, 7, 3, 8
77 %define buf rsp+stack_offset+8 ; shadow space
78 %elif UNIX64
79 DECLARE_REG_TMP 0, 1, 2, 3, 4, 5, 7, 0, 8
80 %define buf rsp-40 ; red zone
81 %else
82 DECLARE_REG_TMP 2, 3, 4, 1, 5, 6, 5, 2, 3
83 %define buf esp+8
84 %endif
85
86 INIT_XMM sse2
87 cglobal msac_decode_symbol_adapt4, 0, 6, 6
88 DECODE_SYMBOL_ADAPT_INIT
89 LEA rax, pw_0xff00
90 movd m2, [t0+msac.rng]
91 movq m1, [t1]
92 movp m3, [t0+msac.dif]
93 mov t3d, [t0+msac.update_cdf]
94 mov t4d, t2d
95 not t2 ; -(n_symbols + 1)
96 pshuflw m2, m2, q0000
97 movd [buf+12], m2
98 pand m2, [rax]
99 mova m0, m1
100 psrlw m1, 6
101 psllw m1, 7
102 pmulhuw m1, m2
103 movq m2, [rax+t2*2]
104 pshuflw m3, m3, c_shuf
105 paddw m1, m2
106 mova [buf+16], m1
107 psubusw m1, m3
108 pxor m2, m2
109 pcmpeqw m1, m2 ; c >= v
110 pmovmskb eax, m1
111 test t3d, t3d
112 jz .renorm ; !allow_update_cdf
113
114 ; update_cdf:
115 movzx t3d, word [t1+t4*2] ; count
116 pcmpeqw m2, m2
117 mov t2d, t3d
118 shr t3d, 4
119 cmp t4d, 3
120 sbb t3d, -5 ; (count >> 4) + (n_symbols > 2) + 4
121 cmp t2d, 32
122 adc t2d, 0 ; count + (count < 32)
123 movd m3, t3d
124 pavgw m2, m1 ; i >= val ? -1 : 32768
125 psubw m2, m0 ; for (i = 0; i < val; i++)
126 psubw m0, m1 ; cdf[i] += (32768 - cdf[i]) >> rate;
127 psraw m2, m3 ; for (; i < n_symbols; i++)
128 paddw m0, m2 ; cdf[i] += (( -1 - cdf[i]) >> rate) + 1;
129 movq [t1], m0
130 mov [t1+t4*2], t2w
131
132 .renorm:
133 tzcnt eax, eax
134 mov t4, [t0+msac.dif]
135 movzx t1d, word [buf+rax+16] ; v
136 movzx t2d, word [buf+rax+14] ; u
137 shr eax, 1
138 .renorm2:
139 %if ARCH_X86_64 == 0
140 %if STACK_ALIGNMENT >= 16
141 add esp, 40
142 %else
143 mov esp, [esp]
144 %endif
145 %endif
146 not t4
147 sub t2d, t1d ; rng
148 shl t1, gprsize*8-16
149 add t4, t1 ; ~dif
150 .renorm3:
151 mov t1d, [t0+msac.cnt]
152 movifnidn t7, t0
153 .renorm4:
154 bsr ecx, t2d
155 xor ecx, 15 ; d
156 .renorm5:
157 shl t2d, cl
158 shl t4, cl
159 mov [t7+msac.rng], t2d
160 not t4
161 sub t1d, ecx
162 jae .end ; no refill required
163
164 ; refill:
165 mov t2, [t7+msac.buf]
166 mov rcx, [t7+msac.end]
167 %if ARCH_X86_64 == 0
168 push t5
169 %endif
170 lea t5, [t2+gprsize]
171 cmp t5, rcx
172 ja .refill_eob
173 mov t2, [t2]
174 lea ecx, [t1+23]
175 add t1d, 16
176 shr ecx, 3 ; shift_bytes
177 bswap t2
178 sub t5, rcx
179 shl ecx, 3 ; shift_bits
180 shr t2, cl
181 sub ecx, t1d ; shift_bits - 16 - cnt
182 mov t1d, gprsize*8-16
183 shl t2, cl
184 mov [t7+msac.buf], t5
185 sub t1d, ecx ; cnt + gprsize*8 - shift_bits
186 xor t4, t2
187 %if ARCH_X86_64 == 0
188 pop t5
189 %endif
190 .end:
191 mov [t7+msac.cnt], t1d
192 mov [t7+msac.dif], t4
193 RET
194 .refill_eob: ; avoid overreading the input buffer
195 mov t5, rcx
196 mov ecx, gprsize*8-24
197 sub ecx, t1d ; c
198 .refill_eob_loop:
199 cmp t2, t5
200 jae .refill_eob_end ; eob reached
201 movzx t1d, byte [t2]
202 inc t2
203 shl t1, cl
204 xor t4, t1
205 sub ecx, 8
206 jge .refill_eob_loop
207 .refill_eob_end:
208 mov t1d, gprsize*8-24
209 %if ARCH_X86_64 == 0
210 pop t5
211 %endif
212 sub t1d, ecx
213 mov [t7+msac.buf], t2
214 mov [t7+msac.dif], t4
215 mov [t7+msac.cnt], t1d
216 RET
217
218 cglobal msac_decode_symbol_adapt8, 0, 6, 6
219 DECODE_SYMBOL_ADAPT_INIT
220 LEA rax, pw_0xff00
221 movd m2, [t0+msac.rng]
222 mova m1, [t1]
223 movp m3, [t0+msac.dif]
224 mov t3d, [t0+msac.update_cdf]
225 mov t4d, t2d
226 not t2
227 pshuflw m2, m2, q0000
228 movd [buf+12], m2
229 punpcklqdq m2, m2
230 mova m0, m1
231 psrlw m1, 6
232 pand m2, [rax]
233 psllw m1, 7
234 pmulhuw m1, m2
235 movu m2, [rax+t2*2]
236 pshuflw m3, m3, c_shuf
237 paddw m1, m2
238 punpcklqdq m3, m3
239 mova [buf+16], m1
240 psubusw m1, m3
241 pxor m2, m2
242 pcmpeqw m1, m2
243 pmovmskb eax, m1
244 test t3d, t3d
245 jz m(msac_decode_symbol_adapt4, SUFFIX).renorm
246 movzx t3d, word [t1+t4*2]
247 pcmpeqw m2, m2
248 mov t2d, t3d
249 shr t3d, 4
250 cmp t4d, 3 ; may be called with n_symbols <= 2
251 sbb t3d, -5
252 cmp t2d, 32
253 adc t2d, 0
254 movd m3, t3d
255 pavgw m2, m1
256 psubw m2, m0
257 psubw m0, m1
258 psraw m2, m3
259 paddw m0, m2
260 mova [t1], m0
261 mov [t1+t4*2], t2w
262 jmp m(msac_decode_symbol_adapt4, SUFFIX).renorm
263
264 cglobal msac_decode_symbol_adapt16, 0, 6, 6
265 DECODE_SYMBOL_ADAPT_INIT
266 LEA rax, pw_0xff00
267 movd m4, [t0+msac.rng]
268 mova m2, [t1]
269 mova m3, [t1+16]
270 movp m5, [t0+msac.dif]
271 mov t3d, [t0+msac.update_cdf]
272 mov t4d, t2d
273 not t2
274 %if WIN64
275 sub rsp, 48 ; need 36 bytes, shadow space is only 32
276 %endif
277 pshuflw m4, m4, q0000
278 movd [buf-4], m4
279 punpcklqdq m4, m4
280 mova m0, m2
281 psrlw m2, 6
282 mova m1, m3
283 psrlw m3, 6
284 pand m4, [rax]
285 psllw m2, 7
286 psllw m3, 7
287 pmulhuw m2, m4
288 pmulhuw m3, m4
289 movu m4, [rax+t2*2]
290 pshuflw m5, m5, c_shuf
291 paddw m2, m4
292 psubw m4, [rax-pw_0xff00+pw_32]
293 punpcklqdq m5, m5
294 paddw m3, m4
295 mova [buf], m2
296 psubusw m2, m5
297 mova [buf+16], m3
298 psubusw m3, m5
299 pxor m4, m4
300 pcmpeqw m2, m4
301 pcmpeqw m3, m4
302 packsswb m5, m2, m3
303 pmovmskb eax, m5
304 test t3d, t3d
305 jz .renorm
306 movzx t3d, word [t1+t4*2]
307 pcmpeqw m4, m4
308 mova m5, m4
309 lea t2d, [t3+80] ; only support n_symbols > 2
310 shr t2d, 4
311 cmp t3d, 32
312 adc t3d, 0
313 pavgw m4, m2
314 pavgw m5, m3
315 psubw m4, m0
316 psubw m0, m2
317 movd m2, t2d
318 psubw m5, m1
319 psubw m1, m3
320 psraw m4, m2
321 psraw m5, m2
322 paddw m0, m4
323 paddw m1, m5
324 mova [t1], m0
325 mova [t1+16], m1
326 mov [t1+t4*2], t3w
327 .renorm:
328 tzcnt eax, eax
329 mov t4, [t0+msac.dif]
330 movzx t1d, word [buf+rax*2]
331 movzx t2d, word [buf+rax*2-2]
332 %if WIN64
333 add rsp, 48
334 %endif
335 jmp m(msac_decode_symbol_adapt4, SUFFIX).renorm2
336
337 cglobal msac_decode_bool_adapt, 0, 6, 0
338 movifnidn t1, r1mp
339 movifnidn t0, r0mp
340 movzx eax, word [t1]
341 movzx t3d, byte [t0+msac.rng+1]
342 mov t4, [t0+msac.dif]
343 mov t2d, [t0+msac.rng]
344 %if ARCH_X86_64
345 mov t5d, eax
346 %endif
347 and eax, ~63
348 imul eax, t3d
349 %if UNIX64
350 mov t6, t4
351 %endif
352 shr eax, 7
353 add eax, 4 ; v
354 mov t3d, eax
355 shl rax, gprsize*8-16 ; vw
356 sub t2d, t3d ; r - v
357 sub t4, rax ; dif - vw
358 setb al
359 cmovb t2d, t3d
360 mov t3d, [t0+msac.update_cdf]
361 %if UNIX64
362 cmovb t4, t6
363 %else
364 cmovb t4, [t0+msac.dif]
365 %endif
366 %if ARCH_X86_64 == 0
367 movzx eax, al
368 %endif
369 not t4
370 test t3d, t3d
371 jz m(msac_decode_symbol_adapt4, SUFFIX).renorm3
372 %if UNIX64 == 0
373 push t6
374 %endif
375 movzx t6d, word [t1+2]
376 %if ARCH_X86_64 == 0
377 push t5
378 movzx t5d, word [t1]
379 %endif
380 movifnidn t7, t0
381 lea ecx, [t6+64]
382 cmp t6d, 32
383 adc t6d, 0
384 mov [t1+2], t6w
385 imul t6d, eax, -32769
386 shr ecx, 4 ; rate
387 add t6d, t5d ; if (bit)
388 sub t5d, eax ; cdf[0] -= ((cdf[0] - 32769) >> rate) + 1;
389 sar t6d, cl ; else
390 sub t5d, t6d ; cdf[0] -= cdf[0] >> rate;
391 mov [t1], t5w
392 %if WIN64
393 mov t1d, [t7+msac.cnt]
394 pop t6
395 jmp m(msac_decode_symbol_adapt4, SUFFIX).renorm4
396 %else
397 %if ARCH_X86_64 == 0
398 pop t5
399 pop t6
400 %endif
401 jmp m(msac_decode_symbol_adapt4, SUFFIX).renorm3
402 %endif
403
404 cglobal msac_decode_bool_equi, 0, 6, 0
405 movifnidn t0, r0mp
406 mov t1d, [t0+msac.rng]
407 mov t4, [t0+msac.dif]
408 mov t2d, t1d
409 mov t1b, 8
410 mov t3, t4
411 mov eax, t1d
412 shr t1d, 1 ; v
413 shl rax, gprsize*8-17 ; vw
414 sub t2d, t1d ; r - v
415 sub t4, rax ; dif - vw
416 cmovb t2d, t1d
417 mov t1d, [t0+msac.cnt]
418 cmovb t4, t3
419 movifnidn t7, t0
420 mov ecx, 0xbfff
421 setb al ; the upper 32 bits contains garbage but that's OK
422 sub ecx, t2d
423 not t4
424 ; In this case of this function, (d =) 16 - clz(v) = 2 - (v >> 14)
425 ; i.e. (0 <= d <= 2) and v < (3 << 14)
426 shr ecx, 14 ; d
427 %if ARCH_X86_64 == 0
428 movzx eax, al
429 %endif
430 jmp m(msac_decode_symbol_adapt4, SUFFIX).renorm5
431
432 cglobal msac_decode_bool, 0, 6, 0
433 movifnidn t0, r0mp
434 movifnidn t1d, r1m
435 movzx eax, byte [t0+msac.rng+1] ; r >> 8
436 mov t4, [t0+msac.dif]
437 mov t2d, [t0+msac.rng]
438 and t1d, ~63
439 imul eax, t1d
440 mov t3, t4
441 shr eax, 7
442 add eax, 4 ; v
443 mov t1d, eax
444 shl rax, gprsize*8-16 ; vw
445 sub t2d, t1d ; r - v
446 sub t4, rax ; dif - vw
447 cmovb t2d, t1d
448 cmovb t4, t3
449 setb al
450 not t4
451 %if ARCH_X86_64 == 0
452 movzx eax, al
453 %endif
454 jmp m(msac_decode_symbol_adapt4, SUFFIX).renorm3
455
456 %macro HI_TOK 1 ; update_cdf
457 %if ARCH_X86_64 == 0
458 mov eax, -24
459 %endif
460 %%loop:
461 %if %1
462 movzx t2d, word [t1+3*2]
463 %endif
464 mova m1, m0
465 pshuflw m2, m2, q0000
466 psrlw m1, 6
467 movd [buf+12], m2
468 pand m2, m4
469 psllw m1, 7
470 pmulhuw m1, m2
471 %if ARCH_X86_64 == 0
472 add eax, 5
473 mov [buf+8], eax
474 %endif
475 pshuflw m3, m3, c_shuf
476 paddw m1, m5
477 movq [buf+16], m1
478 psubusw m1, m3
479 pxor m2, m2
480 pcmpeqw m1, m2
481 pmovmskb eax, m1
482 %if %1
483 lea ecx, [t2+80]
484 pcmpeqw m2, m2
485 shr ecx, 4
486 cmp t2d, 32
487 adc t2d, 0
488 movd m3, ecx
489 pavgw m2, m1
490 psubw m2, m0
491 psubw m0, m1
492 psraw m2, m3
493 paddw m0, m2
494 movq [t1], m0
495 mov [t1+3*2], t2w
496 %endif
497 tzcnt eax, eax
498 movzx ecx, word [buf+rax+16]
499 movzx t2d, word [buf+rax+14]
500 not t4
501 %if ARCH_X86_64
502 add t6d, 5
503 %endif
504 sub eax, 5 ; setup for merging the tok_br and tok branches
505 sub t2d, ecx
506 shl rcx, gprsize*8-16
507 add t4, rcx
508 bsr ecx, t2d
509 xor ecx, 15
510 shl t2d, cl
511 shl t4, cl
512 movd m2, t2d
513 mov [t7+msac.rng], t2d
514 not t4
515 sub t5d, ecx
516 jae %%end
517 mov t2, [t7+msac.buf]
518 mov rcx, [t7+msac.end]
519 %if UNIX64 == 0
520 push t8
521 %endif
522 lea t8, [t2+gprsize]
523 cmp t8, rcx
524 ja %%refill_eob
525 mov t2, [t2]
526 lea ecx, [t5+23]
527 add t5d, 16
528 shr ecx, 3
529 bswap t2
530 sub t8, rcx
531 shl ecx, 3
532 shr t2, cl
533 sub ecx, t5d
534 mov t5d, gprsize*8-16
535 shl t2, cl
536 mov [t7+msac.buf], t8
537 %if UNIX64 == 0
538 pop t8
539 %endif
540 sub t5d, ecx
541 xor t4, t2
542 %%end:
543 movp m3, t4
544 %if ARCH_X86_64
545 add t6d, eax ; CF = tok_br < 3 || tok == 15
546 jnc %%loop
547 lea eax, [t6+30]
548 %else
549 add eax, [buf+8]
550 jnc %%loop
551 add eax, 30
552 %if STACK_ALIGNMENT >= 16
553 add esp, 36
554 %else
555 mov esp, [esp]
556 %endif
557 %endif
558 mov [t7+msac.dif], t4
559 shr eax, 1
560 mov [t7+msac.cnt], t5d
561 RET
562 %%refill_eob:
563 mov t8, rcx
564 mov ecx, gprsize*8-24
565 sub ecx, t5d
566 %%refill_eob_loop:
567 cmp t2, t8
568 jae %%refill_eob_end
569 movzx t5d, byte [t2]
570 inc t2
571 shl t5, cl
572 xor t4, t5
573 sub ecx, 8
574 jge %%refill_eob_loop
575 %%refill_eob_end:
576 %if UNIX64 == 0
577 pop t8
578 %endif
579 mov t5d, gprsize*8-24
580 mov [t7+msac.buf], t2
581 sub t5d, ecx
582 jmp %%end
583 %endmacro
584
585 cglobal msac_decode_hi_tok, 0, 7 + ARCH_X86_64, 6
586 DECODE_SYMBOL_ADAPT_INIT 1
587 %if ARCH_X86_64 == 0 && PIC
588 LEA t2, min_prob+12*2
589 %define base t2-(min_prob+12*2)
590 %else
591 %define base 0
592 %endif
593 movq m0, [t1]
594 movd m2, [t0+msac.rng]
595 mov eax, [t0+msac.update_cdf]
596 movq m4, [base+pw_0xff00]
597 movp m3, [t0+msac.dif]
598 movq m5, [base+min_prob+12*2]
599 mov t4, [t0+msac.dif]
600 mov t5d, [t0+msac.cnt]
601 %if ARCH_X86_64
602 mov t6d, -24
603 %endif
604 movifnidn t7, t0
605 test eax, eax
606 jz .no_update_cdf
607 HI_TOK 1
608 .no_update_cdf:
609 HI_TOK 0
610
611 %if ARCH_X86_64
612 INIT_YMM avx2
613 cglobal msac_decode_symbol_adapt16, 3, 6, 6
614 lea rax, [pw_0xff00]
615 vpbroadcastw m2, [t0+msac.rng]
616 mova m0, [t1]
617 vpbroadcastw m3, [t0+msac.dif+6]
618 vbroadcasti128 m4, [rax]
619 mov t3d, [t0+msac.update_cdf]
620 mov t4d, t2d
621 not t2
622 mov r5, rsp
623 %if WIN64
624 and rsp, ~31
625 sub rsp, 40
626 %else
627 and r5, ~31
628 %define buf r5-32
629 %endif
630 psrlw m1, m0, 6
631 movd [buf-4], xm2
632 pand m2, m4
633 psllw m1, 7
634 pmulhuw m1, m2
635 paddw m1, [rax+t2*2]
636 mova [buf], m1
637 pmaxuw m1, m3
638 pcmpeqw m1, m3
639 pmovmskb eax, m1
640 test t3d, t3d
641 jz .renorm
642 movzx t3d, word [t1+t4*2]
643 pcmpeqw m2, m2
644 lea t2d, [t3+80]
645 shr t2d, 4
646 cmp t3d, 32
647 adc t3d, 0
648 movd xm3, t2d
649 pavgw m2, m1
650 psubw m2, m0
651 psubw m0, m1
652 psraw m2, xm3
653 paddw m0, m2
654 mova [t1], m0
655 mov [t1+t4*2], t3w
656 .renorm:
657 tzcnt eax, eax
658 mov t4, [t0+msac.dif]
659 movzx t1d, word [buf+rax-0]
660 movzx t2d, word [buf+rax-2]
661 shr eax, 1
662 %if WIN64
663 mov rsp, r5
664 %endif
665 vzeroupper
666 jmp m(msac_decode_symbol_adapt4, _sse2).renorm2
667 %endif