Debian Codegraph @

.cargo/vendor/rav1e/src/arm/32/msac.S

raw
1 /*
2 * Copyright © 2019, VideoLAN and dav1d authors
3 * Copyright © 2020, Martin Storsjo
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions are met:
8 *
9 * 1. Redistributions of source code must retain the above copyright notice, this
10 * list of conditions and the following disclaimer.
11 *
12 * 2. Redistributions in binary form must reproduce the above copyright notice,
13 * this list of conditions and the following disclaimer in the documentation
14 * and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
20 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
23 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28 #include "src/arm/asm.S"
29 #include "util.S"
30
31 #define BUF_POS 0
32 #define BUF_END 4
33 #define DIF 8
34 #define RNG 12
35 #define CNT 16
36 #define ALLOW_UPDATE_CDF 20
37
38 const coeffs
39 .short 60, 56, 52, 48, 44, 40, 36, 32, 28, 24, 20, 16, 12, 8, 4, 0
40 .short 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
41 endconst
42
43 const bits, align=4
44 .short 0x1, 0x2, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80
45 .short 0x100, 0x200, 0x400, 0x800, 0x1000, 0x2000, 0x4000, 0x8000
46 endconst
47
48 .macro vld1_align_n d0, q0, q1, src, n
49 .if \n == 4
50 vld1.16 {\d0}, [\src, :64]
51 .elseif \n == 8
52 vld1.16 {\q0}, [\src, :128]
53 .else
54 vld1.16 {\q0, \q1}, [\src, :128]
55 .endif
56 .endm
57
58 .macro vld1_n d0, q0, q1, src, n
59 .if \n == 4
60 vld1.16 {\d0}, [\src]
61 .elseif \n == 8
62 vld1.16 {\q0}, [\src]
63 .else
64 vld1.16 {\q0, \q1}, [\src]
65 .endif
66 .endm
67
68 .macro vst1_align_n d0, q0, q1, src, n
69 .if \n == 4
70 vst1.16 {\d0}, [\src, :64]
71 .elseif \n == 8
72 vst1.16 {\q0}, [\src, :128]
73 .else
74 vst1.16 {\q0, \q1}, [\src, :128]
75 .endif
76 .endm
77
78 .macro vst1_n d0, q0, q1, src, n
79 .if \n == 4
80 vst1.16 {\d0}, [\src]
81 .elseif \n == 8
82 vst1.16 {\q0}, [\src]
83 .else
84 vst1.16 {\q0, \q1}, [\src]
85 .endif
86 .endm
87
88 .macro vshr_n d0, d1, d2, s0, s1, s2, s3, s4, s5, n
89 .if \n == 4
90 vshr.u16 \d0, \s0, \s3
91 .else
92 vshr.u16 \d1, \s1, \s4
93 .if \n == 16
94 vshr.u16 \d2, \s2, \s5
95 .endif
96 .endif
97 .endm
98
99 .macro vadd_n d0, d1, d2, s0, s1, s2, s3, s4, s5, n
100 .if \n == 4
101 vadd.i16 \d0, \s0, \s3
102 .else
103 vadd.i16 \d1, \s1, \s4
104 .if \n == 16
105 vadd.i16 \d2, \s2, \s5
106 .endif
107 .endif
108 .endm
109
110 .macro vsub_n d0, d1, d2, s0, s1, s2, s3, s4, s5, n
111 .if \n == 4
112 vsub.i16 \d0, \s0, \s3
113 .else
114 vsub.i16 \d1, \s1, \s4
115 .if \n == 16
116 vsub.i16 \d2, \s2, \s5
117 .endif
118 .endif
119 .endm
120
121 .macro vand_n d0, d1, d2, s0, s1, s2, s3, s4, s5, n
122 .if \n == 4
123 vand \d0, \s0, \s3
124 .else
125 vand \d1, \s1, \s4
126 .if \n == 16
127 vand \d2, \s2, \s5
128 .endif
129 .endif
130 .endm
131
132 .macro vcge_n d0, d1, d2, s0, s1, s2, s3, s4, s5, n
133 .if \n == 4
134 vcge.u16 \d0, \s0, \s3
135 .else
136 vcge.u16 \d1, \s1, \s4
137 .if \n == 16
138 vcge.u16 \d2, \s2, \s5
139 .endif
140 .endif
141 .endm
142
143 .macro vrhadd_n d0, d1, d2, s0, s1, s2, s3, s4, s5, n
144 .if \n == 4
145 vrhadd.u16 \d0, \s0, \s3
146 .else
147 vrhadd.u16 \d1, \s1, \s4
148 .if \n == 16
149 vrhadd.u16 \d2, \s2, \s5
150 .endif
151 .endif
152 .endm
153
154 .macro vshl_n d0, d1, d2, s0, s1, s2, s3, s4, s5, n
155 .if \n == 4
156 vshl.s16 \d0, \s0, \s3
157 .else
158 vshl.s16 \d1, \s1, \s4
159 .if \n == 16
160 vshl.s16 \d2, \s2, \s5
161 .endif
162 .endif
163 .endm
164
165 .macro vqdmulh_n d0, d1, d2, s0, s1, s2, s3, s4, s5, n
166 .if \n == 4
167 vqdmulh.s16 \d0, \s0, \s3
168 .else
169 vqdmulh.s16 \d1, \s1, \s4
170 .if \n == 16
171 vqdmulh.s16 \d2, \s2, \s5
172 .endif
173 .endif
174 .endm
175
176 // unsigned dav1d_msac_decode_symbol_adapt4_neon(MsacContext *s, uint16_t *cdf,
177 // size_t n_symbols);
178
179 function msac_decode_symbol_adapt4_neon, export=1
180 .macro decode_update n
181 push {r4-r10,lr}
182 sub sp, sp, #48
183 add r8, r0, #RNG
184
185 vld1_align_n d0, q0, q1, r1, \n // cdf
186 vld1.16 {d16[]}, [r8, :16] // rng
187 movrel_local r9, coeffs, 30
188 vmov.i16 d30, #0x7f00 // 0x7f00
189 sub r9, r9, r2, lsl #1
190 vmvn.i16 q14, #0x3f // 0xffc0
191 add r8, sp, #14
192 vand d22, d16, d30 // rng & 0x7f00
193 vst1.16 {d16[0]}, [r8, :16] // store original u = s->rng
194 vand_n d4, q2, q3, d0, q0, q1, d28, q14, q14, \n // cdf & 0xffc0
195 .if \n > 4
196 vmov d23, d22
197 .endif
198
199 vld1_n d16, q8, q9, r9, \n // EC_MIN_PROB * (n_symbols - ret)
200 vqdmulh_n d20, q10, q11, d4, q2, q3, d22, q11, q11, \n // ((cdf >> EC_PROB_SHIFT) * (r - 128)) >> 1
201 add r8, r0, #DIF + 2
202
203 vadd_n d16, q8, q9, d4, q2, q3, d16, q8, q9, \n // v = cdf + EC_MIN_PROB * (n_symbols - ret)
204 .if \n == 4
205 vmov.i16 d17, #0
206 .endif
207 vadd_n d16, q8, q9, d20, q10, q11, d16, q8, q9, \n // v = ((cdf >> EC_PROB_SHIFT) * r) >> 1 + EC_MIN_PROB * (n_symbols - ret)
208
209 add r9, sp, #16
210 vld1.16 {d20[]}, [r8, :16] // dif >> (EC_WIN_SIZE - 16)
211 movrel_local r8, bits
212 vst1_n q8, q8, q9, r9, \n // store v values to allow indexed access
213
214 vmov d21, d20
215 vld1_align_n q12, q12, q13, r8, \n
216 .if \n == 16
217 vmov q11, q10
218 .endif
219
220 vcge_n q2, q2, q3, q10, q10, q11, q8, q8, q9, \n // c >= v
221
222 vand_n q10, q10, q11, q2, q2, q3, q12, q12, q13, \n // One bit per halfword set in the mask
223 .if \n == 16
224 vadd.i16 q10, q10, q11
225 .endif
226 vadd.i16 d20, d20, d21 // Aggregate mask bits
227 ldr r4, [r0, #ALLOW_UPDATE_CDF]
228 vpadd.i16 d20, d20, d20
229 lsl r10, r2, #1
230 vpadd.i16 d20, d20, d20
231 vmov.u16 r3, d20[0]
232 cmp r4, #0
233 rbit r3, r3
234 clz lr, r3 // ret
235
236 beq L(renorm)
237 // update_cdf
238 ldrh r3, [r1, r10] // count = cdf[n_symbols]
239 vmov.i8 q10, #0xff
240 .if \n == 16
241 mov r4, #-5
242 .else
243 mvn r12, r2
244 mov r4, #-4
245 cmn r12, #3 // set C if n_symbols <= 2
246 .endif
247 vrhadd_n d16, q8, q9, d20, q10, q10, d4, q2, q3, \n // i >= val ? -1 : 32768
248 .if \n == 16
249 sub r4, r4, r3, lsr #4 // -((count >> 4) + 5)
250 .else
251 lsr r12, r3, #4 // count >> 4
252 sbc r4, r4, r12 // -((count >> 4) + (n_symbols > 2) + 4)
253 .endif
254 vsub_n d16, q8, q9, d16, q8, q9, d0, q0, q1, \n // (32768 - cdf[i]) or (-1 - cdf[i])
255 .if \n == 4
256 vdup.16 d20, r4 // -rate
257 .else
258 vdup.16 q10, r4 // -rate
259 .endif
260
261 sub r3, r3, r3, lsr #5 // count - (count == 32)
262 vsub_n d0, q0, q1, d0, q0, q1, d4, q2, q3, \n // cdf + (i >= val ? 1 : 0)
263 vshl_n d16, q8, q9, d16, q8, q9, d20, q10, q10, \n // ({32768,-1} - cdf[i]) >> rate
264 add r3, r3, #1 // count + (count < 32)
265 vadd_n d0, q0, q1, d0, q0, q1, d16, q8, q9, \n // cdf + (32768 - cdf[i]) >> rate
266 vst1_align_n d0, q0, q1, r1, \n
267 strh r3, [r1, r10]
268 .endm
269
270 decode_update 4
271
272 L(renorm):
273 add r8, sp, #16
274 add r8, r8, lr, lsl #1
275 ldrh r3, [r8] // v
276 ldrh r4, [r8, #-2] // u
277 ldr r6, [r0, #CNT]
278 ldr r7, [r0, #DIF]
279 sub r4, r4, r3 // rng = u - v
280 clz r5, r4 // clz(rng)
281 eor r5, r5, #16 // d = clz(rng) ^ 16
282 mvn r7, r7 // ~dif
283 add r7, r7, r3, lsl #16 // ~dif + (v << 16)
284 L(renorm2):
285 lsl r4, r4, r5 // rng << d
286 subs r6, r6, r5 // cnt -= d
287 lsl r7, r7, r5 // (~dif + (v << 16)) << d
288 str r4, [r0, #RNG]
289 mvn r7, r7 // ~dif
290 bhs 9f
291
292 // refill
293 ldr r3, [r0, #BUF_POS] // BUF_POS
294 ldr r4, [r0, #BUF_END] // BUF_END
295 add r5, r3, #4
296 cmp r5, r4
297 bgt 2f
298
299 ldr r3, [r3] // next_bits
300 add r8, r6, #23 // shift_bits = cnt + 23
301 add r6, r6, #16 // cnt += 16
302 rev r3, r3 // next_bits = bswap(next_bits)
303 sub r5, r5, r8, lsr #3 // buf_pos -= shift_bits >> 3
304 and r8, r8, #24 // shift_bits &= 24
305 lsr r3, r3, r8 // next_bits >>= shift_bits
306 sub r8, r8, r6 // shift_bits -= 16 + cnt
307 str r5, [r0, #BUF_POS]
308 lsl r3, r3, r8 // next_bits <<= shift_bits
309 rsb r6, r8, #16 // cnt = cnt + 32 - shift_bits
310 eor r7, r7, r3 // dif ^= next_bits
311 b 9f
312
313 2: // refill_eob
314 rsb r5, r6, #8 // c = 8 - cnt
315 3:
316 cmp r3, r4
317 bge 4f
318 ldrb r8, [r3], #1
319 lsl r8, r8, r5
320 eor r7, r7, r8
321 subs r5, r5, #8
322 bge 3b
323
324 4: // refill_eob_end
325 str r3, [r0, #BUF_POS]
326 rsb r6, r5, #8 // cnt = 8 - c
327
328 9:
329 str r6, [r0, #CNT]
330 str r7, [r0, #DIF]
331
332 mov r0, lr
333 add sp, sp, #48
334
335 pop {r4-r10,pc}
336 endfunc
337
338 function msac_decode_symbol_adapt8_neon, export=1
339 decode_update 8
340 b L(renorm)
341 endfunc
342
343 function msac_decode_symbol_adapt16_neon, export=1
344 decode_update 16
345 b L(renorm)
346 endfunc
347
348 function msac_decode_hi_tok_neon, export=1
349 push {r4-r10,lr}
350 vld1.16 {d0}, [r1, :64] // cdf
351 add r4, r0, #RNG
352 vmov.i16 d31, #0x7f00 // 0x7f00
353 movrel_local r5, coeffs, 30-2*3
354 vmvn.i16 d30, #0x3f // 0xffc0
355 ldrh r9, [r1, #6] // count = cdf[n_symbols]
356 vld1.16 {d1[]}, [r4, :16] // rng
357 movrel_local r4, bits
358 vld1.16 {d29}, [r5] // EC_MIN_PROB * (n_symbols - ret)
359 add r5, r0, #DIF + 2
360 vld1.16 {q8}, [r4, :128]
361 mov r2, #-24
362 vand d20, d0, d30 // cdf & 0xffc0
363 ldr r10, [r0, #ALLOW_UPDATE_CDF]
364 vld1.16 {d2[]}, [r5, :16] // dif >> (EC_WIN_SIZE - 16)
365 sub sp, sp, #48
366 ldr r6, [r0, #CNT]
367 ldr r7, [r0, #DIF]
368 vmov d3, d2
369 1:
370 vand d23, d1, d31 // rng & 0x7f00
371 vqdmulh.s16 d18, d20, d23 // ((cdf >> EC_PROB_SHIFT) * (r - 128)) >> 1
372 add r12, sp, #14
373 vadd.i16 d6, d20, d29 // v = cdf + EC_MIN_PROB * (n_symbols - ret)
374 vadd.i16 d6, d18, d6 // v = ((cdf >> EC_PROB_SHIFT) * r) >> 1 + EC_MIN_PROB * (n_symbols - ret)
375 vmov.i16 d7, #0
376 vst1.16 {d1[0]}, [r12, :16] // store original u = s->rng
377 add r12, sp, #16
378 vcge.u16 q2, q1, q3 // c >= v
379 vst1.16 {q3}, [r12] // store v values to allow indexed access
380 vand q9, q2, q8 // One bit per halfword set in the mask
381
382 vadd.i16 d18, d18, d19 // Aggregate mask bits
383 vpadd.i16 d18, d18, d18
384 vpadd.i16 d18, d18, d18
385 vmov.u16 r3, d18[0]
386 cmp r10, #0
387 add r2, r2, #5
388 rbit r3, r3
389 add r8, sp, #16
390 clz lr, r3 // ret
391
392 beq 2f
393 // update_cdf
394 vmov.i8 d22, #0xff
395 mov r4, #-5
396 vrhadd.u16 d6, d22, d4 // i >= val ? -1 : 32768
397 sub r4, r4, r9, lsr #4 // -((count >> 4) + 5)
398 vsub.i16 d6, d6, d0 // (32768 - cdf[i]) or (-1 - cdf[i])
399 vdup.16 d18, r4 // -rate
400
401 sub r9, r9, r9, lsr #5 // count - (count == 32)
402 vsub.i16 d0, d0, d4 // cdf + (i >= val ? 1 : 0)
403 vshl.s16 d6, d6, d18 // ({32768,-1} - cdf[i]) >> rate
404 add r9, r9, #1 // count + (count < 32)
405 vadd.i16 d0, d0, d6 // cdf + (32768 - cdf[i]) >> rate
406 vst1.16 {d0}, [r1, :64]
407 vand d20, d0, d30 // cdf & 0xffc0
408 strh r9, [r1, #6]
409
410 2:
411 add r8, r8, lr, lsl #1
412 ldrh r3, [r8] // v
413 ldrh r4, [r8, #-2] // u
414 sub r4, r4, r3 // rng = u - v
415 clz r5, r4 // clz(rng)
416 eor r5, r5, #16 // d = clz(rng) ^ 16
417 mvn r7, r7 // ~dif
418 add r7, r7, r3, lsl #16 // ~dif + (v << 16)
419 lsl r4, r4, r5 // rng << d
420 subs r6, r6, r5 // cnt -= d
421 lsl r7, r7, r5 // (~dif + (v << 16)) << d
422 str r4, [r0, #RNG]
423 vdup.16 d1, r4
424 mvn r7, r7 // ~dif
425 bhs 9f
426
427 // refill
428 ldr r3, [r0, #BUF_POS] // BUF_POS
429 ldr r4, [r0, #BUF_END] // BUF_END
430 add r5, r3, #4
431 cmp r5, r4
432 bgt 2f
433
434 ldr r3, [r3] // next_bits
435 add r8, r6, #23 // shift_bits = cnt + 23
436 add r6, r6, #16 // cnt += 16
437 rev r3, r3 // next_bits = bswap(next_bits)
438 sub r5, r5, r8, lsr #3 // buf_pos -= shift_bits >> 3
439 and r8, r8, #24 // shift_bits &= 24
440 lsr r3, r3, r8 // next_bits >>= shift_bits
441 sub r8, r8, r6 // shift_bits -= 16 + cnt
442 str r5, [r0, #BUF_POS]
443 lsl r3, r3, r8 // next_bits <<= shift_bits
444 rsb r6, r8, #16 // cnt = cnt + 32 - shift_bits
445 eor r7, r7, r3 // dif ^= next_bits
446 b 9f
447
448 2: // refill_eob
449 rsb r5, r6, #8 // c = 40 - cnt
450 3:
451 cmp r3, r4
452 bge 4f
453 ldrb r8, [r3], #1
454 lsl r8, r8, r5
455 eor r7, r7, r8
456 subs r5, r5, #8
457 bge 3b
458
459 4: // refill_eob_end
460 str r3, [r0, #BUF_POS]
461 rsb r6, r5, #8 // cnt = 40 - c
462
463 9:
464 lsl lr, lr, #1
465 sub lr, lr, #5
466 lsr r12, r7, #16
467 adds r2, r2, lr // carry = tok_br < 3 || tok == 15
468 vdup.16 q1, r12
469 bcc 1b // loop if !carry
470 add r2, r2, #30
471 str r6, [r0, #CNT]
472 add sp, sp, #48
473 str r7, [r0, #DIF]
474 lsr r0, r2, #1
475 pop {r4-r10,pc}
476 endfunc
477
478 function msac_decode_bool_equi_neon, export=1
479 push {r4-r10,lr}
480 ldr r5, [r0, #RNG]
481 ldr r6, [r0, #CNT]
482 sub sp, sp, #48
483 ldr r7, [r0, #DIF]
484 bic r4, r5, #0xff // r &= 0xff00
485 add r4, r4, #8
486 mov r2, #0
487 subs r8, r7, r4, lsl #15 // dif - vw
488 lsr r4, r4, #1 // v
489 sub r5, r5, r4 // r - v
490 itee lo
491 movlo r2, #1
492 movhs r4, r5 // if (ret) v = r - v;
493 movhs r7, r8 // if (ret) dif = dif - vw;
494
495 clz r5, r4 // clz(rng)
496 mvn r7, r7 // ~dif
497 eor r5, r5, #16 // d = clz(rng) ^ 16
498 mov lr, r2
499 b L(renorm2)
500 endfunc
501
502 function msac_decode_bool_neon, export=1
503 push {r4-r10,lr}
504 ldr r5, [r0, #RNG]
505 ldr r6, [r0, #CNT]
506 sub sp, sp, #48
507 ldr r7, [r0, #DIF]
508 lsr r4, r5, #8 // r >> 8
509 bic r1, r1, #0x3f // f &= ~63
510 mul r4, r4, r1
511 mov r2, #0
512 lsr r4, r4, #7
513 add r4, r4, #4 // v
514 subs r8, r7, r4, lsl #16 // dif - vw
515 sub r5, r5, r4 // r - v
516 itee lo
517 movlo r2, #1
518 movhs r4, r5 // if (ret) v = r - v;
519 movhs r7, r8 // if (ret) dif = dif - vw;
520
521 clz r5, r4 // clz(rng)
522 mvn r7, r7 // ~dif
523 eor r5, r5, #16 // d = clz(rng) ^ 16
524 mov lr, r2
525 b L(renorm2)
526 endfunc
527
528 function msac_decode_bool_adapt_neon, export=1
529 push {r4-r10,lr}
530 ldr r9, [r1] // cdf[0-1]
531 ldr r5, [r0, #RNG]
532 movw lr, #0xffc0
533 ldr r6, [r0, #CNT]
534 sub sp, sp, #48
535 ldr r7, [r0, #DIF]
536 lsr r4, r5, #8 // r >> 8
537 and r2, r9, lr // f &= ~63
538 mul r4, r4, r2
539 mov r2, #0
540 lsr r4, r4, #7
541 add r4, r4, #4 // v
542 subs r8, r7, r4, lsl #16 // dif - vw
543 sub r5, r5, r4 // r - v
544 ldr r10, [r0, #ALLOW_UPDATE_CDF]
545 itee lo
546 movlo r2, #1
547 movhs r4, r5 // if (ret) v = r - v;
548 movhs r7, r8 // if (ret) dif = dif - vw;
549
550 cmp r10, #0
551 clz r5, r4 // clz(rng)
552 mvn r7, r7 // ~dif
553 eor r5, r5, #16 // d = clz(rng) ^ 16
554 mov lr, r2
555
556 beq L(renorm2)
557
558 lsr r2, r9, #16 // count = cdf[1]
559 uxth r9, r9 // cdf[0]
560
561 sub r3, r2, r2, lsr #5 // count - (count >= 32)
562 lsr r2, r2, #4 // count >> 4
563 add r10, r3, #1 // count + (count < 32)
564 add r2, r2, #4 // rate = (count >> 4) | 4
565
566 sub r9, r9, lr // cdf[0] -= bit
567 sub r3, r9, lr, lsl #15 // {cdf[0], cdf[0] - 32769}
568 asr r3, r3, r2 // {cdf[0], cdf[0] - 32769} >> rate
569 sub r9, r9, r3 // cdf[0]
570
571 strh r9, [r1]
572 strh r10, [r1, #2]
573
574 b L(renorm2)
575 endfunc