| 1 |
/* |
| 2 |
* Copyright © 2018, VideoLAN and dav1d authors |
| 3 |
* Copyright © 2020, Martin Storsjo |
| 4 |
* All rights reserved. |
| 5 |
* |
| 6 |
* Redistribution and use in source and binary forms, with or without |
| 7 |
* modification, are permitted provided that the following conditions are met: |
| 8 |
* |
| 9 |
* 1. Redistributions of source code must retain the above copyright notice, this |
| 10 |
* list of conditions and the following disclaimer. |
| 11 |
* |
| 12 |
* 2. Redistributions in binary form must reproduce the above copyright notice, |
| 13 |
* this list of conditions and the following disclaimer in the documentation |
| 14 |
* and/or other materials provided with the distribution. |
| 15 |
* |
| 16 |
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND |
| 17 |
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
| 18 |
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE |
| 19 |
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR |
| 20 |
* ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES |
| 21 |
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
| 22 |
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND |
| 23 |
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 24 |
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS |
| 25 |
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 26 |
*/ |
| 27 |
|
| 28 |
#include "src/arm/asm.S" |
| 29 |
#include "util.S" |
| 30 |
|
| 31 |
const right_ext_mask_buf |
| 32 |
.byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 |
| 33 |
.byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 |
| 34 |
.byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 |
| 35 |
.byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 |
| 36 |
right_ext_mask: |
| 37 |
.byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff |
| 38 |
.byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff |
| 39 |
.byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff |
| 40 |
.byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff |
| 41 |
endconst |
| 42 |
|
| 43 |
// void dav1d_wiener_filter_h_16bpc_neon(int16_t *dst, const pixel (*left)[4], |
| 44 |
// const pixel *src, ptrdiff_t stride, |
| 45 |
// const int16_t fh[7], const intptr_t w, |
| 46 |
// int h, enum LrEdgeFlags edges, |
| 47 |
// const int bitdepth_max); |
| 48 |
function wiener_filter_h_16bpc_neon, export=1 |
| 49 |
push {r4-r11,lr} |
| 50 |
vpush {q4-q7} |
| 51 |
ldrd r4, r5, [sp, #100] |
| 52 |
ldrd r6, r7, [sp, #108] |
| 53 |
ldr r8, [sp, #116] // bitdepth_max |
| 54 |
vld1.16 {q0}, [r4, :128] |
| 55 |
clz r8, r8 |
| 56 |
vmov.i32 q14, #1 |
| 57 |
sub r9, r8, #38 // -(bitdepth + 6) |
| 58 |
sub r8, r8, #25 // -round_bits_h |
| 59 |
neg r9, r9 // bitdepth + 6 |
| 60 |
vdup.32 q1, r9 |
| 61 |
vdup.32 q13, r8 // -round_bits_h |
| 62 |
vmov.i16 q15, #8192 |
| 63 |
vshl.u32 q14, q14, q1 // 1 << (bitdepth + 6) |
| 64 |
mov r8, r5 |
| 65 |
// Calculate mid_stride |
| 66 |
add r10, r5, #7 |
| 67 |
bic r10, r10, #7 |
| 68 |
lsl r10, r10, #1 |
| 69 |
|
| 70 |
// Set up pointers for reading/writing alternate rows |
| 71 |
add r12, r0, r10 |
| 72 |
lsl r10, r10, #1 |
| 73 |
add lr, r2, r3 |
| 74 |
lsl r3, r3, #1 |
| 75 |
|
| 76 |
// Subtract the aligned width from mid_stride |
| 77 |
add r11, r5, #7 |
| 78 |
bic r11, r11, #7 |
| 79 |
sub r10, r10, r11, lsl #1 |
| 80 |
|
| 81 |
// Subtract the number of pixels read from the source stride |
| 82 |
add r11, r11, #8 |
| 83 |
sub r3, r3, r11, lsl #1 |
| 84 |
|
| 85 |
// Set up the src pointers to include the left edge, for LR_HAVE_LEFT, left == NULL |
| 86 |
tst r7, #1 // LR_HAVE_LEFT |
| 87 |
beq 2f |
| 88 |
// LR_HAVE_LEFT |
| 89 |
cmp r1, #0 |
| 90 |
bne 0f |
| 91 |
// left == NULL |
| 92 |
sub r2, r2, #6 |
| 93 |
sub lr, lr, #6 |
| 94 |
b 1f |
| 95 |
0: // LR_HAVE_LEFT, left != NULL |
| 96 |
2: // !LR_HAVE_LEFT, increase the stride. |
| 97 |
// For this case we don't read the left 3 pixels from the src pointer, |
| 98 |
// but shift it as if we had done that. |
| 99 |
add r3, r3, #6 |
| 100 |
|
| 101 |
|
| 102 |
1: // Loop vertically |
| 103 |
vld1.16 {q2, q3}, [r2]! |
| 104 |
vld1.16 {q4, q5}, [lr]! |
| 105 |
|
| 106 |
tst r7, #1 // LR_HAVE_LEFT |
| 107 |
beq 0f |
| 108 |
cmp r1, #0 |
| 109 |
beq 2f |
| 110 |
// LR_HAVE_LEFT, left != NULL |
| 111 |
vld1.16 {d3}, [r1]! |
| 112 |
// Move r2/lr back to account for the last 3 pixels we loaded earlier, |
| 113 |
// which we'll shift out. |
| 114 |
sub r2, r2, #6 |
| 115 |
sub lr, lr, #6 |
| 116 |
vld1.16 {d13}, [r1]! |
| 117 |
vext.8 q3, q2, q3, #10 |
| 118 |
vext.8 q2, q1, q2, #10 |
| 119 |
vext.8 q5, q4, q5, #10 |
| 120 |
vext.8 q4, q6, q4, #10 |
| 121 |
b 2f |
| 122 |
0: |
| 123 |
// !LR_HAVE_LEFT, fill q1 with the leftmost pixel |
| 124 |
// and shift q2/q3 to have 3x the first pixel at the front. |
| 125 |
vdup.16 q1, d4[0] |
| 126 |
vdup.16 q6, d8[0] |
| 127 |
// Move r2 back to account for the last 3 pixels we loaded before, |
| 128 |
// which we shifted out. |
| 129 |
sub r2, r2, #6 |
| 130 |
sub lr, lr, #6 |
| 131 |
vext.8 q3, q2, q3, #10 |
| 132 |
vext.8 q2, q1, q2, #10 |
| 133 |
vext.8 q5, q4, q5, #10 |
| 134 |
vext.8 q4, q6, q4, #10 |
| 135 |
|
| 136 |
2: |
| 137 |
|
| 138 |
tst r7, #2 // LR_HAVE_RIGHT |
| 139 |
bne 4f |
| 140 |
// If we'll need to pad the right edge, load that pixel to pad with |
| 141 |
// here since we can find it pretty easily from here. |
| 142 |
sub r9, r5, #14 |
| 143 |
lsl r9, r9, #1 |
| 144 |
ldrh r11, [r2, r9] |
| 145 |
ldrh r9, [lr, r9] |
| 146 |
// Fill q11/q12 with the right padding pixel |
| 147 |
vdup.16 q11, r11 |
| 148 |
vdup.16 q12, r9 |
| 149 |
3: // !LR_HAVE_RIGHT |
| 150 |
|
| 151 |
// Check whether we need to pad the right edge |
| 152 |
cmp r5, #11 |
| 153 |
bge 4f // If w >= 11, all used input pixels are valid |
| 154 |
|
| 155 |
// 1 <= w < 11, w+3 pixels valid in q2-q3. For w=9 or w=10, |
| 156 |
// this ends up called again; it's not strictly needed in those |
| 157 |
// cases (we pad enough here), but keeping the code as simple as possible. |
| 158 |
|
| 159 |
// Insert padding in q2/3.h[w+3] onwards; fuse the +3 (*2) into the |
| 160 |
// buffer pointer. |
| 161 |
movrel_local r4, right_ext_mask, -6 |
| 162 |
sub r4, r4, r5, lsl #1 |
| 163 |
vld1.8 {q9, q10}, [r4] |
| 164 |
|
| 165 |
vbit q2, q11, q9 |
| 166 |
vbit q3, q11, q10 |
| 167 |
vbit q4, q12, q9 |
| 168 |
vbit q5, q12, q10 |
| 169 |
|
| 170 |
4: // Loop horizontally |
| 171 |
vext.8 q7, q2, q3, #4 |
| 172 |
vext.8 q8, q2, q3, #8 |
| 173 |
vext.8 q6, q2, q3, #2 |
| 174 |
vext.8 q9, q2, q3, #10 |
| 175 |
vadd.i16 q8, q8, q7 |
| 176 |
vadd.i16 q9, q9, q6 |
| 177 |
vext.8 q6, q2, q3, #12 |
| 178 |
vext.8 q7, q2, q3, #6 |
| 179 |
vadd.i16 q2, q2, q6 |
| 180 |
vmull.s16 q6, d14, d0[3] |
| 181 |
vmlal.s16 q6, d16, d1[0] |
| 182 |
vmlal.s16 q6, d18, d1[1] |
| 183 |
vmlal.s16 q6, d4, d1[2] |
| 184 |
vmull.s16 q7, d15, d0[3] |
| 185 |
vmlal.s16 q7, d17, d1[0] |
| 186 |
vmlal.s16 q7, d19, d1[1] |
| 187 |
vmlal.s16 q7, d5, d1[2] |
| 188 |
|
| 189 |
vext.8 q8, q4, q5, #4 |
| 190 |
vext.8 q10, q4, q5, #8 |
| 191 |
vext.8 q9, q4, q5, #2 |
| 192 |
vext.8 q2, q4, q5, #10 |
| 193 |
vadd.i16 q10, q10, q8 |
| 194 |
vadd.i16 q2, q2, q9 |
| 195 |
vext.8 q8, q4, q5, #12 |
| 196 |
vext.8 q9, q4, q5, #6 |
| 197 |
vadd.i16 q4, q4, q8 |
| 198 |
vmull.s16 q8, d18, d0[3] |
| 199 |
vmlal.s16 q8, d20, d1[0] |
| 200 |
vmlal.s16 q8, d4, d1[1] |
| 201 |
vmlal.s16 q8, d8, d1[2] |
| 202 |
vmull.s16 q9, d19, d0[3] |
| 203 |
vmlal.s16 q9, d21, d1[0] |
| 204 |
vmlal.s16 q9, d5, d1[1] |
| 205 |
vmlal.s16 q9, d9, d1[2] |
| 206 |
|
| 207 |
vmvn.i16 q10, #0x8000 // 0x7fff = (1 << 15) - 1 |
| 208 |
vadd.i32 q6, q6, q14 |
| 209 |
vadd.i32 q7, q7, q14 |
| 210 |
vadd.i32 q8, q8, q14 |
| 211 |
vadd.i32 q9, q9, q14 |
| 212 |
vrshl.s32 q6, q6, q13 |
| 213 |
vrshl.s32 q7, q7, q13 |
| 214 |
vrshl.s32 q8, q8, q13 |
| 215 |
vrshl.s32 q9, q9, q13 |
| 216 |
vqmovun.s32 d12, q6 |
| 217 |
vqmovun.s32 d13, q7 |
| 218 |
vqmovun.s32 d14, q8 |
| 219 |
vqmovun.s32 d15, q9 |
| 220 |
vmin.u16 q6, q6, q10 |
| 221 |
vmin.u16 q7, q7, q10 |
| 222 |
vsub.i16 q6, q6, q15 |
| 223 |
vsub.i16 q7, q7, q15 |
| 224 |
subs r5, r5, #8 |
| 225 |
vst1.16 {q6}, [r0, :128]! |
| 226 |
vst1.16 {q7}, [r12, :128]! |
| 227 |
|
| 228 |
ble 9f |
| 229 |
tst r7, #2 // LR_HAVE_RIGHT |
| 230 |
vmov q2, q3 |
| 231 |
vmov q4, q5 |
| 232 |
vld1.16 {q3}, [r2]! |
| 233 |
vld1.16 {q5}, [lr]! |
| 234 |
bne 4b // If we don't need to pad, just keep filtering. |
| 235 |
b 3b // If we need to pad, check how many pixels we have left. |
| 236 |
|
| 237 |
9: |
| 238 |
subs r6, r6, #2 |
| 239 |
ble 0f |
| 240 |
// Jump to the next row and loop horizontally |
| 241 |
add r0, r0, r10 |
| 242 |
add r12, r12, r10 |
| 243 |
add r2, r2, r3 |
| 244 |
add lr, lr, r3 |
| 245 |
mov r5, r8 |
| 246 |
b 1b |
| 247 |
0: |
| 248 |
vpop {q4-q7} |
| 249 |
pop {r4-r11,pc} |
| 250 |
endfunc |
| 251 |
|
| 252 |
// void dav1d_wiener_filter_v_16bpc_neon(pixel *dst, ptrdiff_t stride, |
| 253 |
// const int16_t *mid, int w, int h, |
| 254 |
// const int16_t fv[7], enum LrEdgeFlags edges, |
| 255 |
// ptrdiff_t mid_stride, const int bitdepth_max); |
| 256 |
function wiener_filter_v_16bpc_neon, export=1 |
| 257 |
push {r4-r7,lr} |
| 258 |
vpush {q4-q5} |
| 259 |
ldrd r4, r5, [sp, #52] |
| 260 |
ldrd r6, r7, [sp, #60] |
| 261 |
ldr lr, [sp, #68] // bitdepth_max |
| 262 |
vld1.16 {q0}, [r5, :128] |
| 263 |
vdup.16 q5, lr |
| 264 |
clz lr, lr |
| 265 |
sub lr, lr, #11 // round_bits_v |
| 266 |
vdup.32 q4, lr |
| 267 |
mov lr, r4 |
| 268 |
vneg.s32 q4, q4 // -round_bits_v |
| 269 |
|
| 270 |
// Calculate the number of rows to move back when looping vertically |
| 271 |
mov r12, r4 |
| 272 |
tst r6, #4 // LR_HAVE_TOP |
| 273 |
beq 0f |
| 274 |
sub r2, r2, r7, lsl #1 |
| 275 |
add r12, r12, #2 |
| 276 |
0: |
| 277 |
tst r6, #8 // LR_HAVE_BOTTOM |
| 278 |
beq 1f |
| 279 |
add r12, r12, #2 |
| 280 |
|
| 281 |
1: // Start of horizontal loop; start one vertical filter slice. |
| 282 |
// Load rows into q8-q11 and pad properly. |
| 283 |
tst r6, #4 // LR_HAVE_TOP |
| 284 |
vld1.16 {q8}, [r2, :128], r7 |
| 285 |
beq 2f |
| 286 |
// LR_HAVE_TOP |
| 287 |
vld1.16 {q10}, [r2, :128], r7 |
| 288 |
vmov q9, q8 |
| 289 |
vld1.16 {q11}, [r2, :128], r7 |
| 290 |
b 3f |
| 291 |
2: // !LR_HAVE_TOP |
| 292 |
vmov q9, q8 |
| 293 |
vmov q10, q8 |
| 294 |
vmov q11, q8 |
| 295 |
|
| 296 |
3: |
| 297 |
cmp r4, #4 |
| 298 |
blt 5f |
| 299 |
// Start filtering normally; fill in q12-q14 with unique rows. |
| 300 |
vld1.16 {q12}, [r2, :128], r7 |
| 301 |
vld1.16 {q13}, [r2, :128], r7 |
| 302 |
vld1.16 {q14}, [r2, :128], r7 |
| 303 |
|
| 304 |
4: |
| 305 |
.macro filter compare |
| 306 |
subs r4, r4, #1 |
| 307 |
// Interleaving the mul/mla chains actually hurts performance |
| 308 |
// significantly on Cortex A53, thus keeping mul/mla tightly |
| 309 |
// chained like this. |
| 310 |
vmull.s16 q2, d16, d0[0] |
| 311 |
vmlal.s16 q2, d18, d0[1] |
| 312 |
vmlal.s16 q2, d20, d0[2] |
| 313 |
vmlal.s16 q2, d22, d0[3] |
| 314 |
vmlal.s16 q2, d24, d1[0] |
| 315 |
vmlal.s16 q2, d26, d1[1] |
| 316 |
vmlal.s16 q2, d28, d1[2] |
| 317 |
vmull.s16 q3, d17, d0[0] |
| 318 |
vmlal.s16 q3, d19, d0[1] |
| 319 |
vmlal.s16 q3, d21, d0[2] |
| 320 |
vmlal.s16 q3, d23, d0[3] |
| 321 |
vmlal.s16 q3, d25, d1[0] |
| 322 |
vmlal.s16 q3, d27, d1[1] |
| 323 |
vmlal.s16 q3, d29, d1[2] |
| 324 |
vrshl.s32 q2, q2, q4 // round_bits_v |
| 325 |
vrshl.s32 q3, q3, q4 |
| 326 |
vqmovun.s32 d4, q2 |
| 327 |
vqmovun.s32 d5, q3 |
| 328 |
vmin.u16 q2, q2, q5 // bitdepth_max |
| 329 |
vst1.16 {q2}, [r0, :128], r1 |
| 330 |
.if \compare |
| 331 |
cmp r4, #4 |
| 332 |
.else |
| 333 |
ble 9f |
| 334 |
.endif |
| 335 |
vmov q8, q9 |
| 336 |
vmov q9, q10 |
| 337 |
vmov q10, q11 |
| 338 |
vmov q11, q12 |
| 339 |
vmov q12, q13 |
| 340 |
vmov q13, q14 |
| 341 |
.endm |
| 342 |
filter 1 |
| 343 |
blt 7f |
| 344 |
vld1.16 {q14}, [r2, :128], r7 |
| 345 |
b 4b |
| 346 |
|
| 347 |
5: // Less than 4 rows in total; not all of q12-q13 are filled yet. |
| 348 |
tst r6, #8 // LR_HAVE_BOTTOM |
| 349 |
beq 6f |
| 350 |
// LR_HAVE_BOTTOM |
| 351 |
cmp r4, #2 |
| 352 |
// We load at least 2 rows in all cases. |
| 353 |
vld1.16 {q12}, [r2, :128], r7 |
| 354 |
vld1.16 {q13}, [r2, :128], r7 |
| 355 |
bgt 53f // 3 rows in total |
| 356 |
beq 52f // 2 rows in total |
| 357 |
51: // 1 row in total, q11 already loaded, load edge into q12-q14. |
| 358 |
vmov q13, q12 |
| 359 |
b 8f |
| 360 |
52: // 2 rows in total, q11 already loaded, load q12 with content data |
| 361 |
// and 2 rows of edge. |
| 362 |
vld1.16 {q14}, [r2, :128], r7 |
| 363 |
vmov q15, q14 |
| 364 |
b 8f |
| 365 |
53: |
| 366 |
// 3 rows in total, q11 already loaded, load q12 and q13 with content |
| 367 |
// and 2 rows of edge. |
| 368 |
vld1.16 {q14}, [r2, :128], r7 |
| 369 |
vld1.16 {q15}, [r2, :128], r7 |
| 370 |
vmov q1, q15 |
| 371 |
b 8f |
| 372 |
|
| 373 |
6: |
| 374 |
// !LR_HAVE_BOTTOM |
| 375 |
cmp r4, #2 |
| 376 |
bgt 63f // 3 rows in total |
| 377 |
beq 62f // 2 rows in total |
| 378 |
61: // 1 row in total, q11 already loaded, pad that into q12-q14. |
| 379 |
vmov q12, q11 |
| 380 |
vmov q13, q11 |
| 381 |
vmov q14, q11 |
| 382 |
b 8f |
| 383 |
62: // 2 rows in total, q11 already loaded, load q12 and pad that into q12-q15. |
| 384 |
vld1.16 {q12}, [r2, :128], r7 |
| 385 |
vmov q13, q12 |
| 386 |
vmov q14, q12 |
| 387 |
vmov q15, q12 |
| 388 |
b 8f |
| 389 |
63: |
| 390 |
// 3 rows in total, q11 already loaded, load q12 and q13 and pad q13 into q14-q15,q1. |
| 391 |
vld1.16 {q12}, [r2, :128], r7 |
| 392 |
vld1.16 {q13}, [r2, :128], r7 |
| 393 |
vmov q14, q13 |
| 394 |
vmov q15, q13 |
| 395 |
vmov q1, q13 |
| 396 |
b 8f |
| 397 |
|
| 398 |
7: |
| 399 |
// All registers up to q13 are filled already, 3 valid rows left. |
| 400 |
// < 4 valid rows left; fill in padding and filter the last |
| 401 |
// few rows. |
| 402 |
tst r6, #8 // LR_HAVE_BOTTOM |
| 403 |
beq 71f |
| 404 |
// LR_HAVE_BOTTOM; load 2 rows of edge. |
| 405 |
vld1.16 {q14}, [r2, :128], r7 |
| 406 |
vld1.16 {q15}, [r2, :128], r7 |
| 407 |
vmov q1, q15 |
| 408 |
b 8f |
| 409 |
71: |
| 410 |
// !LR_HAVE_BOTTOM, pad 3 rows |
| 411 |
vmov q14, q13 |
| 412 |
vmov q15, q13 |
| 413 |
vmov q1, q13 |
| 414 |
|
| 415 |
8: // At this point, all registers up to q14-q15,q1 are loaded with |
| 416 |
// edge/padding (depending on how many rows are left). |
| 417 |
filter 0 // This branches to 9f when done |
| 418 |
vmov q14, q15 |
| 419 |
vmov q15, q1 |
| 420 |
b 8b |
| 421 |
|
| 422 |
9: // End of one vertical slice. |
| 423 |
subs r3, r3, #8 |
| 424 |
ble 0f |
| 425 |
// Move pointers back up to the top and loop horizontally. |
| 426 |
mls r0, r1, lr, r0 |
| 427 |
mls r2, r7, r12, r2 |
| 428 |
add r0, r0, #16 |
| 429 |
add r2, r2, #16 |
| 430 |
mov r4, lr |
| 431 |
b 1b |
| 432 |
|
| 433 |
0: |
| 434 |
vpop {q4-q5} |
| 435 |
pop {r4-r7,pc} |
| 436 |
.purgem filter |
| 437 |
endfunc |
| 438 |
|
| 439 |
#define SUM_STRIDE (384+16) |
| 440 |
|
| 441 |
#include "looprestoration_tmpl.S" |
| 442 |
|
| 443 |
// void dav1d_sgr_box3_h_16bpc_neon(int32_t *sumsq, int16_t *sum, |
| 444 |
// const pixel (*left)[4], |
| 445 |
// const pixel *src, const ptrdiff_t stride, |
| 446 |
// const int w, const int h, |
| 447 |
// const enum LrEdgeFlags edges); |
| 448 |
function sgr_box3_h_16bpc_neon, export=1 |
| 449 |
push {r4-r11,lr} |
| 450 |
vpush {q4-q7} |
| 451 |
ldrd r4, r5, [sp, #100] |
| 452 |
ldrd r6, r7, [sp, #108] |
| 453 |
add r5, r5, #2 // w += 2 |
| 454 |
|
| 455 |
// Set up pointers for reading/writing alternate rows |
| 456 |
add r10, r0, #(4*SUM_STRIDE) // sumsq |
| 457 |
add r11, r1, #(2*SUM_STRIDE) // sum |
| 458 |
add r12, r3, r4 // src |
| 459 |
lsl r4, r4, #1 |
| 460 |
mov r9, #(2*2*SUM_STRIDE) // double sum stride |
| 461 |
|
| 462 |
// Subtract the aligned width from the output stride. |
| 463 |
add lr, r5, #7 |
| 464 |
bic lr, lr, #7 |
| 465 |
sub r9, r9, lr, lsl #1 |
| 466 |
|
| 467 |
// Store the width for the vertical loop |
| 468 |
mov r8, r5 |
| 469 |
|
| 470 |
// Subtract the number of pixels read from the input from the stride |
| 471 |
add lr, lr, #8 |
| 472 |
sub r4, r4, lr, lsl #1 |
| 473 |
|
| 474 |
// Set up the src pointers to include the left edge, for LR_HAVE_LEFT, left == NULL |
| 475 |
tst r7, #1 // LR_HAVE_LEFT |
| 476 |
beq 2f |
| 477 |
// LR_HAVE_LEFT |
| 478 |
cmp r2, #0 |
| 479 |
bne 0f |
| 480 |
// left == NULL |
| 481 |
sub r3, r3, #4 |
| 482 |
sub r12, r12, #4 |
| 483 |
b 1f |
| 484 |
0: // LR_HAVE_LEFT, left != NULL |
| 485 |
2: // !LR_HAVE_LEFT, increase the stride. |
| 486 |
// For this case we don't read the left 2 pixels from the src pointer, |
| 487 |
// but shift it as if we had done that. |
| 488 |
add r4, r4, #4 |
| 489 |
|
| 490 |
|
| 491 |
1: // Loop vertically |
| 492 |
vld1.16 {q0, q1}, [r3]! |
| 493 |
vld1.16 {q4, q5}, [r12]! |
| 494 |
|
| 495 |
tst r7, #1 // LR_HAVE_LEFT |
| 496 |
beq 0f |
| 497 |
cmp r2, #0 |
| 498 |
beq 2f |
| 499 |
// LR_HAVE_LEFT, left != NULL |
| 500 |
vld1.16 {d5}, [r2]! |
| 501 |
// Move r3/r12 back to account for the last 2 pixels we loaded earlier, |
| 502 |
// which we'll shift out. |
| 503 |
sub r3, r3, #4 |
| 504 |
sub r12, r12, #4 |
| 505 |
vld1.16 {d13}, [r2]! |
| 506 |
vext.8 q1, q0, q1, #12 |
| 507 |
vext.8 q0, q2, q0, #12 |
| 508 |
vext.8 q5, q4, q5, #12 |
| 509 |
vext.8 q4, q6, q4, #12 |
| 510 |
b 2f |
| 511 |
0: |
| 512 |
// !LR_HAVE_LEFT, fill q2 with the leftmost pixel |
| 513 |
// and shift q0 to have 2x the first byte at the front. |
| 514 |
vdup.16 q2, d0[0] |
| 515 |
vdup.16 q6, d8[0] |
| 516 |
// Move r3 back to account for the last 2 pixels we loaded before, |
| 517 |
// which we shifted out. |
| 518 |
sub r3, r3, #4 |
| 519 |
sub r12, r12, #4 |
| 520 |
vext.8 q1, q0, q1, #12 |
| 521 |
vext.8 q0, q2, q0, #12 |
| 522 |
vext.8 q5, q4, q5, #12 |
| 523 |
vext.8 q4, q6, q4, #12 |
| 524 |
|
| 525 |
2: |
| 526 |
tst r7, #2 // LR_HAVE_RIGHT |
| 527 |
bne 4f |
| 528 |
// If we'll need to pad the right edge, load that pixel to pad with |
| 529 |
// here since we can find it pretty easily from here. |
| 530 |
sub lr, r5, #(2 + 16 - 2 + 1) |
| 531 |
lsl lr, lr, #1 |
| 532 |
ldrh r11, [r3, lr] |
| 533 |
ldrh lr, [r12, lr] |
| 534 |
// Fill q14/q15 with the right padding pixel |
| 535 |
vdup.16 q14, r11 |
| 536 |
vdup.16 q15, lr |
| 537 |
// Restore r11 after using it for a temporary value |
| 538 |
add r11, r1, #(2*SUM_STRIDE) |
| 539 |
3: // !LR_HAVE_RIGHT |
| 540 |
|
| 541 |
// Check whether we need to pad the right edge |
| 542 |
cmp r5, #10 |
| 543 |
bge 4f // If w >= 10, all used input pixels are valid |
| 544 |
|
| 545 |
// 1 <= w < 10, w pixels valid in q0-q1. For w=9, this ends up called |
| 546 |
// again; it's not strictly needed in those cases (we pad enough here), |
| 547 |
// but keeping the code as simple as possible. |
| 548 |
|
| 549 |
// Insert padding in q0/1.h[w] onwards |
| 550 |
movrel_local lr, right_ext_mask |
| 551 |
sub lr, lr, r5, lsl #1 |
| 552 |
vld1.8 {q12, q13}, [lr] |
| 553 |
|
| 554 |
vbit q0, q14, q12 |
| 555 |
vbit q1, q14, q13 |
| 556 |
vbit q4, q15, q12 |
| 557 |
vbit q5, q15, q13 |
| 558 |
|
| 559 |
4: // Loop horizontally |
| 560 |
vext.8 q8, q0, q1, #2 |
| 561 |
vext.8 q10, q4, q5, #2 |
| 562 |
vext.8 q9, q0, q1, #4 |
| 563 |
vext.8 q11, q4, q5, #4 |
| 564 |
vadd.i16 q2, q0, q8 |
| 565 |
vadd.i16 q3, q4, q10 |
| 566 |
vadd.i16 q2, q2, q9 |
| 567 |
vadd.i16 q3, q3, q11 |
| 568 |
|
| 569 |
vmull.u16 q6, d0, d0 |
| 570 |
vmlal.u16 q6, d16, d16 |
| 571 |
vmlal.u16 q6, d18, d18 |
| 572 |
vmull.u16 q12, d8, d8 |
| 573 |
vmlal.u16 q12, d20, d20 |
| 574 |
vmlal.u16 q12, d22, d22 |
| 575 |
vmull.u16 q7, d1, d1 |
| 576 |
vmlal.u16 q7, d17, d17 |
| 577 |
vmlal.u16 q7, d19, d19 |
| 578 |
vmull.u16 q13, d9, d9 |
| 579 |
vmlal.u16 q13, d21, d21 |
| 580 |
vmlal.u16 q13, d23, d23 |
| 581 |
subs r5, r5, #8 |
| 582 |
vst1.16 {q2}, [r1, :128]! |
| 583 |
vst1.16 {q3}, [r11, :128]! |
| 584 |
vst1.32 {q6, q7}, [r0, :128]! |
| 585 |
vst1.32 {q12, q13}, [r10, :128]! |
| 586 |
|
| 587 |
ble 9f |
| 588 |
tst r7, #2 // LR_HAVE_RIGHT |
| 589 |
vmov q0, q1 |
| 590 |
vmov q4, q5 |
| 591 |
vld1.16 {q1}, [r3]! |
| 592 |
vld1.16 {q5}, [r12]! |
| 593 |
|
| 594 |
bne 4b // If we don't need to pad, just keep summing. |
| 595 |
b 3b // If we need to pad, check how many pixels we have left. |
| 596 |
|
| 597 |
9: |
| 598 |
subs r6, r6, #2 |
| 599 |
ble 0f |
| 600 |
// Jump to the next row and loop horizontally |
| 601 |
add r0, r0, r9, lsl #1 |
| 602 |
add r10, r10, r9, lsl #1 |
| 603 |
add r1, r1, r9 |
| 604 |
add r11, r11, r9 |
| 605 |
add r3, r3, r4 |
| 606 |
add r12, r12, r4 |
| 607 |
mov r5, r8 |
| 608 |
b 1b |
| 609 |
0: |
| 610 |
vpop {q4-q7} |
| 611 |
pop {r4-r11,pc} |
| 612 |
endfunc |
| 613 |
|
| 614 |
// void dav1d_sgr_box5_h_16bpc_neon(int32_t *sumsq, int16_t *sum, |
| 615 |
// const pixel (*left)[4], |
| 616 |
// const pixel *src, const ptrdiff_t stride, |
| 617 |
// const int w, const int h, |
| 618 |
// const enum LrEdgeFlags edges); |
| 619 |
function sgr_box5_h_16bpc_neon, export=1 |
| 620 |
push {r4-r11,lr} |
| 621 |
vpush {q4-q7} |
| 622 |
ldrd r4, r5, [sp, #100] |
| 623 |
ldrd r6, r7, [sp, #108] |
| 624 |
add r5, r5, #2 // w += 2 |
| 625 |
|
| 626 |
// Set up pointers for reading/writing alternate rows |
| 627 |
add r10, r0, #(4*SUM_STRIDE) // sumsq |
| 628 |
add r11, r1, #(2*SUM_STRIDE) // sum |
| 629 |
add r12, r3, r4 // src |
| 630 |
lsl r4, r4, #1 |
| 631 |
mov r9, #(2*2*SUM_STRIDE) // double sum stride |
| 632 |
|
| 633 |
// Subtract the aligned width from the output stride. |
| 634 |
add lr, r5, #7 |
| 635 |
bic lr, lr, #7 |
| 636 |
sub r9, r9, lr, lsl #1 |
| 637 |
add lr, lr, #8 |
| 638 |
sub r4, r4, lr, lsl #1 |
| 639 |
|
| 640 |
// Store the width for the vertical loop |
| 641 |
mov r8, r5 |
| 642 |
|
| 643 |
// Set up the src pointers to include the left edge, for LR_HAVE_LEFT, left == NULL |
| 644 |
tst r7, #1 // LR_HAVE_LEFT |
| 645 |
beq 2f |
| 646 |
// LR_HAVE_LEFT |
| 647 |
cmp r2, #0 |
| 648 |
bne 0f |
| 649 |
// left == NULL |
| 650 |
sub r3, r3, #6 |
| 651 |
sub r12, r12, #6 |
| 652 |
b 1f |
| 653 |
0: // LR_HAVE_LEFT, left != NULL |
| 654 |
2: // !LR_HAVE_LEFT, increase the stride. |
| 655 |
// For this case we don't read the left 3 pixels from the src pointer, |
| 656 |
// but shift it as if we had done that. |
| 657 |
add r4, r4, #6 |
| 658 |
|
| 659 |
1: // Loop vertically |
| 660 |
vld1.16 {q0, q1}, [r3]! |
| 661 |
vld1.16 {q4, q5}, [r12]! |
| 662 |
|
| 663 |
tst r7, #1 // LR_HAVE_LEFT |
| 664 |
beq 0f |
| 665 |
cmp r2, #0 |
| 666 |
beq 2f |
| 667 |
// LR_HAVE_LEFT, left != NULL |
| 668 |
vld1.16 {d5}, [r2]! |
| 669 |
// Move r3/r12 back to account for the last 3 pixels we loaded earlier, |
| 670 |
// which we'll shift out. |
| 671 |
sub r3, r3, #6 |
| 672 |
sub r12, r12, #6 |
| 673 |
vld1.16 {d13}, [r2]! |
| 674 |
vext.8 q1, q0, q1, #10 |
| 675 |
vext.8 q0, q2, q0, #10 |
| 676 |
vext.8 q5, q4, q5, #10 |
| 677 |
vext.8 q4, q6, q4, #10 |
| 678 |
b 2f |
| 679 |
0: |
| 680 |
// !LR_HAVE_LEFT, fill q2 with the leftmost pixel |
| 681 |
// and shift q0 to have 3x the first pixel at the front. |
| 682 |
vdup.16 q2, d0[0] |
| 683 |
vdup.16 q6, d8[0] |
| 684 |
// Move r3 back to account for the last 3 pixels we loaded before, |
| 685 |
// which we shifted out. |
| 686 |
sub r3, r3, #6 |
| 687 |
sub r12, r12, #6 |
| 688 |
vext.8 q1, q0, q1, #10 |
| 689 |
vext.8 q0, q2, q0, #10 |
| 690 |
vext.8 q5, q4, q5, #10 |
| 691 |
vext.8 q4, q6, q4, #10 |
| 692 |
|
| 693 |
2: |
| 694 |
tst r7, #2 // LR_HAVE_RIGHT |
| 695 |
bne 4f |
| 696 |
// If we'll need to pad the right edge, load that pixel to pad with |
| 697 |
// here since we can find it pretty easily from here. |
| 698 |
sub lr, r5, #(2 + 16 - 3 + 1) |
| 699 |
lsl lr, lr, #1 |
| 700 |
ldrh r11, [r3, lr] |
| 701 |
ldrh lr, [r12, lr] |
| 702 |
// Fill q14/q15 with the right padding pixel |
| 703 |
vdup.16 q14, r11 |
| 704 |
vdup.16 q15, lr |
| 705 |
// Restore r11 after using it for a temporary value |
| 706 |
add r11, r1, #(2*SUM_STRIDE) |
| 707 |
3: // !LR_HAVE_RIGHT |
| 708 |
|
| 709 |
// Check whether we need to pad the right edge |
| 710 |
cmp r5, #11 |
| 711 |
bge 4f // If w >= 11, all used input pixels are valid |
| 712 |
|
| 713 |
// 1 <= w < 11, w+1 pixels valid in q0-q1. For w=9 or w=10, |
| 714 |
// this ends up called again; it's not strictly needed in those |
| 715 |
// cases (we pad enough here), but keeping the code as simple as possible. |
| 716 |
|
| 717 |
// Insert padding in q0/1.h[w+1] onwards; fuse the +1 into the |
| 718 |
// buffer pointer. |
| 719 |
movrel_local lr, right_ext_mask, -2 |
| 720 |
sub lr, lr, r5, lsl #1 |
| 721 |
vld1.8 {q12, q13}, [lr] |
| 722 |
|
| 723 |
vbit q0, q14, q12 |
| 724 |
vbit q1, q14, q13 |
| 725 |
vbit q4, q15, q12 |
| 726 |
vbit q5, q15, q13 |
| 727 |
|
| 728 |
4: // Loop horizontally |
| 729 |
vext.8 q8, q0, q1, #2 |
| 730 |
vext.8 q10, q4, q5, #2 |
| 731 |
vext.8 q9, q0, q1, #4 |
| 732 |
vext.8 q11, q4, q5, #4 |
| 733 |
vadd.i16 q2, q0, q8 |
| 734 |
vadd.i16 q3, q4, q10 |
| 735 |
vadd.i16 q2, q2, q9 |
| 736 |
vadd.i16 q3, q3, q11 |
| 737 |
|
| 738 |
vmull.u16 q6, d0, d0 |
| 739 |
vmlal.u16 q6, d16, d16 |
| 740 |
vmlal.u16 q6, d18, d18 |
| 741 |
vmull.u16 q12, d8, d8 |
| 742 |
vmlal.u16 q12, d20, d20 |
| 743 |
vmlal.u16 q12, d22, d22 |
| 744 |
vmull.u16 q7, d1, d1 |
| 745 |
vmlal.u16 q7, d17, d17 |
| 746 |
vmlal.u16 q7, d19, d19 |
| 747 |
vmull.u16 q13, d9, d9 |
| 748 |
vmlal.u16 q13, d21, d21 |
| 749 |
vmlal.u16 q13, d23, d23 |
| 750 |
|
| 751 |
vext.8 q8, q0, q1, #6 |
| 752 |
vext.8 q10, q4, q5, #6 |
| 753 |
vext.8 q9, q0, q1, #8 |
| 754 |
vext.8 q11, q4, q5, #8 |
| 755 |
vadd.i16 q2, q2, q8 |
| 756 |
vadd.i16 q3, q3, q10 |
| 757 |
vadd.i16 q2, q2, q9 |
| 758 |
vadd.i16 q3, q3, q11 |
| 759 |
|
| 760 |
vmlal.u16 q6, d16, d16 |
| 761 |
vmlal.u16 q6, d1, d1 |
| 762 |
vmlal.u16 q12, d20, d20 |
| 763 |
vmlal.u16 q12, d9, d9 |
| 764 |
vmlal.u16 q7, d17, d17 |
| 765 |
vmlal.u16 q7, d19, d19 |
| 766 |
vmlal.u16 q13, d21, d21 |
| 767 |
vmlal.u16 q13, d23, d23 |
| 768 |
|
| 769 |
subs r5, r5, #8 |
| 770 |
vst1.16 {q2}, [r1, :128]! |
| 771 |
vst1.16 {q3}, [r11, :128]! |
| 772 |
vst1.32 {q6, q7}, [r0, :128]! |
| 773 |
vst1.32 {q12, q13}, [r10, :128]! |
| 774 |
|
| 775 |
ble 9f |
| 776 |
tst r7, #2 // LR_HAVE_RIGHT |
| 777 |
vmov q0, q1 |
| 778 |
vmov q4, q5 |
| 779 |
vld1.16 {q1}, [r3]! |
| 780 |
vld1.16 {q5}, [r12]! |
| 781 |
bne 4b // If we don't need to pad, just keep summing. |
| 782 |
b 3b // If we need to pad, check how many pixels we have left. |
| 783 |
|
| 784 |
9: |
| 785 |
subs r6, r6, #2 |
| 786 |
ble 0f |
| 787 |
// Jump to the next row and loop horizontally |
| 788 |
add r0, r0, r9, lsl #1 |
| 789 |
add r10, r10, r9, lsl #1 |
| 790 |
add r1, r1, r9 |
| 791 |
add r11, r11, r9 |
| 792 |
add r3, r3, r4 |
| 793 |
add r12, r12, r4 |
| 794 |
mov r5, r8 |
| 795 |
b 1b |
| 796 |
0: |
| 797 |
vpop {q4-q7} |
| 798 |
pop {r4-r11,pc} |
| 799 |
endfunc |
| 800 |
|
| 801 |
sgr_funcs 16 |