| 1 |
/* Copyright (c) 2023, The rav1e contributors. All rights reserved |
| 2 |
* |
| 3 |
* This source code is subject to the terms of the BSD 2 Clause License and |
| 4 |
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License |
| 5 |
* was not distributed with this source code in the LICENSE file, you can |
| 6 |
* obtain it at www.aomedia.org/license/software. If the Alliance for Open |
| 7 |
* Media Patent License 1.0 was not distributed with this source code in the |
| 8 |
* PATENTS file, you can obtain it at www.aomedia.org/license/patent. |
| 9 |
*/ |
| 10 |
|
| 11 |
#include "src/arm/asm.S" |
| 12 |
#include "util.S" |
| 13 |
|
| 14 |
// v0: tmp register |
| 15 |
// v1: src input |
| 16 |
// v2: dst input |
| 17 |
// v3 = sum(src_{i,j}) |
| 18 |
// v4 = sum(src_{i,j}^2) |
| 19 |
// v5 = sum(dst_{i,j}) |
| 20 |
// v6 = sum(dst_{i,j}^2) |
| 21 |
// v7 = sum(src_{i,j} * dst_{i,j}) |
| 22 |
// v16: zero register |
| 23 |
.macro CDEF_DIST_W8 |
| 24 |
uabal v3.8h, v1.8b, v16.8b // sum pixel values |
| 25 |
umull v0.8h, v1.8b, v1.8b // square |
| 26 |
uabal v4.4s, v0.4h, v16.4h // accumulate |
| 27 |
uabal2 v4.4s, v0.8h, v16.8h |
| 28 |
uabal v5.8h, v2.8b, v16.8b // same as above, but for dst |
| 29 |
umull v0.8h, v2.8b, v2.8b |
| 30 |
uabal v6.4s, v0.4h, v16.4h |
| 31 |
uabal2 v6.4s, v0.8h, v16.8h |
| 32 |
umull v0.8h, v1.8b, v2.8b // src_{i,j} * dst_{i,j} |
| 33 |
uabal v7.4s, v0.4h, v16.4h |
| 34 |
uabal2 v7.4s, v0.8h, v16.8h |
| 35 |
.endm |
| 36 |
|
| 37 |
.macro CDEF_DIST_REFINE shift=0 |
| 38 |
addv h3, v3.8h |
| 39 |
umull v3.4s, v3.4h, v3.4h |
| 40 |
urshr v3.4s, v3.4s, #(6-\shift) // s3: sum(src_{i,j})^2 / N |
| 41 |
addv s4, v4.4s // s4: sum(src_{i,j}^2) |
| 42 |
addv h5, v5.8h |
| 43 |
umull v5.4s, v5.4h, v5.4h |
| 44 |
urshr v5.4s, v5.4s, #(6-\shift) // s5: sum(dst_{i,j})^2 / N |
| 45 |
addv s6, v6.4s // s6: sum(dst_{i,j}^2) |
| 46 |
addv s7, v7.4s |
| 47 |
add v0.4s, v4.4s, v6.4s |
| 48 |
sub v0.4s, v0.4s, v7.4s |
| 49 |
sub v0.4s, v0.4s, v7.4s // s0: sse |
| 50 |
uqsub v4.4s, v4.4s, v3.4s // s4: svar |
| 51 |
uqsub v6.4s, v6.4s, v5.4s // s6: dvar |
| 52 |
.if \shift != 0 |
| 53 |
shl v4.4s, v4.4s, #\shift |
| 54 |
shl v6.4s, v6.4s, #\shift |
| 55 |
.endif |
| 56 |
str s4, [x4] |
| 57 |
str s6, [x4, #4] |
| 58 |
str s0, [x4, #8] |
| 59 |
.endm |
| 60 |
|
| 61 |
.macro LOAD_ROW |
| 62 |
ldr d1, [x0] |
| 63 |
ldr d2, [x2] |
| 64 |
add x0, x0, x1 |
| 65 |
add x2, x2, x3 |
| 66 |
.endm |
| 67 |
|
| 68 |
.macro LOAD_ROWS |
| 69 |
ldr s1, [x0] |
| 70 |
ldr s2, [x2] |
| 71 |
ldr s0, [x0, x1] |
| 72 |
ldr s17, [x2, x3] |
| 73 |
add x0, x0, x1, lsl 1 |
| 74 |
add x2, x2, x3, lsl 1 |
| 75 |
zip1 v1.2s, v1.2s, v0.2s |
| 76 |
zip1 v2.2s, v2.2s, v17.2s |
| 77 |
.endm |
| 78 |
|
| 79 |
.macro CDEF_DIST_INIT width, height |
| 80 |
.irp i, v3.8h, v4.8h, v5.8h, v6.8h, v7.8h, v16.8h |
| 81 |
movi \i, #0 |
| 82 |
.endr |
| 83 |
.if \width == 4 |
| 84 |
mov w5, #(\height / 2) |
| 85 |
.else |
| 86 |
mov w5, #\height |
| 87 |
.endif |
| 88 |
.endm |
| 89 |
|
| 90 |
// x0: src: *const u8, |
| 91 |
// x1: src_stride: isize, |
| 92 |
// x2: dst: *const u8, |
| 93 |
// x3: dst_stride: isize, |
| 94 |
// x4: ret_ptr: *mut u32, |
| 95 |
function cdef_dist_kernel_4x4_neon, export=1 |
| 96 |
CDEF_DIST_INIT 4, 4 |
| 97 |
L(cdk_4x4): |
| 98 |
LOAD_ROWS |
| 99 |
CDEF_DIST_W8 |
| 100 |
subs w5, w5, #1 |
| 101 |
bne L(cdk_4x4) |
| 102 |
CDEF_DIST_REFINE 2 |
| 103 |
ret |
| 104 |
endfunc |
| 105 |
|
| 106 |
function cdef_dist_kernel_4x8_neon, export=1 |
| 107 |
CDEF_DIST_INIT 4, 8 |
| 108 |
L(cdk_4x8): |
| 109 |
LOAD_ROWS |
| 110 |
CDEF_DIST_W8 |
| 111 |
subs w5, w5, #1 |
| 112 |
bne L(cdk_4x8) |
| 113 |
CDEF_DIST_REFINE 1 |
| 114 |
ret |
| 115 |
endfunc |
| 116 |
|
| 117 |
function cdef_dist_kernel_8x4_neon, export=1 |
| 118 |
CDEF_DIST_INIT 8, 4 |
| 119 |
L(cdk_8x4): |
| 120 |
LOAD_ROW |
| 121 |
CDEF_DIST_W8 |
| 122 |
subs w5, w5, #1 |
| 123 |
bne L(cdk_8x4) |
| 124 |
CDEF_DIST_REFINE 1 |
| 125 |
ret |
| 126 |
endfunc |
| 127 |
|
| 128 |
function cdef_dist_kernel_8x8_neon, export=1 |
| 129 |
CDEF_DIST_INIT 8, 8 |
| 130 |
L(cdk_8x8): |
| 131 |
LOAD_ROW |
| 132 |
CDEF_DIST_W8 |
| 133 |
subs w5, w5, #1 |
| 134 |
bne L(cdk_8x8) |
| 135 |
CDEF_DIST_REFINE |
| 136 |
ret |
| 137 |
endfunc |
| 138 |
|
| 139 |
// v0: tmp register |
| 140 |
// v1: src input |
| 141 |
// v2: dst input |
| 142 |
// v3 = sum(src_{i,j}) |
| 143 |
// v4 = sum(src_{i,j}^2) |
| 144 |
// v5 = sum(dst_{i,j}) |
| 145 |
// v6 = sum(dst_{i,j}^2) |
| 146 |
// v7 = sum(src_{i,j} * dst_{i,j}) |
| 147 |
// v16: zero register |
| 148 |
.macro CDEF_DIST_HBD_W8 |
| 149 |
uabal v3.4s, v1.4h, v16.4h // sum pixel values |
| 150 |
uabal2 v3.4s, v1.8h, v16.8h |
| 151 |
umlal v4.4s, v1.4h, v1.4h // square and accumulate |
| 152 |
umlal2 v4.4s, v1.8h, v1.8h |
| 153 |
uabal v5.4s, v2.4h, v16.4h // same as above, but for dst |
| 154 |
uabal2 v5.4s, v2.8h, v16.8h |
| 155 |
umlal v6.4s, v2.4h, v2.4h |
| 156 |
umlal2 v6.4s, v2.8h, v2.8h |
| 157 |
umlal v7.4s, v1.4h, v2.4h // src_{i,j} * dst_{i,j} |
| 158 |
umlal2 v7.4s, v1.8h, v2.8h |
| 159 |
.endm |
| 160 |
|
| 161 |
.macro CDEF_DIST_HBD_REFINE shift=0 |
| 162 |
addv s3, v3.4s |
| 163 |
umull v3.2d, v3.2s, v3.2s |
| 164 |
urshr d3, d3, #(6-\shift) // d3: sum(src_{i,j})^2 / N |
| 165 |
uaddlv d4, v4.4s // d4: sum(src_{i,j}^2) |
| 166 |
addv s5, v5.4s |
| 167 |
umull v5.2d, v5.2s, v5.2s |
| 168 |
urshr d5, d5, #(6-\shift) // d5: sum(dst_{i,j})^2 / N |
| 169 |
uaddlv d6, v6.4s // d6: sum(dst_{i,j}^2) |
| 170 |
uaddlv d7, v7.4s |
| 171 |
add d0, d4, d6 |
| 172 |
sub d0, d0, d7 |
| 173 |
sub d0, d0, d7 // d0: sse |
| 174 |
uqsub d4, d4, d3 // d4: svar |
| 175 |
uqsub d6, d6, d5 // d6: dvar |
| 176 |
.if \shift != 0 |
| 177 |
shl d4, d4, #\shift |
| 178 |
shl d6, d6, #\shift |
| 179 |
.endif |
| 180 |
str s4, [x4] |
| 181 |
str s6, [x4, #4] |
| 182 |
str s0, [x4, #8] |
| 183 |
.endm |
| 184 |
|
| 185 |
.macro LOAD_ROW_HBD |
| 186 |
ldr q1, [x0] |
| 187 |
ldr q2, [x2] |
| 188 |
add x0, x0, x1 |
| 189 |
add x2, x2, x3 |
| 190 |
.endm |
| 191 |
|
| 192 |
.macro LOAD_ROWS_HBD |
| 193 |
ldr d1, [x0] |
| 194 |
ldr d2, [x2] |
| 195 |
ldr d0, [x0, x1] |
| 196 |
ldr d17, [x2, x3] |
| 197 |
add x0, x0, x1, lsl 1 |
| 198 |
add x2, x2, x3, lsl 1 |
| 199 |
mov v1.d[1], v0.d[0] |
| 200 |
mov v2.d[1], v17.d[0] |
| 201 |
.endm |
| 202 |
|
| 203 |
// x0: src: *const u16, |
| 204 |
// x1: src_stride: isize, |
| 205 |
// x2: dst: *const u16, |
| 206 |
// x3: dst_stride: isize, |
| 207 |
// x4: ret_ptr: *mut u32, |
| 208 |
function cdef_dist_kernel_4x4_hbd_neon, export=1 |
| 209 |
CDEF_DIST_INIT 4, 4 |
| 210 |
L(cdk_hbd_4x4): |
| 211 |
LOAD_ROWS_HBD |
| 212 |
CDEF_DIST_HBD_W8 |
| 213 |
subs w5, w5, #1 |
| 214 |
bne L(cdk_hbd_4x4) |
| 215 |
CDEF_DIST_HBD_REFINE 2 |
| 216 |
ret |
| 217 |
endfunc |
| 218 |
|
| 219 |
function cdef_dist_kernel_4x8_hbd_neon, export=1 |
| 220 |
CDEF_DIST_INIT 4, 8 |
| 221 |
L(cdk_hbd_4x8): |
| 222 |
LOAD_ROWS_HBD |
| 223 |
CDEF_DIST_HBD_W8 |
| 224 |
subs w5, w5, #1 |
| 225 |
bne L(cdk_hbd_4x8) |
| 226 |
CDEF_DIST_HBD_REFINE 1 |
| 227 |
ret |
| 228 |
endfunc |
| 229 |
|
| 230 |
function cdef_dist_kernel_8x4_hbd_neon, export=1 |
| 231 |
CDEF_DIST_INIT 8, 4 |
| 232 |
L(cdk_hbd_8x4): |
| 233 |
LOAD_ROW_HBD |
| 234 |
CDEF_DIST_HBD_W8 |
| 235 |
subs w5, w5, #1 |
| 236 |
bne L(cdk_hbd_8x4) |
| 237 |
CDEF_DIST_HBD_REFINE 1 |
| 238 |
ret |
| 239 |
endfunc |
| 240 |
|
| 241 |
function cdef_dist_kernel_8x8_hbd_neon, export=1 |
| 242 |
CDEF_DIST_INIT 8, 8 |
| 243 |
L(cdk_hbd_8x8): |
| 244 |
LOAD_ROW_HBD |
| 245 |
CDEF_DIST_HBD_W8 |
| 246 |
subs w5, w5, #1 |
| 247 |
bne L(cdk_hbd_8x8) |
| 248 |
CDEF_DIST_HBD_REFINE |
| 249 |
ret |
| 250 |
endfunc |