Debian Codegraph @

.cargo/vendor/rav1e/src/arm/64/cdef_dist.S

raw
1 /* Copyright (c) 2023, The rav1e contributors. All rights reserved
2 *
3 * This source code is subject to the terms of the BSD 2 Clause License and
4 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
5 * was not distributed with this source code in the LICENSE file, you can
6 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
7 * Media Patent License 1.0 was not distributed with this source code in the
8 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
9 */
10
11 #include "src/arm/asm.S"
12 #include "util.S"
13
14 // v0: tmp register
15 // v1: src input
16 // v2: dst input
17 // v3 = sum(src_{i,j})
18 // v4 = sum(src_{i,j}^2)
19 // v5 = sum(dst_{i,j})
20 // v6 = sum(dst_{i,j}^2)
21 // v7 = sum(src_{i,j} * dst_{i,j})
22 // v16: zero register
23 .macro CDEF_DIST_W8
24 uabal v3.8h, v1.8b, v16.8b // sum pixel values
25 umull v0.8h, v1.8b, v1.8b // square
26 uabal v4.4s, v0.4h, v16.4h // accumulate
27 uabal2 v4.4s, v0.8h, v16.8h
28 uabal v5.8h, v2.8b, v16.8b // same as above, but for dst
29 umull v0.8h, v2.8b, v2.8b
30 uabal v6.4s, v0.4h, v16.4h
31 uabal2 v6.4s, v0.8h, v16.8h
32 umull v0.8h, v1.8b, v2.8b // src_{i,j} * dst_{i,j}
33 uabal v7.4s, v0.4h, v16.4h
34 uabal2 v7.4s, v0.8h, v16.8h
35 .endm
36
37 .macro CDEF_DIST_REFINE shift=0
38 addv h3, v3.8h
39 umull v3.4s, v3.4h, v3.4h
40 urshr v3.4s, v3.4s, #(6-\shift) // s3: sum(src_{i,j})^2 / N
41 addv s4, v4.4s // s4: sum(src_{i,j}^2)
42 addv h5, v5.8h
43 umull v5.4s, v5.4h, v5.4h
44 urshr v5.4s, v5.4s, #(6-\shift) // s5: sum(dst_{i,j})^2 / N
45 addv s6, v6.4s // s6: sum(dst_{i,j}^2)
46 addv s7, v7.4s
47 add v0.4s, v4.4s, v6.4s
48 sub v0.4s, v0.4s, v7.4s
49 sub v0.4s, v0.4s, v7.4s // s0: sse
50 uqsub v4.4s, v4.4s, v3.4s // s4: svar
51 uqsub v6.4s, v6.4s, v5.4s // s6: dvar
52 .if \shift != 0
53 shl v4.4s, v4.4s, #\shift
54 shl v6.4s, v6.4s, #\shift
55 .endif
56 str s4, [x4]
57 str s6, [x4, #4]
58 str s0, [x4, #8]
59 .endm
60
61 .macro LOAD_ROW
62 ldr d1, [x0]
63 ldr d2, [x2]
64 add x0, x0, x1
65 add x2, x2, x3
66 .endm
67
68 .macro LOAD_ROWS
69 ldr s1, [x0]
70 ldr s2, [x2]
71 ldr s0, [x0, x1]
72 ldr s17, [x2, x3]
73 add x0, x0, x1, lsl 1
74 add x2, x2, x3, lsl 1
75 zip1 v1.2s, v1.2s, v0.2s
76 zip1 v2.2s, v2.2s, v17.2s
77 .endm
78
79 .macro CDEF_DIST_INIT width, height
80 .irp i, v3.8h, v4.8h, v5.8h, v6.8h, v7.8h, v16.8h
81 movi \i, #0
82 .endr
83 .if \width == 4
84 mov w5, #(\height / 2)
85 .else
86 mov w5, #\height
87 .endif
88 .endm
89
90 // x0: src: *const u8,
91 // x1: src_stride: isize,
92 // x2: dst: *const u8,
93 // x3: dst_stride: isize,
94 // x4: ret_ptr: *mut u32,
95 function cdef_dist_kernel_4x4_neon, export=1
96 CDEF_DIST_INIT 4, 4
97 L(cdk_4x4):
98 LOAD_ROWS
99 CDEF_DIST_W8
100 subs w5, w5, #1
101 bne L(cdk_4x4)
102 CDEF_DIST_REFINE 2
103 ret
104 endfunc
105
106 function cdef_dist_kernel_4x8_neon, export=1
107 CDEF_DIST_INIT 4, 8
108 L(cdk_4x8):
109 LOAD_ROWS
110 CDEF_DIST_W8
111 subs w5, w5, #1
112 bne L(cdk_4x8)
113 CDEF_DIST_REFINE 1
114 ret
115 endfunc
116
117 function cdef_dist_kernel_8x4_neon, export=1
118 CDEF_DIST_INIT 8, 4
119 L(cdk_8x4):
120 LOAD_ROW
121 CDEF_DIST_W8
122 subs w5, w5, #1
123 bne L(cdk_8x4)
124 CDEF_DIST_REFINE 1
125 ret
126 endfunc
127
128 function cdef_dist_kernel_8x8_neon, export=1
129 CDEF_DIST_INIT 8, 8
130 L(cdk_8x8):
131 LOAD_ROW
132 CDEF_DIST_W8
133 subs w5, w5, #1
134 bne L(cdk_8x8)
135 CDEF_DIST_REFINE
136 ret
137 endfunc
138
139 // v0: tmp register
140 // v1: src input
141 // v2: dst input
142 // v3 = sum(src_{i,j})
143 // v4 = sum(src_{i,j}^2)
144 // v5 = sum(dst_{i,j})
145 // v6 = sum(dst_{i,j}^2)
146 // v7 = sum(src_{i,j} * dst_{i,j})
147 // v16: zero register
148 .macro CDEF_DIST_HBD_W8
149 uabal v3.4s, v1.4h, v16.4h // sum pixel values
150 uabal2 v3.4s, v1.8h, v16.8h
151 umlal v4.4s, v1.4h, v1.4h // square and accumulate
152 umlal2 v4.4s, v1.8h, v1.8h
153 uabal v5.4s, v2.4h, v16.4h // same as above, but for dst
154 uabal2 v5.4s, v2.8h, v16.8h
155 umlal v6.4s, v2.4h, v2.4h
156 umlal2 v6.4s, v2.8h, v2.8h
157 umlal v7.4s, v1.4h, v2.4h // src_{i,j} * dst_{i,j}
158 umlal2 v7.4s, v1.8h, v2.8h
159 .endm
160
161 .macro CDEF_DIST_HBD_REFINE shift=0
162 addv s3, v3.4s
163 umull v3.2d, v3.2s, v3.2s
164 urshr d3, d3, #(6-\shift) // d3: sum(src_{i,j})^2 / N
165 uaddlv d4, v4.4s // d4: sum(src_{i,j}^2)
166 addv s5, v5.4s
167 umull v5.2d, v5.2s, v5.2s
168 urshr d5, d5, #(6-\shift) // d5: sum(dst_{i,j})^2 / N
169 uaddlv d6, v6.4s // d6: sum(dst_{i,j}^2)
170 uaddlv d7, v7.4s
171 add d0, d4, d6
172 sub d0, d0, d7
173 sub d0, d0, d7 // d0: sse
174 uqsub d4, d4, d3 // d4: svar
175 uqsub d6, d6, d5 // d6: dvar
176 .if \shift != 0
177 shl d4, d4, #\shift
178 shl d6, d6, #\shift
179 .endif
180 str s4, [x4]
181 str s6, [x4, #4]
182 str s0, [x4, #8]
183 .endm
184
185 .macro LOAD_ROW_HBD
186 ldr q1, [x0]
187 ldr q2, [x2]
188 add x0, x0, x1
189 add x2, x2, x3
190 .endm
191
192 .macro LOAD_ROWS_HBD
193 ldr d1, [x0]
194 ldr d2, [x2]
195 ldr d0, [x0, x1]
196 ldr d17, [x2, x3]
197 add x0, x0, x1, lsl 1
198 add x2, x2, x3, lsl 1
199 mov v1.d[1], v0.d[0]
200 mov v2.d[1], v17.d[0]
201 .endm
202
203 // x0: src: *const u16,
204 // x1: src_stride: isize,
205 // x2: dst: *const u16,
206 // x3: dst_stride: isize,
207 // x4: ret_ptr: *mut u32,
208 function cdef_dist_kernel_4x4_hbd_neon, export=1
209 CDEF_DIST_INIT 4, 4
210 L(cdk_hbd_4x4):
211 LOAD_ROWS_HBD
212 CDEF_DIST_HBD_W8
213 subs w5, w5, #1
214 bne L(cdk_hbd_4x4)
215 CDEF_DIST_HBD_REFINE 2
216 ret
217 endfunc
218
219 function cdef_dist_kernel_4x8_hbd_neon, export=1
220 CDEF_DIST_INIT 4, 8
221 L(cdk_hbd_4x8):
222 LOAD_ROWS_HBD
223 CDEF_DIST_HBD_W8
224 subs w5, w5, #1
225 bne L(cdk_hbd_4x8)
226 CDEF_DIST_HBD_REFINE 1
227 ret
228 endfunc
229
230 function cdef_dist_kernel_8x4_hbd_neon, export=1
231 CDEF_DIST_INIT 8, 4
232 L(cdk_hbd_8x4):
233 LOAD_ROW_HBD
234 CDEF_DIST_HBD_W8
235 subs w5, w5, #1
236 bne L(cdk_hbd_8x4)
237 CDEF_DIST_HBD_REFINE 1
238 ret
239 endfunc
240
241 function cdef_dist_kernel_8x8_hbd_neon, export=1
242 CDEF_DIST_INIT 8, 8
243 L(cdk_hbd_8x8):
244 LOAD_ROW_HBD
245 CDEF_DIST_HBD_W8
246 subs w5, w5, #1
247 bne L(cdk_hbd_8x8)
248 CDEF_DIST_HBD_REFINE
249 ret
250 endfunc