Debian Codegraph @

.cargo/vendor/rav1e/src/x86/cdef_dist.asm

raw
1 ; Copyright (c) 2022, The rav1e contributors. All rights reserved
2 ;
3 ; This source code is subject to the terms of the BSD 2 Clause License and
4 ; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
5 ; was not distributed with this source code in the LICENSE file, you can
6 ; obtain it at www.aomedia.org/license/software. If the Alliance for Open
7 ; Media Patent License 1.0 was not distributed with this source code in the
8 ; PATENTS file, you can obtain it at www.aomedia.org/license/patent.
9
10 %include "config.asm"
11 %include "ext/x86/x86inc.asm"
12
13 SECTION .text
14
15 %if ARCH_X86_64
16
17 ; m0: zero register
18 ; m1: src input
19 ; m2: dst input
20 ; m3 = sum(src_{i,j})
21 ; m4 = sum(src_{i,j}^2)
22 ; m5 = sum(dst_{i,j})
23 ; m6 = sum(dst_{i,j}^2)
24 ; m7 = sum(src_{i,j} * dst_{i,j})
25 ; m8: tmp register
26 %macro CDEF_DIST_W8_SSE2 0
27 psadbw m8, m1, m0 ; sum pixel values
28 paddd m3, m8 ; accumulate
29 punpcklbw m1, m0 ; convert to 16-bits
30 pmaddwd m8, m1, m1 ; square and horz add
31 paddd m4, m8 ; accumulate
32 psadbw m8, m2, m0 ; same as above, but for dst
33 paddd m5, m8
34 punpcklbw m2, m0
35 pmaddwd m8, m2, m2
36 paddd m6, m8
37 pmaddwd m8, m1, m2 ; src_{i,j} * dst_{i,j} (and horz add)
38 paddd m7, m8
39 %endmacro
40
41 ; Refine sums into variances and sse
42 ; parameter: scale log2 relative to 8x8
43 %macro CDEF_DIST_REFINE_SSE2 1
44 ; Compute [sum(src)^2, sum(dst)^2]
45 punpckldq m3, m5 ; store sums in a single vector
46 pcmpeqd m0, m0 ; -1 (for rounding)
47 pmaddwd m3, m3
48
49 ; Divide by area and round
50 pslld m0, 5 - %1
51 psubd m3, m0 ; + (1 << (5 - %1))
52 psrld m3, 6 - %1
53
54 pshufd m0, m7, q3232 ; reduce sum(src * dst)
55 punpckldq m1, m4, m6 ; reduce [sum(src^2), sum(dst^2)]
56 paddd m7, m0
57 punpckhdq m4, m6
58 paddd m1, m4
59 paddd m7, m7 ; 2 * sum(src * dst) [Partially reduced; len 2]
60 pshufd m0, m1, q3232
61
62 ; Equivelent to:
63 ; paddd m1, m0
64 ; psubd m0, m1, m7 ; sse = sum(src^2) + sum(dst^2) - sum(src * dst)
65 ; but with fewer dependancies
66 psubd m7, m1
67 paddd m1, m0 ; [sum(src^2), sum(dst^2)]
68 psubd m0, m7 ; sse (Needs reducing; len 2)
69
70 psubd m1, m3 ; [src variance, dst variance]
71 ; Scale up the variances up to 8x8
72 ; TODO: this can be handled inside ssim boost in the future
73 %if %1 != 0
74 pslld m1, %1
75 %endif
76 movq [ret_ptrq], m1
77
78 ; Final reduce for sse
79 pshuflw m2, m0, q3232
80 paddd m0, m2
81 movd [ret_ptrq+8], m0
82 %endmacro
83
84 INIT_XMM sse2
85 cglobal cdef_dist_kernel_4x4, 5, 5, 9, \
86 src, src_stride, dst, dst_stride, ret_ptr
87 pxor m0, m0
88 movd m1, [srcq]
89 movd m2, [srcq+src_strideq]
90 punpckldq m1, m2
91 movd m2, [dstq]
92 movd m8, [dstq+dst_strideq]
93 lea srcq, [srcq+2*src_strideq]
94 lea dstq, [dstq+2*dst_strideq]
95 punpckldq m2, m8
96 psadbw m3, m1, m0
97 punpcklbw m1, m0
98 pmaddwd m4, m1, m1
99 psadbw m5, m2, m0
100 punpcklbw m2, m0
101 pmaddwd m6, m2, m2
102 pmaddwd m7, m1, m2
103
104 movd m1, [srcq]
105 movd m2, [srcq+src_strideq]
106 punpckldq m1, m2
107 movd m2, [dstq]
108 movd m8, [dstq+dst_strideq]
109 punpckldq m2, m8
110 CDEF_DIST_W8_SSE2
111
112 CDEF_DIST_REFINE_SSE2 2
113 RET
114
115 cglobal cdef_dist_kernel_4x8, 5, 7, 9, \
116 src, src_stride, dst, dst_stride, ret_ptr, src_stride3, dst_stride3
117 lea src_stride3q, [src_strideq*3]
118 lea dst_stride3q, [dst_strideq*3]
119
120 pxor m0, m0
121 movd m1, [srcq]
122 movd m2, [srcq+src_strideq]
123 punpckldq m1, m2
124 movd m2, [dstq]
125 movd m8, [dstq+dst_strideq]
126 punpckldq m2, m8
127 psadbw m3, m1, m0
128 punpcklbw m1, m0
129 pmaddwd m4, m1, m1
130 psadbw m5, m2, m0
131 punpcklbw m2, m0
132 pmaddwd m6, m2, m2
133 pmaddwd m7, m1, m2
134
135 movd m1, [srcq+2*src_strideq]
136 movd m2, [srcq+src_stride3q]
137 punpckldq m1, m2
138 movd m2, [dstq+2*dst_strideq]
139 movd m8, [dstq+dst_stride3q]
140 lea srcq, [srcq+4*src_strideq]
141 lea dstq, [dstq+4*dst_strideq]
142 punpckldq m2, m8
143 CDEF_DIST_W8_SSE2
144 movd m1, [srcq]
145 movd m2, [srcq+src_strideq]
146 punpckldq m1, m2
147 movd m2, [dstq]
148 movd m8, [dstq+dst_strideq]
149 punpckldq m2, m8
150 CDEF_DIST_W8_SSE2
151 movd m1, [srcq+2*src_strideq]
152 movd m2, [srcq+src_stride3q]
153 punpckldq m1, m2
154 movd m2, [dstq+2*dst_strideq]
155 movd m8, [dstq+dst_stride3q]
156 punpckldq m2, m8
157 CDEF_DIST_W8_SSE2
158
159 CDEF_DIST_REFINE_SSE2 1
160 RET
161
162 cglobal cdef_dist_kernel_8x4, 5, 5, 9, \
163 src, src_stride, dst, dst_stride, ret_ptr
164 pxor m0, m0
165 movq m1, [srcq]
166 psadbw m3, m1, m0
167 punpcklbw m1, m0
168 pmaddwd m4, m1, m1
169 movq m2, [dstq]
170 psadbw m5, m2, m0
171 punpcklbw m2, m0
172 pmaddwd m6, m2, m2
173 pmaddwd m7, m1, m2
174
175 movq m1, [srcq+src_strideq]
176 movq m2, [dstq+dst_strideq]
177 lea srcq, [srcq+2*src_strideq]
178 lea dstq, [dstq+2*dst_strideq]
179 CDEF_DIST_W8_SSE2
180 movq m1, [srcq]
181 movq m2, [dstq]
182 CDEF_DIST_W8_SSE2
183 movq m1, [srcq+src_strideq]
184 movq m2, [dstq+dst_strideq]
185 CDEF_DIST_W8_SSE2
186
187 CDEF_DIST_REFINE_SSE2 1
188 RET
189
190 cglobal cdef_dist_kernel_8x8, 5, 7, 9, \
191 src, src_stride, dst, dst_stride, ret_ptr, src_stride3, dst_stride3
192 lea src_stride3q, [src_strideq*3]
193 lea dst_stride3q, [dst_strideq*3]
194
195 pxor m0, m0
196 movq m1, [srcq]
197 psadbw m3, m1, m0
198 punpcklbw m1, m0
199 pmaddwd m4, m1, m1
200 movq m2, [dstq]
201 psadbw m5, m2, m0
202 punpcklbw m2, m0
203 pmaddwd m6, m2, m2
204 pmaddwd m7, m1, m2
205
206 movq m1, [srcq+src_strideq]
207 movq m2, [dstq+dst_strideq]
208 CDEF_DIST_W8_SSE2
209 movq m1, [srcq+2*src_strideq]
210 movq m2, [dstq+2*dst_strideq]
211 CDEF_DIST_W8_SSE2
212 movq m1, [srcq+src_stride3q]
213 movq m2, [dstq+dst_stride3q]
214 CDEF_DIST_W8_SSE2
215 lea srcq, [srcq+src_strideq*4]
216 lea dstq, [dstq+dst_strideq*4]
217
218 movq m1, [srcq]
219 movq m2, [dstq]
220 CDEF_DIST_W8_SSE2
221 movq m1, [srcq+src_strideq]
222 movq m2, [dstq+dst_strideq]
223 CDEF_DIST_W8_SSE2
224 movq m1, [srcq+2*src_strideq]
225 movq m2, [dstq+2*dst_strideq]
226 CDEF_DIST_W8_SSE2
227 movq m1, [srcq+src_stride3q]
228 movq m2, [dstq+dst_stride3q]
229 CDEF_DIST_W8_SSE2
230
231 CDEF_DIST_REFINE_SSE2 0
232 RET
233 %endif ; ARCH_X86_64