Debian Codegraph @

.cargo/vendor/rav1e/src/x86/sse.asm

raw
1 ; Copyright (c) 2020-2022, The rav1e contributors. All rights reserved
2 ;
3 ; This source code is subject to the terms of the BSD 2 Clause License and
4 ; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
5 ; was not distributed with this source code in the LICENSE file, you can
6 ; obtain it at www.aomedia.org/license/software. If the Alliance for Open
7 ; Media Patent License 1.0 was not distributed with this source code in the
8 ; PATENTS file, you can obtain it at www.aomedia.org/license/patent.
9
10 %include "config.asm"
11 %include "ext/x86/x86inc.asm"
12
13 ; Must match crate::dist::rust::GET_WEIGHTED_SSE_SHIFT
14 %define get_weighted_sse_shift 8
15 %define get_weighted_sse_round (1 << get_weighted_sse_shift >> 1)
16
17 SECTION_RODATA 32
18 addsub: times 16 db 1, -1
19 rounding: times 4 dq get_weighted_sse_round
20
21 SECTION .text
22
23 %define m(x) mangle(private_prefix %+ _ %+ x %+ SUFFIX)
24
25 ; Consolidate scaling and rounding to one place so that it is easier to change.
26
27 %macro SSE_SCALE_4X4 0
28 ; Multiply and shift using scalar code
29 mov scaled, [scaleq]
30 imul rax, scaleq
31 add rax, get_weighted_sse_round
32 shr rax, get_weighted_sse_shift
33 %endmacro
34
35 ; 1 is the input and output register.
36 ; 2-3 are tmp registers.
37 %macro SSE_SCALE 2-3
38 ; Reduce 32-bit sums to 64-bits sums.
39 pshufd m%2, m%1, q3311
40 paddd m%1, m%2
41
42 LOAD_SCALES %2, %3
43
44 ; Multiply and shift with rounding.
45 pmuludq m%1, m%2
46 mova m%2, [rounding]
47 paddq m%1, m%2
48 psrlq m%1, get_weighted_sse_shift
49 %endmacro
50
51 %macro LOAD_SCALES_4X8 2
52 ; Load 1 scale from each of the 2 rows.
53 movd m%1, [scaleq]
54 movd m%2, [scaleq+scale_strideq]
55 ; 64-bit unpack since our loads have only one value each.
56 punpcklqdq m%1, m%2
57 %endmacro
58
59 ; 2 is unused
60 %macro LOAD_SCALES_8X4 2
61 ; Convert to 64-bits.
62 ; It doesn't matter that the upper halves are full of garbage.
63 movq m%1, [scaleq]
64 pshufd m%1, m%1, q1100
65 %endmacro
66
67 ; 2 is unused
68 %macro LOAD_SCALES_16X4 2
69 pmovzxdq m%1, [scaleq]
70 %endmacro
71
72 ; Separate from other scale macros, since it uses 2 inputs.
73 ; 1-2 are inputs regs and 1 is the output reg.
74 ; 3-4 are tmp registers
75 %macro SSE_SCALE_32X4 4
76 pshufd m%3, m%1, q3311
77 paddd m%1, m%3
78 pshufd m%3, m%2, q3311
79 paddd m%2, m%3
80
81 ; Load scale for 4x4 blocks and convert to 64-bits.
82 ; It doesn't matter if the upper halves are full of garbage.
83 ; raw load: 0, 1, 2, 3 | 4, 5, 6, 7
84 ; unpack low: 0, 1 | 4, 5
85 ; unpack high: 2, 3, | 6, 7
86 mova m%4, [scaleq]
87 punpckldq m%3, m%4, m%4
88 punpckhdq m%4, m%4
89
90 pmuludq m%1, m%3
91 pmuludq m%2, m%4
92 mova m%3, [rounding]
93 paddq m%1, m%3
94 paddq m%2, m%3
95 psrlq m%1, get_weighted_sse_shift
96 psrlq m%2, get_weighted_sse_shift
97 paddq m%1, m%2
98 %endmacro
99
100 INIT_XMM ssse3
101 ; Use scale_stride's register to store src_stride3
102 cglobal weighted_sse_4x4, 6, 7, 5, \
103 src, src_stride, dst, dst_stride, scale, \
104 src_stride3, dst_stride3
105 lea src_stride3q, [src_strideq*3]
106 lea dst_stride3q, [dst_strideq*3]
107 movq m0, [addsub]
108 movd m1, [srcq]
109 movd m2, [dstq]
110 punpcklbw m1, m2
111 movd m2, [srcq+src_strideq]
112 movd m3, [dstq+dst_strideq]
113 punpcklbw m2, m3
114 pmaddubsw m1, m0
115 pmaddubsw m2, m0
116 pmaddwd m1, m1
117 pmaddwd m2, m2
118 paddd m1, m2
119 movd m2, [srcq+src_strideq*2]
120 movd m3, [dstq+dst_strideq*2]
121 punpcklbw m2, m3
122 movd m3, [srcq+src_stride3q]
123 movd m4, [dstq+dst_stride3q]
124 punpcklbw m3, m4
125 pmaddubsw m2, m0
126 pmaddubsw m3, m0
127 pmaddwd m2, m2
128 pmaddwd m3, m3
129 paddd m2, m3
130 paddd m1, m2
131
132 pshuflw m0, m1, q3232
133 paddd m0, m1
134 movd eax, m0
135
136 ; Multiply and shift using scalar code.
137 SSE_SCALE_4X4
138 RET
139
140 %macro WEIGHTED_SSE_4X8_KERNEL 0
141 movd m1, [srcq]
142 movd m2, [srcq+src_strideq*4]
143 punpckldq m1, m2
144 movd m2, [dstq]
145 movd m3, [dstq+dst_strideq*4]
146 add srcq, src_strideq
147 add dstq, dst_strideq
148 punpckldq m2, m3
149 punpcklbw m1, m2
150 movd m2, [srcq]
151 movd m3, [srcq+src_strideq*4]
152 punpckldq m2, m3
153 movd m3, [dstq]
154 movd m4, [dstq+dst_strideq*4]
155 add srcq, src_strideq
156 add dstq, dst_strideq
157 punpckldq m3, m4
158 punpcklbw m2, m3
159 pmaddubsw m1, m0
160 pmaddubsw m2, m0
161 pmaddwd m1, m1
162 pmaddwd m2, m2
163 paddd m1, m2
164 movd m2, [srcq]
165 movd m3, [srcq+src_strideq*4]
166 punpckldq m2, m3
167 movd m3, [dstq]
168 movd m4, [dstq+dst_strideq*4]
169 add srcq, src_strideq
170 add dstq, dst_strideq
171 punpckldq m3, m4
172 punpcklbw m2, m3
173 movd m3, [srcq]
174 movd m4, [srcq+src_strideq*4]
175 punpckldq m3, m4
176 movd m4, [dstq]
177 movd m5, [dstq+dst_strideq*4]
178 punpckldq m4, m5
179 punpcklbw m3, m4
180 pmaddubsw m2, m0
181 pmaddubsw m3, m0
182 pmaddwd m2, m2
183 pmaddwd m3, m3
184 paddd m2, m3
185 paddd m1, m2
186
187 %define LOAD_SCALES LOAD_SCALES_4X8
188 SSE_SCALE 1, 2, 3
189 %endmacro
190
191 INIT_XMM ssse3
192 cglobal weighted_sse_4x8, 6, 6, 6, \
193 src, src_stride, dst, dst_stride, scale, scale_stride
194 mova m0, [addsub]
195 WEIGHTED_SSE_4X8_KERNEL
196
197 pshufd m0, m1, q3232
198 paddq m1, m0
199 movq rax, m1
200 RET
201
202 INIT_XMM ssse3
203 cglobal weighted_sse_4x16, 6, 6, 7, \
204 src, src_stride, dst, dst_stride, scale, scale_stride
205 mova m0, [addsub]
206
207 WEIGHTED_SSE_4X8_KERNEL
208 ; Swap so the use of this macro will use m6 as the result
209 SWAP 1, 6
210
211 lea scaleq, [scaleq+scale_strideq*2]
212 ; Already incremented by stride 3 times, but must go up 5 more to get to 8
213 add srcq, src_strideq
214 add dstq, dst_strideq
215 lea srcq, [srcq+src_strideq*4]
216 lea dstq, [dstq+dst_strideq*4]
217 WEIGHTED_SSE_4X8_KERNEL
218 paddq m6, m1
219
220 pshufd m0, m6, q3232
221 paddq m6, m0
222 movq rax, m6
223 RET
224
225 %macro WEIGHTED_SSE_8X4_KERNEL 0
226 movq m1, [srcq]
227 movq m2, [dstq]
228 punpcklbw m1, m2
229 movq m2, [srcq+src_strideq]
230 movq m3, [dstq+dst_strideq]
231 punpcklbw m2, m3
232 pmaddubsw m1, m0
233 pmaddubsw m2, m0
234 pmaddwd m1, m1
235 pmaddwd m2, m2
236 paddd m1, m2
237 movq m2, [srcq+src_strideq*2]
238 movq m3, [dstq+dst_strideq*2]
239 punpcklbw m2, m3
240 movq m3, [srcq+src_stride3q]
241 movq m4, [dstq+dst_stride3q]
242 punpcklbw m3, m4
243 pmaddubsw m2, m0
244 pmaddubsw m3, m0
245 pmaddwd m2, m2
246 pmaddwd m3, m3
247 paddd m2, m3
248 paddd m1, m2
249
250 %define LOAD_SCALES LOAD_SCALES_8X4
251 SSE_SCALE 1, 2
252 %endmacro
253
254 %macro WEIGHTED_SSE_16X4_KERNEL 0
255 pmovzxbw m0, [srcq]
256 pmovzxbw m1, [dstq]
257 psubw m0, m1
258 pmaddwd m0, m0
259 pmovzxbw m1, [srcq+src_strideq]
260 pmovzxbw m2, [dstq+dst_strideq]
261 psubw m1, m2
262 pmaddwd m1, m1
263 paddd m0, m1
264 pmovzxbw m1, [srcq+src_strideq*2]
265 pmovzxbw m2, [dstq+dst_strideq*2]
266 psubw m1, m2
267 pmaddwd m1, m1
268 pmovzxbw m2, [srcq+src_stride3q]
269 pmovzxbw m3, [dstq+dst_stride3q]
270 psubw m2, m3
271 pmaddwd m2, m2
272 paddd m1, m2
273 paddd m1, m0
274
275 %define LOAD_SCALES LOAD_SCALES_16X4
276 SSE_SCALE 1, 2
277 %endmacro
278
279 %macro WEIGHTED_SSE_32X4_KERNEL 0
280 ; Unpacking high and low results in sums that are 8 samples apart. To
281 ; correctly apply weights, two separate registers are needed to accumulate.
282 mova m2, [srcq]
283 mova m3, [dstq]
284 punpcklbw m1, m2, m3
285 punpckhbw m2, m3
286 mova m4, [srcq+src_strideq]
287 mova m5, [dstq+dst_strideq]
288 punpcklbw m3, m4, m5
289 punpckhbw m4, m5
290 pmaddubsw m1, m0
291 pmaddubsw m2, m0
292 pmaddubsw m3, m0
293 pmaddubsw m4, m0
294 pmaddwd m1, m1
295 pmaddwd m2, m2
296 pmaddwd m3, m3
297 pmaddwd m4, m4
298 ; Accumulate
299 paddd m1, m3
300 paddd m2, m4
301 mova m4, [srcq+src_strideq*2]
302 mova m5, [dstq+dst_strideq*2]
303 punpcklbw m3, m4, m5
304 punpckhbw m4, m5
305 mova m6, [srcq+src_stride3q]
306 mova m7, [dstq+dst_stride3q]
307 punpcklbw m5, m6, m7
308 punpckhbw m6, m7
309 pmaddubsw m3, m0
310 pmaddubsw m4, m0
311 pmaddubsw m5, m0
312 pmaddubsw m6, m0
313 pmaddwd m3, m3
314 pmaddwd m4, m4
315 pmaddwd m5, m5
316 pmaddwd m6, m6
317 paddd m3, m5
318 paddd m4, m6
319 paddd m1, m3
320 paddd m2, m4
321
322 SSE_SCALE_32X4 1, 2, 3, 4
323 %endmacro
324
325 %macro WEIGHTED_SSE 2 ; w, h
326 %if %1 == 8
327 %if %2 == 4
328 ; Use scale_stride's register to store src_stride3
329 cglobal weighted_sse_%1x%2, 6, 7, 5, \
330 src, src_stride, dst, dst_stride, scale, \
331 src_stride3, dst_stride3
332 %else
333 cglobal weighted_sse_%1x%2, 6, 9, 6, \
334 src, src_stride, dst, dst_stride, scale, scale_stride, \
335 src_stride3, dst_stride3, h
336 %endif
337 %elif %1 == 16
338 %if %2 == 4
339 ; Use scale_stride's register to store src_stride3
340 cglobal weighted_sse_%1x%2, 6, 7, 4, \
341 src, src_stride, dst, dst_stride, scale, \
342 src_stride3, dst_stride3
343 %else
344 cglobal weighted_sse_%1x%2, 6, 9, 5, \
345 src, src_stride, dst, dst_stride, scale, scale_stride, \
346 src_stride3, dst_stride3, h
347 %endif
348 %elif %1 == 32
349 cglobal weighted_sse_%1x%2, 6, 9, 9, \
350 src, src_stride, dst, dst_stride, scale, scale_stride, \
351 src_stride3, dst_stride3, h
352 %else ; > 32
353 cglobal weighted_sse_%1x%2, 6, 10, 9, \
354 src, src_stride, dst, dst_stride, scale, scale_stride, \
355 src_stride3, dst_stride3, h, w
356 %endif
357 ; === Setup ===
358 ; kernel_width/kernel_height: number of elements that the kernel processes.
359 ; m0: except for when w == 16, m0 is used to hold a constant 1, -1... vector
360 ; register for diffing the two sources.
361 ; sum: The kernel stores it's results on m1. The last vector register is used
362 ; unless only one iteration is done.
363
364 ; Default the kernel width to the width of this function.
365 %define kernel_width %1
366 %define kernel_height 4
367 %if %1 == 8
368 mova m0, [addsub]
369 %endif
370
371 %if %1 >= 32
372 mova m0, [addsub]
373 ; Iterate multiple times when w > 32.
374 %define kernel_width 32
375 %endif
376
377 %if %1 > kernel_width || %2 > kernel_height
378 ; Add onto the last used vector register.
379 %assign sum xmm_regs_used-1
380 %else
381 ; Use the result from the kernel
382 %define sum 1
383 %endif
384
385 lea src_stride3q, [src_strideq*3]
386 lea dst_stride3q, [dst_strideq*3]
387 %if %1 > kernel_width || %2 > kernel_height
388 pxor m%[sum], m%[sum]
389 %endif
390 %if %2 > kernel_height
391 mov hd, %2/kernel_height-1
392 .loop:
393 %endif
394
395 %if %1 > kernel_width
396 mov wd, %1/kernel_width-1
397 .loop_horiz:
398 %endif
399
400 WEIGHTED_SSE_%[kernel_width]X%[kernel_height]_KERNEL
401 %if %2 > kernel_height || %1 > kernel_width
402 paddq m%[sum], m1
403 %endif
404
405 %if %1 > kernel_width
406 add scaleq, kernel_width*4/4
407 add srcq, kernel_width
408 add dstq, kernel_width
409 dec wq
410 jge .loop_horiz
411 %endif
412
413 %if %2 > kernel_height
414 ; Move down 4 rows.
415 %if %1 > kernel_width
416 ; src/dst is incremented by width when processing multi iteration rows.
417 ; Reduce the offset by the width of the row.
418 lea srcq, [srcq+src_strideq*4 - %1]
419 lea dstq, [dstq+dst_strideq*4 - %1]
420 ; The behaviour for scale is similar
421 lea scaleq, [scaleq+scale_strideq - %1*4/4]
422 %else
423 lea srcq, [srcq+src_strideq*4]
424 lea dstq, [dstq+dst_strideq*4]
425 add scaleq, scale_strideq
426 %endif
427 dec hq
428 jge .loop
429 %endif
430
431 %if mmsize == 16
432 pshufd m2, m%[sum], q3232
433 paddq m%[sum], m2
434 movq rax, m%[sum]
435 %elif mmsize == 32
436 vextracti128 xm2, m%[sum], 1
437 paddq xm%[sum], xm2
438 pshufd xm2, xm%[sum], q3232
439 paddq xm%[sum], xm2
440 movq rax, xm%[sum]
441 %endif
442 RET
443
444 %undef sum
445 %undef kernel_width
446 %endmacro
447
448 INIT_XMM ssse3
449 WEIGHTED_SSE 8, 4
450 %if ARCH_X86_64
451 WEIGHTED_SSE 8, 8
452 WEIGHTED_SSE 8, 16
453 WEIGHTED_SSE 8, 32
454 %endif ; ARCH_X86_64
455
456 INIT_YMM avx2
457 WEIGHTED_SSE 16, 4
458 %if ARCH_X86_64
459 WEIGHTED_SSE 16, 8
460 WEIGHTED_SSE 16, 16
461 WEIGHTED_SSE 16, 32
462 WEIGHTED_SSE 16, 64
463
464 WEIGHTED_SSE 32, 8
465 WEIGHTED_SSE 32, 16
466 WEIGHTED_SSE 32, 32
467 WEIGHTED_SSE 32, 64
468
469 WEIGHTED_SSE 64, 16
470 WEIGHTED_SSE 64, 32
471 WEIGHTED_SSE 64, 64
472 WEIGHTED_SSE 64, 128
473
474 WEIGHTED_SSE 128, 64
475 WEIGHTED_SSE 128, 128
476 %endif ; ARCH_X86_64
477
478 INIT_XMM sse2
479
480 cglobal weighted_sse_4x4_hbd, 6, 8, 4, \
481 src, src_stride, dst, dst_stride, scale, scale_stride, \
482 src_stride3, dst_stride3
483 lea src_stride3q, [src_strideq*3]
484 lea dst_stride3q, [dst_strideq*3]
485 movq m0, [srcq]
486 movq m1, [dstq]
487 psubw m0, m1
488 pmaddwd m0, m0
489 movq m1, [srcq+src_strideq]
490 movq m2, [dstq+dst_strideq]
491 psubw m1, m2
492 pmaddwd m1, m1
493 paddd m0, m1
494 movq m1, [srcq+src_strideq*2]
495 movq m2, [dstq+dst_strideq*2]
496 psubw m1, m2
497 pmaddwd m1, m1
498 movq m2, [srcq+src_stride3q]
499 movq m3, [dstq+dst_stride3q]
500 psubw m2, m3
501 pmaddwd m2, m2
502 paddd m1, m2
503 paddd m0, m1
504
505 pshuflw m1, m0, q3232
506 paddd m0, m1
507 movd eax, m0
508
509 ; Multiply and shift using scalar code.
510 SSE_SCALE_4X4
511 RET