Debian Codegraph @

src/libmpg123/dct64_avx.S

raw
1 /*
2 dct36_sse: AVX optimized dct64 for x86-64
3
4 copyright 1995-2026 by the mpg123 project - free software under the terms of the LGPL 2.1
5 see COPYING and AUTHORS files in distribution or http://mpg123.org
6 initially written by Taihei Monma
7 */
8
9 #include "x86_64_defs.h"
10 #include "mangle.h"
11
12 #define samples %rdx
13 #define costab %rcx
14 #define out0 %rdi
15 #define out1 %rsi
16
17 /*
18 void dct64_avx(short *out0, short *out1, real *samples);
19 */
20
21 RODATA
22 ALIGN32
23 costab_avx:
24 .long 1056974725
25 .long 1057056395
26 .long 1057223771
27 .long 1057485416
28 .long 1057855544
29 .long 1058356026
30 .long 1059019886
31 .long 1059897405
32 .long 1061067246
33 .long 1062657950
34 .long 1064892987
35 .long 1066774581
36 .long 1069414683
37 .long 1073984175
38 .long 1079645762
39 .long 1092815430
40 .long 1057005197
41 .long 1057342072
42 .long 1058087743
43 .long 1059427869
44 .long 1061799040
45 .long 1065862217
46 .long 1071413542
47 .long 1084439708
48 .long 1057128951
49 .long 1058664893
50 .long 1063675095
51 .long 1076102863
52 .long 1057655764
53 .long 1067924853
54 .long 1060439283
55 .long 0
56 .text
57 ALIGN16
58 .globl ASM_NAME(INT123_dct64_avx)
59 ASM_NAME(INT123_dct64_avx):
60 #ifdef IS_MSABI
61 push %rbp
62 mov %rsp, %rbp
63 sub $112, %rsp
64 movaps %xmm6, (%rsp)
65 movaps %xmm7, 16(%rsp)
66 movaps %xmm8, 32(%rsp)
67 movaps %xmm9, 48(%rsp)
68 movaps %xmm10, 64(%rsp)
69 movaps %xmm11, 80(%rsp)
70 movaps %xmm12, 96(%rsp)
71 push %rdi
72 push %rsi
73 mov %rcx, %rdi
74 mov %rdx, %rsi
75 mov %r8, %rdx
76 #endif
77 leaq costab_avx(%rip), costab
78
79 vmovups (samples), %ymm0 # input[0,1,2,3,4,5,6,7]
80 vmovups 32(samples), %ymm1 # input[8,9,10,11,12,13,14,15]
81 vperm2f128 $0x23, 64(samples), %ymm2, %ymm2
82 vperm2f128 $0x23, 96(samples), %ymm3, %ymm3
83 vshufps $0x1b, %ymm2, %ymm2, %ymm2 # input[23,22,21,20,19,18,17,16]
84 vshufps $0x1b, %ymm3, %ymm3, %ymm3 # input[31,30,29,28,27,26,25,24]
85 vsubps %ymm2, %ymm1, %ymm6
86 vsubps %ymm3, %ymm0, %ymm7
87 vaddps %ymm0, %ymm3, %ymm4 # bufs[0,1,2,3,4,5,6,7]
88 vaddps %ymm1, %ymm2, %ymm5 # bufs[8,9,10,11,12,13,14,15]
89 vmulps (costab), %ymm7, %ymm7 # bufs[31,30,29,28,27,26,25,24] cos64[0,1,2,3,4,5,6,7]
90 vmulps 32(costab), %ymm6, %ymm6 # bufs[23,22,21,20,19,18,17,16] cos64[8,9,10,11,12,13,14,15]
91
92 vmovaps 64(costab), %ymm8 # cos32[0,1,2,3,4,5,6,7]
93
94 vshufps $0x1b, %ymm5, %ymm5, %ymm5
95 vshufps $0x1b, %ymm6, %ymm6, %ymm6
96 vperm2f128 $0x01, %ymm5, %ymm5, %ymm5 # bufs[15,14,13,12,11,10,9,8]
97 vperm2f128 $0x01, %ymm6, %ymm6, %ymm6 # bufs[16,17,18,19,20,21,22,23]
98 vsubps %ymm5, %ymm4, %ymm1
99 vsubps %ymm6, %ymm7, %ymm3
100 vaddps %ymm5, %ymm4, %ymm0 # bufs[32,33,34,35,36,37,38,39]
101 vaddps %ymm6, %ymm7, %ymm2 # bufs[48,49,50,51,52,53,54,55]
102 vmulps %ymm1, %ymm8, %ymm1 # bufs[47,46,45,44,43,42,41,40]
103 vmulps %ymm3, %ymm8, %ymm3 # bufs[63,62,61,60,59,58,57,56]
104
105 vmovaps 96(costab), %ymm8 # cos16[0,1,2,3]:cos8[0,1]:cos4[0]:-
106 vperm2f128 $0x00, %ymm8, %ymm8, %ymm9 # cos16[0,1,2,3,0,1,2,3]
107
108 vperm2f128 $0x20, %ymm1, %ymm0, %ymm4 # bufs[32,33,34,35,47,46,45,44]
109 vperm2f128 $0x31, %ymm1, %ymm0, %ymm5
110 vshufps $0x1b, %ymm5, %ymm5, %ymm5 # bufs[39,38,37,36,40,41,42,43]
111 vperm2f128 $0x20, %ymm3, %ymm2, %ymm6 # bufs[48,49,50,51,63,62,61,60]
112 vperm2f128 $0x31, %ymm3, %ymm2, %ymm7
113 vshufps $0x1b, %ymm7, %ymm7, %ymm7 # bufs[55,54,53,52,56,57,58,59]
114 vsubps %ymm5, %ymm4, %ymm1
115 vsubps %ymm7, %ymm6, %ymm3
116 vaddps %ymm5, %ymm4, %ymm0 # bufs[0,1,2,3,8,9,10,11]
117 vaddps %ymm7, %ymm6, %ymm2 # bufs[16,17,18,19,24,25,26,27]
118 vmulps %ymm1, %ymm9, %ymm1 # bufs[7,6,5,4,15,14,13,12]
119 vmulps %ymm3, %ymm9, %ymm3 # bufs[23,22,21,20,31,30,29,28]
120
121 vperm2f128 $0x11, %ymm8, %ymm8, %ymm8 # cos8[0,1]:cos4[0]:-:cos8[0,1]:cos4[0]:-
122 vmovddup %ymm8, %ymm9 # cos8[0,1,0,1,0,1,0,1]
123
124 vunpcklps %ymm1, %ymm0, %ymm4 # bufs[0,7,1,6,8,15,9,14]
125 vunpckhps %ymm1, %ymm0, %ymm5 # bufs[2,5,3,4,10,13,11,12]
126 vunpcklps %ymm3, %ymm2, %ymm6 # bufs[16,23,17,22,24,31,25,30]
127 vunpckhps %ymm3, %ymm2, %ymm7 # bufs[18,21,19,20,26,29,27,28]
128 vshufps $0xd8, %ymm4, %ymm4, %ymm4 # bufs[0,1,7,6,8,9,15,14]
129 vshufps $0x72, %ymm5, %ymm5, %ymm5 # bufs[3,2,4,5,11,10,12,13]
130 vshufps $0xd8, %ymm6, %ymm6, %ymm6 # bufs[16,17,23,22,24,25,31,30]
131 vshufps $0x72, %ymm7, %ymm7, %ymm7 # bufs[19,18,20,21,27,26,28,29]
132 vsubps %ymm5, %ymm4, %ymm1
133 vsubps %ymm7, %ymm6, %ymm3
134 vaddps %ymm5, %ymm4, %ymm0 # bufs[32,33,36,37,40,41,44,45]
135 vaddps %ymm7, %ymm6, %ymm2 # bufs[48,49,52,53,56,57,60,61]
136 vmulps %ymm1, %ymm9, %ymm1 # bufs[35,34,39,38,43,42,47,46]
137 vmulps %ymm3, %ymm9, %ymm3 # bufs[51,50,55,54,59,58,63,62]
138
139 vpermilps $0xaa, %ymm8, %ymm8 # cos4[0,0,0,0,0,0,0,0]
140
141 vshufps $0xd8, %ymm0, %ymm0, %ymm0 # bufs[32,36,33,37,40,44,41,45]
142 vshufps $0xd8, %ymm1, %ymm1, %ymm1 # bufs[35,39,34,38,43,47,42,46]
143 vshufps $0xd8, %ymm2, %ymm2, %ymm2 # bufs[48,52,49,53,56,60,57,61]
144 vshufps $0xd8, %ymm3, %ymm3, %ymm3 # bufs[51,55,50,54,59,63,58,62]
145 vunpcklps %ymm1, %ymm0, %ymm4 # bufs[32,35,36,39,40,43,44,47]
146 vunpckhps %ymm1, %ymm0, %ymm5 # bufs[33,34,37,38,41,42,45,46]
147 vunpcklps %ymm3, %ymm2, %ymm6 # bufs[48,51,52,55,56,59,60,63]
148 vunpckhps %ymm3, %ymm2, %ymm7 # bufs[49,50,53,54,57,58,61,62]
149 vsubps %ymm5, %ymm4, %ymm1
150 vsubps %ymm7, %ymm6, %ymm3
151 vaddps %ymm5, %ymm4, %ymm0 # bufs[0,2,4,6,8,10,12,14]
152 vaddps %ymm7, %ymm6, %ymm2 # bufs[16,18,20,22,24,26,28,30]
153 vmulps %ymm1, %ymm8, %ymm1 # bufs[1,3,5,7,9,11,13,15]
154 vmulps %ymm3, %ymm8, %ymm3 # bufs[17,19,21,23,25,27,29,31]
155
156 vxorps %ymm8, %ymm8, %ymm8
157 vblendps $0xaa, %ymm1, %ymm8, %ymm5
158 vblendps $0xaa, %ymm3, %ymm8, %ymm6
159 vaddps %ymm5, %ymm0, %ymm0
160 vaddps %ymm6, %ymm2, %ymm2
161 vunpcklps %ymm1, %ymm0, %ymm4 # bufs[0,1,2,3,8,9,10,11]
162 vunpckhps %ymm1, %ymm0, %ymm5 # bufs[4,5,6,7,12,13,14,15]
163 vunpcklps %ymm3, %ymm2, %ymm6 # bufs[16,17,18,19,24,25,26,27]
164 vunpckhps %ymm3, %ymm2, %ymm7 # bufs[20,21,22,23,28,29,30,31]
165
166 vextractf128 $0x1, %ymm4, %xmm0 # bufs[8,9,10,11]
167 vextractf128 $0x1, %ymm5, %xmm1 # bufs[12,13,14,15]
168 vextractf128 $0x1, %ymm6, %xmm2 # bufs[24,25,26,27]
169 vextractf128 $0x1, %ymm7, %xmm3 # bufs[28,29,30,31]
170
171 vshufps $0x1e, %xmm5, %xmm5, %xmm9 # bufs[6,7,5,4]
172 vshufps $0x1e, %xmm1, %xmm1, %xmm10 # bufs[14,15,13,12]
173 vshufps $0x1e, %xmm7, %xmm7, %xmm11 # bufs[22,23,21,20]
174 vshufps $0x1e, %xmm3, %xmm3, %xmm12 # bufs[30,31,29,28]
175 vblendps $0x7, %xmm9, %xmm8, %xmm9 # bufs[6,7,5,-]
176 vblendps $0x7, %xmm10, %xmm8, %xmm10 # bufs[14,15,13,-]
177 vblendps $0x7, %xmm11, %xmm8, %xmm11 # bufs[22,23,21,-]
178 vblendps $0x7, %xmm12, %xmm8, %xmm12 # bufs[30,31,29,-]
179 vaddps %xmm5, %xmm9, %xmm5
180 vaddps %xmm1, %xmm10, %xmm1
181 vaddps %xmm7, %xmm11, %xmm7
182 vaddps %xmm3, %xmm12, %xmm3
183
184 prefetcht0 512(out0)
185
186 vshufps $0x1e, %xmm0, %xmm0, %xmm9 # bufs[10,11,9,8]
187 vshufps $0x1e, %xmm2, %xmm2, %xmm10 # bufs[26,27,25,24]
188 vaddps %xmm1, %xmm0, %xmm0
189 vaddps %xmm3, %xmm2, %xmm2
190 vblendps $0x7, %xmm9, %xmm8, %xmm9 # bufs[10,11,9,-]
191 vblendps $0x7, %xmm10, %xmm8, %xmm10 # bufs[26,27,25,-]
192 vaddps %xmm1, %xmm9, %xmm1
193 vaddps %xmm3, %xmm10, %xmm3
194
195 vzeroupper
196 prefetcht0 512(out1)
197
198 cvtps2dq %xmm4, %xmm4
199 cvtps2dq %xmm0, %xmm0
200 cvtps2dq %xmm5, %xmm5
201 cvtps2dq %xmm1, %xmm1
202 packssdw %xmm5, %xmm4
203 packssdw %xmm1, %xmm0
204 movq %xmm4, %rcx
205 pshufd $0x4e, %xmm4, %xmm5
206 movq %xmm0, %rdx
207 pshufd $0x4e, %xmm0, %xmm1
208 movq %xmm5, %r8
209 movq %xmm1, %r9
210
211 addq $512, out0
212 movq $-64, %rax
213
214 movw %cx, (out0)
215 movw %dx, (out0,%rax,1)
216 movw %r8w, (out0,%rax,2)
217 movw %r9w, -64(out0,%rax,2)
218 leaq (out0,%rax,4), out0
219 shr $16, %rcx
220 shr $16, %rdx
221 shr $16, %r8
222 shr $16, %r9
223 movw %cx, (out0,%rax,4)
224 negq %rax
225 movw %cx, (out1)
226 movw %dx, (out1,%rax,1)
227 movw %r8w, (out1,%rax,2)
228 movw %r9w, 64(out1,%rax,2)
229 leaq (out1,%rax,4), out1
230 shr $16, %rcx
231 shr $16, %rdx
232 shr $16, %r8
233 shr $16, %r9
234 negq %rax
235 movw %cx, (out0)
236 movw %dx, (out0,%rax,1)
237 movw %r8w, (out0,%rax,2)
238 movw %r9w, -64(out0,%rax,2)
239 shr $16, %rcx
240 shr $16, %rdx
241 shr $16, %r8
242 shr $16, %r9
243 negq %rax
244 movw %cx, (out1)
245 movw %dx, (out1,%rax,1)
246 movw %r8w, (out1,%rax,2)
247 movw %r9w, 64(out1,%rax,2)
248
249 leaq -32(out0,%rax,4), out0
250 negq %rax
251 leaq 32(out1,%rax,4), out1
252
253 vshufps $0x1e, %xmm6, %xmm6, %xmm0
254 vblendps $0x7, %xmm0, %xmm8, %xmm0
255 addps %xmm2, %xmm6
256 addps %xmm7, %xmm2
257 addps %xmm3, %xmm7
258 addps %xmm0, %xmm3
259 cvtps2dq %xmm6, %xmm6
260 cvtps2dq %xmm2, %xmm2
261 cvtps2dq %xmm7, %xmm7
262 cvtps2dq %xmm3, %xmm3
263 packssdw %xmm7, %xmm6
264 packssdw %xmm3, %xmm2
265 movq %xmm6, %rcx
266 pshufd $0x4e, %xmm6, %xmm7
267 movq %xmm2, %rdx
268 pshufd $0x4e, %xmm2, %xmm3
269 movq %xmm7, %r8
270 movq %xmm3, %r9
271
272 movw %cx, (out0)
273 movw %dx, (out0,%rax,1)
274 movw %r8w, (out0,%rax,2)
275 movw %r9w, -64(out0,%rax,2)
276 leaq (out0,%rax,4), out0
277 shr $16, %rcx
278 shr $16, %rdx
279 shr $16, %r8
280 shr $16, %r9
281 negq %rax
282 movw %cx, (out1)
283 movw %dx, (out1,%rax,1)
284 movw %r8w, (out1,%rax,2)
285 movw %r9w, 64(out1,%rax,2)
286 leaq (out1,%rax,4), out1
287 shr $16, %rcx
288 shr $16, %rdx
289 shr $16, %r8
290 shr $16, %r9
291 negq %rax
292 movw %cx, (out0)
293 movw %dx, (out0,%rax,1)
294 movw %r8w, (out0,%rax,2)
295 movw %r9w, -64(out0,%rax,2)
296 shr $16, %rcx
297 shr $16, %rdx
298 shr $16, %r8
299 shr $16, %r9
300 negq %rax
301 movw %cx, (out1)
302 movw %dx, (out1,%rax,1)
303 movw %r8w, (out1,%rax,2)
304 movw %r9w, 64(out1,%rax,2)
305
306 #ifdef IS_MSABI
307 pop %rsi
308 pop %rdi
309 movaps (%rsp), %xmm6
310 movaps 16(%rsp), %xmm7
311 movaps 32(%rsp), %xmm8
312 movaps 48(%rsp), %xmm9
313 movaps 64(%rsp), %xmm10
314 movaps 80(%rsp), %xmm11
315 movaps 96(%rsp), %xmm12
316 mov %rbp, %rsp
317 pop %rbp
318 #endif
319 ret
320
321 NONEXEC_STACK