Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/x86/sse/fastconv/parse.h

raw
1 /*
2 * Copyright (C) 2020 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2020 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 31 мар. 2020 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_X86_SSE_FASTCONV_PARSE_H_
23 #define PRIVATE_DSP_ARCH_X86_SSE_FASTCONV_PARSE_H_
24
25 #ifndef PRIVATE_DSP_ARCH_X86_SSE_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_X86_SSE_IMPL */
28
29 namespace lsp
30 {
31 namespace sse
32 {
33 void fastconv_parse(float *dst, const float *src, size_t rank)
34 {
35 // Prepare for butterflies
36 const float *wk = &XFFT_W[(rank-3) << 3];
37 const float *ak = &XFFT_A[(rank-3) << 3];
38 size_t items = size_t(1) << (rank + 1);
39 size_t bs = items;
40 size_t n = bs >> 1;
41
42 // Iterate first cycle
43 if (n > 4)
44 {
45 // ONE LARGE CYCLE
46 // Set initial values of pointers
47 float *a = dst;
48 float *b = &a[n];
49 size_t k = n;
50
51 ARCH_X86_ASM
52 (
53 __ASM_EMIT("movaps 0x00(%[ak]), %%xmm6") /* xmm6 = rA[i] */
54 __ASM_EMIT("movaps 0x10(%[ak]), %%xmm7") /* xmm7 = iA[i] */
55 __ASM_EMIT("xorps %%xmm5, %%xmm5") /* xmm5 = 0 */
56 :
57 : [ak] "r"(ak)
58 :
59 "%xmm5", "%xmm6", "%xmm7"
60 );
61
62 ARCH_X86_ASM
63 (
64 // __ASM_EMIT(".align 16")
65 __ASM_EMIT("1:")
66
67 __ASM_EMIT("movups 0x00(%[src]), %%xmm0") /* xmm0 = s[i] */
68 __ASM_EMIT("xorps %%xmm3, %%xmm3") /* xmm3 = 0 */
69 __ASM_EMIT("movaps %%xmm0, %%xmm4") /* xmm4 = s[i] */
70 __ASM_EMIT("movaps %%xmm0, %%xmm2") /* xmm2 = s[i] */
71 __ASM_EMIT("mulps %%xmm7, %%xmm4") /* xmm4 = s[i]*iA[i] */
72 __ASM_EMIT("mulps %%xmm6, %%xmm2") /* xmm2 = s[i]*rA[i] */
73 __ASM_EMIT("subps %%xmm4, %%xmm3") /* xmm3 = -s[i]*iA[i] */
74
75 __ASM_EMIT("movups %%xmm0, 0x00(%[a])")
76 __ASM_EMIT("movups %%xmm5, 0x10(%[a])")
77 __ASM_EMIT("movups %%xmm2, 0x00(%[b])")
78 __ASM_EMIT("movups %%xmm3, 0x10(%[b])")
79
80 __ASM_EMIT("add $0x10, %[src]")
81 __ASM_EMIT("add $0x20, %[a]")
82 __ASM_EMIT("add $0x20, %[b]")
83
84 /* Repeat loop */
85 __ASM_EMIT("sub $8, %[k]")
86 __ASM_EMIT("jz 2f")
87
88 /* Rotate angle */
89 __ASM_EMIT("movaps 0x00(%[wk]), %%xmm0") /* xmm0 = rw[i] */
90 __ASM_EMIT("movaps 0x10(%[wk]), %%xmm1") /* xmm1 = iw[i] */
91 __ASM_EMIT("movaps %%xmm0, %%xmm2") /* xmm2 = rw[i] */
92 __ASM_EMIT("movaps %%xmm1, %%xmm3") /* xmm3 = iw[i] */
93 __ASM_EMIT("mulps %%xmm6, %%xmm3") /* xmm3 = rA[i] * iw[i] */
94 __ASM_EMIT("mulps %%xmm7, %%xmm1") /* xmm1 = iA[i] * iw[i] */
95 __ASM_EMIT("mulps %%xmm0, %%xmm6") /* xmm6 = rA[i] * rw[i] */
96 __ASM_EMIT("mulps %%xmm2, %%xmm7") /* xmm7 = iA[i] * rw[i] */
97 __ASM_EMIT("subps %%xmm1, %%xmm6") /* xmm6 = rA[i] * rw[i] - iA[i] * iw[i] */
98 __ASM_EMIT("addps %%xmm3, %%xmm7") /* xmm7 = rA[i] * iw[i] + iA[i] * rw[i] */
99 __ASM_EMIT("jmp 1b")
100
101 __ASM_EMIT("2:")
102
103 : [a] "+r" (a), [b] "+r" (b), [src] "+r" (src), [k] "+r" (k)
104 : [wk] "r"(wk)
105 : "cc", "memory",
106 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
107 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
108 );
109
110 wk -= 8;
111 ak -= 8;
112 n >>= 1;
113 bs >>= 1;
114 }
115 else
116 {
117 // Unpack 4x real to 4x split complex
118 ARCH_X86_ASM
119 (
120 __ASM_EMIT("movups 0x00(%[src]), %%xmm0") /* xmm0 = s[i] */
121 __ASM_EMIT("xorps %%xmm1, %%xmm1") /* xmm1 = 0 */
122 __ASM_EMIT("movups %%xmm0, 0x00(%[dst])")
123 __ASM_EMIT("movups %%xmm1, 0x10(%[dst])")
124 :
125 : [items] "r" (items), [src] "r" (src), [dst] "r" (dst)
126 : "cc", "memory",
127 "%xmm0", "%xmm1"
128 );
129 }
130
131 // Iterate butterflies
132 for (; n > 4; n >>= 1, bs >>= 1)
133 {
134 for (size_t p=0; p<items; p += bs)
135 {
136 // Set initial values of pointers
137 float *a = &dst[p];
138 float *b = &a[n];
139 size_t k = n;
140
141 ARCH_X86_ASM
142 (
143 __ASM_EMIT("movaps 0x00(%[ak]), %%xmm6") /* xmm6 = rA[i] */
144 __ASM_EMIT("movaps 0x10(%[ak]), %%xmm7") /* xmm7 = iA[i] */
145 :
146 : [ak] "r"(ak)
147 : "%xmm6", "%xmm7"
148 );
149
150 ARCH_X86_ASM
151 (
152 // __ASM_EMIT(".align 16")
153 __ASM_EMIT("1:")
154
155 __ASM_EMIT("movups 0x00(%[a]), %%xmm0") /* xmm0 = ra[i] */
156 __ASM_EMIT("movups 0x10(%[a]), %%xmm1") /* xmm1 = ia[i] */
157 __ASM_EMIT("movups 0x00(%[b]), %%xmm2") /* xmm2 = rb[i] */
158 __ASM_EMIT("movups 0x10(%[b]), %%xmm3") /* xmm3 = ib[i] */
159
160 __ASM_EMIT("movaps %%xmm0, %%xmm4") /* xmm4 = ra[i] */
161 __ASM_EMIT("movaps %%xmm1, %%xmm5") /* xmm5 = ia[i] */
162 __ASM_EMIT("addps %%xmm2, %%xmm0") /* xmm0 = ra[i]+rb[i] */
163 __ASM_EMIT("addps %%xmm3, %%xmm1") /* xmm1 = ia[i]+ib[i] */
164 __ASM_EMIT("subps %%xmm2, %%xmm4") /* xmm4 = ra[i]-rb[i] = rc[i] */
165 __ASM_EMIT("subps %%xmm3, %%xmm5") /* xmm5 = ia[i]-ib[i] = ic[i] */
166 __ASM_EMIT("movaps %%xmm4, %%xmm2") /* xmm2 = rc[i] */
167 __ASM_EMIT("movaps %%xmm5, %%xmm3") /* xmm3 = ic[i] */
168 __ASM_EMIT("mulps %%xmm6, %%xmm4") /* xmm4 = rA[i]*rc[i] */
169 __ASM_EMIT("mulps %%xmm7, %%xmm5") /* xmm5 = iA[i]*ic[i] */
170 __ASM_EMIT("mulps %%xmm6, %%xmm3") /* xmm3 = rA[i]*ic[i] */
171 __ASM_EMIT("mulps %%xmm7, %%xmm2") /* xmm2 = iA[i]*rc[i] */
172 __ASM_EMIT("addps %%xmm5, %%xmm4") /* xmm4 = rA[i]*rc[i] + iA[i]*ic[i] */
173 __ASM_EMIT("subps %%xmm2, %%xmm3") /* xmm3 = rA[i]*ic[i] - iA[i]*rc[i] */
174
175 __ASM_EMIT("movups %%xmm0, 0x00(%[a])")
176 __ASM_EMIT("movups %%xmm1, 0x10(%[a])")
177 __ASM_EMIT("movups %%xmm4, 0x00(%[b])")
178 __ASM_EMIT("movups %%xmm3, 0x10(%[b])")
179
180 /* Repeat loop */
181 __ASM_EMIT("add $0x20, %[a]")
182 __ASM_EMIT("add $0x20, %[b]")
183 __ASM_EMIT("sub $8, %[k]")
184 __ASM_EMIT("jz 2f")
185
186 /* Rotate angle */
187 __ASM_EMIT("movaps 0x00(%[wk]), %%xmm0") /* xmm0 = rw */
188 __ASM_EMIT("movaps 0x10(%[wk]), %%xmm1") /* xmm1 = iw */
189 __ASM_EMIT("movaps %%xmm0, %%xmm2") /* xmm2 = rw */
190 __ASM_EMIT("movaps %%xmm1, %%xmm3") /* xmm3 = iw */
191 __ASM_EMIT("mulps %%xmm6, %%xmm3") /* xmm3 = ra * iw */
192 __ASM_EMIT("mulps %%xmm7, %%xmm1") /* xmm1 = ia * iw */
193 __ASM_EMIT("mulps %%xmm0, %%xmm6") /* xmm6 = ra * rw */
194 __ASM_EMIT("mulps %%xmm2, %%xmm7") /* xmm7 = ia * rw */
195 __ASM_EMIT("subps %%xmm1, %%xmm6") /* xmm6 = ra * rw - ia * iw */
196 __ASM_EMIT("addps %%xmm3, %%xmm7") /* xmm7 = ra * iw + ia * rw */
197 __ASM_EMIT("jmp 1b")
198
199 __ASM_EMIT("2:")
200
201 : [a] "+r" (a), [b] "+r" (b), [k] "+r" (k)
202 : [wk] "r"(wk)
203 : "cc", "memory",
204 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
205 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
206 );
207 }
208
209 ak -= 8;
210 wk -= 8;
211 }
212
213 ARCH_X86_ASM
214 (
215 // __ASM_EMIT(".align 16")
216 __ASM_EMIT("1:")
217
218 __ASM_EMIT("movups 0x00(%[dst]), %%xmm0") /* xmm0 = r0 r1 r2 r3 */
219 __ASM_EMIT("movups 0x10(%[dst]), %%xmm1") /* xmm1 = i0 i1 i2 i3 */
220 __ASM_EMIT("movups 0x20(%[dst]), %%xmm4")
221 __ASM_EMIT("movups 0x30(%[dst]), %%xmm5")
222
223 __ASM_EMIT("movaps %%xmm0, %%xmm2") /* xmm2 = r0 r1 r2 r3 */
224 __ASM_EMIT("movaps %%xmm4, %%xmm6")
225 __ASM_EMIT("unpcklps %%xmm1, %%xmm0") /* xmm0 = r0 i0 r1 i1 */
226 __ASM_EMIT("unpcklps %%xmm5, %%xmm4")
227 __ASM_EMIT("unpckhps %%xmm1, %%xmm2") /* xmm2 = r2 i2 r3 i3 */
228 __ASM_EMIT("unpckhps %%xmm5, %%xmm6")
229 __ASM_EMIT("movaps %%xmm0, %%xmm1") /* xmm1 = r0 i0 r1 i1 */
230 __ASM_EMIT("movaps %%xmm4, %%xmm5")
231
232 __ASM_EMIT("addps %%xmm2, %%xmm0") /* xmm0 = r0+r2 i0+i2 r1+r3 i1+i3 = r0k i0k r2k i2k */
233 __ASM_EMIT("addps %%xmm6, %%xmm4")
234 __ASM_EMIT("subps %%xmm2, %%xmm1") /* xmm1 = r0-r2 i0-i2 r1-r3 i1-i3 = r1k i1k r3k i3k */
235 __ASM_EMIT("subps %%xmm6, %%xmm5")
236 __ASM_EMIT("movaps %%xmm0, %%xmm2") /* xmm2 = r0k i0k r2k i2k */
237 __ASM_EMIT("movaps %%xmm4, %%xmm6")
238 __ASM_EMIT("unpcklps %%xmm1, %%xmm0") /* xmm0 = r0k r1k i0k i1k */
239 __ASM_EMIT("unpcklps %%xmm5, %%xmm4")
240 __ASM_EMIT("unpckhps %%xmm1, %%xmm2") /* xmm2 = r2k r3k i2k i3k */
241 __ASM_EMIT("unpckhps %%xmm5, %%xmm6")
242 __ASM_EMIT("movaps %%xmm0, %%xmm1") /* xmm1 = r0k r1k i0k i1k */
243 __ASM_EMIT("movaps %%xmm4, %%xmm5")
244 __ASM_EMIT("shufps $0x6c, %%xmm2, %%xmm2") /* xmm2 = r2k i3k i2k r3k */
245 __ASM_EMIT("shufps $0x6c, %%xmm6, %%xmm6")
246
247 __ASM_EMIT("addps %%xmm2, %%xmm0") /* xmm0 = r0k+r2k r1k+i3k i0k+i2k i1k+r3k */
248 __ASM_EMIT("addps %%xmm6, %%xmm4")
249 __ASM_EMIT("subps %%xmm2, %%xmm1") /* xmm1 = r0k-r2k r1k-i3k i0k-i2k i1k-r3k */
250 __ASM_EMIT("subps %%xmm6, %%xmm5")
251 __ASM_EMIT("movaps %%xmm0, %%xmm2") /* xmm2 = r0k+r2k r1k+i3k i0k+i2k i1k+r3k */
252 __ASM_EMIT("movaps %%xmm4, %%xmm6")
253 __ASM_EMIT("unpcklps %%xmm1, %%xmm0") /* xmm0 = r0k+r2k r0k-r2k r1k+i3k r1k-i3k */
254 __ASM_EMIT("unpcklps %%xmm5, %%xmm4")
255 __ASM_EMIT("unpckhps %%xmm1, %%xmm2") /* xmm2 = i0k+i2k i0k-i2k i1k+r3k i1k-r3k */
256 __ASM_EMIT("unpckhps %%xmm5, %%xmm6")
257 __ASM_EMIT("shufps $0xb4, %%xmm2, %%xmm2") /* xmm2 = i0k+i2k i0k-i2k i1k-r3k i1k+r3k */
258 __ASM_EMIT("shufps $0xb4, %%xmm6, %%xmm6")
259
260 __ASM_EMIT("movups %%xmm0, 0x00(%[dst])")
261 __ASM_EMIT("movups %%xmm2, 0x10(%[dst])")
262 __ASM_EMIT("movups %%xmm4, 0x20(%[dst])")
263 __ASM_EMIT("movups %%xmm6, 0x30(%[dst])")
264
265 __ASM_EMIT("add $0x40, %[dst]")
266 __ASM_EMIT("sub $16, %[k]")
267 __ASM_EMIT("jnz 1b")
268
269 : [dst] "+r" (dst), [k] "+r" (items)
270 :
271 : "cc", "memory",
272 "%xmm0", "%xmm1", "%xmm2",
273 "%xmm4", "%xmm5", "%xmm6"
274 );
275 }
276
277 static inline void fastconv_parse_internal(float *dst, const float *src, size_t rank)
278 {
279 // Prepare for butterflies
280 const float *wk = &XFFT_W[(rank-3) << 3];
281 const float *ak = &XFFT_A[(rank-3) << 3];
282 size_t items = size_t(1) << (rank + 1);
283 size_t bs = items;
284 size_t n = bs >> 1;
285
286 // Iterate first cycle
287 if (n > 4)
288 {
289 // ONE LARGE CYCLE
290 // Set initial values of pointers
291 float *a = dst;
292 float *b = &a[n];
293 size_t k = n;
294
295 ARCH_X86_ASM
296 (
297 __ASM_EMIT("movaps 0x00(%[ak]), %%xmm6") /* xmm6 = rA[i] */
298 __ASM_EMIT("movaps 0x10(%[ak]), %%xmm7") /* xmm7 = iA[i] */
299 __ASM_EMIT("xorps %%xmm5, %%xmm5") /* xmm5 = 0 */
300 :
301 : [ak] "r"(ak)
302 :
303 "%xmm5", "%xmm6", "%xmm7"
304 );
305
306 ARCH_X86_ASM
307 (
308 // __ASM_EMIT(".align 16")
309 __ASM_EMIT("1:")
310
311 __ASM_EMIT("movups 0x00(%[src]), %%xmm0") /* xmm0 = s[i] */
312
313 __ASM_EMIT("xorps %%xmm3, %%xmm3") /* xmm3 = 0 */
314 __ASM_EMIT("movaps %%xmm0, %%xmm4") /* xmm4 = s[i] */
315 __ASM_EMIT("movaps %%xmm0, %%xmm2") /* xmm2 = s[i] */
316 __ASM_EMIT("mulps %%xmm7, %%xmm4") /* xmm4 = s[i]*iA[i] */
317 __ASM_EMIT("mulps %%xmm6, %%xmm2") /* xmm2 = s[i]*rA[i] */
318 __ASM_EMIT("subps %%xmm4, %%xmm3") /* xmm3 = -s[i]*iA[i] */
319
320 __ASM_EMIT("movups %%xmm0, 0x00(%[a])")
321 __ASM_EMIT("movups %%xmm5, 0x10(%[a])")
322 __ASM_EMIT("movups %%xmm2, 0x00(%[b])")
323 __ASM_EMIT("movups %%xmm3, 0x10(%[b])")
324
325 __ASM_EMIT("add $0x10, %[src]")
326 __ASM_EMIT("add $0x20, %[a]")
327 __ASM_EMIT("add $0x20, %[b]")
328
329 /* Repeat loop */
330 __ASM_EMIT32("subl $8, %[k]")
331 __ASM_EMIT64("sub $8, %[k]")
332 __ASM_EMIT("jz 2f")
333
334 /* Rotate angle */
335 __ASM_EMIT("movaps 0x00(%[wk]), %%xmm0") /* xmm0 = rw[i] */
336 __ASM_EMIT("movaps 0x10(%[wk]), %%xmm1") /* xmm1 = iw[i] */
337 __ASM_EMIT("movaps %%xmm0, %%xmm2") /* xmm2 = rw[i] */
338 __ASM_EMIT("movaps %%xmm1, %%xmm3") /* xmm3 = iw[i] */
339 __ASM_EMIT("mulps %%xmm6, %%xmm3") /* xmm3 = rA[i] * iw[i] */
340 __ASM_EMIT("mulps %%xmm7, %%xmm1") /* xmm1 = iA[i] * iw[i] */
341 __ASM_EMIT("mulps %%xmm0, %%xmm6") /* xmm6 = rA[i] * rw[i] */
342 __ASM_EMIT("mulps %%xmm2, %%xmm7") /* xmm7 = iA[i] * rw[i] */
343 __ASM_EMIT("subps %%xmm1, %%xmm6") /* xmm6 = rA[i] * rw[i] - iA[i] * iw[i] */
344 __ASM_EMIT("addps %%xmm3, %%xmm7") /* xmm7 = rA[i] * iw[i] + iA[i] * rw[i] */
345 __ASM_EMIT("jmp 1b")
346
347 __ASM_EMIT("2:")
348
349 : [a] "+r" (a), [b] "+r" (b), [src] "+r" (src), [k] "+r" (k)
350 : [wk] "r" (wk)
351 : "cc", "memory",
352 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
353 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
354 );
355
356 wk -= 8;
357 ak -= 8;
358 n >>= 1;
359 bs >>= 1;
360 }
361 else
362 {
363 // Unpack 4x real to 4x split complex
364 ARCH_X86_ASM
365 (
366 __ASM_EMIT("movups 0x00(%[src]), %%xmm0") /* xmm0 = s[i] */
367 __ASM_EMIT("xorps %%xmm1, %%xmm1") /* xmm1 = 0 */
368 __ASM_EMIT("movups %%xmm0, 0x00(%[dst])")
369 __ASM_EMIT("movups %%xmm1, 0x10(%[dst])")
370 :
371 : [src] "r" (src), [dst] "r" (dst)
372 : "cc", "memory",
373 "%xmm0", "%xmm1"
374 );
375 }
376
377 // Iterate butterflies
378 for (; n > 4; n >>= 1, bs >>= 1)
379 {
380 for (size_t p=0; p<items; p += bs)
381 {
382 // Set initial values of pointers
383 float *a = &dst[p];
384 float *b = &a[n];
385 size_t k = n;
386
387 ARCH_X86_ASM
388 (
389 __ASM_EMIT("movaps 0x00(%[ak]), %%xmm6") /* xmm6 = rA[i] */
390 __ASM_EMIT("movaps 0x10(%[ak]), %%xmm7") /* xmm7 = iA[i] */
391 :
392 : [ak] "r"(ak)
393 : "%xmm6", "%xmm7"
394 );
395
396 ARCH_X86_ASM
397 (
398 __ASM_EMIT("1:")
399
400 __ASM_EMIT("movups 0x00(%[a]), %%xmm0") /* xmm0 = ra[i] */
401 __ASM_EMIT("movups 0x10(%[a]), %%xmm1") /* xmm1 = ia[i] */
402 __ASM_EMIT("movups 0x00(%[b]), %%xmm2") /* xmm2 = rb[i] */
403 __ASM_EMIT("movups 0x10(%[b]), %%xmm3") /* xmm3 = ib[i] */
404
405 __ASM_EMIT("movaps %%xmm0, %%xmm4") /* xmm4 = ra[i] */
406 __ASM_EMIT("movaps %%xmm1, %%xmm5") /* xmm5 = ia[i] */
407 __ASM_EMIT("addps %%xmm2, %%xmm0") /* xmm0 = ra[i]+rb[i] */
408 __ASM_EMIT("addps %%xmm3, %%xmm1") /* xmm1 = ia[i]+ib[i] */
409 __ASM_EMIT("subps %%xmm2, %%xmm4") /* xmm4 = ra[i]-rb[i] = rc[i] */
410 __ASM_EMIT("subps %%xmm3, %%xmm5") /* xmm5 = ia[i]-ib[i] = ic[i] */
411 __ASM_EMIT("movaps %%xmm4, %%xmm2") /* xmm2 = rc[i] */
412 __ASM_EMIT("movaps %%xmm5, %%xmm3") /* xmm3 = ic[i] */
413 __ASM_EMIT("mulps %%xmm6, %%xmm4") /* xmm4 = rA[i]*rc[i] */
414 __ASM_EMIT("mulps %%xmm7, %%xmm5") /* xmm5 = iA[i]*ic[i] */
415 __ASM_EMIT("mulps %%xmm6, %%xmm3") /* xmm3 = rA[i]*ic[i] */
416 __ASM_EMIT("mulps %%xmm7, %%xmm2") /* xmm2 = iA[i]*rc[i] */
417 __ASM_EMIT("addps %%xmm5, %%xmm4") /* xmm4 = rA[i]*rc[i] + iA[i]*ic[i] */
418 __ASM_EMIT("subps %%xmm2, %%xmm3") /* xmm3 = rA[i]*ic[i] - iA[i]*rc[i] */
419
420 __ASM_EMIT("movups %%xmm0, 0x00(%[a])")
421 __ASM_EMIT("movups %%xmm1, 0x10(%[a])")
422 __ASM_EMIT("movups %%xmm4, 0x00(%[b])")
423 __ASM_EMIT("movups %%xmm3, 0x10(%[b])")
424
425 /* Repeat loop */
426 __ASM_EMIT("add $0x20, %[a]")
427 __ASM_EMIT("add $0x20, %[b]")
428 __ASM_EMIT("sub $8, %[k]")
429 __ASM_EMIT("jz 2f")
430
431 /* Rotate angle */
432 __ASM_EMIT("movaps 0x00(%[wk]), %%xmm0") /* xmm0 = rw */
433 __ASM_EMIT("movaps 0x10(%[wk]), %%xmm1") /* xmm1 = iw */
434 __ASM_EMIT("movaps %%xmm0, %%xmm2") /* xmm2 = rw */
435 __ASM_EMIT("movaps %%xmm1, %%xmm3") /* xmm3 = iw */
436 __ASM_EMIT("mulps %%xmm6, %%xmm3") /* xmm3 = ra * iw */
437 __ASM_EMIT("mulps %%xmm7, %%xmm1") /* xmm1 = ia * iw */
438 __ASM_EMIT("mulps %%xmm0, %%xmm6") /* xmm6 = ra * rw */
439 __ASM_EMIT("mulps %%xmm2, %%xmm7") /* xmm7 = ia * rw */
440 __ASM_EMIT("subps %%xmm1, %%xmm6") /* xmm6 = ra * rw - ia * iw */
441 __ASM_EMIT("addps %%xmm3, %%xmm7") /* xmm7 = ra * iw + ia * rw */
442 __ASM_EMIT("jmp 1b")
443
444 __ASM_EMIT("2:")
445
446 : [a] "+r" (a), [b] "+r" (b), [k] "+r" (k)
447 : [wk] "r"(wk)
448 : "cc", "memory",
449 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
450 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
451 );
452 }
453
454 ak -= 8;
455 wk -= 8;
456 }
457 }
458 }
459 }
460
461 #endif /* PRIVATE_DSP_ARCH_X86_SSE_FASTCONV_PARSE_H_ */