Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/x86/sse/fft/p_scramble.h

raw
1 /*
2 * Copyright (C) 2020 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2020 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 31 мар. 2020 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_X86_SSE_IMPL
23 #error "This header should not be included directly"
24 #endif /* PRIVATE_DSP_ARCH_X86_SSE_IMPL */
25
26 namespace lsp
27 {
28 namespace sse
29 {
30 // This is the SSE implementation of the scrambling functions for self data
31 static inline void FFT_SCRAMBLE_SELF_DIRECT_NAME(float *dst, const float *src, size_t rank)
32 {
33 // Calculate number of items
34 size_t items = (1 << rank) - 1;
35
36 for (size_t i = 1; i < items; ++i)
37 {
38 size_t j = reverse_bits(FFT_TYPE(i), rank); /* Reverse the order of the bits */
39 if (i >= j)
40 continue;
41
42 /* Copy the values from the reversed position */
43 ARCH_X86_ASM
44 (
45 __ASM_EMIT("movlps (%[dst], %[i], 8), %%xmm0")
46 __ASM_EMIT("movlps (%[dst], %[j], 8), %%xmm1")
47 __ASM_EMIT("movlps %%xmm0, (%[dst], %[j], 8)")
48 __ASM_EMIT("movlps %%xmm1, (%[dst], %[i], 8)")
49 :
50 : [dst] "r"(dst), [i] "r"(i), [j] "r"(j)
51 : "memory",
52 "%xmm0", "%xmm1"
53 );
54 }
55
56 // Perform butterfly 8x
57 items = 1 << (rank - 3);
58
59 // Perform 4-element butterflies
60 ARCH_X86_ASM
61 (
62 /* Loop */
63 __ASM_EMIT(".align 16")
64 __ASM_EMIT("1:")
65
66 /* Load data to registers */
67 __ASM_EMIT("movups 0x00(%[dst]), %%xmm0") /* xmm0 = r0 i0 r1 i1 */
68 __ASM_EMIT("movups 0x10(%[dst]), %%xmm2") /* xmm2 = r2 i2 r3 i3 */
69 __ASM_EMIT("movups 0x20(%[dst]), %%xmm4") /* xmm4 = r4 i4 r5 i5 */
70 __ASM_EMIT("movups 0x30(%[dst]), %%xmm6") /* xmm6 = r6 i6 r7 i7 */
71
72 /* 1st-order 4x butterfly */
73 __ASM_EMIT("movaps %%xmm0, %%xmm1") /* xmm1 = r0 i0 r1 i1 */
74 __ASM_EMIT("movaps %%xmm4, %%xmm5") /* xmm5 = r4 i4 r5 i5 */
75 __ASM_EMIT("shufps $0x44, %%xmm2, %%xmm0") /* xmm0 = r0 i0 r2 i2 */
76 __ASM_EMIT("shufps $0x44, %%xmm6, %%xmm4") /* xmm4 = r4 i4 r6 i6 */
77 __ASM_EMIT("shufps $0xee, %%xmm2, %%xmm1") /* xmm1 = r1 i1 r3 i3 */
78 __ASM_EMIT("shufps $0xee, %%xmm6, %%xmm5") /* xmm5 = r5 i5 r7 i7 */
79
80 __ASM_EMIT("movaps %%xmm0, %%xmm2") /* xmm2 = r0 i0 r2 i2 */
81 __ASM_EMIT("movaps %%xmm4, %%xmm6") /* xmm6 = r4 i4 r6 i6 */
82 __ASM_EMIT("addps %%xmm1, %%xmm0") /* xmm0 = r0+r1 i0+i1 r2+r3 i2+i3 = r0' i0' r2' i2' */
83 __ASM_EMIT("addps %%xmm5, %%xmm4") /* xmm4 = r4+r5 i4+i5 r6+r7 i6+i7 = r4' i4' r6' i6' */
84 __ASM_EMIT("subps %%xmm1, %%xmm2") /* xmm2 = r0-r1 i0-i1 r2-r3 i2-i3 = r1' i1' r3' i3' */
85 __ASM_EMIT("subps %%xmm5, %%xmm6") /* xmm6 = r4-r5 i4-i5 r6-r7 i6-i7 = r5' i5' r7' i7' */
86
87 /* 2nd-order 4x butterfly */
88 __ASM_EMIT("movaps %%xmm0, %%xmm1") /* xmm1 = r0' i0' r2' i2' */
89 __ASM_EMIT("movaps %%xmm4, %%xmm5") /* xmm5 = r4' i4' r6' i6' */
90 __ASM_EMIT("shufps $0x44, %%xmm2, %%xmm0") /* xmm0 = r0' i0' r1' i1' */
91 __ASM_EMIT("shufps $0x44, %%xmm6, %%xmm4") /* xmm4 = r4' i4' r5' i5' */
92 __ASM_EMIT("shufps $0xbe, %%xmm2, %%xmm1") /* xmm1 = r2' i2' i3' r3' */
93 __ASM_EMIT("shufps $0xbe, %%xmm6, %%xmm5") /* xmm5 = r6' i6' i7' r7' */
94
95 __ASM_EMIT("movaps %%xmm0, %%xmm2") /* xmm2 = r0' i0' r1' i1' */
96 __ASM_EMIT("movaps %%xmm4, %%xmm6") /* xmm6 = r4' i4' r5' i5' */
97 __ASM_EMIT("addps %%xmm1, %%xmm0") /* xmm0 = r0'+r2' i0'+i2' r1'+i3' i1'+r3' = r0" i0" r1" i3" */
98 __ASM_EMIT("addps %%xmm5, %%xmm4") /* xmm4 = r4'+r6' i4'+i6' r5'+i7' i5'+r7' = r4" i4" r5" i7" */
99 __ASM_EMIT("subps %%xmm1, %%xmm2") /* xmm2 = r0'-r2' i0'-i2' r1'-i3' i1'-r3' = r2" i2" r3" i1" */
100 __ASM_EMIT("subps %%xmm5, %%xmm6") /* xmm6 = r4'-r6' i4'-i6' r5'-i7' i5'-r7' = r6" i6" r7" i5" */
101
102 /* Reorder and store */
103 __ASM_EMIT("movaps %%xmm0, %%xmm1") /* xmm1 = r0" i0" r1" i3" */
104 __ASM_EMIT("movaps %%xmm4, %%xmm5") /* xmm5 = r4" i4" r5" i7" */
105 __ASM_EMIT("shufps $0x88, %%xmm2, %%xmm0") /* xmm0 = r0" r1" r2" r3" */
106 __ASM_EMIT("shufps $0x88, %%xmm6, %%xmm4") /* xmm4 = r4" r5" r6" r7" */
107 __ASM_EMIT("shufps $0xdd, %%xmm2, %%xmm1") /* xmm1 = i0" i3" i2" i1" */
108 __ASM_EMIT("shufps $0xdd, %%xmm6, %%xmm5") /* xmm5 = i4" i7" i6" i5" */
109 __ASM_EMIT("shufps $0x6c, %%xmm1, %%xmm1") /* xmm1 = i0" i1" i2" i3" */
110 __ASM_EMIT("shufps $0x6c, %%xmm5, %%xmm5") /* xmm5 = i4" i5" i6" i7" */
111
112 __ASM_EMIT("movups %%xmm0, 0x00(%[dst])")
113 __ASM_EMIT("movups %%xmm1, 0x10(%[dst])")
114 __ASM_EMIT("movups %%xmm4, 0x20(%[dst])")
115 __ASM_EMIT("movups %%xmm5, 0x30(%[dst])")
116
117 /* Move pointers and repeat cycle */
118 __ASM_EMIT("add $0x40, %[dst]")
119 __ASM_EMIT("dec %[items]")
120 __ASM_EMIT("jnz 1b")
121
122 : [dst] "+r"(dst), [items] "+r"(items)
123 :
124 : "cc", "memory",
125 "%xmm0", "%xmm1", "%xmm2", "%xmm4", "%xmm5", "%xmm6"
126 );
127 }
128
129 static inline void FFT_SCRAMBLE_SELF_REVERSE_NAME(float *dst, const float *src, size_t rank)
130 {
131 // Calculate number of items
132 size_t items = (1 << rank) - 1;
133
134 for (size_t i = 1; i < items; ++i)
135 {
136 size_t j = reverse_bits(FFT_TYPE(i), rank); /* Reverse the order of the bits */
137 if (i >= j)
138 continue;
139
140 /* Copy the values from the reversed position */
141 ARCH_X86_ASM
142 (
143 __ASM_EMIT("movlps (%[dst], %[i], 8), %%xmm0")
144 __ASM_EMIT("movlps (%[dst], %[j], 8), %%xmm1")
145 __ASM_EMIT("movlps %%xmm0, (%[dst], %[j], 8)")
146 __ASM_EMIT("movlps %%xmm1, (%[dst], %[i], 8)")
147 :
148 : [dst] "r"(dst), [i] "r"(i), [j] "r"(j)
149 : "memory",
150 "%xmm0", "%xmm1"
151 );
152 }
153
154 // Perform butterfly 8x
155 items = 1 << (rank - 3);
156
157 // Perform 4-element butterflies
158 ARCH_X86_ASM
159 (
160 /* Loop */
161 __ASM_EMIT(".align 16")
162 __ASM_EMIT("1:")
163
164 /* Load data to registers */
165 __ASM_EMIT("movups 0x00(%[dst]), %%xmm0") /* xmm0 = r0 i0 r1 i1 */
166 __ASM_EMIT("movups 0x10(%[dst]), %%xmm2") /* xmm2 = r2 i2 r3 i3 */
167 __ASM_EMIT("movups 0x20(%[dst]), %%xmm4") /* xmm4 = r4 i4 r5 i5 */
168 __ASM_EMIT("movups 0x30(%[dst]), %%xmm6") /* xmm6 = r6 i6 r7 i7 */
169
170 /* 1st-order 4x butterfly */
171 __ASM_EMIT("movaps %%xmm0, %%xmm1") /* xmm1 = r0 i0 r1 i1 */
172 __ASM_EMIT("movaps %%xmm4, %%xmm5") /* xmm5 = r4 i4 r5 i5 */
173 __ASM_EMIT("shufps $0x44, %%xmm2, %%xmm0") /* xmm0 = r0 i0 r2 i2 */
174 __ASM_EMIT("shufps $0x44, %%xmm6, %%xmm4") /* xmm4 = r4 i4 r6 i6 */
175 __ASM_EMIT("shufps $0xee, %%xmm2, %%xmm1") /* xmm1 = r1 i1 r3 i3 */
176 __ASM_EMIT("shufps $0xee, %%xmm6, %%xmm5") /* xmm5 = r5 i5 r7 i7 */
177
178 __ASM_EMIT("movaps %%xmm0, %%xmm2") /* xmm2 = r0 i0 r2 i2 */
179 __ASM_EMIT("movaps %%xmm4, %%xmm6") /* xmm6 = r4 i4 r6 i6 */
180 __ASM_EMIT("addps %%xmm1, %%xmm0") /* xmm0 = r0+r1 i0+i1 r2+r3 i2+i3 = r0' i0' r2' i2' */
181 __ASM_EMIT("addps %%xmm5, %%xmm4") /* xmm4 = r4+r5 i4+i5 r6+r7 i6+i7 = r4' i4' r6' i6' */
182 __ASM_EMIT("subps %%xmm1, %%xmm2") /* xmm2 = r0-r1 i0-i1 r2-r3 i2-i3 = r1' i1' r3' i3' */
183 __ASM_EMIT("subps %%xmm5, %%xmm6") /* xmm6 = r4-r5 i4-i5 r6-r7 i6-i7 = r5' i5' r7' i7' */
184
185 /* 2nd-order 4x butterfly */
186 __ASM_EMIT("movaps %%xmm0, %%xmm1") /* xmm1 = r0' i0' r2' i2' */
187 __ASM_EMIT("movaps %%xmm4, %%xmm5") /* xmm5 = r4' i4' r6' i6' */
188 __ASM_EMIT("shufps $0x44, %%xmm2, %%xmm0") /* xmm0 = r0' i0' r1' i1' */
189 __ASM_EMIT("shufps $0xbe, %%xmm2, %%xmm1") /* xmm1 = r2' i2' i3' r3' */
190 __ASM_EMIT("shufps $0x44, %%xmm6, %%xmm4") /* xmm4 = r4' i4' r5' i5' */
191 __ASM_EMIT("shufps $0xbe, %%xmm6, %%xmm5") /* xmm5 = r6' i6' i7' r7' */
192
193 __ASM_EMIT("movaps %%xmm0, %%xmm2") /* xmm2 = r0' i0' r1' i1' */
194 __ASM_EMIT("movaps %%xmm4, %%xmm6") /* xmm6 = r4' i4' r5' i5' */
195 __ASM_EMIT("addps %%xmm1, %%xmm0") /* xmm0 = r0'+r2' i0'+i2' r1'+i3' i1'+r3' = r0" i0" r3" i1" */
196 __ASM_EMIT("addps %%xmm5, %%xmm4") /* xmm4 = r4'+r6' i4'+i6' r5'+i7' i5'+r7' = r4" i4" r7" i5" */
197 __ASM_EMIT("subps %%xmm1, %%xmm2") /* xmm2 = r0'-r2' i0'-i2' r1'-i3' i1'-r3' = r2" i2" r1" i3" */
198 __ASM_EMIT("subps %%xmm5, %%xmm6") /* xmm6 = r4'-r6' i4'-i6' r5'-i7' i5'-r7' = r6" i6" r5" i7" */
199
200 /* Reorder and store */
201 __ASM_EMIT("movaps %%xmm0, %%xmm1") /* xmm1 = r0" i0" r3" i1" */
202 __ASM_EMIT("movaps %%xmm4, %%xmm5") /* xmm5 = r4" i4" r7" i5" */
203 __ASM_EMIT("shufps $0x88, %%xmm2, %%xmm0") /* xmm0 = r0" r3" r2" r1" */
204 __ASM_EMIT("shufps $0x88, %%xmm6, %%xmm4") /* xmm4 = r4" r7" r6" r5" */
205 __ASM_EMIT("shufps $0xdd, %%xmm2, %%xmm1") /* xmm1 = i0" i1" i2" i3" */
206 __ASM_EMIT("shufps $0xdd, %%xmm6, %%xmm5") /* xmm5 = i4" i5" i6" i7" */
207 __ASM_EMIT("shufps $0x6c, %%xmm0, %%xmm0") /* xmm0 = r0" r1" r2" r3" */
208 __ASM_EMIT("shufps $0x6c, %%xmm4, %%xmm4") /* xmm4 = r4" r5" r6" r7" */
209
210 __ASM_EMIT("movups %%xmm0, 0x00(%[dst])")
211 __ASM_EMIT("movups %%xmm1, 0x10(%[dst])")
212 __ASM_EMIT("movups %%xmm4, 0x20(%[dst])")
213 __ASM_EMIT("movups %%xmm5, 0x30(%[dst])")
214
215 /* Move pointers and repeat cycle */
216 __ASM_EMIT("add $0x40, %[dst]")
217 __ASM_EMIT("dec %[items]")
218 __ASM_EMIT("jnz 1b")
219
220 : [dst] "+r"(dst), [items] "+r"(items)
221 :
222 : "cc", "memory",
223 "%xmm0", "%xmm1", "%xmm2", "%xmm4", "%xmm5", "%xmm6"
224 );
225 }
226
227 static inline void FFT_SCRAMBLE_COPY_DIRECT_NAME(float *dst, const float *src, size_t rank)
228 {
229 size_t regs = 1 << rank;
230
231 for (size_t i=0; i<regs; ++i)
232 {
233 size_t index = reverse_bits(FFT_TYPE(i), rank);
234
235 // Perform 4-element butterflies
236 ARCH_X86_ASM
237 (
238 /* Load data to registers */
239 __ASM_EMIT("movlps (%[src], %[index], 8), %%xmm0") /* xmm0 = r0 i0 x x */
240 __ASM_EMIT("add %[regs], %[index]")
241 __ASM_EMIT("movlps (%[src], %[index], 8), %%xmm4") /* xmm4 = r4 i4 x x */
242 __ASM_EMIT("add %[regs], %[index]")
243
244 __ASM_EMIT("movlps (%[src], %[index], 8), %%xmm2") /* xmm2 = r2 i2 x x */
245 __ASM_EMIT("add %[regs], %[index]")
246 __ASM_EMIT("movlhps %%xmm2, %%xmm0") /* xmm0 = r0 i0 r2 i2 */
247
248 __ASM_EMIT("movlps (%[src], %[index], 8), %%xmm6") /* xmm6 = r6 i6 x x */
249 __ASM_EMIT("movaps %%xmm0, %%xmm2") /* xmm2 = r0 i0 r2 i2 */
250 __ASM_EMIT("add %[regs], %[index]")
251 __ASM_EMIT("movlhps %%xmm6, %%xmm4") /* xmm4 = r4 i4 r6 i6 */
252
253 __ASM_EMIT("movlps (%[src], %[index], 8), %%xmm1") /* xmm0 = r1 i1 x x */
254 __ASM_EMIT("add %[regs], %[index]")
255 __ASM_EMIT("movaps %%xmm4, %%xmm6") /* xmm6 = r4 i4 r6 i6 */
256 __ASM_EMIT("movlps (%[src], %[index], 8), %%xmm5") /* xmm4 = r5 i5 x x */
257 __ASM_EMIT("add %[regs], %[index]")
258
259 __ASM_EMIT("movlps (%[src], %[index], 8), %%xmm3") /* xmm1 = r3 i3 x x */
260 __ASM_EMIT("add %[regs], %[index]")
261 __ASM_EMIT("movlhps %%xmm3, %%xmm1") /* xmm1 = r1 i1 r3 i3 */
262
263 __ASM_EMIT("movlps (%[src], %[index], 8), %%xmm7") /* xmm5 = r7 i7 x x */
264 __ASM_EMIT("addps %%xmm1, %%xmm0") /* xmm0 = r0+r1 i0+i1 r2+r3 i2+i3 = r0' i0' r2' i2' */
265 __ASM_EMIT("add %[regs], %[index]")
266 __ASM_EMIT("movlhps %%xmm7, %%xmm5") /* xmm5 = r5 i5 r7 i7 */
267
268 /* 1st-order 4x butterfly */
269 /* xmm0 = r0 i0 r2 i2 */
270 /* xmm4 = r4 i4 r6 i6 */
271 /* xmm1 = r1 i1 r3 i3 */
272 /* xmm5 = r5 i5 r7 i7 */
273 __ASM_EMIT("subps %%xmm1, %%xmm2") /* xmm2 = r0-r1 i0-i1 r2-r3 i2-i3 = r1' i1' r3' i3' */
274 __ASM_EMIT("addps %%xmm5, %%xmm4") /* xmm4 = r4+r5 i4+i5 r6+r7 i6+i7 = r4' i4' r6' i6' */
275 __ASM_EMIT("subps %%xmm5, %%xmm6") /* xmm6 = r4-r5 i4-i5 r6-r7 i6-i7 = r5' i5' r7' i7' */
276
277 /* 2nd-order 4x butterfly */
278 __ASM_EMIT("movaps %%xmm0, %%xmm1") /* xmm1 = r0' i0' r2' i2' */
279 __ASM_EMIT("movaps %%xmm4, %%xmm5") /* xmm5 = r4' i4' r6' i6' */
280 __ASM_EMIT("shufps $0x44, %%xmm2, %%xmm0") /* xmm0 = r0' i0' r1' i1' */
281 __ASM_EMIT("shufps $0xbe, %%xmm2, %%xmm1") /* xmm1 = r2' i2' i3' r3' */
282 __ASM_EMIT("shufps $0x44, %%xmm6, %%xmm4") /* xmm4 = r4' i4' r5' i5' */
283 __ASM_EMIT("shufps $0xbe, %%xmm6, %%xmm5") /* xmm5 = r6' i6' i7' r7' */
284
285 __ASM_EMIT("movaps %%xmm0, %%xmm2") /* xmm2 = r0' i0' r1' i1' */
286 __ASM_EMIT("movaps %%xmm4, %%xmm6") /* xmm6 = r4' i4' r5' i5' */
287 __ASM_EMIT("addps %%xmm1, %%xmm0") /* xmm0 = r0'+r2' i0'+i2' r1'+i3' i1'+r3' = r0" i0" r1" i3" */
288 __ASM_EMIT("addps %%xmm5, %%xmm4") /* xmm4 = r4'+r6' i4'+i6' r5'+i7' i5'+r7' = r4" i4" r5" i7" */
289 __ASM_EMIT("subps %%xmm1, %%xmm2") /* xmm2 = r0'-r2' i0'-i2' r1'-i3' i1'-r3' = r2" i2" r3" i1" */
290 __ASM_EMIT("subps %%xmm5, %%xmm6") /* xmm6 = r4'-r6' i4'-i6' r5'-i7' i5'-r7' = r6" i6" r7" i5" */
291
292 /* Reorder and store */
293 __ASM_EMIT("movaps %%xmm0, %%xmm1") /* xmm1 = r0" i0" r1" i3" */
294 __ASM_EMIT("movaps %%xmm4, %%xmm5") /* xmm5 = r4" i4" r5" i7" */
295 __ASM_EMIT("shufps $0x88, %%xmm2, %%xmm0") /* xmm0 = r0" r1" r2" r3" */
296 __ASM_EMIT("shufps $0x88, %%xmm6, %%xmm4") /* xmm4 = r4" r5" r6" r7" */
297 __ASM_EMIT("shufps $0xdd, %%xmm2, %%xmm1") /* xmm1 = i0" i3" i2" i1" */
298 __ASM_EMIT("shufps $0xdd, %%xmm6, %%xmm5") /* xmm5 = i4" i7" i6" i5" */
299 __ASM_EMIT("shufps $0x6c, %%xmm1, %%xmm1") /* xmm1 = i0" i1" i2" i3" */
300 __ASM_EMIT("shufps $0x6c, %%xmm5, %%xmm5") /* xmm5 = i4" i5" i6" i7" */
301
302 __ASM_EMIT("movups %%xmm0, 0x00(%[dst])")
303 __ASM_EMIT("movups %%xmm1, 0x10(%[dst])")
304 __ASM_EMIT("movups %%xmm4, 0x20(%[dst])")
305 __ASM_EMIT("movups %%xmm5, 0x30(%[dst])")
306
307 /* Move pointers and repeat cycle */
308 __ASM_EMIT("add $0x40, %[dst]")
309
310 : [dst] "+r"(dst), [src] "+r"(src), [index] "+r"(index)
311 : [regs] "r" (regs)
312 : "cc", "memory",
313 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
314 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
315 );
316 }
317 }
318
319 static inline void FFT_SCRAMBLE_COPY_REVERSE_NAME(float *dst, const float *src, size_t rank)
320 {
321 size_t regs = 1 << rank;
322
323 for (size_t i=0; i<regs; ++i)
324 {
325 size_t index = reverse_bits(FFT_TYPE(i), rank);
326
327 // Perform 4-element butterflies
328 ARCH_X86_ASM
329 (
330 /* Load data to registers */
331 __ASM_EMIT("movlps (%[src], %[index], 8), %%xmm0") /* xmm0 = r0 i0 x x */
332 __ASM_EMIT("add %[regs], %[index]")
333
334 __ASM_EMIT("movlps (%[src], %[index], 8), %%xmm4") /* xmm4 = r4 i4 x x */
335 __ASM_EMIT("add %[regs], %[index]")
336
337 __ASM_EMIT("movlps (%[src], %[index], 8), %%xmm2") /* xmm2 = r2 i2 x x */
338 __ASM_EMIT("add %[regs], %[index]")
339 __ASM_EMIT("movlhps %%xmm2, %%xmm0") /* xmm0 = r0 i0 r2 i2 */
340
341 __ASM_EMIT("movlps (%[src], %[index], 8), %%xmm6") /* xmm6 = r6 i6 x x */
342 __ASM_EMIT("movaps %%xmm0, %%xmm2") /* xmm2 = r0 i0 r2 i2 */
343 __ASM_EMIT("add %[regs], %[index]")
344 __ASM_EMIT("movlhps %%xmm6, %%xmm4") /* xmm4 = r4 i4 r6 i6 */
345
346 __ASM_EMIT("movlps (%[src], %[index], 8), %%xmm1") /* xmm1 = r1 i1 x x */
347 __ASM_EMIT("add %[regs], %[index]")
348 __ASM_EMIT("movaps %%xmm4, %%xmm6") /* xmm6 = r4 i4 r6 i6 */
349 __ASM_EMIT("movlps (%[src], %[index], 8), %%xmm5") /* xmm5 = r5 i5 x x */
350 __ASM_EMIT("add %[regs], %[index]")
351
352 __ASM_EMIT("movlps (%[src], %[index], 8), %%xmm3") /* xmm3 = r3 i3 x x */
353 __ASM_EMIT("add %[regs], %[index]")
354 __ASM_EMIT("movlhps %%xmm3, %%xmm1") /* xmm1 = r1 i1 r3 i3 */
355
356 __ASM_EMIT("movlps (%[src], %[index], 8), %%xmm7") /* xmm7 = r7 i7 x x */
357 __ASM_EMIT("addps %%xmm1, %%xmm0") /* xmm0 = r0+r1 i0+i1 r2+r3 i2+i3 = r0' i0' r2' i2' */
358 __ASM_EMIT("add %[regs], %[index]")
359 __ASM_EMIT("movlhps %%xmm7, %%xmm5") /* xmm5 = r5 i5 r7 i7 */
360
361 /* 1st-order 4x butterfly */
362 /* xmm0 = r0 i0 r2 i2 */
363 /* xmm4 = r4 i4 r6 i6 */
364 /* xmm1 = r1 i1 r3 i3 */
365 /* xmm5 = r5 i5 r7 i7 */
366 __ASM_EMIT("subps %%xmm1, %%xmm2") /* xmm2 = r0-r1 i0-i1 r2-r3 i2-i3 = r1' i1' r3' i3' */
367 __ASM_EMIT("addps %%xmm5, %%xmm4") /* xmm4 = r4+r5 i4+i5 r6+r7 i6+i7 = r4' i4' r6' i6' */
368 __ASM_EMIT("subps %%xmm5, %%xmm6") /* xmm6 = r4-r5 i4-i5 r6-r7 i6-i7 = r5' i5' r7' i7' */
369
370 /* 2nd-order 4x butterfly */
371 __ASM_EMIT("movaps %%xmm0, %%xmm1") /* xmm1 = r0' i0' r2' i2' */
372 __ASM_EMIT("movaps %%xmm4, %%xmm5") /* xmm5 = r4' i4' r6' i6' */
373 __ASM_EMIT("shufps $0x44, %%xmm2, %%xmm0") /* xmm0 = r0' i0' r1' i1' */
374 __ASM_EMIT("shufps $0xbe, %%xmm2, %%xmm1") /* xmm1 = r2' i2' i3' r3' */
375 __ASM_EMIT("shufps $0x44, %%xmm6, %%xmm4") /* xmm4 = r4' i4' r5' i5' */
376 __ASM_EMIT("shufps $0xbe, %%xmm6, %%xmm5") /* xmm5 = r6' i6' i7' r7' */
377
378 __ASM_EMIT("movaps %%xmm0, %%xmm2") /* xmm2 = r0' i0' r1' i1' */
379 __ASM_EMIT("movaps %%xmm4, %%xmm6") /* xmm6 = r4' i4' r5' i5' */
380 __ASM_EMIT("addps %%xmm1, %%xmm0") /* xmm0 = r0'+r2' i0'+i2' r1'+i3' i1'+r3' = r0" i0" r3" i1" */
381 __ASM_EMIT("addps %%xmm5, %%xmm4") /* xmm4 = r4'+r6' i4'+i6' r5'+i7' i5'+r7' = r4" i4" r7" i5" */
382 __ASM_EMIT("subps %%xmm1, %%xmm2") /* xmm2 = r0'-r2' i0'-i2' r1'-i3' i1'-r3' = r2" i2" r1" i3" */
383 __ASM_EMIT("subps %%xmm5, %%xmm6") /* xmm6 = r4'-r6' i4'-i6' r5'-i7' i5'-r7' = r6" i6" r5" i7" */
384
385 /* Reorder and store */
386 __ASM_EMIT("movaps %%xmm0, %%xmm1") /* xmm1 = r0" i0" r3" i1" */
387 __ASM_EMIT("movaps %%xmm4, %%xmm5") /* xmm5 = r4" i4" r7" i5" */
388 __ASM_EMIT("shufps $0x88, %%xmm2, %%xmm0") /* xmm0 = r0" r3" r2" r1" */
389 __ASM_EMIT("shufps $0x88, %%xmm6, %%xmm4") /* xmm4 = r4" r7" r6" r5" */
390 __ASM_EMIT("shufps $0xdd, %%xmm2, %%xmm1") /* xmm1 = i0" i1" i2" i3" */
391 __ASM_EMIT("shufps $0xdd, %%xmm6, %%xmm5") /* xmm5 = i4" i5" i6" i7" */
392 __ASM_EMIT("shufps $0x6c, %%xmm0, %%xmm0") /* xmm0 = r0" r1" r2" r3" */
393 __ASM_EMIT("shufps $0x6c, %%xmm4, %%xmm4") /* xmm4 = r4" r5" r6" r7" */
394
395 __ASM_EMIT("movups %%xmm0, 0x00(%[dst])")
396 __ASM_EMIT("movups %%xmm1, 0x10(%[dst])")
397 __ASM_EMIT("movups %%xmm4, 0x20(%[dst])")
398 __ASM_EMIT("movups %%xmm5, 0x30(%[dst])")
399
400 /* Move pointers and repeat cycle */
401 __ASM_EMIT("add $0x40, %[dst]")
402
403 : [dst] "+r"(dst), [src] "+r"(src), [index] "+r"(index)
404 : [regs] "r" (regs)
405 : "cc", "memory",
406 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
407 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
408 );
409 }
410 }
411 }
412 }
413
414 #undef FFT_SCRAMBLE_SELF_DIRECT_NAME
415 #undef FFT_SCRAMBLE_COPY_DIRECT_NAME
416 #undef FFT_SCRAMBLE_SELF_REVERSE_NAME
417 #undef FFT_SCRAMBLE_COPY_REVERSE_NAME
418 #undef FFT_REPACK
419 #undef FFT_TYPE
420