Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/x86/sse/fft/scramble.h

raw
1 /*
2 * Copyright (C) 2020 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2020 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 31 мар. 2020 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_X86_SSE_IMPL
23 #error "This header should not be included directly"
24 #endif /* PRIVATE_DSP_ARCH_X86_SSE_IMPL */
25
26 namespace lsp
27 {
28 namespace sse
29 {
30 static inline void FFT_SCRAMBLE_SELF_DIRECT_NAME(float *dst_re, float *dst_im, const float *src_re, const float *src_im, size_t rank)
31 {
32 // Calculate number of items
33 size_t items = (1 << rank) - 1;
34
35 for (size_t i = 1; i < items; ++i)
36 {
37 size_t j = reverse_bits(FFT_TYPE(i), rank); /* Reverse the order of the bits */
38 if (i >= j)
39 continue;
40
41 /* Copy the values from the reversed position */
42 ARCH_X86_ASM
43 (
44 __ASM_EMIT("movss (%[dst_re], %[i], 4), %%xmm0")
45 __ASM_EMIT("movss (%[dst_im], %[i], 4), %%xmm1")
46 __ASM_EMIT("movss (%[dst_re], %[j], 4), %%xmm2")
47 __ASM_EMIT("movss (%[dst_im], %[j], 4), %%xmm3")
48 __ASM_EMIT("movss %%xmm2, (%[dst_re], %[i], 4)")
49 __ASM_EMIT("movss %%xmm3, (%[dst_im], %[i], 4)")
50 __ASM_EMIT("movss %%xmm0, (%[dst_re], %[j], 4)")
51 __ASM_EMIT("movss %%xmm1, (%[dst_im], %[j], 4)")
52 :
53 : [dst_re] "r"(dst_re), [dst_im] "r"(dst_im), [i] "r"(i), [j] "r"(j)
54 : "memory",
55 "%xmm0", "%xmm1", "%xmm2", "%xmm3"
56 );
57 }
58
59 // Perform butterfly 8x
60 items = 1 << (rank - 3);
61
62 // Perform 4-element butterflies
63 ARCH_X86_ASM
64 (
65 /* Loop */
66 __ASM_EMIT(".align 16")
67 __ASM_EMIT("1:")
68
69 /* Load data to registers */
70 __ASM_EMIT("movups 0x00(%[dst_re]), %%xmm0") /* xmm0 = r0 r1 r2 r3 */
71 __ASM_EMIT("movups 0x10(%[dst_re]), %%xmm4") /* xmm4 = r4 r5 r6 r7 */
72 __ASM_EMIT("movups 0x00(%[dst_im]), %%xmm2") /* xmm2 = i0 i1 i2 i3 */
73 __ASM_EMIT("movups 0x10(%[dst_im]), %%xmm6") /* xmm6 = i4 i5 i6 i7 */
74
75 /* 1st-order 4x butterfly */
76 __ASM_EMIT("movaps %%xmm0, %%xmm1") /* xmm1 = r0 r1 r2 r3 */
77 __ASM_EMIT("movaps %%xmm2, %%xmm3") /* xmm3 = i0 i1 i2 i3 */
78 __ASM_EMIT("shufps $0x88, %%xmm4, %%xmm0") /* xmm0 = r0 r2 r4 r6 */
79 __ASM_EMIT("shufps $0x88, %%xmm6, %%xmm2") /* xmm2 = i0 i2 i4 i6 */
80 __ASM_EMIT("shufps $0xdd, %%xmm4, %%xmm1") /* xmm1 = r1 r3 r5 r7 */
81 __ASM_EMIT("shufps $0xdd, %%xmm6, %%xmm3") /* xmm3 = i1 i3 i5 i7 */
82
83 __ASM_EMIT("movaps %%xmm0, %%xmm4") /* xmm4 = r0 r2 r4 r6 */
84 __ASM_EMIT("movaps %%xmm2, %%xmm6") /* xmm5 = i0 i2 i4 i6 */
85 __ASM_EMIT("addps %%xmm1, %%xmm0") /* xmm0 = r0+r1 r2+r3 r4+r5 r6+r7 = r0' r2' r4' r6' */
86 __ASM_EMIT("addps %%xmm3, %%xmm2") /* xmm2 = i0+i1 i2+i3 i4+i5 i6+i7 = i0' i2' i4' i6' */
87 __ASM_EMIT("subps %%xmm1, %%xmm4") /* xmm4 = r0-r1 r2-r3 r4-r5 r6-r7 = r1' r3' r5' r7' */
88 __ASM_EMIT("subps %%xmm3, %%xmm6") /* xmm6 = i0-i1 i2-i3 i4-i5 i6-i7 = i1' i3' i5' i7' */
89
90 /* 2nd-order 4x butterfly */
91 __ASM_EMIT("movaps %%xmm0, %%xmm1") /* xmm1 = r0' r2' r4' r6' */
92 __ASM_EMIT("movaps %%xmm2, %%xmm3") /* xmm3 = i0' i2' i4' i6' */
93 __ASM_EMIT("shufps $0x88, %%xmm4, %%xmm0") /* xmm0 = r0' r4' r1' r5' */
94 __ASM_EMIT("shufps $0xdd, %%xmm6, %%xmm1") /* xmm1 = r2' r6' i3' i7' */
95 __ASM_EMIT("shufps $0xdd, %%xmm4, %%xmm3") /* xmm3 = i2' i6' r3' r7' */
96 __ASM_EMIT("shufps $0x88, %%xmm6, %%xmm2") /* xmm2 = i0' i4' i1' i5' */
97
98 __ASM_EMIT("movaps %%xmm0, %%xmm4") /* xmm4 = r0' r4' r1' r5' */
99 __ASM_EMIT("movaps %%xmm2, %%xmm6") /* xmm5 = i0' i4' i1' i5' */
100 __ASM_EMIT("addps %%xmm1, %%xmm0") /* xmm0 = r0'+r2' r4'+r6' r1'+i3' r5'+i7' = r0" r4" r1" r5" */
101 __ASM_EMIT("addps %%xmm3, %%xmm2") /* xmm2 = i0'+i2' i4'+i6' i1'+r3' i5'+r7' = i0" i4" i3" i7" */
102 __ASM_EMIT("subps %%xmm1, %%xmm4") /* xmm4 = r0'-r2' r4'-r6' r1'-i3' r5'-i7' = r2" r6" r3" r7" */
103 __ASM_EMIT("subps %%xmm3, %%xmm6") /* xmm6 = i0'-i2' i4'-i6' i1'-r3' i5'-r7' = i2" i6" i1" i5" */
104
105 /* Reorder and store */
106 __ASM_EMIT("movaps %%xmm0, %%xmm1") /* xmm1 = r0" r4" r1" r5" */
107 __ASM_EMIT("movaps %%xmm2, %%xmm3") /* xmm3 = i0" i4" i3" i7" */
108 __ASM_EMIT("shufps $0x88, %%xmm4, %%xmm0") /* xmm0 = r0" r1" r2" r3" */
109 __ASM_EMIT("shufps $0x88, %%xmm6, %%xmm2") /* xmm2 = i0" i3" i2" i1" */
110 __ASM_EMIT("shufps $0xdd, %%xmm4, %%xmm1") /* xmm1 = r4" r5" r6" r7" */
111 __ASM_EMIT("shufps $0xdd, %%xmm6, %%xmm3") /* xmm3 = i4" i7" i6" i5" */
112 __ASM_EMIT("shufps $0x6c, %%xmm2, %%xmm2") /* xmm2 = i0" i1" i2" i3" */
113 __ASM_EMIT("shufps $0x6c, %%xmm3, %%xmm3") /* xmm3 = i4" i5" i6" i7" */
114
115 __ASM_EMIT("movups %%xmm0, 0x00(%[dst_re])")
116 __ASM_EMIT("movups %%xmm1, 0x10(%[dst_re])")
117 __ASM_EMIT("movups %%xmm2, 0x00(%[dst_im])")
118 __ASM_EMIT("movups %%xmm3, 0x10(%[dst_im])")
119
120 /* Move pointers */
121 __ASM_EMIT("add $0x20, %[dst_re]")
122 __ASM_EMIT("add $0x20, %[dst_im]")
123
124 /* Repeat cycle */
125 __ASM_EMIT("dec %[items]")
126 __ASM_EMIT("jnz 1b")
127 __ASM_EMIT("2:")
128 : [dst_re] "+r"(dst_re), [dst_im] "+r"(dst_im), [items] "+r"(items)
129 :
130 : "cc", "memory",
131 "%xmm0", "%xmm1", "%xmm2"
132 );
133 }
134
135 static inline void FFT_SCRAMBLE_COPY_DIRECT_NAME(float *dst_re, float *dst_im, const float *src_re, const float *src_im, size_t rank)
136 {
137 size_t regs = 1 << rank;
138
139 for (size_t i=0; i<regs; ++i)
140 {
141 size_t index = reverse_bits(FFT_TYPE(i), rank);
142
143 ARCH_X86_ASM
144 (
145 /* Load scalar values */
146 __ASM_EMIT("movss (%[src_re], %[index], 4), %%xmm0") /* xmm0 = r0 x x x */
147 __ASM_EMIT("movss (%[src_im], %[index], 4), %%xmm2") /* xmm2 = i0 x x x */
148 __ASM_EMIT("add %[regs], %[index]")
149
150 __ASM_EMIT("movss (%[src_re], %[index], 4), %%xmm1") /* xmm1 = r4 x x x */
151 __ASM_EMIT("movss (%[src_im], %[index], 4), %%xmm3") /* xmm3 = i4 x x x */
152 __ASM_EMIT("add %[regs], %[index]")
153
154 __ASM_EMIT("movss (%[src_re], %[index], 4), %%xmm4") /* xmm4 = r2 x x x */
155 __ASM_EMIT("movss (%[src_im], %[index], 4), %%xmm6") /* xmm6 = i2 x x x */
156 __ASM_EMIT("add %[regs], %[index]")
157 __ASM_EMIT("movlhps %%xmm4, %%xmm0") /* xmm0 = r0 x r2 x */
158 __ASM_EMIT("movlhps %%xmm6, %%xmm2") /* xmm2 = i0 x i2 x */
159
160 __ASM_EMIT("movss (%[src_re], %[index], 4), %%xmm4") /* xmm4 = r6 x x x */
161 __ASM_EMIT("movss (%[src_im], %[index], 4), %%xmm6") /* xmm6 = i6 x x x */
162 __ASM_EMIT("add %[regs], %[index]")
163 __ASM_EMIT("movlhps %%xmm4, %%xmm1") /* xmm1 = r4 x r6 x */
164 __ASM_EMIT("movlhps %%xmm6, %%xmm3") /* xmm3 = i4 x i6 x */
165 __ASM_EMIT("shufps $0x88, %%xmm1, %%xmm0") /* xmm0 = r0 r2 r4 r6 */
166 __ASM_EMIT("shufps $0x88, %%xmm3, %%xmm2") /* xmm2 = i0 i2 i4 i6 */
167
168 __ASM_EMIT("movss (%[src_re], %[index], 4), %%xmm1") /* xmm1 = r1 x x x */
169 __ASM_EMIT("movss (%[src_im], %[index], 4), %%xmm3") /* xmm3 = i1 x x x */
170 __ASM_EMIT("add %[regs], %[index]")
171
172 __ASM_EMIT("movss (%[src_re], %[index], 4), %%xmm4") /* xmm4 = r5 x x x */
173 __ASM_EMIT("movss (%[src_im], %[index], 4), %%xmm6") /* xmm6 = i5 x x x */
174 __ASM_EMIT("add %[regs], %[index]")
175
176 __ASM_EMIT("movss (%[src_re], %[index], 4), %%xmm5") /* xmm5 = r3 x x x */
177 __ASM_EMIT("movss (%[src_im], %[index], 4), %%xmm7") /* xmm7 = i3 x x x */
178 __ASM_EMIT("add %[regs], %[index]")
179 __ASM_EMIT("movlhps %%xmm5, %%xmm1") /* xmm1 = r1 x r3 x */
180 __ASM_EMIT("movlhps %%xmm7, %%xmm3") /* xmm3 = i1 x i3 x */
181
182 __ASM_EMIT("movss (%[src_re], %[index], 4), %%xmm5") /* xmm5 = r7 x x x */
183 __ASM_EMIT("movss (%[src_im], %[index], 4), %%xmm7") /* xmm7 = i7 x x x */
184 __ASM_EMIT("movlhps %%xmm5, %%xmm4") /* xmm4 = r5 x r7 x */
185 __ASM_EMIT("movlhps %%xmm7, %%xmm6") /* xmm6 = i5 x i7 x */
186 __ASM_EMIT("shufps $0x88, %%xmm4, %%xmm1") /* xmm1 = r1 r3 r5 r7 */
187 __ASM_EMIT("shufps $0x88, %%xmm6, %%xmm3") /* xmm3 = i1 i3 i5 i7 */
188
189 /* 1st-order 4x butterfly */
190 /* xmm0 = r0 r2 r4 r6 */
191 /* xmm2 = i0 i2 i4 i6 */
192 /* xmm1 = r1 r3 r5 r7 */
193 /* xmm3 = i1 i3 i5 i7 */
194
195 __ASM_EMIT("movaps %%xmm0, %%xmm4") /* xmm4 = r0 r2 r4 r6 */
196 __ASM_EMIT("movaps %%xmm2, %%xmm6") /* xmm5 = i0 i2 i4 i6 */
197 __ASM_EMIT("addps %%xmm1, %%xmm0") /* xmm0 = r0+r1 r2+r3 r4+r5 r6+r7 = r0' r2' r4' r6' */
198 __ASM_EMIT("addps %%xmm3, %%xmm2") /* xmm2 = i0+i1 i2+i3 i4+i5 i6+i7 = i0' i2' i4' i6' */
199 __ASM_EMIT("subps %%xmm1, %%xmm4") /* xmm4 = r0-r1 r2-r3 r4-r5 r6-r7 = r1' r3' r5' r7' */
200 __ASM_EMIT("subps %%xmm3, %%xmm6") /* xmm6 = i0-i1 i2-i3 i4-i5 i6-i7 = i1' i3' i5' i7' */
201
202 /* 2nd-order 4x butterfly */
203 __ASM_EMIT("movaps %%xmm0, %%xmm1") /* xmm1 = r0' r2' r4' r6' */
204 __ASM_EMIT("movaps %%xmm2, %%xmm3") /* xmm3 = i0' i2' i4' i6' */
205 __ASM_EMIT("shufps $0x88, %%xmm4, %%xmm0") /* xmm0 = r0' r4' r1' r5' */
206 __ASM_EMIT("shufps $0xdd, %%xmm6, %%xmm1") /* xmm1 = r2' r6' i3' i7' */
207 __ASM_EMIT("shufps $0xdd, %%xmm4, %%xmm3") /* xmm3 = i2' i6' r3' r7' */
208 __ASM_EMIT("shufps $0x88, %%xmm6, %%xmm2") /* xmm2 = i0' i4' i1' i5' */
209
210 __ASM_EMIT("movaps %%xmm0, %%xmm4") /* xmm4 = r0' r4' r1' r5' */
211 __ASM_EMIT("movaps %%xmm2, %%xmm6") /* xmm5 = i0' i4' i1' i5' */
212 __ASM_EMIT("addps %%xmm1, %%xmm0") /* xmm0 = r0'+r2' r4'+r6' r1'+i3' r5'+i7' = r0" r4" r1" r5" */
213 __ASM_EMIT("addps %%xmm3, %%xmm2") /* xmm2 = i0'+i2' i4'+i6' i1'+i3' i5'+i7' = i0" i4" i3" i7" */
214 __ASM_EMIT("subps %%xmm1, %%xmm4") /* xmm4 = r0'-r2' r4'-r6' r1'-i3' r5'-i7' = r2" r6" r3" r7" */
215 __ASM_EMIT("subps %%xmm3, %%xmm6") /* xmm6 = i0'-i2' i4'-i6' i1'-i3' i5'-i7' = i2" i6" i1" i5" */
216
217 /* Reorder and store */
218 __ASM_EMIT("movaps %%xmm0, %%xmm1") /* xmm1 = r0" r4" r1" r5" */
219 __ASM_EMIT("movaps %%xmm2, %%xmm3") /* xmm3 = i0" i4" i3" i7" */
220 __ASM_EMIT("shufps $0x88, %%xmm4, %%xmm0") /* xmm0 = r0" r1" r2" r3" */
221 __ASM_EMIT("shufps $0x88, %%xmm6, %%xmm2") /* xmm2 = i0" i3" i2" i1" */
222 __ASM_EMIT("shufps $0xdd, %%xmm4, %%xmm1") /* xmm1 = r4" r5" r6" r7" */
223 __ASM_EMIT("shufps $0xdd, %%xmm6, %%xmm3") /* xmm3 = i4" i7" i6" i5" */
224 __ASM_EMIT("shufps $0x6c, %%xmm2, %%xmm2") /* xmm2 = i0" i1" i2" i3" */
225 __ASM_EMIT("shufps $0x6c, %%xmm3, %%xmm3") /* xmm3 = i4" i5" i6" i7" */
226
227 __ASM_EMIT("movups %%xmm0, 0x00(%[dst_re])")
228 __ASM_EMIT("movups %%xmm1, 0x10(%[dst_re])")
229 __ASM_EMIT("movups %%xmm2, 0x00(%[dst_im])")
230 __ASM_EMIT("movups %%xmm3, 0x10(%[dst_im])")
231
232 /* Update pointers */
233 __ASM_EMIT("add $0x20, %[dst_re]")
234 __ASM_EMIT("add $0x20, %[dst_im]")
235
236 : [dst_re] "+r" (dst_re), [dst_im] "+r"(dst_im), [index] "+r"(index)
237 : [src_re] "r"(src_re), [src_im] "r"(src_im), [regs] __ASM_ARG_RO(regs)
238 : "cc", "memory",
239 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
240 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
241 );
242 }
243 }
244
245 static inline void FFT_SCRAMBLE_SELF_REVERSE_NAME(float *dst_re, float *dst_im, const float *src_re, const float *src_im, size_t rank)
246 {
247 // Calculate number of items
248 size_t items = (1 << rank) - 1;
249
250 for (size_t i = 1; i < items; ++i)
251 {
252 size_t j = reverse_bits(FFT_TYPE(i), rank); /* Reverse the order of the bits */
253 if (i >= j)
254 continue;
255
256 /* Copy the values from the reversed position */
257 ARCH_X86_ASM
258 (
259 __ASM_EMIT("movss (%[dst_re], %[i], 4), %%xmm0")
260 __ASM_EMIT("movss (%[dst_im], %[i], 4), %%xmm1")
261 __ASM_EMIT("movss (%[dst_re], %[j], 4), %%xmm2")
262 __ASM_EMIT("movss (%[dst_im], %[j], 4), %%xmm3")
263 __ASM_EMIT("movss %%xmm2, (%[dst_re], %[i], 4)")
264 __ASM_EMIT("movss %%xmm3, (%[dst_im], %[i], 4)")
265 __ASM_EMIT("movss %%xmm0, (%[dst_re], %[j], 4)")
266 __ASM_EMIT("movss %%xmm1, (%[dst_im], %[j], 4)")
267 :
268 : [dst_re] "r"(dst_re), [dst_im] "r"(dst_im), [i] "r"(i), [j] "r"(j)
269 : "memory",
270 "%xmm0", "%xmm1", "%xmm2", "%xmm3"
271 );
272 }
273
274 // Perform butterfly 4x
275 items = 1 << (rank - 3);
276
277 // Perform 4-element butterflies
278 ARCH_X86_ASM
279 (
280 /* Loop */
281 __ASM_EMIT(".align 16")
282 __ASM_EMIT("1:")
283
284 /* Load data to registers */
285 __ASM_EMIT("movups 0x00(%[dst_re]), %%xmm0") /* xmm0 = r0 r1 r2 r3 */
286 __ASM_EMIT("movups 0x10(%[dst_re]), %%xmm4") /* xmm4 = r4 r5 r6 r7 */
287 __ASM_EMIT("movups 0x00(%[dst_im]), %%xmm2") /* xmm2 = i0 i1 i2 i3 */
288 __ASM_EMIT("movups 0x10(%[dst_im]), %%xmm6") /* xmm6 = i4 i5 i6 i7 */
289
290 /* 1st-order 4x butterfly */
291 __ASM_EMIT("movaps %%xmm0, %%xmm1") /* xmm1 = r0 r1 r2 r3 */
292 __ASM_EMIT("movaps %%xmm2, %%xmm3") /* xmm3 = i0 i1 i2 i3 */
293 __ASM_EMIT("shufps $0x88, %%xmm4, %%xmm0") /* xmm0 = r0 r2 r4 r6 */
294 __ASM_EMIT("shufps $0x88, %%xmm6, %%xmm2") /* xmm2 = i0 i2 i4 i6 */
295 __ASM_EMIT("shufps $0xdd, %%xmm4, %%xmm1") /* xmm1 = r1 r3 r5 r7 */
296 __ASM_EMIT("shufps $0xdd, %%xmm6, %%xmm3") /* xmm3 = i1 i3 i5 i7 */
297
298 __ASM_EMIT("movaps %%xmm0, %%xmm4") /* xmm4 = r0 r2 r4 r6 */
299 __ASM_EMIT("movaps %%xmm2, %%xmm6") /* xmm5 = i0 i2 i4 i6 */
300 __ASM_EMIT("addps %%xmm1, %%xmm0") /* xmm0 = r0+r1 r2+r3 r4+r5 r6+r7 = r0' r2' r4' r6' */
301 __ASM_EMIT("addps %%xmm3, %%xmm2") /* xmm2 = i0+i1 i2+i3 i4+i5 i6+i7 = i0' i2' i4' i6' */
302 __ASM_EMIT("subps %%xmm1, %%xmm4") /* xmm4 = r0-r1 r2-r3 r4-r5 r6-r7 = r1' r3' r5' r7' */
303 __ASM_EMIT("subps %%xmm3, %%xmm6") /* xmm6 = i0-i1 i2-i3 i4-i5 i6-i7 = i1' i3' i5' i7' */
304
305 /* 2nd-order 4x butterfly */
306 __ASM_EMIT("movaps %%xmm0, %%xmm1") /* xmm1 = r0' r2' r4' r6' */
307 __ASM_EMIT("movaps %%xmm2, %%xmm3") /* xmm3 = i0' i2' i4' i6' */
308 __ASM_EMIT("shufps $0x88, %%xmm4, %%xmm0") /* xmm0 = r0' r4' r1' r5' */
309 __ASM_EMIT("shufps $0xdd, %%xmm6, %%xmm1") /* xmm1 = r2' r6' i3' i7' */
310 __ASM_EMIT("shufps $0xdd, %%xmm4, %%xmm3") /* xmm3 = i2' i6' r3' r7' */
311 __ASM_EMIT("shufps $0x88, %%xmm6, %%xmm2") /* xmm2 = i0' i4' i1' i5' */
312
313 __ASM_EMIT("movaps %%xmm0, %%xmm4") /* xmm4 = r0' r4' r1' r5' */
314 __ASM_EMIT("movaps %%xmm2, %%xmm6") /* xmm5 = i0' i4' i1' i5' */
315 __ASM_EMIT("addps %%xmm1, %%xmm0") /* xmm0 = r0'+r2' r4'+r6' r1'+i3' r5'+i7' = r0" r4" r3" r7" */
316 __ASM_EMIT("addps %%xmm3, %%xmm2") /* xmm2 = i0'+i2' i4'+i6' i1'+i3' i5'+i7' = i0" i4" i1" i5" */
317 __ASM_EMIT("subps %%xmm1, %%xmm4") /* xmm4 = r0'-r2' r4'-r6' r1'-i3' r5'-i7' = r2" r6" r1" r5" */
318 __ASM_EMIT("subps %%xmm3, %%xmm6") /* xmm6 = i0'-i2' i4'-i6' i1'-i3' i5'-i7' = i2" i6" i3" i7" */
319
320 /* Reorder and store */
321 __ASM_EMIT("movaps %%xmm0, %%xmm1") /* xmm1 = r0" r4" r3" r7" */
322 __ASM_EMIT("movaps %%xmm2, %%xmm3") /* xmm3 = i0" i4" i1" i5" */
323 __ASM_EMIT("shufps $0x88, %%xmm4, %%xmm0") /* xmm0 = r0" r3" r2" r1" */
324 __ASM_EMIT("shufps $0x88, %%xmm6, %%xmm2") /* xmm2 = i0" i1" i2" i3" */
325 __ASM_EMIT("shufps $0xdd, %%xmm4, %%xmm1") /* xmm1 = r4" r7" r6" r5" */
326 __ASM_EMIT("shufps $0xdd, %%xmm6, %%xmm3") /* xmm3 = i4" i5" i6" i7" */
327 __ASM_EMIT("shufps $0x6c, %%xmm0, %%xmm0") /* xmm0 = r0" r1" r2" r3" */
328 __ASM_EMIT("shufps $0x6c, %%xmm1, %%xmm1") /* xmm1 = r4" r5" r6" r7" */
329
330 __ASM_EMIT("movups %%xmm0, 0x00(%[dst_re])")
331 __ASM_EMIT("movups %%xmm1, 0x10(%[dst_re])")
332 __ASM_EMIT("movups %%xmm2, 0x00(%[dst_im])")
333 __ASM_EMIT("movups %%xmm3, 0x10(%[dst_im])")
334
335 /* Move pointers */
336 __ASM_EMIT("add $0x20, %[dst_re]")
337 __ASM_EMIT("add $0x20, %[dst_im]")
338
339 /* Repeat cycle */
340 __ASM_EMIT("dec %[items]")
341 __ASM_EMIT("jnz 1b")
342 __ASM_EMIT("2:")
343 : [dst_re] "+r"(dst_re), [dst_im] "+r"(dst_im), [items] "+r"(items)
344 :
345 : "cc", "memory",
346 "%xmm0", "%xmm1", "%xmm2"
347 );
348 }
349
350 static inline void FFT_SCRAMBLE_COPY_REVERSE_NAME(float *dst_re, float *dst_im, const float *src_re, const float *src_im, size_t rank)
351 {
352 size_t regs = 1 << rank;
353
354 for (size_t i=0; i<regs; ++i)
355 {
356 size_t index = reverse_bits(FFT_TYPE(i), rank);
357
358 ARCH_X86_ASM
359 (
360 /* Load scalar values */
361 __ASM_EMIT("movss (%[src_re], %[index], 4), %%xmm0") /* xmm0 = r0 x x x */
362 __ASM_EMIT("movss (%[src_im], %[index], 4), %%xmm2") /* xmm2 = i0 x x x */
363 __ASM_EMIT("add %[regs], %[index]")
364
365 __ASM_EMIT("movss (%[src_re], %[index], 4), %%xmm1") /* xmm1 = r4 x x x */
366 __ASM_EMIT("movss (%[src_im], %[index], 4), %%xmm3") /* xmm3 = i4 x x x */
367 __ASM_EMIT("add %[regs], %[index]")
368
369 __ASM_EMIT("movss (%[src_re], %[index], 4), %%xmm4") /* xmm4 = r2 x x x */
370 __ASM_EMIT("movss (%[src_im], %[index], 4), %%xmm6") /* xmm6 = i2 x x x */
371 __ASM_EMIT("add %[regs], %[index]")
372 __ASM_EMIT("movlhps %%xmm4, %%xmm0") /* xmm0 = r0 x r2 x */
373 __ASM_EMIT("movlhps %%xmm6, %%xmm2") /* xmm2 = i0 x i2 x */
374
375 __ASM_EMIT("movss (%[src_re], %[index], 4), %%xmm4") /* xmm4 = r6 x x x */
376 __ASM_EMIT("movss (%[src_im], %[index], 4), %%xmm6") /* xmm6 = i6 x x x */
377 __ASM_EMIT("add %[regs], %[index]")
378 __ASM_EMIT("movlhps %%xmm4, %%xmm1") /* xmm1 = r4 x r6 x */
379 __ASM_EMIT("movlhps %%xmm6, %%xmm3") /* xmm3 = i4 x i6 x */
380 __ASM_EMIT("shufps $0x88, %%xmm1, %%xmm0") /* xmm0 = r0 r2 r4 r6 */
381 __ASM_EMIT("shufps $0x88, %%xmm3, %%xmm2") /* xmm2 = i0 i2 i4 i6 */
382
383 __ASM_EMIT("movss (%[src_re], %[index], 4), %%xmm1") /* xmm1 = r1 x x x */
384 __ASM_EMIT("movss (%[src_im], %[index], 4), %%xmm3") /* xmm3 = i1 x x x */
385 __ASM_EMIT("add %[regs], %[index]")
386
387 __ASM_EMIT("movss (%[src_re], %[index], 4), %%xmm4") /* xmm4 = r5 x x x */
388 __ASM_EMIT("movss (%[src_im], %[index], 4), %%xmm6") /* xmm6 = i5 x x x */
389 __ASM_EMIT("add %[regs], %[index]")
390
391 __ASM_EMIT("movss (%[src_re], %[index], 4), %%xmm5") /* xmm5 = r3 x x x */
392 __ASM_EMIT("movss (%[src_im], %[index], 4), %%xmm7") /* xmm7 = i3 x x x */
393 __ASM_EMIT("add %[regs], %[index]")
394 __ASM_EMIT("movlhps %%xmm5, %%xmm1") /* xmm1 = r1 x r3 x */
395 __ASM_EMIT("movlhps %%xmm7, %%xmm3") /* xmm3 = i1 x i3 x */
396
397 __ASM_EMIT("movss (%[src_re], %[index], 4), %%xmm5") /* xmm5 = r7 x x x */
398 __ASM_EMIT("movss (%[src_im], %[index], 4), %%xmm7") /* xmm7 = i7 x x x */
399 __ASM_EMIT("movlhps %%xmm5, %%xmm4") /* xmm4 = r5 x r7 x */
400 __ASM_EMIT("movlhps %%xmm7, %%xmm6") /* xmm6 = i5 x i7 x */
401 __ASM_EMIT("shufps $0x88, %%xmm4, %%xmm1") /* xmm1 = r1 r3 r5 r7 */
402 __ASM_EMIT("shufps $0x88, %%xmm6, %%xmm3") /* xmm3 = i1 i3 i5 i7 */
403
404 /* 1st-order 4x butterfly */
405 /* xmm0 = r0 r2 r4 r6 */
406 /* xmm2 = i0 i2 i4 i6 */
407 /* xmm1 = r1 r3 r5 r7 */
408 /* xmm3 = i1 i3 i5 i7 */
409
410 __ASM_EMIT("movaps %%xmm0, %%xmm4") /* xmm4 = r0 r2 r4 r6 */
411 __ASM_EMIT("movaps %%xmm2, %%xmm6") /* xmm5 = i0 i2 i4 i6 */
412 __ASM_EMIT("addps %%xmm1, %%xmm0") /* xmm0 = r0+r1 r2+r3 r4+r5 r6+r7 = r0' r2' r4' r6' */
413 __ASM_EMIT("addps %%xmm3, %%xmm2") /* xmm2 = i0+i1 i2+i3 i4+i5 i6+i7 = i0' i2' i4' i6' */
414 __ASM_EMIT("subps %%xmm1, %%xmm4") /* xmm4 = r0-r1 r2-r3 r4-r5 r6-r7 = r1' r3' r5' r7' */
415 __ASM_EMIT("subps %%xmm3, %%xmm6") /* xmm6 = i0-i1 i2-i3 i4-i5 i6-i7 = i1' i3' i5' i7' */
416
417 /* 2nd-order 4x butterfly */
418 __ASM_EMIT("movaps %%xmm0, %%xmm1") /* xmm1 = r0' r2' r4' r6' */
419 __ASM_EMIT("movaps %%xmm2, %%xmm3") /* xmm3 = i0' i2' i4' i6' */
420 __ASM_EMIT("shufps $0x88, %%xmm4, %%xmm0") /* xmm0 = r0' r4' r1' r5' */
421 __ASM_EMIT("shufps $0xdd, %%xmm6, %%xmm1") /* xmm1 = r2' r6' i3' i7' */
422 __ASM_EMIT("shufps $0xdd, %%xmm4, %%xmm3") /* xmm3 = i2' i6' r3' r7' */
423 __ASM_EMIT("shufps $0x88, %%xmm6, %%xmm2") /* xmm2 = i0' i4' i1' i5' */
424
425 __ASM_EMIT("movaps %%xmm0, %%xmm4") /* xmm4 = r0' r4' r1' r5' */
426 __ASM_EMIT("movaps %%xmm2, %%xmm6") /* xmm5 = i0' i4' i1' i5' */
427 __ASM_EMIT("addps %%xmm1, %%xmm0") /* xmm0 = r0'+r2' r4'+r6' r1'+i3' r5'+i7' = r0" r4" r3" r7" */
428 __ASM_EMIT("addps %%xmm3, %%xmm2") /* xmm2 = i0'+i2' i4'+i6' i1'+r3' i5'+i7' = i0" i4" i1" i5" */
429 __ASM_EMIT("subps %%xmm1, %%xmm4") /* xmm4 = r0'-r2' r4'-r6' r1'-i3' r5'-i7' = r2" r6" r1" r5" */
430 __ASM_EMIT("subps %%xmm3, %%xmm6") /* xmm6 = i0'-i2' i4'-i6' i1'-r3' i5'-i7' = i2" i6" i3" i7" */
431
432 /* Reorder and store */
433 __ASM_EMIT("movaps %%xmm0, %%xmm1") /* xmm1 = r0" r4" r3" r7" */
434 __ASM_EMIT("movaps %%xmm2, %%xmm3") /* xmm3 = i0" i4" i1" i5" */
435 __ASM_EMIT("shufps $0x88, %%xmm4, %%xmm0") /* xmm0 = r0" r3" r2" r1" */
436 __ASM_EMIT("shufps $0x88, %%xmm6, %%xmm2") /* xmm2 = i0" i1" i2" i3" */
437 __ASM_EMIT("shufps $0xdd, %%xmm4, %%xmm1") /* xmm1 = r4" r7" r6" r5" */
438 __ASM_EMIT("shufps $0xdd, %%xmm6, %%xmm3") /* xmm3 = i4" i5" i6" i7" */
439 __ASM_EMIT("shufps $0x6c, %%xmm0, %%xmm0") /* xmm0 = r0" r1" r2" r3" */
440 __ASM_EMIT("shufps $0x6c, %%xmm1, %%xmm1") /* xmm1 = r4" r5" r6" r7" */
441
442 __ASM_EMIT("movups %%xmm0, 0x00(%[dst_re])")
443 __ASM_EMIT("movups %%xmm1, 0x10(%[dst_re])")
444 __ASM_EMIT("movups %%xmm2, 0x00(%[dst_im])")
445 __ASM_EMIT("movups %%xmm3, 0x10(%[dst_im])")
446
447 /* Update pointers */
448 __ASM_EMIT("add $0x20, %[dst_re]")
449 __ASM_EMIT("add $0x20, %[dst_im]")
450
451 : [dst_re] "+r" (dst_re), [dst_im] "+r"(dst_im), [index] "+r"(index)
452 : [src_re] "r"(src_re), [src_im] "r"(src_im), [regs] __ASM_ARG_RO(regs)
453 : "cc", "memory",
454 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
455 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
456 );
457 }
458 }
459 }
460 }
461
462 #undef FFT_SCRAMBLE_SELF_DIRECT_NAME
463 #undef FFT_SCRAMBLE_SELF_REVERSE_NAME
464 #undef FFT_SCRAMBLE_COPY_DIRECT_NAME
465 #undef FFT_SCRAMBLE_COPY_REVERSE_NAME
466 #undef FFT_TYPE
467