Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/x86/avx/fft/scramble.h

raw
1 /*
2 * Copyright (C) 2020 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2020 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 31 мар. 2020 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_X86_AVX_IMPL
23 #error "This header should not be included directly"
24 #endif /* PRIVATE_DSP_ARCH_X86_AVX_IMPL */
25
26 namespace lsp
27 {
28 namespace avx
29 {
30 static inline void FFT_SCRAMBLE_SELF_DIRECT_NAME(float *dst_re, float *dst_im, size_t rank)
31 {
32 // Calculate number of items
33 size_t items = (1 << rank) - 1;
34
35 for (size_t i = 1; i < items; ++i)
36 {
37 size_t j = reverse_bits(FFT_TYPE(i), rank); /* Reverse the order of the bits */
38 if (i >= j)
39 continue;
40
41 /* Copy the values from the reversed position */
42 ARCH_X86_ASM
43 (
44 __ASM_EMIT("vmovss (%[dst_re], %[i], 4), %%xmm0")
45 __ASM_EMIT("vmovss (%[dst_im], %[i], 4), %%xmm1")
46 __ASM_EMIT("vmovss (%[dst_re], %[j], 4), %%xmm2")
47 __ASM_EMIT("vmovss (%[dst_im], %[j], 4), %%xmm3")
48 __ASM_EMIT("vmovss %%xmm2, (%[dst_re], %[i], 4)")
49 __ASM_EMIT("vmovss %%xmm3, (%[dst_im], %[i], 4)")
50 __ASM_EMIT("vmovss %%xmm0, (%[dst_re], %[j], 4)")
51 __ASM_EMIT("vmovss %%xmm1, (%[dst_im], %[j], 4)")
52 :
53 : [dst_re] "r" (dst_re), [dst_im] "r" (dst_im),
54 [i] "r"(i), [j] "r"(j)
55 : "memory",
56 "%xmm0", "%xmm1", "%xmm2", "%xmm3"
57 );
58 }
59
60 // Perform butterfly 8x
61 size_t off = 0;
62 items = 1 << (rank - 3);
63
64 // Perform 4-element butterflies
65 ARCH_X86_ASM
66 (
67 /* Loop 2x 4-element butterflies */
68 __ASM_EMIT("sub $2, %[items]")
69 __ASM_EMIT("jb 2f")
70 __ASM_EMIT("1:")
71 /* Load data to registers */
72 __ASM_EMIT("vmovups 0x00(%[dst_re], %[off]), %%xmm0") /* xmm0 = r0 r1 r2 r3 */
73 __ASM_EMIT("vmovups 0x10(%[dst_re], %[off]), %%xmm4") /* xmm4 = r4 r5 r6 r7 */
74 __ASM_EMIT("vinsertf128 $1, 0x20(%[dst_re], %[off]), %%ymm0, %%ymm0") /* ymm0 = r0 r1 r2 r3 */
75 __ASM_EMIT("vinsertf128 $1, 0x30(%[dst_re], %[off]), %%ymm4, %%ymm4") /* ymm4 = r4 r5 r6 r7 */
76 __ASM_EMIT("vmovups 0x00(%[dst_im], %[off]), %%xmm2") /* xmm2 = i0 i1 i2 i3 */
77 __ASM_EMIT("vmovups 0x10(%[dst_im], %[off]), %%xmm6") /* xmm6 = i4 i5 i6 i7 */
78 __ASM_EMIT("vinsertf128 $1, 0x20(%[dst_im], %[off]), %%ymm2, %%ymm2") /* ymm2 = i0 i1 i2 i3 */
79 __ASM_EMIT("vinsertf128 $1, 0x30(%[dst_im], %[off]), %%ymm6, %%ymm6") /* ymm6 = i4 i5 i6 i7 */
80 /* 1st-order 4x butterfly */
81 __ASM_EMIT("vhsubps %%ymm4, %%ymm0, %%ymm1") /* ymm1 = r0-r1 r2-r3 r4-r5 r6-r7 = r1' r3' r5' r7' */
82 __ASM_EMIT("vhsubps %%ymm6, %%ymm2, %%ymm3") /* ymm3 = i0-i1 i2-i3 i4-i5 i6-i7 = i1' i3' i5' i7' */
83 __ASM_EMIT("vhaddps %%ymm4, %%ymm0, %%ymm0") /* ymm0 = r0+r1 r2+r3 r4+r5 r6+r7 = r0' r2' r4' r6' */
84 __ASM_EMIT("vhaddps %%ymm6, %%ymm2, %%ymm2") /* ymm2 = i0+i1 i2+i3 i4+i5 i6+i7 = i0' i2' i4' i6' */
85 /* 2nd-order 4x butterfly */
86 __ASM_EMIT("vblendps $0xaa, %%ymm3, %%ymm1, %%ymm4") /* ymm4 = r1' i3' r5' i7' */
87 __ASM_EMIT("vblendps $0xaa, %%ymm1, %%ymm3, %%ymm5") /* ymm5 = i1' r3' i5' r7' */
88 __ASM_EMIT("vhsubps %%ymm4, %%ymm0, %%ymm1") /* ymm1 = r0'-r2' r4'-r6' r1'-i3' r5'-i7' = r2" r6" r3" r7" */
89 __ASM_EMIT("vhsubps %%ymm5, %%ymm2, %%ymm3") /* ymm3 = i0'-i2' i4'-i6' i1'-r3' i5'-r7' = i2" i6" i1" i5" */
90 __ASM_EMIT("vhaddps %%ymm4, %%ymm0, %%ymm0") /* ymm0 = r0'+r2' r4'+r6' r1'+i3' r5'+i7' = r0" r4" r1" r5" */
91 __ASM_EMIT("vhaddps %%ymm5, %%ymm2, %%ymm2") /* ymm2 = i0'+i2' i4'+i6' i1'+r3' i5'+r7' = i0" i4" i3" i7" */
92 __ASM_EMIT("vblendps $0xcc, %%ymm3, %%ymm2, %%ymm4") /* ymm4 = i0" i4" i1" i5" */
93 __ASM_EMIT("vblendps $0xcc, %%ymm2, %%ymm3, %%ymm5") /* ymm5 = i2" i6" i3" i7" */
94 __ASM_EMIT("vshufps $0x88, %%ymm1, %%ymm0, %%ymm2") /* ymm2 = r0" r1" r2" r3" */
95 __ASM_EMIT("vshufps $0xdd, %%ymm1, %%ymm0, %%ymm3") /* ymm3 = r4" r5" r6" r7" */
96 __ASM_EMIT("vshufps $0x88, %%ymm5, %%ymm4, %%ymm6") /* ymm6 = i0" i1" i2" i3" */
97 __ASM_EMIT("vshufps $0xdd, %%ymm5, %%ymm4, %%ymm7") /* ymm7 = i4" i5" i6" i7" */
98 /* 3rd-order 8x butterfly */
99 __ASM_EMIT("vmulps 0x20 + %[FFT_A], %%ymm3, %%ymm4") /* ymm4 = x_im * b_re */ \
100 __ASM_EMIT("vmulps 0x20 + %[FFT_A], %%ymm7, %%ymm5") /* ymm5 = x_im * b_im */ \
101 __ASM_EMIT(FFT_FMA("vmulps 0x00 + %[FFT_A], %%ymm3, %%ymm3", "")) /* ymm3 = x_re * b_re */ \
102 __ASM_EMIT(FFT_FMA("vmulps 0x00 + %[FFT_A], %%ymm7, %%ymm7", "")) /* ymm7 = x_re * b_im */ \
103 __ASM_EMIT(FFT_FMA("vaddps %%ymm5, %%ymm3, %%ymm5", "vfmadd231ps 0x00 + %[FFT_A], %%ymm3, %%ymm5")) /* ymm5 = c_re = x_re * b_re + x_im * b_im */ \
104 __ASM_EMIT(FFT_FMA("vsubps %%ymm4, %%ymm7, %%ymm4", "vfmsub231ps 0x00 + %[FFT_A], %%ymm7, %%ymm4")) /* ymm4 = c_im = x_re * b_im - x_im * b_re */ \
105 __ASM_EMIT("vsubps %%ymm5, %%ymm2, %%ymm0") /* ymm0 = a_re - c_re */ \
106 __ASM_EMIT("vsubps %%ymm4, %%ymm6, %%ymm1") /* ymm1 = a_im - c_im */ \
107 __ASM_EMIT("vaddps %%ymm5, %%ymm2, %%ymm2") /* ymm2 = a_re + c_re */ \
108 __ASM_EMIT("vaddps %%ymm4, %%ymm6, %%ymm3") /* ymm3 = a_im + c_im */ \
109 /* Store */
110 __ASM_EMIT("vmovups %%xmm2, 0x00(%[dst_re], %[off])")
111 __ASM_EMIT("vmovups %%xmm0, 0x10(%[dst_re], %[off])")
112 __ASM_EMIT("vextractf128 $1, %%ymm2, 0x20(%[dst_re], %[off])")
113 __ASM_EMIT("vextractf128 $1, %%ymm0, 0x30(%[dst_re], %[off])")
114 __ASM_EMIT("vmovups %%xmm3, 0x00(%[dst_im], %[off])")
115 __ASM_EMIT("vmovups %%xmm1, 0x10(%[dst_im], %[off])")
116 __ASM_EMIT("vextractf128 $1, %%ymm3, 0x20(%[dst_im], %[off])")
117 __ASM_EMIT("vextractf128 $1, %%ymm1, 0x30(%[dst_im], %[off])")
118 /* Move pointers and repeat*/
119 __ASM_EMIT("add $0x40, %[off]")
120 __ASM_EMIT("sub $2, %[items]")
121 __ASM_EMIT("jae 1b")
122 __ASM_EMIT("2:")
123 /* x4 scramble block */
124 __ASM_EMIT("add $1, %[items]")
125 __ASM_EMIT("jl 4f")
126 __ASM_EMIT("vmovups 0x00(%[dst_re], %[off]), %%xmm0") /* xmm0 = r0 r1 r2 r3 */
127 __ASM_EMIT("vmovups 0x10(%[dst_re], %[off]), %%xmm4") /* xmm4 = r4 r5 r6 r7 */
128 __ASM_EMIT("vmovups 0x00(%[dst_im], %[off]), %%xmm2") /* xmm2 = i0 i1 i2 i3 */
129 __ASM_EMIT("vmovups 0x10(%[dst_im], %[off]), %%xmm6") /* xmm6 = i4 i5 i6 i7 */
130 /* 1st-order 4x butterfly */
131 __ASM_EMIT("vhsubps %%xmm4, %%xmm0, %%xmm1") /* xmm1 = r0-r1 r2-r3 r4-r5 r6-r7 = r1' r3' r5' r7' */
132 __ASM_EMIT("vhsubps %%xmm6, %%xmm2, %%xmm3") /* xmm3 = i0-i1 i2-i3 i4-i5 i6-i7 = i1' i3' i5' i7' */
133 __ASM_EMIT("vhaddps %%xmm4, %%xmm0, %%xmm0") /* xmm0 = r0+r1 r2+r3 r4+r5 r6+r7 = r0' r2' r4' r6' */
134 __ASM_EMIT("vhaddps %%xmm6, %%xmm2, %%xmm2") /* xmm2 = i0+i1 i2+i3 i4+i5 i6+i7 = i0' i2' i4' i6' */
135 /* 2nd-order 4x butterfly */
136 __ASM_EMIT("vblendps $0xaa, %%xmm3, %%xmm1, %%xmm4") /* xmm4 = r1' i3' r5' i7' */
137 __ASM_EMIT("vblendps $0xaa, %%xmm1, %%xmm3, %%xmm5") /* xmm5 = i1' r3' i5' r7' */
138 __ASM_EMIT("vhsubps %%xmm4, %%xmm0, %%xmm1") /* xmm1 = r0'-r2' r4'-r6' r1'-i3' r5'-i7' = r2" r6" r3" r7" */
139 __ASM_EMIT("vhsubps %%xmm5, %%xmm2, %%xmm3") /* xmm3 = i0'-i2' i4'-i6' i1'-r3' i5'-r7' = i2" i6" i1" i5" */
140 __ASM_EMIT("vhaddps %%xmm4, %%xmm0, %%xmm0") /* xmm0 = r0'+r2' r4'+r6' r1'+i3' r5'+i7' = r0" r4" r1" r5" */
141 __ASM_EMIT("vhaddps %%xmm5, %%xmm2, %%xmm2") /* xmm2 = i0'+i2' i4'+i6' i1'+r3' i5'+r7' = i0" i4" i3" i7" */
142 __ASM_EMIT("vblendps $0xcc, %%xmm3, %%xmm2, %%xmm4") /* xmm4 = i0" i4" i1" i5" */
143 __ASM_EMIT("vblendps $0xcc, %%xmm2, %%xmm3, %%xmm5") /* xmm5 = i2" i6" i3" i7" */
144 __ASM_EMIT("vshufps $0x88, %%xmm1, %%xmm0, %%xmm2") /* xmm2 = r0" r1" r2" r3" */
145 __ASM_EMIT("vshufps $0xdd, %%xmm1, %%xmm0, %%xmm3") /* xmm3 = r4" r5" r6" r7" */
146 __ASM_EMIT("vshufps $0x88, %%xmm5, %%xmm4, %%xmm6") /* xmm6 = i0" i1" i2" i3" */
147 __ASM_EMIT("vshufps $0xdd, %%xmm5, %%xmm4, %%xmm7") /* xmm7 = i4" i5" i6" i7" */
148 /* 3rd-order 8x butterfly */
149 __ASM_EMIT("vmulps 0x20 + %[FFT_A], %%xmm3, %%xmm4") /* xmm4 = x_im * b_re */ \
150 __ASM_EMIT("vmulps 0x20 + %[FFT_A], %%xmm7, %%xmm5") /* xmm5 = x_im * b_im */ \
151 __ASM_EMIT(FFT_FMA("vmulps 0x00 + %[FFT_A], %%xmm3, %%xmm3", "")) /* xmm3 = x_re * b_re */ \
152 __ASM_EMIT(FFT_FMA("vmulps 0x00 + %[FFT_A], %%xmm7, %%xmm7", "")) /* xmm7 = x_re * b_im */ \
153 __ASM_EMIT(FFT_FMA("vaddps %%xmm5, %%xmm3, %%xmm5", "vfmadd231ps 0x00 + %[FFT_A], %%xmm3, %%xmm5")) /* xmm5 = c_re = x_re * b_re + x_im * b_im */ \
154 __ASM_EMIT(FFT_FMA("vsubps %%xmm4, %%xmm7, %%xmm4", "vfmsub231ps 0x00 + %[FFT_A], %%xmm7, %%xmm4")) /* xmm4 = c_im = x_re * b_im - x_im * b_re */ \
155 __ASM_EMIT("vsubps %%xmm5, %%xmm2, %%xmm0") /* xmm0 = a_re - c_re */ \
156 __ASM_EMIT("vsubps %%xmm4, %%xmm6, %%xmm1") /* xmm1 = a_im - c_im */ \
157 __ASM_EMIT("vaddps %%xmm5, %%xmm2, %%xmm2") /* xmm2 = a_re + c_re */ \
158 __ASM_EMIT("vaddps %%xmm4, %%xmm6, %%xmm3") /* xmm3 = a_im + c_im */ \
159 /* Store */
160 __ASM_EMIT("vmovups %%xmm2, 0x00(%[dst_re], %[off])")
161 __ASM_EMIT("vmovups %%xmm0, 0x10(%[dst_re], %[off])")
162 __ASM_EMIT("vmovups %%xmm3, 0x00(%[dst_im], %[off])")
163 __ASM_EMIT("vmovups %%xmm1, 0x10(%[dst_im], %[off])")
164 __ASM_EMIT("4:")
165
166 : [dst_re] "+r"(dst_re), [dst_im] "+r"(dst_im),
167 [off] "+r" (off), [items] "+r"(items)
168 : [FFT_A] "o" (FFT_A)
169 : "cc", "memory",
170 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
171 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
172 );
173 }
174
175 static inline void FFT_SCRAMBLE_SELF_REVERSE_NAME(float *dst_re, float *dst_im, size_t rank)
176 {
177 // Calculate number of items
178 size_t items = (1 << rank) - 1;
179
180 for (size_t i = 1; i < items; ++i)
181 {
182 size_t j = reverse_bits(FFT_TYPE(i), rank); /* Reverse the order of the bits */
183 if (i >= j)
184 continue;
185
186 /* Copy the values from the reversed position */
187 ARCH_X86_ASM
188 (
189 __ASM_EMIT("vmovss (%[dst_re], %[i], 4), %%xmm0")
190 __ASM_EMIT("vmovss (%[dst_im], %[i], 4), %%xmm1")
191 __ASM_EMIT("vmovss (%[dst_re], %[j], 4), %%xmm2")
192 __ASM_EMIT("vmovss (%[dst_im], %[j], 4), %%xmm3")
193 __ASM_EMIT("vmovss %%xmm2, (%[dst_re], %[i], 4)")
194 __ASM_EMIT("vmovss %%xmm3, (%[dst_im], %[i], 4)")
195 __ASM_EMIT("vmovss %%xmm0, (%[dst_re], %[j], 4)")
196 __ASM_EMIT("vmovss %%xmm1, (%[dst_im], %[j], 4)")
197 :
198 : [dst_re] "r"(dst_re), [dst_im] "r"(dst_im),
199 [i] "r"(i), [j] "r"(j)
200 : "memory",
201 "%xmm0", "%xmm1", "%xmm2", "%xmm3"
202 );
203 }
204
205 // Perform butterfly 8x
206 size_t off = 0;
207 items = 1 << (rank - 3);
208
209 // Perform 4-element butterflies
210 ARCH_X86_ASM
211 (
212 /* Loop 2x 4-element butterflies */
213 __ASM_EMIT("sub $2, %[items]")
214 __ASM_EMIT("jb 2f")
215 __ASM_EMIT("1:")
216 /* Load data to registers */
217 __ASM_EMIT("vmovups 0x00(%[dst_re], %[off]), %%xmm0") /* xmm0 = r0 r1 r2 r3 */
218 __ASM_EMIT("vmovups 0x10(%[dst_re], %[off]), %%xmm4") /* xmm4 = r4 r5 r6 r7 */
219 __ASM_EMIT("vinsertf128 $1, 0x20(%[dst_re], %[off]), %%ymm0, %%ymm0") /* ymm0 = r0 r1 r2 r3 */
220 __ASM_EMIT("vinsertf128 $1, 0x30(%[dst_re], %[off]), %%ymm4, %%ymm4") /* ymm4 = r4 r5 r6 r7 */
221 __ASM_EMIT("vmovups 0x00(%[dst_im], %[off]), %%xmm2") /* xmm2 = i0 i1 i2 i3 */
222 __ASM_EMIT("vmovups 0x10(%[dst_im], %[off]), %%xmm6") /* xmm6 = i4 i5 i6 i7 */
223 __ASM_EMIT("vinsertf128 $1, 0x20(%[dst_im], %[off]), %%ymm2, %%ymm2") /* ymm2 = i0 i1 i2 i3 */
224 __ASM_EMIT("vinsertf128 $1, 0x30(%[dst_im], %[off]), %%ymm6, %%ymm6") /* ymm6 = i4 i5 i6 i7 */
225 /* 1st-order 4x butterfly */
226 __ASM_EMIT("vhsubps %%ymm4, %%ymm0, %%ymm1") /* ymm1 = r0-r1 r2-r3 r4-r5 r6-r7 = r1' r3' r5' r7' */
227 __ASM_EMIT("vhsubps %%ymm6, %%ymm2, %%ymm3") /* ymm3 = i0-i1 i2-i3 i4-i5 i6-i7 = i1' i3' i5' i7' */
228 __ASM_EMIT("vhaddps %%ymm4, %%ymm0, %%ymm0") /* ymm0 = r0+r1 r2+r3 r4+r5 r6+r7 = r0' r2' r4' r6' */
229 __ASM_EMIT("vhaddps %%ymm6, %%ymm2, %%ymm2") /* ymm2 = i0+i1 i2+i3 i4+i5 i6+i7 = i0' i2' i4' i6' */
230 /* 2nd-order 4x butterfly */
231 __ASM_EMIT("vblendps $0xaa, %%ymm3, %%ymm1, %%ymm4") /* ymm4 = r1' i3' r5' i7' */
232 __ASM_EMIT("vblendps $0xaa, %%ymm1, %%ymm3, %%ymm5") /* ymm5 = i1' r3' i5' r7' */
233 __ASM_EMIT("vhsubps %%ymm4, %%ymm0, %%ymm1") /* ymm1 = r0'-r2' r4'-r6' r1'-i3' r5'-i7' = r2" r6" r1" r5" */
234 __ASM_EMIT("vhsubps %%ymm5, %%ymm2, %%ymm3") /* ymm3 = i0'-i2' i4'-i6' i1'-r3' i5'-r7' = i2" i6" i3" i7" */
235 __ASM_EMIT("vhaddps %%ymm4, %%ymm0, %%ymm0") /* ymm0 = r0'+r2' r4'+r6' r1'+i3' r5'+i7' = r0" r4" r3" r7" */
236 __ASM_EMIT("vhaddps %%ymm5, %%ymm2, %%ymm2") /* ymm2 = i0'+i2' i4'+i6' i1'+r3' i5'+r7' = i0" i4" i1" i5" */
237 __ASM_EMIT("vblendps $0xcc, %%ymm1, %%ymm0, %%ymm4") /* ymm4 = r0" i4" r1" r5" */
238 __ASM_EMIT("vblendps $0xcc, %%ymm0, %%ymm1, %%ymm5") /* ymm5 = r2" r6" r3" r7" */
239 __ASM_EMIT("vshufps $0x88, %%ymm3, %%ymm2, %%ymm6") /* ymm6 = i0" i1" i2" i3" */
240 __ASM_EMIT("vshufps $0xdd, %%ymm3, %%ymm2, %%ymm7") /* ymm7 = i4" i5" i6" i7" */
241 __ASM_EMIT("vshufps $0x88, %%ymm5, %%ymm4, %%ymm2") /* ymm2 = r0" r1" r2" r3" */
242 __ASM_EMIT("vshufps $0xdd, %%ymm5, %%ymm4, %%ymm3") /* ymm3 = r4" r5" r6" r7" */
243 /* 3rd-order 8x butterfly */
244 __ASM_EMIT("vmulps 0x20 + %[FFT_A], %%ymm3, %%ymm4") /* ymm4 = x_im * b_re */ \
245 __ASM_EMIT("vmulps 0x20 + %[FFT_A], %%ymm7, %%ymm5") /* ymm5 = x_im * b_im */ \
246 __ASM_EMIT(FFT_FMA("vmulps 0x00 + %[FFT_A], %%ymm3, %%ymm3", "")) /* ymm3 = x_re * b_re */ \
247 __ASM_EMIT(FFT_FMA("vmulps 0x00 + %[FFT_A], %%ymm7, %%ymm7", "")) /* ymm7 = x_re * b_im */ \
248 __ASM_EMIT(FFT_FMA("vsubps %%ymm5, %%ymm3, %%ymm5", "vfmsub231ps 0x00 + %[FFT_A], %%ymm3, %%ymm5")) /* ymm5 = c_re = x_re * b_re - x_im * b_im */ \
249 __ASM_EMIT(FFT_FMA("vaddps %%ymm4, %%ymm7, %%ymm4", "vfmadd231ps 0x00 + %[FFT_A], %%ymm7, %%ymm4")) /* ymm4 = c_im = x_re * b_im + x_im * b_re */ \
250 __ASM_EMIT("vsubps %%ymm5, %%ymm2, %%ymm0") /* ymm0 = a_re - c_re */ \
251 __ASM_EMIT("vsubps %%ymm4, %%ymm6, %%ymm1") /* ymm1 = a_im - c_im */ \
252 __ASM_EMIT("vaddps %%ymm5, %%ymm2, %%ymm2") /* ymm2 = a_re + c_re */ \
253 __ASM_EMIT("vaddps %%ymm4, %%ymm6, %%ymm3") /* ymm3 = a_im + c_im */ \
254 /* Store */
255 __ASM_EMIT("vmovups %%xmm2, 0x00(%[dst_re], %[off])")
256 __ASM_EMIT("vmovups %%xmm0, 0x10(%[dst_re], %[off])")
257 __ASM_EMIT("vextractf128 $1, %%ymm2, 0x20(%[dst_re], %[off])")
258 __ASM_EMIT("vextractf128 $1, %%ymm0, 0x30(%[dst_re], %[off])")
259 __ASM_EMIT("vmovups %%xmm3, 0x00(%[dst_im], %[off])")
260 __ASM_EMIT("vmovups %%xmm1, 0x10(%[dst_im], %[off])")
261 __ASM_EMIT("vextractf128 $1, %%ymm3, 0x20(%[dst_im], %[off])")
262 __ASM_EMIT("vextractf128 $1, %%ymm1, 0x30(%[dst_im], %[off])")
263 /* Move pointers and repeat*/
264 __ASM_EMIT("add $0x40, %[off]")
265 __ASM_EMIT("sub $2, %[items]")
266 __ASM_EMIT("jae 1b")
267 __ASM_EMIT("2:")
268 /* x4 scramble block */
269 __ASM_EMIT("add $1, %[items]")
270 __ASM_EMIT("jl 4f")
271 __ASM_EMIT("vmovups 0x00(%[dst_re], %[off]), %%xmm0") /* xmm0 = r0 r1 r2 r3 */
272 __ASM_EMIT("vmovups 0x10(%[dst_re], %[off]), %%xmm4") /* xmm4 = r4 r5 r6 r7 */
273 __ASM_EMIT("vmovups 0x00(%[dst_im], %[off]), %%xmm2") /* xmm2 = i0 i1 i2 i3 */
274 __ASM_EMIT("vmovups 0x10(%[dst_im], %[off]), %%xmm6") /* xmm6 = i4 i5 i6 i7 */
275 /* 1st-order 4x butterfly */
276 __ASM_EMIT("vhsubps %%xmm4, %%xmm0, %%xmm1") /* xmm1 = r0-r1 r2-r3 r4-r5 r6-r7 = r1' r3' r5' r7' */
277 __ASM_EMIT("vhsubps %%xmm6, %%xmm2, %%xmm3") /* xmm3 = i0-i1 i2-i3 i4-i5 i6-i7 = i1' i3' i5' i7' */
278 __ASM_EMIT("vhaddps %%xmm4, %%xmm0, %%xmm0") /* xmm0 = r0+r1 r2+r3 r4+r5 r6+r7 = r0' r2' r4' r6' */
279 __ASM_EMIT("vhaddps %%xmm6, %%xmm2, %%xmm2") /* xmm2 = i0+i1 i2+i3 i4+i5 i6+i7 = i0' i2' i4' i6' */
280 /* 2nd-order 4x butterfly */
281 __ASM_EMIT("vblendps $0xaa, %%xmm3, %%xmm1, %%xmm4") /* xmm4 = r1' i3' r5' i7' */
282 __ASM_EMIT("vblendps $0xaa, %%xmm1, %%xmm3, %%xmm5") /* xmm5 = i1' r3' i5' r7' */
283 __ASM_EMIT("vhsubps %%xmm4, %%xmm0, %%xmm1") /* xmm1 = r0'-r2' r4'-r6' r1'-i3' r5'-i7' = r2" r6" r1" r5" */
284 __ASM_EMIT("vhsubps %%xmm5, %%xmm2, %%xmm3") /* xmm3 = i0'-i2' i4'-i6' i1'-r3' i5'-r7' = i2" i6" i3" i7" */
285 __ASM_EMIT("vhaddps %%xmm4, %%xmm0, %%xmm0") /* xmm0 = r0'+r2' r4'+r6' r1'+i3' r5'+i7' = r0" r4" r3" r7" */
286 __ASM_EMIT("vhaddps %%xmm5, %%xmm2, %%xmm2") /* xmm2 = i0'+i2' i4'+i6' i1'+r3' i5'+r7' = i0" i4" i1" i5" */
287 __ASM_EMIT("vblendps $0xcc, %%xmm1, %%xmm0, %%xmm4") /* xmm4 = r0" i4" r1" r5" */
288 __ASM_EMIT("vblendps $0xcc, %%xmm0, %%xmm1, %%xmm5") /* xmm5 = r2" r6" r3" r7" */
289 __ASM_EMIT("vshufps $0x88, %%xmm3, %%xmm2, %%xmm6") /* xmm6 = i0" i1" i2" i3" */
290 __ASM_EMIT("vshufps $0xdd, %%xmm3, %%xmm2, %%xmm7") /* xmm7 = i4" i5" i6" i7" */
291 __ASM_EMIT("vshufps $0x88, %%xmm5, %%xmm4, %%xmm2") /* xmm2 = r0" r1" r2" r3" */
292 __ASM_EMIT("vshufps $0xdd, %%xmm5, %%xmm4, %%xmm3") /* xmm3 = r4" r5" r6" r7" */
293 /* 3rd-order 8x butterfly */
294 __ASM_EMIT("vmulps 0x20 + %[FFT_A], %%xmm3, %%xmm4") /* xmm4 = x_im * b_re */ \
295 __ASM_EMIT("vmulps 0x20 + %[FFT_A], %%xmm7, %%xmm5") /* xmm5 = x_im * b_im */ \
296 __ASM_EMIT(FFT_FMA("vmulps 0x00 + %[FFT_A], %%xmm3, %%xmm3", "")) /* xmm3 = x_re * b_re */ \
297 __ASM_EMIT(FFT_FMA("vmulps 0x00 + %[FFT_A], %%xmm7, %%xmm7", "")) /* xmm7 = x_re * b_im */ \
298 __ASM_EMIT(FFT_FMA("vsubps %%xmm5, %%xmm3, %%xmm5", "vfmsub231ps 0x00 + %[FFT_A], %%xmm3, %%xmm5")) /* xmm5 = c_re = x_re * b_re - x_im * b_im */ \
299 __ASM_EMIT(FFT_FMA("vaddps %%xmm4, %%xmm7, %%xmm4", "vfmadd231ps 0x00 + %[FFT_A], %%xmm7, %%xmm4")) /* xmm4 = c_im = x_re * b_im + x_im * b_re */ \
300 __ASM_EMIT("vsubps %%xmm5, %%xmm2, %%xmm0") /* xmm0 = a_re - c_re */ \
301 __ASM_EMIT("vsubps %%xmm4, %%xmm6, %%xmm1") /* xmm1 = a_im - c_im */ \
302 __ASM_EMIT("vaddps %%xmm5, %%xmm2, %%xmm2") /* xmm2 = a_re + c_re */ \
303 __ASM_EMIT("vaddps %%xmm4, %%xmm6, %%xmm3") /* xmm3 = a_im + c_im */ \
304 /* Store */
305 __ASM_EMIT("vmovups %%xmm2, 0x00(%[dst_re], %[off])")
306 __ASM_EMIT("vmovups %%xmm0, 0x10(%[dst_re], %[off])")
307 __ASM_EMIT("vmovups %%xmm3, 0x00(%[dst_im], %[off])")
308 __ASM_EMIT("vmovups %%xmm1, 0x10(%[dst_im], %[off])")
309 __ASM_EMIT("4:")
310
311 : [dst_re] "+r"(dst_re), [dst_im] "+r"(dst_im),
312 [off] "+r" (off), [items] "+r"(items)
313 : [FFT_A] "o" (FFT_A)
314 : "cc", "memory",
315 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
316 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
317 );
318 }
319
320 static inline void FFT_SCRAMBLE_COPY_DIRECT_NAME(float *dst_re, float *dst_im, const float *src_re, const float *src_im, size_t rank)
321 {
322 size_t regs = 1 << rank;
323
324 for (size_t i=0; i<regs; ++i)
325 {
326 size_t index = reverse_bits(FFT_TYPE(i), rank);
327
328 ARCH_X86_ASM
329 (
330 /* Load scalar values */
331 __ASM_EMIT("vinsertps $0x00, (%[src_re], %[index], 4), %%xmm0, %%xmm0") /* xmm0 = r0 x x x */
332 __ASM_EMIT("vinsertps $0x00, (%[src_im], %[index], 4), %%xmm2, %%xmm2") /* xmm2 = i0 x x x */
333 __ASM_EMIT("add %[regs], %[index]")
334 __ASM_EMIT("vinsertps $0x00, (%[src_re], %[index], 4), %%xmm1, %%xmm1") /* xmm1 = r8 x x x */
335 __ASM_EMIT("vinsertps $0x00, (%[src_im], %[index], 4), %%xmm3, %%xmm3") /* xmm3 = i8 x x x */
336 __ASM_EMIT("add %[regs], %[index]")
337 __ASM_EMIT("vinsertps $0x00, (%[src_re], %[index], 4), %%xmm4, %%xmm4") /* xmm4 = r4 x x x */
338 __ASM_EMIT("vinsertps $0x00, (%[src_im], %[index], 4), %%xmm6, %%xmm6") /* xmm6 = i4 x x x */
339 __ASM_EMIT("add %[regs], %[index]")
340 __ASM_EMIT("vinsertps $0x00, (%[src_re], %[index], 4), %%xmm5, %%xmm5") /* xmm5 = r12 x x x */
341 __ASM_EMIT("vinsertps $0x00, (%[src_im], %[index], 4), %%xmm7, %%xmm7") /* xmm7 = i12 x x x */
342 __ASM_EMIT("add %[regs], %[index]")
343
344 __ASM_EMIT("vinsertps $0x20, (%[src_re], %[index], 4), %%xmm0, %%xmm0") /* xmm0 = r0 x r2 x */
345 __ASM_EMIT("vinsertps $0x20, (%[src_im], %[index], 4), %%xmm2, %%xmm2") /* xmm2 = i0 x i2 x */
346 __ASM_EMIT("add %[regs], %[index]")
347 __ASM_EMIT("vinsertps $0x20, (%[src_re], %[index], 4), %%xmm1, %%xmm1") /* xmm1 = r8 x r10 x */
348 __ASM_EMIT("vinsertps $0x20, (%[src_im], %[index], 4), %%xmm3, %%xmm3") /* xmm3 = i8 x i10 x */
349 __ASM_EMIT("add %[regs], %[index]")
350 __ASM_EMIT("vinsertps $0x20, (%[src_re], %[index], 4), %%xmm4, %%xmm4") /* xmm4 = r4 x r6 x */
351 __ASM_EMIT("vinsertps $0x20, (%[src_im], %[index], 4), %%xmm6, %%xmm6") /* xmm6 = i4 x i6 x */
352 __ASM_EMIT("add %[regs], %[index]")
353 __ASM_EMIT("vinsertps $0x20, (%[src_re], %[index], 4), %%xmm5, %%xmm5") /* xmm5 = r12 x r14 x */
354 __ASM_EMIT("vinsertps $0x20, (%[src_im], %[index], 4), %%xmm7, %%xmm7") /* xmm7 = i12 x i14 x */
355 __ASM_EMIT("add %[regs], %[index]")
356
357 __ASM_EMIT("vinsertps $0x10, (%[src_re], %[index], 4), %%xmm0, %%xmm0") /* xmm0 = r0 r1 r2 x */
358 __ASM_EMIT("vinsertps $0x10, (%[src_im], %[index], 4), %%xmm2, %%xmm2") /* xmm2 = i0 i1 i2 x */
359 __ASM_EMIT("add %[regs], %[index]")
360 __ASM_EMIT("vinsertps $0x10, (%[src_re], %[index], 4), %%xmm1, %%xmm1") /* xmm1 = r8 r9 r10 x */
361 __ASM_EMIT("vinsertps $0x10, (%[src_im], %[index], 4), %%xmm3, %%xmm3") /* xmm3 = i8 i9 i10 x */
362 __ASM_EMIT("add %[regs], %[index]")
363 __ASM_EMIT("vinsertps $0x10, (%[src_re], %[index], 4), %%xmm4, %%xmm4") /* xmm4 = r4 r5 r6 x */
364 __ASM_EMIT("vinsertps $0x10, (%[src_im], %[index], 4), %%xmm6, %%xmm6") /* xmm6 = i4 i5 i6 x */
365 __ASM_EMIT("add %[regs], %[index]")
366 __ASM_EMIT("vinsertps $0x10, (%[src_re], %[index], 4), %%xmm5, %%xmm5") /* xmm5 = r12 r13 r14 x */
367 __ASM_EMIT("vinsertps $0x10, (%[src_im], %[index], 4), %%xmm7, %%xmm7") /* xmm7 = i12 i13 i14 x */
368 __ASM_EMIT("add %[regs], %[index]")
369
370 __ASM_EMIT("vinsertps $0x30, (%[src_re], %[index], 4), %%xmm0, %%xmm0") /* xmm0 = r0 r1 r2 r3 */
371 __ASM_EMIT("vinsertps $0x30, (%[src_im], %[index], 4), %%xmm2, %%xmm2") /* xmm2 = i0 i1 i2 i3 */
372 __ASM_EMIT("add %[regs], %[index]")
373 __ASM_EMIT("vinsertps $0x30, (%[src_re], %[index], 4), %%xmm1, %%xmm1") /* xmm1 = r8 r9 r10 r11 */
374 __ASM_EMIT("vinsertps $0x30, (%[src_im], %[index], 4), %%xmm3, %%xmm3") /* xmm3 = i8 i9 i10 i11 */
375 __ASM_EMIT("add %[regs], %[index]")
376 __ASM_EMIT("vinsertps $0x30, (%[src_re], %[index], 4), %%xmm4, %%xmm4") /* xmm4 = r4 r5 r6 r7 */
377 __ASM_EMIT("vinsertps $0x30, (%[src_im], %[index], 4), %%xmm6, %%xmm6") /* xmm6 = i4 i5 i6 i7 */
378 __ASM_EMIT("add %[regs], %[index]")
379 __ASM_EMIT("vinsertps $0x30, (%[src_re], %[index], 4), %%xmm5, %%xmm5") /* xmm5 = r12 r13 r14 r15 */
380 __ASM_EMIT("vinsertps $0x30, (%[src_im], %[index], 4), %%xmm7, %%xmm7") /* xmm7 = i12 i13 i14 i15 */
381 __ASM_EMIT("add %[regs], %[index]")
382
383 __ASM_EMIT("vinsertf128 $1, %%xmm1, %%ymm0, %%ymm0") /* ymm0 = r0 r1 r2 r3 ... */
384 __ASM_EMIT("vinsertf128 $1, %%xmm3, %%ymm2, %%ymm2") /* ymm2 = i0 i1 i2 i3 ... */
385 __ASM_EMIT("vinsertf128 $1, %%xmm5, %%ymm4, %%ymm4") /* ymm4 = r4 r5 r6 r7 ... */
386 __ASM_EMIT("vinsertf128 $1, %%xmm7, %%ymm6, %%ymm6") /* ymm0 = i4 i5 i6 i7 ... */
387 /* 1st-order 4x butterfly */
388 __ASM_EMIT("vhsubps %%ymm4, %%ymm0, %%ymm1") /* ymm1 = r0-r1 r2-r3 r4-r5 r6-r7 = r1' r3' r5' r7' */
389 __ASM_EMIT("vhsubps %%ymm6, %%ymm2, %%ymm3") /* ymm3 = i0-i1 i2-i3 i4-i5 i6-i7 = i1' i3' i5' i7' */
390 __ASM_EMIT("vhaddps %%ymm4, %%ymm0, %%ymm0") /* ymm0 = r0+r1 r2+r3 r4+r5 r6+r7 = r0' r2' r4' r6' */
391 __ASM_EMIT("vhaddps %%ymm6, %%ymm2, %%ymm2") /* ymm2 = i0+i1 i2+i3 i4+i5 i6+i7 = i0' i2' i4' i6' */
392 /* 2nd-order 4x butterfly */
393 __ASM_EMIT("vblendps $0xaa, %%ymm3, %%ymm1, %%ymm4") /* ymm4 = r1' i3' r5' i7' */
394 __ASM_EMIT("vblendps $0xaa, %%ymm1, %%ymm3, %%ymm5") /* ymm5 = i1' r3' i5' r7' */
395 __ASM_EMIT("vhsubps %%ymm4, %%ymm0, %%ymm1") /* ymm1 = r0'-r2' r4'-r6' r1'-i3' r5'-i7' = r2" r6" r3" r7" */
396 __ASM_EMIT("vhsubps %%ymm5, %%ymm2, %%ymm3") /* ymm3 = i0'-i2' i4'-i6' i1'-r3' i5'-r7' = i2" i6" i1" i5" */
397 __ASM_EMIT("vhaddps %%ymm4, %%ymm0, %%ymm0") /* ymm0 = r0'+r2' r4'+r6' r1'+i3' r5'+i7' = r0" r4" r1" r5" */
398 __ASM_EMIT("vhaddps %%ymm5, %%ymm2, %%ymm2") /* ymm2 = i0'+i2' i4'+i6' i1'+r3' i5'+r7' = i0" i4" i3" i7" */
399 __ASM_EMIT("vblendps $0xcc, %%ymm3, %%ymm2, %%ymm4") /* ymm4 = i0" i4" i1" i5" */
400 __ASM_EMIT("vblendps $0xcc, %%ymm2, %%ymm3, %%ymm5") /* ymm5 = i2" i6" i3" i7" */
401 __ASM_EMIT("vshufps $0x88, %%ymm1, %%ymm0, %%ymm2") /* ymm2 = r0" r1" r2" r3" */
402 __ASM_EMIT("vshufps $0xdd, %%ymm1, %%ymm0, %%ymm3") /* ymm3 = r4" r5" r6" r7" */
403 __ASM_EMIT("vshufps $0x88, %%ymm5, %%ymm4, %%ymm6") /* ymm6 = i0" i1" i2" i3" */
404 __ASM_EMIT("vshufps $0xdd, %%ymm5, %%ymm4, %%ymm7") /* ymm7 = i4" i5" i6" i7" */
405 /* 3rd-order 8x butterfly */
406 __ASM_EMIT("vmulps 0x20 + %[FFT_A], %%ymm3, %%ymm4") /* ymm4 = x_im * b_re */ \
407 __ASM_EMIT("vmulps 0x20 + %[FFT_A], %%ymm7, %%ymm5") /* ymm5 = x_im * b_im */ \
408 __ASM_EMIT(FFT_FMA("vmulps 0x00 + %[FFT_A], %%ymm3, %%ymm3", "")) /* ymm3 = x_re * b_re */ \
409 __ASM_EMIT(FFT_FMA("vmulps 0x00 + %[FFT_A], %%ymm7, %%ymm7", "")) /* ymm7 = x_re * b_im */ \
410 __ASM_EMIT(FFT_FMA("vaddps %%ymm5, %%ymm3, %%ymm5", "vfmadd231ps 0x00 + %[FFT_A], %%ymm3, %%ymm5")) /* ymm5 = c_re = x_re * b_re + x_im * b_im */ \
411 __ASM_EMIT(FFT_FMA("vsubps %%ymm4, %%ymm7, %%ymm4", "vfmsub231ps 0x00 + %[FFT_A], %%ymm7, %%ymm4")) /* ymm4 = c_im = x_re * b_im - x_im * b_re */ \
412 __ASM_EMIT("vsubps %%ymm5, %%ymm2, %%ymm0") /* ymm0 = a_re - c_re */ \
413 __ASM_EMIT("vsubps %%ymm4, %%ymm6, %%ymm1") /* ymm1 = a_im - c_im */ \
414 __ASM_EMIT("vaddps %%ymm5, %%ymm2, %%ymm2") /* ymm2 = a_re + c_re */ \
415 __ASM_EMIT("vaddps %%ymm4, %%ymm6, %%ymm3") /* ymm3 = a_im + c_im */ \
416 /* Store */
417 __ASM_EMIT("vmovups %%xmm2, 0x00(%[dst_re])")
418 __ASM_EMIT("vmovups %%xmm0, 0x10(%[dst_re])")
419 __ASM_EMIT("vextractf128 $1, %%ymm2, 0x20(%[dst_re])")
420 __ASM_EMIT("vextractf128 $1, %%ymm0, 0x30(%[dst_re])")
421 __ASM_EMIT("vmovups %%xmm3, 0x00(%[dst_im])")
422 __ASM_EMIT("vmovups %%xmm1, 0x10(%[dst_im])")
423 __ASM_EMIT("vextractf128 $1, %%ymm3, 0x20(%[dst_im])")
424 __ASM_EMIT("vextractf128 $1, %%ymm1, 0x30(%[dst_im])")
425 __ASM_EMIT("add $0x40, %[dst_re]")
426 __ASM_EMIT("add $0x40, %[dst_im]")
427
428 : [dst_re] "+r" (dst_re), [dst_im] "+r"(dst_im), [index] "+r"(index)
429 : [src_re] "r" (src_re), [src_im] "r"(src_im),
430 [regs] X86_GREG (regs),
431 [FFT_A] "o" (FFT_A)
432 : "cc", "memory",
433 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
434 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
435 );
436 }
437 }
438
439 static inline void FFT_SCRAMBLE_COPY_REVERSE_NAME(float *dst_re, float *dst_im, const float *src_re, const float *src_im, size_t rank)
440 {
441 size_t regs = 1 << rank;
442
443 for (size_t i=0; i<regs; ++i)
444 {
445 size_t index = reverse_bits(FFT_TYPE(i), rank);
446
447 ARCH_X86_ASM
448 (
449 /* Load scalar values */
450 __ASM_EMIT("vinsertps $0x00, (%[src_re], %[index], 4), %%xmm0, %%xmm0") /* xmm0 = r0 x x x */
451 __ASM_EMIT("vinsertps $0x00, (%[src_im], %[index], 4), %%xmm2, %%xmm2") /* xmm2 = i0 x x x */
452 __ASM_EMIT("add %[regs], %[index]")
453 __ASM_EMIT("vinsertps $0x00, (%[src_re], %[index], 4), %%xmm1, %%xmm1") /* xmm1 = r8 x x x */
454 __ASM_EMIT("vinsertps $0x00, (%[src_im], %[index], 4), %%xmm3, %%xmm3") /* xmm3 = i8 x x x */
455 __ASM_EMIT("add %[regs], %[index]")
456 __ASM_EMIT("vinsertps $0x00, (%[src_re], %[index], 4), %%xmm4, %%xmm4") /* xmm4 = r4 x x x */
457 __ASM_EMIT("vinsertps $0x00, (%[src_im], %[index], 4), %%xmm6, %%xmm6") /* xmm6 = i4 x x x */
458 __ASM_EMIT("add %[regs], %[index]")
459 __ASM_EMIT("vinsertps $0x00, (%[src_re], %[index], 4), %%xmm5, %%xmm5") /* xmm5 = r12 x x x */
460 __ASM_EMIT("vinsertps $0x00, (%[src_im], %[index], 4), %%xmm7, %%xmm7") /* xmm7 = i12 x x x */
461 __ASM_EMIT("add %[regs], %[index]")
462
463 __ASM_EMIT("vinsertps $0x20, (%[src_re], %[index], 4), %%xmm0, %%xmm0") /* xmm0 = r0 x r2 x */
464 __ASM_EMIT("vinsertps $0x20, (%[src_im], %[index], 4), %%xmm2, %%xmm2") /* xmm2 = i0 x i2 x */
465 __ASM_EMIT("add %[regs], %[index]")
466 __ASM_EMIT("vinsertps $0x20, (%[src_re], %[index], 4), %%xmm1, %%xmm1") /* xmm1 = r8 x r10 x */
467 __ASM_EMIT("vinsertps $0x20, (%[src_im], %[index], 4), %%xmm3, %%xmm3") /* xmm3 = i8 x i10 x */
468 __ASM_EMIT("add %[regs], %[index]")
469 __ASM_EMIT("vinsertps $0x20, (%[src_re], %[index], 4), %%xmm4, %%xmm4") /* xmm4 = r4 x r6 x */
470 __ASM_EMIT("vinsertps $0x20, (%[src_im], %[index], 4), %%xmm6, %%xmm6") /* xmm6 = i4 x i6 x */
471 __ASM_EMIT("add %[regs], %[index]")
472 __ASM_EMIT("vinsertps $0x20, (%[src_re], %[index], 4), %%xmm5, %%xmm5") /* xmm5 = r12 x r14 x */
473 __ASM_EMIT("vinsertps $0x20, (%[src_im], %[index], 4), %%xmm7, %%xmm7") /* xmm7 = i12 x i14 x */
474 __ASM_EMIT("add %[regs], %[index]")
475
476 __ASM_EMIT("vinsertps $0x10, (%[src_re], %[index], 4), %%xmm0, %%xmm0") /* xmm0 = r0 r1 r2 x */
477 __ASM_EMIT("vinsertps $0x10, (%[src_im], %[index], 4), %%xmm2, %%xmm2") /* xmm2 = i0 i1 i2 x */
478 __ASM_EMIT("add %[regs], %[index]")
479 __ASM_EMIT("vinsertps $0x10, (%[src_re], %[index], 4), %%xmm1, %%xmm1") /* xmm1 = r8 r9 r10 x */
480 __ASM_EMIT("vinsertps $0x10, (%[src_im], %[index], 4), %%xmm3, %%xmm3") /* xmm3 = i8 i9 i10 x */
481 __ASM_EMIT("add %[regs], %[index]")
482 __ASM_EMIT("vinsertps $0x10, (%[src_re], %[index], 4), %%xmm4, %%xmm4") /* xmm4 = r4 r5 r6 x */
483 __ASM_EMIT("vinsertps $0x10, (%[src_im], %[index], 4), %%xmm6, %%xmm6") /* xmm6 = i4 i5 i6 x */
484 __ASM_EMIT("add %[regs], %[index]")
485 __ASM_EMIT("vinsertps $0x10, (%[src_re], %[index], 4), %%xmm5, %%xmm5") /* xmm5 = r12 r13 r14 x */
486 __ASM_EMIT("vinsertps $0x10, (%[src_im], %[index], 4), %%xmm7, %%xmm7") /* xmm7 = i12 i13 i14 x */
487 __ASM_EMIT("add %[regs], %[index]")
488
489 __ASM_EMIT("vinsertps $0x30, (%[src_re], %[index], 4), %%xmm0, %%xmm0") /* xmm0 = r0 r1 r2 r3 */
490 __ASM_EMIT("vinsertps $0x30, (%[src_im], %[index], 4), %%xmm2, %%xmm2") /* xmm2 = i0 i1 i2 i3 */
491 __ASM_EMIT("add %[regs], %[index]")
492 __ASM_EMIT("vinsertps $0x30, (%[src_re], %[index], 4), %%xmm1, %%xmm1") /* xmm1 = r8 r9 r10 r11 */
493 __ASM_EMIT("vinsertps $0x30, (%[src_im], %[index], 4), %%xmm3, %%xmm3") /* xmm3 = i8 i9 i10 i11 */
494 __ASM_EMIT("add %[regs], %[index]")
495 __ASM_EMIT("vinsertps $0x30, (%[src_re], %[index], 4), %%xmm4, %%xmm4") /* xmm4 = r4 r5 r6 r7 */
496 __ASM_EMIT("vinsertps $0x30, (%[src_im], %[index], 4), %%xmm6, %%xmm6") /* xmm6 = i4 i5 i6 i7 */
497 __ASM_EMIT("add %[regs], %[index]")
498 __ASM_EMIT("vinsertps $0x30, (%[src_re], %[index], 4), %%xmm5, %%xmm5") /* xmm5 = r12 r13 r14 r15 */
499 __ASM_EMIT("vinsertps $0x30, (%[src_im], %[index], 4), %%xmm7, %%xmm7") /* xmm7 = i12 i13 i14 i15 */
500 __ASM_EMIT("add %[regs], %[index]")
501
502 __ASM_EMIT("vinsertf128 $1, %%xmm1, %%ymm0, %%ymm0") /* ymm0 = r0 r1 r2 r3 ... */
503 __ASM_EMIT("vinsertf128 $1, %%xmm3, %%ymm2, %%ymm2") /* ymm2 = i0 i1 i2 i3 ... */
504 __ASM_EMIT("vinsertf128 $1, %%xmm5, %%ymm4, %%ymm4") /* ymm4 = r4 r5 r6 r7 ... */
505 __ASM_EMIT("vinsertf128 $1, %%xmm7, %%ymm6, %%ymm6") /* ymm0 = i4 i5 i6 i7 ... */
506 /* 1st-order 4x butterfly */
507 __ASM_EMIT("vhsubps %%ymm4, %%ymm0, %%ymm1") /* ymm1 = r0-r1 r2-r3 r4-r5 r6-r7 = r1' r3' r5' r7' */
508 __ASM_EMIT("vhsubps %%ymm6, %%ymm2, %%ymm3") /* ymm3 = i0-i1 i2-i3 i4-i5 i6-i7 = i1' i3' i5' i7' */
509 __ASM_EMIT("vhaddps %%ymm4, %%ymm0, %%ymm0") /* ymm0 = r0+r1 r2+r3 r4+r5 r6+r7 = r0' r2' r4' r6' */
510 __ASM_EMIT("vhaddps %%ymm6, %%ymm2, %%ymm2") /* ymm2 = i0+i1 i2+i3 i4+i5 i6+i7 = i0' i2' i4' i6' */
511 /* 2nd-order 4x butterfly */
512 __ASM_EMIT("vblendps $0xaa, %%ymm3, %%ymm1, %%ymm4") /* ymm4 = r1' i3' r5' i7' */
513 __ASM_EMIT("vblendps $0xaa, %%ymm1, %%ymm3, %%ymm5") /* ymm5 = i1' r3' i5' r7' */
514 __ASM_EMIT("vhsubps %%ymm4, %%ymm0, %%ymm1") /* ymm1 = r0'-r2' r4'-r6' r1'-i3' r5'-i7' = r2" r6" r1" r5" */
515 __ASM_EMIT("vhsubps %%ymm5, %%ymm2, %%ymm3") /* ymm3 = i0'-i2' i4'-i6' i1'-r3' i5'-r7' = i2" i6" i3" i7" */
516 __ASM_EMIT("vhaddps %%ymm4, %%ymm0, %%ymm0") /* ymm0 = r0'+r2' r4'+r6' r1'+i3' r5'+i7' = r0" r4" r3" r7" */
517 __ASM_EMIT("vhaddps %%ymm5, %%ymm2, %%ymm2") /* ymm2 = i0'+i2' i4'+i6' i1'+r3' i5'+r7' = i0" i4" i1" i5" */
518 __ASM_EMIT("vblendps $0xcc, %%ymm1, %%ymm0, %%ymm4") /* ymm4 = r0" i4" r1" r5" */
519 __ASM_EMIT("vblendps $0xcc, %%ymm0, %%ymm1, %%ymm5") /* ymm5 = r2" r6" r3" r7" */
520 __ASM_EMIT("vshufps $0x88, %%ymm3, %%ymm2, %%ymm6") /* ymm6 = i0" i1" i2" i3" */
521 __ASM_EMIT("vshufps $0xdd, %%ymm3, %%ymm2, %%ymm7") /* ymm7 = i4" i5" i6" i7" */
522 __ASM_EMIT("vshufps $0x88, %%ymm5, %%ymm4, %%ymm2") /* ymm2 = r0" r1" r2" r3" */
523 __ASM_EMIT("vshufps $0xdd, %%ymm5, %%ymm4, %%ymm3") /* ymm3 = r4" r5" r6" r7" */
524 /* 3rd-order 8x butterfly */
525 __ASM_EMIT("vmulps 0x20 + %[FFT_A], %%ymm3, %%ymm4") /* ymm4 = x_im * b_re */ \
526 __ASM_EMIT("vmulps 0x20 + %[FFT_A], %%ymm7, %%ymm5") /* ymm5 = x_im * b_im */ \
527 __ASM_EMIT(FFT_FMA("vmulps 0x00 + %[FFT_A], %%ymm3, %%ymm3", "")) /* ymm3 = x_re * b_re */ \
528 __ASM_EMIT(FFT_FMA("vmulps 0x00 + %[FFT_A], %%ymm7, %%ymm7", "")) /* ymm7 = x_re * b_im */ \
529 __ASM_EMIT(FFT_FMA("vsubps %%ymm5, %%ymm3, %%ymm5", "vfmsub231ps 0x00 + %[FFT_A], %%ymm3, %%ymm5")) /* ymm5 = c_re = x_re * b_re - x_im * b_im */ \
530 __ASM_EMIT(FFT_FMA("vaddps %%ymm4, %%ymm7, %%ymm4", "vfmadd231ps 0x00 + %[FFT_A], %%ymm7, %%ymm4")) /* ymm4 = c_im = x_re * b_im + x_im * b_re */ \
531 __ASM_EMIT("vsubps %%ymm5, %%ymm2, %%ymm0") /* ymm0 = a_re - c_re */ \
532 __ASM_EMIT("vsubps %%ymm4, %%ymm6, %%ymm1") /* ymm1 = a_im - c_im */ \
533 __ASM_EMIT("vaddps %%ymm5, %%ymm2, %%ymm2") /* ymm2 = a_re + c_re */ \
534 __ASM_EMIT("vaddps %%ymm4, %%ymm6, %%ymm3") /* ymm3 = a_im + c_im */ \
535 /* Store */
536 __ASM_EMIT("vmovups %%xmm2, 0x00(%[dst_re])")
537 __ASM_EMIT("vmovups %%xmm0, 0x10(%[dst_re])")
538 __ASM_EMIT("vextractf128 $1, %%ymm2, 0x20(%[dst_re])")
539 __ASM_EMIT("vextractf128 $1, %%ymm0, 0x30(%[dst_re])")
540 __ASM_EMIT("vmovups %%xmm3, 0x00(%[dst_im])")
541 __ASM_EMIT("vmovups %%xmm1, 0x10(%[dst_im])")
542 __ASM_EMIT("vextractf128 $1, %%ymm3, 0x20(%[dst_im])")
543 __ASM_EMIT("vextractf128 $1, %%ymm1, 0x30(%[dst_im])")
544 __ASM_EMIT("add $0x40, %[dst_re]")
545 __ASM_EMIT("add $0x40, %[dst_im]")
546
547 : [dst_re] "+r" (dst_re), [dst_im] "+r"(dst_im), [index] "+r"(index)
548 : [src_re] "r"(src_re), [src_im] "r"(src_im),
549 [regs] X86_GREG (regs),
550 [FFT_A] "o" (FFT_A)
551 : "cc", "memory",
552 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
553 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
554 );
555 }
556 }
557 } /* namespace avx */
558 } /* namespace lsp */
559
560 #undef FFT_SCRAMBLE_SELF_DIRECT_NAME
561 #undef FFT_SCRAMBLE_SELF_REVERSE_NAME
562 #undef FFT_SCRAMBLE_COPY_DIRECT_NAME
563 #undef FFT_SCRAMBLE_COPY_REVERSE_NAME
564 #undef FFT_TYPE
565 #undef FFT_FMA
566