Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/x86/avx/fft/p_scramble.h

raw
1 /*
2 * Copyright (C) 2020 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2020 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 31 мар. 2020 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_X86_AVX_IMPL
23 #error "This header should not be included directly"
24 #endif /* PRIVATE_DSP_ARCH_X86_AVX_IMPL */
25
26 namespace lsp
27 {
28 namespace avx
29 {
30 static inline void FFT_PSCRAMBLE_SELF_DIRECT_NAME(float *dst, size_t rank)
31 {
32 // Calculate number of items
33 size_t items = (1 << rank) - 1;
34
35 for (size_t i = 1; i < items; ++i)
36 {
37 size_t j = reverse_bits(FFT_TYPE(i), rank); /* Reverse the order of the bits */
38 if (i >= j)
39 continue;
40
41 /* Copy the values from the reversed position */
42 /* Copy the values from the reversed position */
43 ARCH_X86_ASM
44 (
45 __ASM_EMIT("vmovlps (%[dst], %[i], 8), %%xmm0, %%xmm0")
46 __ASM_EMIT("vmovlps (%[dst], %[j], 8), %%xmm1, %%xmm1")
47 __ASM_EMIT("vmovlps %%xmm0, (%[dst], %[j], 8)")
48 __ASM_EMIT("vmovlps %%xmm1, (%[dst], %[i], 8)")
49 :
50 : [dst] "r"(dst), [i] "r"(i), [j] "r"(j)
51 : "memory",
52 "%xmm0", "%xmm1"
53 );
54 }
55
56 // Perform butterfly 8x
57 items = 1 << (rank - 3);
58
59 // Perform 4-element butterflies
60 ARCH_X86_ASM
61 (
62 /* Loop 2x 4-element butterflies */
63 __ASM_EMIT("sub $2, %[items]")
64 __ASM_EMIT("jb 2f")
65 __ASM_EMIT("1:")
66 /* Load data to registers */
67 __ASM_EMIT("vmovups 0x00(%[dst]), %%xmm0") /* xmm0 = r0 i0 r1 i1 */
68 __ASM_EMIT("vmovups 0x10(%[dst]), %%xmm1") /* xmm1 = r2 i2 r3 i3 */
69 __ASM_EMIT("vmovups 0x20(%[dst]), %%xmm4") /* xmm4 = r4 i4 r5 i5 */
70 __ASM_EMIT("vmovups 0x30(%[dst]), %%xmm5") /* xmm5 = r6 i6 r7 i7 */
71 __ASM_EMIT("vinsertf128 $1, 0x40(%[dst]), %%ymm0, %%ymm0") /* ymm0 = r0 i0 r1 i1 r8 i8 r9 i9 */
72 __ASM_EMIT("vinsertf128 $1, 0x50(%[dst]), %%ymm1, %%ymm1") /* ymm1 = r2 i2 r3 i3 r10 i10 r11 i11 */
73 __ASM_EMIT("vinsertf128 $1, 0x60(%[dst]), %%ymm4, %%ymm4") /* ymm4 = r4 i4 r5 i5 r12 i12 r13 i13 */
74 __ASM_EMIT("vinsertf128 $1, 0x70(%[dst]), %%ymm5, %%ymm5") /* ymm5 = r6 i6 r7 i7 r14 i14 r15 i15 */
75 __ASM_EMIT("vshufps $0xdd, %%ymm1, %%ymm0, %%ymm2") /* ymm2 = i0 i1 i2 i3 i8 i9 i10 i11 */
76 __ASM_EMIT("vshufps $0x88, %%ymm1, %%ymm0, %%ymm0") /* ymm0 = r0 r1 r2 r3 r8 r9 r10 r11 */
77 __ASM_EMIT("vshufps $0xdd, %%ymm5, %%ymm4, %%ymm6") /* ymm6 = i4 i5 i6 i7 i12 i13 i14 i15 */
78 __ASM_EMIT("vshufps $0x88, %%ymm5, %%ymm4, %%ymm4") /* ymm4 = r4 r5 r6 r7 r12 r13 r14 r15 */
79 /* 1st-order 4x butterfly */
80 __ASM_EMIT("vhsubps %%ymm4, %%ymm0, %%ymm1") /* ymm1 = r0-r1 r2-r3 r4-r5 r6-r7 = r1' r3' r5' r7' */
81 __ASM_EMIT("vhsubps %%ymm6, %%ymm2, %%ymm3") /* ymm3 = i0-i1 i2-i3 i4-i5 i6-i7 = i1' i3' i5' i7' */
82 __ASM_EMIT("vhaddps %%ymm4, %%ymm0, %%ymm0") /* ymm0 = r0+r1 r2+r3 r4+r5 r6+r7 = r0' r2' r4' r6' */
83 __ASM_EMIT("vhaddps %%ymm6, %%ymm2, %%ymm2") /* ymm2 = i0+i1 i2+i3 i4+i5 i6+i7 = i0' i2' i4' i6' */
84 /* 2nd-order 4x butterfly */
85 __ASM_EMIT("vblendps $0xaa, %%ymm3, %%ymm1, %%ymm4") /* ymm4 = r1' i3' r5' i7' */
86 __ASM_EMIT("vblendps $0xaa, %%ymm1, %%ymm3, %%ymm5") /* ymm5 = i1' r3' i5' r7' */
87 __ASM_EMIT("vhsubps %%ymm4, %%ymm0, %%ymm1") /* ymm1 = r0'-r2' r4'-r6' r1'-i3' r5'-i7' = r2" r6" r3" r7" */
88 __ASM_EMIT("vhsubps %%ymm5, %%ymm2, %%ymm3") /* ymm3 = i0'-i2' i4'-i6' i1'-r3' i5'-r7' = i2" i6" i1" i5" */
89 __ASM_EMIT("vhaddps %%ymm4, %%ymm0, %%ymm0") /* ymm0 = r0'+r2' r4'+r6' r1'+i3' r5'+i7' = r0" r4" r1" r5" */
90 __ASM_EMIT("vhaddps %%ymm5, %%ymm2, %%ymm2") /* ymm2 = i0'+i2' i4'+i6' i1'+r3' i5'+r7' = i0" i4" i3" i7" */
91 __ASM_EMIT("vblendps $0xcc, %%ymm3, %%ymm2, %%ymm4") /* ymm4 = i0" i4" i1" i5" */
92 __ASM_EMIT("vblendps $0xcc, %%ymm2, %%ymm3, %%ymm5") /* ymm5 = i2" i6" i3" i7" */
93 __ASM_EMIT("vshufps $0x88, %%ymm1, %%ymm0, %%ymm2") /* ymm2 = r0" r1" r2" r3" */
94 __ASM_EMIT("vshufps $0xdd, %%ymm1, %%ymm0, %%ymm3") /* ymm3 = r4" r5" r6" r7" */
95 __ASM_EMIT("vshufps $0x88, %%ymm5, %%ymm4, %%ymm6") /* ymm6 = i0" i1" i2" i3" */
96 __ASM_EMIT("vshufps $0xdd, %%ymm5, %%ymm4, %%ymm7") /* ymm7 = i4" i5" i6" i7" */
97 /* 3rd-order 8x butterfly */
98 __ASM_EMIT("vmulps 0x20 + %[FFT_A], %%ymm3, %%ymm4") /* ymm4 = x_im * b_re */ \
99 __ASM_EMIT("vmulps 0x20 + %[FFT_A], %%ymm7, %%ymm5") /* ymm5 = x_im * b_im */ \
100 __ASM_EMIT(FFT_FMA("vmulps 0x00 + %[FFT_A], %%ymm3, %%ymm3", "")) /* ymm3 = x_re * b_re */ \
101 __ASM_EMIT(FFT_FMA("vmulps 0x00 + %[FFT_A], %%ymm7, %%ymm7", "")) /* ymm7 = x_re * b_im */ \
102 __ASM_EMIT(FFT_FMA("vaddps %%ymm5, %%ymm3, %%ymm5", "vfmadd231ps 0x00 + %[FFT_A], %%ymm3, %%ymm5")) /* ymm5 = c_re = x_re * b_re + x_im * b_im */ \
103 __ASM_EMIT(FFT_FMA("vsubps %%ymm4, %%ymm7, %%ymm4", "vfmsub231ps 0x00 + %[FFT_A], %%ymm7, %%ymm4")) /* ymm4 = c_im = x_re * b_im - x_im * b_re */ \
104 __ASM_EMIT("vsubps %%ymm5, %%ymm2, %%ymm0") /* ymm0 = a_re - c_re */ \
105 __ASM_EMIT("vsubps %%ymm4, %%ymm6, %%ymm1") /* ymm1 = a_im - c_im */ \
106 __ASM_EMIT("vaddps %%ymm5, %%ymm2, %%ymm2") /* ymm2 = a_re + c_re */ \
107 __ASM_EMIT("vaddps %%ymm4, %%ymm6, %%ymm3") /* ymm3 = a_im + c_im */ \
108 /* Store */
109 __ASM_EMIT("vmovups %%xmm2, 0x00(%[dst])")
110 __ASM_EMIT("vmovups %%xmm0, 0x10(%[dst])")
111 __ASM_EMIT("vmovups %%xmm3, 0x20(%[dst])")
112 __ASM_EMIT("vmovups %%xmm1, 0x30(%[dst])")
113 __ASM_EMIT("vextractf128 $1, %%ymm2, 0x40(%[dst])")
114 __ASM_EMIT("vextractf128 $1, %%ymm0, 0x50(%[dst])")
115 __ASM_EMIT("vextractf128 $1, %%ymm3, 0x60(%[dst])")
116 __ASM_EMIT("vextractf128 $1, %%ymm1, 0x70(%[dst])")
117 /* Move pointers and repeat*/
118 __ASM_EMIT("add $0x80, %[dst]")
119 __ASM_EMIT("sub $2, %[items]")
120 __ASM_EMIT("jae 1b")
121 __ASM_EMIT("2:")
122 /* x4 scramble block */
123 __ASM_EMIT("add $1, %[items]")
124 __ASM_EMIT("jl 4f")
125 __ASM_EMIT("vmovups 0x00(%[dst]), %%xmm0") /* xmm0 = r0 i0 r1 i1 */
126 __ASM_EMIT("vmovups 0x10(%[dst]), %%xmm1") /* xmm1 = r2 i2 r3 i3 */
127 __ASM_EMIT("vmovups 0x20(%[dst]), %%xmm4") /* xmm4 = r4 i4 r5 i5 */
128 __ASM_EMIT("vmovups 0x30(%[dst]), %%xmm5") /* xmm5 = r6 i6 r7 i7 */
129 __ASM_EMIT("vshufps $0xdd, %%xmm1, %%xmm0, %%xmm2") /* xmm2 = i0 i1 i2 i3 */
130 __ASM_EMIT("vshufps $0x88, %%xmm1, %%xmm0, %%xmm0") /* xmm0 = r0 r1 r2 r3 */
131 __ASM_EMIT("vshufps $0xdd, %%xmm5, %%xmm4, %%xmm6") /* xmm4 = i4 i5 i6 i7 */
132 __ASM_EMIT("vshufps $0x88, %%xmm5, %%xmm4, %%xmm4") /* xmm6 = r4 r5 r6 r7 */
133 /* 1st-order 4x butterfly */
134 __ASM_EMIT("vhsubps %%xmm4, %%xmm0, %%xmm1") /* xmm1 = r0-r1 r2-r3 r4-r5 r6-r7 = r1' r3' r5' r7' */
135 __ASM_EMIT("vhsubps %%xmm6, %%xmm2, %%xmm3") /* xmm3 = i0-i1 i2-i3 i4-i5 i6-i7 = i1' i3' i5' i7' */
136 __ASM_EMIT("vhaddps %%xmm4, %%xmm0, %%xmm0") /* xmm0 = r0+r1 r2+r3 r4+r5 r6+r7 = r0' r2' r4' r6' */
137 __ASM_EMIT("vhaddps %%xmm6, %%xmm2, %%xmm2") /* xmm2 = i0+i1 i2+i3 i4+i5 i6+i7 = i0' i2' i4' i6' */
138 /* 2nd-order 4x butterfly */
139 __ASM_EMIT("vblendps $0xaa, %%xmm3, %%xmm1, %%xmm4") /* xmm4 = r1' i3' r5' i7' */
140 __ASM_EMIT("vblendps $0xaa, %%xmm1, %%xmm3, %%xmm5") /* xmm5 = i1' r3' i5' r7' */
141 __ASM_EMIT("vhsubps %%xmm4, %%xmm0, %%xmm1") /* xmm1 = r0'-r2' r4'-r6' r1'-i3' r5'-i7' = r2" r6" r3" r7" */
142 __ASM_EMIT("vhsubps %%xmm5, %%xmm2, %%xmm3") /* xmm3 = i0'-i2' i4'-i6' i1'-r3' i5'-r7' = i2" i6" i1" i5" */
143 __ASM_EMIT("vhaddps %%xmm4, %%xmm0, %%xmm0") /* xmm0 = r0'+r2' r4'+r6' r1'+i3' r5'+i7' = r0" r4" r1" r5" */
144 __ASM_EMIT("vhaddps %%xmm5, %%xmm2, %%xmm2") /* xmm2 = i0'+i2' i4'+i6' i1'+r3' i5'+r7' = i0" i4" i3" i7" */
145 __ASM_EMIT("vblendps $0xcc, %%xmm3, %%xmm2, %%xmm4") /* xmm4 = i0" i4" i1" i5" */
146 __ASM_EMIT("vblendps $0xcc, %%xmm2, %%xmm3, %%xmm5") /* xmm5 = i2" i6" i3" i7" */
147 __ASM_EMIT("vshufps $0x88, %%xmm1, %%xmm0, %%xmm2") /* xmm2 = r0" r1" r2" r3" */
148 __ASM_EMIT("vshufps $0xdd, %%xmm1, %%xmm0, %%xmm3") /* xmm3 = r4" r5" r6" r7" */
149 __ASM_EMIT("vshufps $0x88, %%xmm5, %%xmm4, %%xmm6") /* xmm6 = i0" i1" i2" i3" */
150 __ASM_EMIT("vshufps $0xdd, %%xmm5, %%xmm4, %%xmm7") /* xmm7 = i4" i5" i6" i7" */
151 /* 3rd-order 8x butterfly */
152 __ASM_EMIT("vmulps 0x20 + %[FFT_A], %%xmm3, %%xmm4") /* xmm4 = x_im * b_re */ \
153 __ASM_EMIT("vmulps 0x20 + %[FFT_A], %%xmm7, %%xmm5") /* xmm5 = x_im * b_im */ \
154 __ASM_EMIT(FFT_FMA("vmulps 0x00 + %[FFT_A], %%xmm3, %%xmm3", "")) /* xmm3 = x_re * b_re */ \
155 __ASM_EMIT(FFT_FMA("vmulps 0x00 + %[FFT_A], %%xmm7, %%xmm7", "")) /* xmm7 = x_re * b_im */ \
156 __ASM_EMIT(FFT_FMA("vaddps %%xmm5, %%xmm3, %%xmm5", "vfmadd231ps 0x00 + %[FFT_A], %%xmm3, %%xmm5")) /* xmm5 = c_re = x_re * b_re + x_im * b_im */ \
157 __ASM_EMIT(FFT_FMA("vsubps %%xmm4, %%xmm7, %%xmm4", "vfmsub231ps 0x00 + %[FFT_A], %%xmm7, %%xmm4")) /* xmm4 = c_im = x_re * b_im - x_im * b_re */ \
158 __ASM_EMIT("vsubps %%xmm5, %%xmm2, %%xmm0") /* xmm0 = a_re - c_re */ \
159 __ASM_EMIT("vsubps %%xmm4, %%xmm6, %%xmm1") /* xmm1 = a_im - c_im */ \
160 __ASM_EMIT("vaddps %%xmm5, %%xmm2, %%xmm2") /* xmm2 = a_re + c_re */ \
161 __ASM_EMIT("vaddps %%xmm4, %%xmm6, %%xmm3") /* xmm3 = a_im + c_im */ \
162 /* Store */
163 __ASM_EMIT("vmovups %%xmm2, 0x00(%[dst])")
164 __ASM_EMIT("vmovups %%xmm0, 0x10(%[dst])")
165 __ASM_EMIT("vmovups %%xmm3, 0x20(%[dst])")
166 __ASM_EMIT("vmovups %%xmm1, 0x30(%[dst])")
167 __ASM_EMIT("4:")
168
169 : [dst] "+r" (dst), [items] "+r" (items)
170 : [FFT_A] "o" (FFT_A)
171 : "cc", "memory",
172 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
173 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
174 );
175 }
176
177 static inline void FFT_PSCRAMBLE_SELF_REVERSE_NAME(float *dst, size_t rank)
178 {
179 // Calculate number of items
180 size_t items = (1 << rank) - 1;
181
182 for (size_t i = 1; i < items; ++i)
183 {
184 size_t j = reverse_bits(FFT_TYPE(i), rank); /* Reverse the order of the bits */
185 if (i >= j)
186 continue;
187
188 /* Copy the values from the reversed position */
189 /* Copy the values from the reversed position */
190 ARCH_X86_ASM
191 (
192 __ASM_EMIT("vmovlps (%[dst], %[i], 8), %%xmm0, %%xmm0")
193 __ASM_EMIT("vmovlps (%[dst], %[j], 8), %%xmm1, %%xmm1")
194 __ASM_EMIT("vmovlps %%xmm0, (%[dst], %[j], 8)")
195 __ASM_EMIT("vmovlps %%xmm1, (%[dst], %[i], 8)")
196 :
197 : [dst] "r"(dst), [i] "r"(i), [j] "r"(j)
198 : "memory",
199 "%xmm0", "%xmm1"
200 );
201 }
202
203 // Perform butterfly 8x
204 items = 1 << (rank - 3);
205
206 // Perform 4-element butterflies
207 ARCH_X86_ASM
208 (
209 /* Loop 2x 4-element butterflies */
210 __ASM_EMIT("sub $2, %[items]")
211 __ASM_EMIT("jb 2f")
212 __ASM_EMIT("1:")
213 /* Load data to registers */
214 __ASM_EMIT("vmovups 0x00(%[dst]), %%xmm0") /* xmm0 = r0 i0 r1 i1 */
215 __ASM_EMIT("vmovups 0x10(%[dst]), %%xmm1") /* xmm1 = r2 i2 r3 i3 */
216 __ASM_EMIT("vmovups 0x20(%[dst]), %%xmm4") /* xmm4 = r4 i4 r5 i5 */
217 __ASM_EMIT("vmovups 0x30(%[dst]), %%xmm5") /* xmm5 = r6 i6 r7 i7 */
218 __ASM_EMIT("vinsertf128 $1, 0x40(%[dst]), %%ymm0, %%ymm0") /* ymm0 = r0 i0 r1 i1 r8 i8 r9 i9 */
219 __ASM_EMIT("vinsertf128 $1, 0x50(%[dst]), %%ymm1, %%ymm1") /* ymm1 = r2 i2 r3 i3 r10 i10 r11 i11 */
220 __ASM_EMIT("vinsertf128 $1, 0x60(%[dst]), %%ymm4, %%ymm4") /* ymm4 = r4 i4 r5 i5 r12 i12 r13 i13 */
221 __ASM_EMIT("vinsertf128 $1, 0x70(%[dst]), %%ymm5, %%ymm5") /* ymm5 = r6 i6 r7 i7 r14 i14 r15 i15 */
222 __ASM_EMIT("vshufps $0xdd, %%ymm1, %%ymm0, %%ymm2") /* ymm2 = i0 i1 i2 i3 i8 i9 i10 i11 */
223 __ASM_EMIT("vshufps $0x88, %%ymm1, %%ymm0, %%ymm0") /* ymm0 = r0 r1 r2 r3 r8 r9 r10 r11 */
224 __ASM_EMIT("vshufps $0xdd, %%ymm5, %%ymm4, %%ymm6") /* ymm6 = i4 i5 i6 i7 i12 i13 i14 i15 */
225 __ASM_EMIT("vshufps $0x88, %%ymm5, %%ymm4, %%ymm4") /* ymm4 = r4 r5 r6 r7 r12 r13 r14 r15 */
226 /* 1st-order 4x butterfly */
227 __ASM_EMIT("vhsubps %%ymm4, %%ymm0, %%ymm1") /* ymm1 = r0-r1 r2-r3 r4-r5 r6-r7 = r1' r3' r5' r7' */
228 __ASM_EMIT("vhsubps %%ymm6, %%ymm2, %%ymm3") /* ymm3 = i0-i1 i2-i3 i4-i5 i6-i7 = i1' i3' i5' i7' */
229 __ASM_EMIT("vhaddps %%ymm4, %%ymm0, %%ymm0") /* ymm0 = r0+r1 r2+r3 r4+r5 r6+r7 = r0' r2' r4' r6' */
230 __ASM_EMIT("vhaddps %%ymm6, %%ymm2, %%ymm2") /* ymm2 = i0+i1 i2+i3 i4+i5 i6+i7 = i0' i2' i4' i6' */
231 /* 2nd-order 4x butterfly */
232 __ASM_EMIT("vblendps $0xaa, %%ymm3, %%ymm1, %%ymm4") /* ymm4 = r1' i3' r5' i7' */
233 __ASM_EMIT("vblendps $0xaa, %%ymm1, %%ymm3, %%ymm5") /* ymm5 = i1' r3' i5' r7' */
234 __ASM_EMIT("vhsubps %%ymm4, %%ymm0, %%ymm1") /* ymm1 = r0'-r2' r4'-r6' r1'-i3' r5'-i7' = r2" r6" r1" r5" */
235 __ASM_EMIT("vhsubps %%ymm5, %%ymm2, %%ymm3") /* ymm3 = i0'-i2' i4'-i6' i1'-r3' i5'-r7' = i2" i6" i3" i7" */
236 __ASM_EMIT("vhaddps %%ymm4, %%ymm0, %%ymm0") /* ymm0 = r0'+r2' r4'+r6' r1'+i3' r5'+i7' = r0" r4" r3" r7" */
237 __ASM_EMIT("vhaddps %%ymm5, %%ymm2, %%ymm2") /* ymm2 = i0'+i2' i4'+i6' i1'+r3' i5'+r7' = i0" i4" i1" i5" */
238 __ASM_EMIT("vblendps $0xcc, %%ymm1, %%ymm0, %%ymm4") /* ymm4 = r0" i4" r1" r5" */
239 __ASM_EMIT("vblendps $0xcc, %%ymm0, %%ymm1, %%ymm5") /* ymm5 = r2" r6" r3" r7" */
240 __ASM_EMIT("vshufps $0x88, %%ymm3, %%ymm2, %%ymm6") /* ymm6 = i0" i1" i2" i3" */
241 __ASM_EMIT("vshufps $0xdd, %%ymm3, %%ymm2, %%ymm7") /* ymm7 = i4" i5" i6" i7" */
242 __ASM_EMIT("vshufps $0x88, %%ymm5, %%ymm4, %%ymm2") /* ymm2 = r0" r1" r2" r3" */
243 __ASM_EMIT("vshufps $0xdd, %%ymm5, %%ymm4, %%ymm3") /* ymm3 = r4" r5" r6" r7" */
244 /* 3rd-order 8x butterfly */
245 __ASM_EMIT("vmulps 0x20 + %[FFT_A], %%ymm3, %%ymm4") /* ymm4 = x_im * b_re */ \
246 __ASM_EMIT("vmulps 0x20 + %[FFT_A], %%ymm7, %%ymm5") /* ymm5 = x_im * b_im */ \
247 __ASM_EMIT(FFT_FMA("vmulps 0x00 + %[FFT_A], %%ymm3, %%ymm3", "")) /* ymm3 = x_re * b_re */ \
248 __ASM_EMIT(FFT_FMA("vmulps 0x00 + %[FFT_A], %%ymm7, %%ymm7", "")) /* ymm7 = x_re * b_im */ \
249 __ASM_EMIT(FFT_FMA("vsubps %%ymm5, %%ymm3, %%ymm5", "vfmsub231ps 0x00 + %[FFT_A], %%ymm3, %%ymm5")) /* ymm5 = c_re = x_re * b_re - x_im * b_im */ \
250 __ASM_EMIT(FFT_FMA("vaddps %%ymm4, %%ymm7, %%ymm4", "vfmadd231ps 0x00 + %[FFT_A], %%ymm7, %%ymm4")) /* ymm4 = c_im = x_re * b_im + x_im * b_re */ \
251 __ASM_EMIT("vsubps %%ymm5, %%ymm2, %%ymm0") /* ymm0 = a_re - c_re */ \
252 __ASM_EMIT("vsubps %%ymm4, %%ymm6, %%ymm1") /* ymm1 = a_im - c_im */ \
253 __ASM_EMIT("vaddps %%ymm5, %%ymm2, %%ymm2") /* ymm2 = a_re + c_re */ \
254 __ASM_EMIT("vaddps %%ymm4, %%ymm6, %%ymm3") /* ymm3 = a_im + c_im */ \
255 /* Store */
256 __ASM_EMIT("vmovups %%xmm2, 0x00(%[dst])")
257 __ASM_EMIT("vmovups %%xmm0, 0x10(%[dst])")
258 __ASM_EMIT("vmovups %%xmm3, 0x20(%[dst])")
259 __ASM_EMIT("vmovups %%xmm1, 0x30(%[dst])")
260 __ASM_EMIT("vextractf128 $1, %%ymm2, 0x40(%[dst])")
261 __ASM_EMIT("vextractf128 $1, %%ymm0, 0x50(%[dst])")
262 __ASM_EMIT("vextractf128 $1, %%ymm3, 0x60(%[dst])")
263 __ASM_EMIT("vextractf128 $1, %%ymm1, 0x70(%[dst])")
264 /* Move pointers and repeat*/
265 __ASM_EMIT("add $0x80, %[dst]")
266 __ASM_EMIT("sub $2, %[items]")
267 __ASM_EMIT("jae 1b")
268 __ASM_EMIT("2:")
269 /* x4 scramble block */
270 __ASM_EMIT("add $1, %[items]")
271 __ASM_EMIT("jl 4f")
272 __ASM_EMIT("vmovups 0x00(%[dst]), %%xmm0") /* xmm0 = r0 i0 r1 i1 */
273 __ASM_EMIT("vmovups 0x10(%[dst]), %%xmm1") /* xmm1 = r2 i2 r3 i3 */
274 __ASM_EMIT("vmovups 0x20(%[dst]), %%xmm4") /* xmm4 = r4 i4 r5 i5 */
275 __ASM_EMIT("vmovups 0x30(%[dst]), %%xmm5") /* xmm5 = r6 i6 r7 i7 */
276 __ASM_EMIT("vshufps $0xdd, %%xmm1, %%xmm0, %%xmm2") /* xmm2 = i0 i1 i2 i3 */
277 __ASM_EMIT("vshufps $0x88, %%xmm1, %%xmm0, %%xmm0") /* xmm0 = r0 r1 r2 r3 */
278 __ASM_EMIT("vshufps $0xdd, %%xmm5, %%xmm4, %%xmm6") /* xmm4 = i4 i5 i6 i7 */
279 __ASM_EMIT("vshufps $0x88, %%xmm5, %%xmm4, %%xmm4") /* xmm6 = r4 r5 r6 r7 */
280 /* 1st-order 4x butterfly */
281 __ASM_EMIT("vhsubps %%xmm4, %%xmm0, %%xmm1") /* xmm1 = r0-r1 r2-r3 r4-r5 r6-r7 = r1' r3' r5' r7' */
282 __ASM_EMIT("vhsubps %%xmm6, %%xmm2, %%xmm3") /* xmm3 = i0-i1 i2-i3 i4-i5 i6-i7 = i1' i3' i5' i7' */
283 __ASM_EMIT("vhaddps %%xmm4, %%xmm0, %%xmm0") /* xmm0 = r0+r1 r2+r3 r4+r5 r6+r7 = r0' r2' r4' r6' */
284 __ASM_EMIT("vhaddps %%xmm6, %%xmm2, %%xmm2") /* xmm2 = i0+i1 i2+i3 i4+i5 i6+i7 = i0' i2' i4' i6' */
285 /* 2nd-order 4x butterfly */
286 __ASM_EMIT("vblendps $0xaa, %%xmm3, %%xmm1, %%xmm4") /* xmm4 = r1' i3' r5' i7' */
287 __ASM_EMIT("vblendps $0xaa, %%xmm1, %%xmm3, %%xmm5") /* xmm5 = i1' r3' i5' r7' */
288 __ASM_EMIT("vhsubps %%xmm4, %%xmm0, %%xmm1") /* xmm1 = r0'-r2' r4'-r6' r1'-i3' r5'-i7' = r2" r6" r1" r5" */
289 __ASM_EMIT("vhsubps %%xmm5, %%xmm2, %%xmm3") /* xmm3 = i0'-i2' i4'-i6' i1'-r3' i5'-r7' = i2" i6" i3" i7" */
290 __ASM_EMIT("vhaddps %%xmm4, %%xmm0, %%xmm0") /* xmm0 = r0'+r2' r4'+r6' r1'+i3' r5'+i7' = r0" r4" r3" r7" */
291 __ASM_EMIT("vhaddps %%xmm5, %%xmm2, %%xmm2") /* xmm2 = i0'+i2' i4'+i6' i1'+r3' i5'+r7' = i0" i4" i1" i5" */
292 __ASM_EMIT("vblendps $0xcc, %%xmm1, %%xmm0, %%xmm4") /* xmm4 = r0" i4" r1" r5" */
293 __ASM_EMIT("vblendps $0xcc, %%xmm0, %%xmm1, %%xmm5") /* xmm5 = r2" r6" r3" r7" */
294 __ASM_EMIT("vshufps $0x88, %%xmm3, %%xmm2, %%xmm6") /* xmm6 = i0" i1" i2" i3" */
295 __ASM_EMIT("vshufps $0xdd, %%xmm3, %%xmm2, %%xmm7") /* xmm7 = i4" i5" i6" i7" */
296 __ASM_EMIT("vshufps $0x88, %%xmm5, %%xmm4, %%xmm2") /* xmm2 = r0" r1" r2" r3" */
297 __ASM_EMIT("vshufps $0xdd, %%xmm5, %%xmm4, %%xmm3") /* xmm3 = r4" r5" r6" r7" */
298 /* 3rd-order 8x butterfly */
299 __ASM_EMIT("vmulps 0x20 + %[FFT_A], %%xmm3, %%xmm4") /* xmm4 = x_im * b_re */ \
300 __ASM_EMIT("vmulps 0x20 + %[FFT_A], %%xmm7, %%xmm5") /* xmm5 = x_im * b_im */ \
301 __ASM_EMIT(FFT_FMA("vmulps 0x00 + %[FFT_A], %%xmm3, %%xmm3", "")) /* xmm3 = x_re * b_re */ \
302 __ASM_EMIT(FFT_FMA("vmulps 0x00 + %[FFT_A], %%xmm7, %%xmm7", "")) /* xmm7 = x_re * b_im */ \
303 __ASM_EMIT(FFT_FMA("vsubps %%xmm5, %%xmm3, %%xmm5", "vfmsub231ps 0x00 + %[FFT_A], %%xmm3, %%xmm5")) /* xmm5 = c_re = x_re * b_re - x_im * b_im */ \
304 __ASM_EMIT(FFT_FMA("vaddps %%xmm4, %%xmm7, %%xmm4", "vfmadd231ps 0x00 + %[FFT_A], %%xmm7, %%xmm4")) /* xmm4 = c_im = x_re * b_im + x_im * b_re */ \
305 __ASM_EMIT("vsubps %%xmm5, %%xmm2, %%xmm0") /* xmm0 = a_re - c_re */ \
306 __ASM_EMIT("vsubps %%xmm4, %%xmm6, %%xmm1") /* xmm1 = a_im - c_im */ \
307 __ASM_EMIT("vaddps %%xmm5, %%xmm2, %%xmm2") /* xmm2 = a_re + c_re */ \
308 __ASM_EMIT("vaddps %%xmm4, %%xmm6, %%xmm3") /* xmm3 = a_im + c_im */ \
309 /* Store */
310 __ASM_EMIT("vmovups %%xmm2, 0x00(%[dst])")
311 __ASM_EMIT("vmovups %%xmm0, 0x10(%[dst])")
312 __ASM_EMIT("vmovups %%xmm3, 0x20(%[dst])")
313 __ASM_EMIT("vmovups %%xmm1, 0x30(%[dst])")
314 __ASM_EMIT("4:")
315
316 : [dst] "+r" (dst), [items] "+r" (items)
317 : [FFT_A] "o" (FFT_A)
318 : "cc", "memory",
319 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
320 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
321 );
322 }
323
324 static inline void FFT_PSCRAMBLE_COPY_DIRECT_NAME(float *dst, const float *src, size_t rank)
325 {
326 size_t regs = 1 << rank;
327
328 for (size_t i=0; i<regs; ++i)
329 {
330 size_t index = reverse_bits(FFT_TYPE(i), rank);
331
332 ARCH_X86_ASM
333 (
334 /* Load scalar values */
335 __ASM_EMIT("vmovlps (%[src], %[index], 8), %%xmm0, %%xmm0") /* xmm0 = r0 i0 x x */
336 __ASM_EMIT("add %[regs], %[index]")
337 __ASM_EMIT("vmovlps (%[src], %[index], 8), %%xmm2, %%xmm2") /* xmm2 = r8 i8 x x */
338 __ASM_EMIT("add %[regs], %[index]")
339 __ASM_EMIT("vmovlps (%[src], %[index], 8), %%xmm4, %%xmm4") /* xmm4 = r4 i4 x x */
340 __ASM_EMIT("add %[regs], %[index]")
341 __ASM_EMIT("vmovlps (%[src], %[index], 8), %%xmm6, %%xmm6") /* xmm6 = r12 i12 x x */
342 __ASM_EMIT("add %[regs], %[index]")
343 __ASM_EMIT("vmovlps (%[src], %[index], 8), %%xmm1, %%xmm1") /* xmm1 = r2 i2 x x */
344 __ASM_EMIT("add %[regs], %[index]")
345 __ASM_EMIT("vmovlps (%[src], %[index], 8), %%xmm3, %%xmm3") /* xmm3 = r10 i10 x x */
346 __ASM_EMIT("add %[regs], %[index]")
347 __ASM_EMIT("vmovlps (%[src], %[index], 8), %%xmm5, %%xmm5") /* xmm5 = r6 i6 x x */
348 __ASM_EMIT("add %[regs], %[index]")
349 __ASM_EMIT("vmovlps (%[src], %[index], 8), %%xmm7, %%xmm7") /* xmm7 = r14 i14 x x */
350 __ASM_EMIT("add %[regs], %[index]")
351
352 __ASM_EMIT("vmovhps (%[src], %[index], 8), %%xmm0, %%xmm0") /* xmm0 = r0 i0 r1 i1 */
353 __ASM_EMIT("add %[regs], %[index]")
354 __ASM_EMIT("vmovhps (%[src], %[index], 8), %%xmm2, %%xmm2") /* xmm2 = r8 i8 r9 i9 */
355 __ASM_EMIT("add %[regs], %[index]")
356 __ASM_EMIT("vmovhps (%[src], %[index], 8), %%xmm4, %%xmm4") /* xmm4 = r4 i4 r5 i5 */
357 __ASM_EMIT("add %[regs], %[index]")
358 __ASM_EMIT("vmovhps (%[src], %[index], 8), %%xmm6, %%xmm6") /* xmm6 = r12 i12 r13 r14 */
359 __ASM_EMIT("add %[regs], %[index]")
360 __ASM_EMIT("vmovhps (%[src], %[index], 8), %%xmm1, %%xmm1") /* xmm1 = r2 i2 r3 i3 */
361 __ASM_EMIT("add %[regs], %[index]")
362 __ASM_EMIT("vmovhps (%[src], %[index], 8), %%xmm3, %%xmm3") /* xmm3 = r10 i10 r11 i11 */
363 __ASM_EMIT("add %[regs], %[index]")
364 __ASM_EMIT("vmovhps (%[src], %[index], 8), %%xmm5, %%xmm5") /* xmm5 = r6 i6 r7 i7 */
365 __ASM_EMIT("add %[regs], %[index]")
366 __ASM_EMIT("vmovhps (%[src], %[index], 8), %%xmm7, %%xmm7") /* xmm7 = r14 i14 r15 i15 */
367 __ASM_EMIT("add %[regs], %[index]")
368
369 __ASM_EMIT("vinsertf128 $1, %%xmm2, %%ymm0, %%ymm0") /* ymm0 = r0 i0 r1 i1 r8 i8 r9 i9 */
370 __ASM_EMIT("vinsertf128 $1, %%xmm3, %%ymm1, %%ymm1") /* ymm1 = r2 i2 r3 i3 r10 i10 r11 i11 */
371 __ASM_EMIT("vinsertf128 $1, %%xmm6, %%ymm4, %%ymm4") /* ymm4 = r4 i4 r5 i5 r12 i12 r13 r14 */
372 __ASM_EMIT("vinsertf128 $1, %%xmm7, %%ymm5, %%ymm5") /* ymm5 = r6 i6 r7 i7 r14 i14 r15 i15 */
373 __ASM_EMIT("vshufps $0xdd, %%ymm1, %%ymm0, %%ymm2") /* ymm2 = i0 i1 i2 i3 i8 i9 i10 i11 */
374 __ASM_EMIT("vshufps $0x88, %%ymm1, %%ymm0, %%ymm0") /* ymm0 = r0 r1 r2 r3 r8 r9 r10 r11 */
375 __ASM_EMIT("vshufps $0xdd, %%ymm5, %%ymm4, %%ymm6") /* ymm6 = i4 i5 i6 i7 i12 i13 i14 i15 */
376 __ASM_EMIT("vshufps $0x88, %%ymm5, %%ymm4, %%ymm4") /* ymm4 = r4 r5 r6 r7 r12 r13 r14 r15 */
377 /* 1st-order 4x butterfly */
378 __ASM_EMIT("vhsubps %%ymm4, %%ymm0, %%ymm1") /* ymm1 = r0-r1 r2-r3 r4-r5 r6-r7 = r1' r3' r5' r7' */
379 __ASM_EMIT("vhsubps %%ymm6, %%ymm2, %%ymm3") /* ymm3 = i0-i1 i2-i3 i4-i5 i6-i7 = i1' i3' i5' i7' */
380 __ASM_EMIT("vhaddps %%ymm4, %%ymm0, %%ymm0") /* ymm0 = r0+r1 r2+r3 r4+r5 r6+r7 = r0' r2' r4' r6' */
381 __ASM_EMIT("vhaddps %%ymm6, %%ymm2, %%ymm2") /* ymm2 = i0+i1 i2+i3 i4+i5 i6+i7 = i0' i2' i4' i6' */
382 /* 2nd-order 4x butterfly */
383 __ASM_EMIT("vblendps $0xaa, %%ymm3, %%ymm1, %%ymm4") /* ymm4 = r1' i3' r5' i7' */
384 __ASM_EMIT("vblendps $0xaa, %%ymm1, %%ymm3, %%ymm5") /* ymm5 = i1' r3' i5' r7' */
385 __ASM_EMIT("vhsubps %%ymm4, %%ymm0, %%ymm1") /* ymm1 = r0'-r2' r4'-r6' r1'-i3' r5'-i7' = r2" r6" r3" r7" */
386 __ASM_EMIT("vhsubps %%ymm5, %%ymm2, %%ymm3") /* ymm3 = i0'-i2' i4'-i6' i1'-r3' i5'-r7' = i2" i6" i1" i5" */
387 __ASM_EMIT("vhaddps %%ymm4, %%ymm0, %%ymm0") /* ymm0 = r0'+r2' r4'+r6' r1'+i3' r5'+i7' = r0" r4" r1" r5" */
388 __ASM_EMIT("vhaddps %%ymm5, %%ymm2, %%ymm2") /* ymm2 = i0'+i2' i4'+i6' i1'+r3' i5'+r7' = i0" i4" i3" i7" */
389 __ASM_EMIT("vblendps $0xcc, %%ymm3, %%ymm2, %%ymm4") /* ymm4 = i0" i4" i1" i5" */
390 __ASM_EMIT("vblendps $0xcc, %%ymm2, %%ymm3, %%ymm5") /* ymm5 = i2" i6" i3" i7" */
391 __ASM_EMIT("vshufps $0x88, %%ymm1, %%ymm0, %%ymm2") /* ymm2 = r0" r1" r2" r3" */
392 __ASM_EMIT("vshufps $0xdd, %%ymm1, %%ymm0, %%ymm3") /* ymm3 = r4" r5" r6" r7" */
393 __ASM_EMIT("vshufps $0x88, %%ymm5, %%ymm4, %%ymm6") /* ymm6 = i0" i1" i2" i3" */
394 __ASM_EMIT("vshufps $0xdd, %%ymm5, %%ymm4, %%ymm7") /* ymm7 = i4" i5" i6" i7" */
395 /* 3rd-order 8x butterfly */
396 __ASM_EMIT("vmulps 0x20 + %[FFT_A], %%ymm3, %%ymm4") /* ymm4 = x_im * b_re */ \
397 __ASM_EMIT("vmulps 0x20 + %[FFT_A], %%ymm7, %%ymm5") /* ymm5 = x_im * b_im */ \
398 __ASM_EMIT(FFT_FMA("vmulps 0x00 + %[FFT_A], %%ymm3, %%ymm3", "")) /* ymm3 = x_re * b_re */ \
399 __ASM_EMIT(FFT_FMA("vmulps 0x00 + %[FFT_A], %%ymm7, %%ymm7", "")) /* ymm7 = x_re * b_im */ \
400 __ASM_EMIT(FFT_FMA("vaddps %%ymm5, %%ymm3, %%ymm5", "vfmadd231ps 0x00 + %[FFT_A], %%ymm3, %%ymm5")) /* ymm5 = c_re = x_re * b_re + x_im * b_im */ \
401 __ASM_EMIT(FFT_FMA("vsubps %%ymm4, %%ymm7, %%ymm4", "vfmsub231ps 0x00 + %[FFT_A], %%ymm7, %%ymm4")) /* ymm4 = c_im = x_re * b_im - x_im * b_re */ \
402 __ASM_EMIT("vsubps %%ymm5, %%ymm2, %%ymm0") /* ymm0 = a_re - c_re */ \
403 __ASM_EMIT("vsubps %%ymm4, %%ymm6, %%ymm1") /* ymm1 = a_im - c_im */ \
404 __ASM_EMIT("vaddps %%ymm5, %%ymm2, %%ymm2") /* ymm2 = a_re + c_re */ \
405 __ASM_EMIT("vaddps %%ymm4, %%ymm6, %%ymm3") /* ymm3 = a_im + c_im */ \
406 /* Store */
407 __ASM_EMIT("vmovups %%xmm2, 0x00(%[dst])")
408 __ASM_EMIT("vmovups %%xmm0, 0x10(%[dst])")
409 __ASM_EMIT("vmovups %%xmm3, 0x20(%[dst])")
410 __ASM_EMIT("vmovups %%xmm1, 0x30(%[dst])")
411 __ASM_EMIT("vextractf128 $1, %%ymm2, 0x40(%[dst])")
412 __ASM_EMIT("vextractf128 $1, %%ymm0, 0x50(%[dst])")
413 __ASM_EMIT("vextractf128 $1, %%ymm3, 0x60(%[dst])")
414 __ASM_EMIT("vextractf128 $1, %%ymm1, 0x70(%[dst])")
415 __ASM_EMIT("add $0x80, %[dst]")
416
417 : [dst] "+r" (dst), [index] "+r"(index)
418 : [src] "r" (src), [regs] "r" (regs),
419 [FFT_A] "o" (FFT_A)
420 : "cc", "memory",
421 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
422 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
423 );
424 }
425 }
426
427 static inline void FFT_PSCRAMBLE_COPY_REVERSE_NAME(float *dst, const float *src, size_t rank)
428 {
429 size_t regs = 1 << rank;
430
431 for (size_t i=0; i<regs; ++i)
432 {
433 size_t index = reverse_bits(FFT_TYPE(i), rank);
434
435 ARCH_X86_ASM
436 (
437 /* Load scalar values */
438 __ASM_EMIT("vmovlps (%[src], %[index], 8), %%xmm0, %%xmm0") /* xmm0 = r0 i0 x x */
439 __ASM_EMIT("add %[regs], %[index]")
440 __ASM_EMIT("vmovlps (%[src], %[index], 8), %%xmm2, %%xmm2") /* xmm2 = r8 i8 x x */
441 __ASM_EMIT("add %[regs], %[index]")
442 __ASM_EMIT("vmovlps (%[src], %[index], 8), %%xmm4, %%xmm4") /* xmm4 = r4 i4 x x */
443 __ASM_EMIT("add %[regs], %[index]")
444 __ASM_EMIT("vmovlps (%[src], %[index], 8), %%xmm6, %%xmm6") /* xmm6 = r12 i12 x x */
445 __ASM_EMIT("add %[regs], %[index]")
446 __ASM_EMIT("vmovlps (%[src], %[index], 8), %%xmm1, %%xmm1") /* xmm1 = r2 i2 x x */
447 __ASM_EMIT("add %[regs], %[index]")
448 __ASM_EMIT("vmovlps (%[src], %[index], 8), %%xmm3, %%xmm3") /* xmm3 = r10 i10 x x */
449 __ASM_EMIT("add %[regs], %[index]")
450 __ASM_EMIT("vmovlps (%[src], %[index], 8), %%xmm5, %%xmm5") /* xmm5 = r6 i6 x x */
451 __ASM_EMIT("add %[regs], %[index]")
452 __ASM_EMIT("vmovlps (%[src], %[index], 8), %%xmm7, %%xmm7") /* xmm7 = r14 i14 x x */
453 __ASM_EMIT("add %[regs], %[index]")
454
455 __ASM_EMIT("vmovhps (%[src], %[index], 8), %%xmm0, %%xmm0") /* xmm0 = r0 i0 r1 i1 */
456 __ASM_EMIT("add %[regs], %[index]")
457 __ASM_EMIT("vmovhps (%[src], %[index], 8), %%xmm2, %%xmm2") /* xmm2 = r8 i8 r9 i9 */
458 __ASM_EMIT("add %[regs], %[index]")
459 __ASM_EMIT("vmovhps (%[src], %[index], 8), %%xmm4, %%xmm4") /* xmm4 = r4 i4 r5 i5 */
460 __ASM_EMIT("add %[regs], %[index]")
461 __ASM_EMIT("vmovhps (%[src], %[index], 8), %%xmm6, %%xmm6") /* xmm6 = r12 i12 r13 r14 */
462 __ASM_EMIT("add %[regs], %[index]")
463 __ASM_EMIT("vmovhps (%[src], %[index], 8), %%xmm1, %%xmm1") /* xmm1 = r2 i2 r3 i3 */
464 __ASM_EMIT("add %[regs], %[index]")
465 __ASM_EMIT("vmovhps (%[src], %[index], 8), %%xmm3, %%xmm3") /* xmm3 = r10 i10 r11 i11 */
466 __ASM_EMIT("add %[regs], %[index]")
467 __ASM_EMIT("vmovhps (%[src], %[index], 8), %%xmm5, %%xmm5") /* xmm5 = r6 i6 r7 i7 */
468 __ASM_EMIT("add %[regs], %[index]")
469 __ASM_EMIT("vmovhps (%[src], %[index], 8), %%xmm7, %%xmm7") /* xmm7 = r14 i14 r15 i15 */
470 __ASM_EMIT("add %[regs], %[index]")
471
472 __ASM_EMIT("vinsertf128 $1, %%xmm2, %%ymm0, %%ymm0") /* ymm0 = r0 i0 r1 i1 r8 i8 r9 i9 */
473 __ASM_EMIT("vinsertf128 $1, %%xmm3, %%ymm1, %%ymm1") /* ymm1 = r2 i2 r3 i3 r10 i10 r11 i11 */
474 __ASM_EMIT("vinsertf128 $1, %%xmm6, %%ymm4, %%ymm4") /* ymm4 = r4 i4 r5 i5 r12 i12 r13 r14 */
475 __ASM_EMIT("vinsertf128 $1, %%xmm7, %%ymm5, %%ymm5") /* ymm5 = r6 i6 r7 i7 r14 i14 r15 i15 */
476 __ASM_EMIT("vshufps $0xdd, %%ymm1, %%ymm0, %%ymm2") /* ymm2 = i0 i1 i2 i3 i8 i9 i10 i11 */
477 __ASM_EMIT("vshufps $0x88, %%ymm1, %%ymm0, %%ymm0") /* ymm0 = r0 r1 r2 r3 r8 r9 r10 r11 */
478 __ASM_EMIT("vshufps $0xdd, %%ymm5, %%ymm4, %%ymm6") /* ymm6 = i4 i5 i6 i7 i12 i13 i14 i15 */
479 __ASM_EMIT("vshufps $0x88, %%ymm5, %%ymm4, %%ymm4") /* ymm4 = r4 r5 r6 r7 r12 r13 r14 r15 */
480 /* 1st-order 4x butterfly */
481 __ASM_EMIT("vhsubps %%ymm4, %%ymm0, %%ymm1") /* ymm1 = r0-r1 r2-r3 r4-r5 r6-r7 = r1' r3' r5' r7' */
482 __ASM_EMIT("vhsubps %%ymm6, %%ymm2, %%ymm3") /* ymm3 = i0-i1 i2-i3 i4-i5 i6-i7 = i1' i3' i5' i7' */
483 __ASM_EMIT("vhaddps %%ymm4, %%ymm0, %%ymm0") /* ymm0 = r0+r1 r2+r3 r4+r5 r6+r7 = r0' r2' r4' r6' */
484 __ASM_EMIT("vhaddps %%ymm6, %%ymm2, %%ymm2") /* ymm2 = i0+i1 i2+i3 i4+i5 i6+i7 = i0' i2' i4' i6' */
485 /* 2nd-order 4x butterfly */
486 __ASM_EMIT("vblendps $0xaa, %%ymm3, %%ymm1, %%ymm4") /* ymm4 = r1' i3' r5' i7' */
487 __ASM_EMIT("vblendps $0xaa, %%ymm1, %%ymm3, %%ymm5") /* ymm5 = i1' r3' i5' r7' */
488 __ASM_EMIT("vhsubps %%ymm4, %%ymm0, %%ymm1") /* ymm1 = r0'-r2' r4'-r6' r1'-i3' r5'-i7' = r2" r6" r1" r5" */
489 __ASM_EMIT("vhsubps %%ymm5, %%ymm2, %%ymm3") /* ymm3 = i0'-i2' i4'-i6' i1'-r3' i5'-r7' = i2" i6" i3" i7" */
490 __ASM_EMIT("vhaddps %%ymm4, %%ymm0, %%ymm0") /* ymm0 = r0'+r2' r4'+r6' r1'+i3' r5'+i7' = r0" r4" r3" r7" */
491 __ASM_EMIT("vhaddps %%ymm5, %%ymm2, %%ymm2") /* ymm2 = i0'+i2' i4'+i6' i1'+r3' i5'+r7' = i0" i4" i1" i5" */
492 __ASM_EMIT("vblendps $0xcc, %%ymm1, %%ymm0, %%ymm4") /* ymm4 = r0" i4" r1" r5" */
493 __ASM_EMIT("vblendps $0xcc, %%ymm0, %%ymm1, %%ymm5") /* ymm5 = r2" r6" r3" r7" */
494 __ASM_EMIT("vshufps $0x88, %%ymm3, %%ymm2, %%ymm6") /* ymm6 = i0" i1" i2" i3" */
495 __ASM_EMIT("vshufps $0xdd, %%ymm3, %%ymm2, %%ymm7") /* ymm7 = i4" i5" i6" i7" */
496 __ASM_EMIT("vshufps $0x88, %%ymm5, %%ymm4, %%ymm2") /* ymm2 = r0" r1" r2" r3" */
497 __ASM_EMIT("vshufps $0xdd, %%ymm5, %%ymm4, %%ymm3") /* ymm3 = r4" r5" r6" r7" */
498 /* 3rd-order 8x butterfly */
499 __ASM_EMIT("vmulps 0x20 + %[FFT_A], %%ymm3, %%ymm4") /* ymm4 = x_im * b_re */ \
500 __ASM_EMIT("vmulps 0x20 + %[FFT_A], %%ymm7, %%ymm5") /* ymm5 = x_im * b_im */ \
501 __ASM_EMIT(FFT_FMA("vmulps 0x00 + %[FFT_A], %%ymm3, %%ymm3", "")) /* ymm3 = x_re * b_re */ \
502 __ASM_EMIT(FFT_FMA("vmulps 0x00 + %[FFT_A], %%ymm7, %%ymm7", "")) /* ymm7 = x_re * b_im */ \
503 __ASM_EMIT(FFT_FMA("vsubps %%ymm5, %%ymm3, %%ymm5", "vfmsub231ps 0x00 + %[FFT_A], %%ymm3, %%ymm5")) /* ymm5 = c_re = x_re * b_re - x_im * b_im */ \
504 __ASM_EMIT(FFT_FMA("vaddps %%ymm4, %%ymm7, %%ymm4", "vfmadd231ps 0x00 + %[FFT_A], %%ymm7, %%ymm4")) /* ymm4 = c_im = x_re * b_im + x_im * b_re */ \
505 __ASM_EMIT("vsubps %%ymm5, %%ymm2, %%ymm0") /* ymm0 = a_re - c_re */ \
506 __ASM_EMIT("vsubps %%ymm4, %%ymm6, %%ymm1") /* ymm1 = a_im - c_im */ \
507 __ASM_EMIT("vaddps %%ymm5, %%ymm2, %%ymm2") /* ymm2 = a_re + c_re */ \
508 __ASM_EMIT("vaddps %%ymm4, %%ymm6, %%ymm3") /* ymm3 = a_im + c_im */ \
509 /* Store */
510 __ASM_EMIT("vmovups %%xmm2, 0x00(%[dst])")
511 __ASM_EMIT("vmovups %%xmm0, 0x10(%[dst])")
512 __ASM_EMIT("vmovups %%xmm3, 0x20(%[dst])")
513 __ASM_EMIT("vmovups %%xmm1, 0x30(%[dst])")
514 __ASM_EMIT("vextractf128 $1, %%ymm2, 0x40(%[dst])")
515 __ASM_EMIT("vextractf128 $1, %%ymm0, 0x50(%[dst])")
516 __ASM_EMIT("vextractf128 $1, %%ymm3, 0x60(%[dst])")
517 __ASM_EMIT("vextractf128 $1, %%ymm1, 0x70(%[dst])")
518 __ASM_EMIT("add $0x80, %[dst]")
519
520 : [dst] "+r" (dst), [index] "+r"(index)
521 : [src] "r" (src), [regs] "r" (regs),
522 [FFT_A] "o" (FFT_A)
523 : "cc", "memory",
524 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
525 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
526 );
527 }
528 }
529 }
530 }
531
532 #undef FFT_PSCRAMBLE_SELF_DIRECT_NAME
533 #undef FFT_PSCRAMBLE_SELF_REVERSE_NAME
534 #undef FFT_PSCRAMBLE_COPY_DIRECT_NAME
535 #undef FFT_PSCRAMBLE_COPY_REVERSE_NAME
536 #undef FFT_TYPE
537 #undef FFT_FMA