Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/x86/avx512/fft/scramble.h

raw
1 /*
2 * Copyright (C) 2025 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2025 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 21 февр. 2025 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_X86_AVX512_IMPL
23 #error "This header should not be included directly"
24 #endif /* PRIVATE_DSP_ARCH_X86_AVX512_IMPL */
25
26 #define FFT_SCRAMBLE_LOAD4X2(re, im) \
27 __ASM_EMIT("vinsertps $0x00, (%[src_re], %[index], 4), " re ", " re) /* re = R0 x x x */ \
28 __ASM_EMIT("vinsertps $0x00, (%[src_im], %[index], 4), " im ", " im) /* im = I0 x x x */ \
29 __ASM_EMIT("add %[regs], %[index]") \
30 __ASM_EMIT("vinsertps $0x10, (%[src_re], %[index], 4), " re ", " re) /* re = R0 R1 x x */ \
31 __ASM_EMIT("vinsertps $0x10, (%[src_im], %[index], 4), " im ", " im) /* im = I0 I1 x x */ \
32 __ASM_EMIT("add %[regs], %[index]") \
33 __ASM_EMIT("vinsertps $0x20, (%[src_re], %[index], 4), " re ", " re) /* re = R0 R1 R2 x */ \
34 __ASM_EMIT("vinsertps $0x20, (%[src_im], %[index], 4), " im ", " im) /* im = I0 I1 I2 x */ \
35 __ASM_EMIT("add %[regs], %[index]") \
36 __ASM_EMIT("vinsertps $0x30, (%[src_re], %[index], 4), " re ", " re) /* re = R0 R1 R2 R3 */ \
37 __ASM_EMIT("vinsertps $0x30, (%[src_im], %[index], 4), " im ", " im) /* im = I0 I1 I2 I3 */ \
38 __ASM_EMIT("add %[regs], %[index]")
39
40 namespace lsp
41 {
42 namespace avx512
43 {
44 static inline void FFT_SCRAMBLE_SELF_DIRECT_NAME(float *dst_re, float *dst_im, size_t rank)
45 {
46 // Calculate number of items
47 size_t items = (1 << rank) - 1;
48
49 for (size_t i = 1; i < items; ++i)
50 {
51 size_t j = reverse_bits(FFT_TYPE(i), rank); /* Reverse the order of the bits */
52 if (i >= j)
53 continue;
54
55 /* Copy the values from the reversed position */
56 ARCH_X86_ASM
57 (
58 __ASM_EMIT("vmovss (%[dst_re], %[i], 4), %%xmm0")
59 __ASM_EMIT("vmovss (%[dst_im], %[i], 4), %%xmm1")
60 __ASM_EMIT("vmovss (%[dst_re], %[j], 4), %%xmm2")
61 __ASM_EMIT("vmovss (%[dst_im], %[j], 4), %%xmm3")
62 __ASM_EMIT("vmovss %%xmm2, (%[dst_re], %[i], 4)")
63 __ASM_EMIT("vmovss %%xmm3, (%[dst_im], %[i], 4)")
64 __ASM_EMIT("vmovss %%xmm0, (%[dst_re], %[j], 4)")
65 __ASM_EMIT("vmovss %%xmm1, (%[dst_im], %[j], 4)")
66 :
67 : [dst_re] "r" (dst_re), [dst_im] "r" (dst_im),
68 [i] "r"(i), [j] "r"(j)
69 : "memory",
70 "%xmm0", "%xmm1", "%xmm2", "%xmm3"
71 );
72 }
73
74 // Perform butterfly 8x
75 size_t off = 0;
76 items = 1 << (rank - 3);
77
78 // Perform 4-element butterflies
79 ARCH_X86_ASM
80 (
81 /* Loop 4x 4-element butterflies */
82 __ASM_EMIT("sub $4, %[items]")
83 __ASM_EMIT("jb 2f")
84 __ASM_EMIT("1:")
85 /* Load data to registers */
86 __ASM_EMIT("vmovups 0x00(%[dst_re], %[off]), %%zmm0") /* zmm0 = r0 r1 r2 r3 r4 r5 r6 r7 r8 r9 r10 r11 r12 r13 r14 r15 */
87 __ASM_EMIT("vmovups 0x40(%[dst_re], %[off]), %%zmm4") /* zmm4 = r16 r17 r18 r19 r20 r21 r22 r23 r24 r25 r26 r27 r28 r29 r30 r31 */
88 __ASM_EMIT("vmovups 0x00(%[dst_im], %[off]), %%zmm2") /* zmm2 = i0 i1 i2 i3 i4 i5 i6 i7 i8 i9 i10 i11 i12 i13 i14 i15 */
89 __ASM_EMIT("vmovups 0x40(%[dst_im], %[off]), %%zmm5") /* zmm5 = i16 i17 i18 i19 i20 i21 i22 i23 i24 i25 i26 i27 i28 i29 i30 i31 */
90 /* 1st-order 16x butterfly */
91 __ASM_EMIT("vmovaps %%zmm0, %%zmm1") /* zmm1 = zmm0 = r0 r1 r2 r3 r4 r5 r6 r7 r8 r9 r10 r11 r12 r13 r14 r15 */
92 __ASM_EMIT("vmovaps %%zmm2, %%zmm3") /* zmm3 = zmm2 = i0 i1 i2 i3 i4 i5 i6 i7 i8 i9 i10 i11 i12 i13 i14 i15 */
93 __ASM_EMIT("vmovaps 0x00 + %[FFT_I], %%zmm6") /* zmm6 = shuffle_a[0] */
94 __ASM_EMIT("vmovaps 0x40 + %[FFT_I], %%zmm7") /* zmm7 = shuffle_b[0] */
95 __ASM_EMIT("vpermt2ps %%zmm4, %%zmm6, %%zmm0") /* zmm0 = r0 r2 r4 r6 r8 r10 r12 r14 r16 r18 r20 r22 r24 r26 r28 r30 */
96 __ASM_EMIT("vpermt2ps %%zmm4, %%zmm7, %%zmm1") /* zmm1 = r1 r3 r5 r7 r9 r11 r13 r15 r17 r19 r21 r23 r25 r27 r29 r31 */
97 __ASM_EMIT("vpermt2ps %%zmm5, %%zmm6, %%zmm2") /* zmm2 = i0 i2 i4 i6 i8 i10 i12 i14 i16 i18 i20 i22 i24 i26 i28 i30 */
98 __ASM_EMIT("vpermt2ps %%zmm5, %%zmm7, %%zmm3") /* zmm3 = i1 i3 i5 i7 i9 i11 i13 i15 i17 i19 i21 i23 i25 i27 i29 i31 */
99 __ASM_EMIT("vsubps %%zmm1, %%zmm0, %%zmm4") /* zmm4 = r0-r1 r2-r3 r4-r5 r6-r7 ... = r1' r3' r5' r7' r9' r11' r13' r15' r17' r19' r21' r23' r25' r27' r29' r31' */
100 __ASM_EMIT("vsubps %%zmm3, %%zmm2, %%zmm5") /* zmm5 = i0-i1 i2-i3 i4-i5 i6-i7 ... = i1' i3' i5' i7' i9' i11' i13' i15' i17' i19' i21' i23' i25' i27' i29' i31' */
101 __ASM_EMIT("vaddps %%zmm1, %%zmm0, %%zmm0") /* zmm0 = r0+r1 r2+r3 r4+r5 r6+r7 ... = r0' r2' r4' r6' r8' r10' r12' r14' r16' r18' r20' r22' r24' r26' r28' r30' */
102 __ASM_EMIT("vaddps %%zmm3, %%zmm2, %%zmm2") /* zmm2 = i0+i1 i2+i3 i4+i5 i6+i7 ... = i0' i2' i4' i6' i8' i10' i12' i14' i16' i18' i20' i22' i24' i26' i28' i30' */
103 /* 2nd-order 16x butterfly */
104 __ASM_EMIT("vmovaps %%zmm0, %%zmm1") /* zmm1 = zmm0 = r0' r2' r4' r6' r8' r10' r12' r14' r16' r18' r20' r22' r24' r26' r28' r30' */
105 __ASM_EMIT("vmovaps %%zmm2, %%zmm3") /* zmm3 = zmm2 = i0' i2' i4' i6' i8' i10' i12' i14' i16' i18' i20' i22' i24' i26' i28' i30' */
106 __ASM_EMIT("vmovaps 0x80 + %[FFT_I], %%zmm6") /* zmm6 = shuffle_a[1] */
107 __ASM_EMIT("vmovaps 0xc0 + %[FFT_I], %%zmm7") /* zmm7 = shuffle_b[1] */
108 __ASM_EMIT("vpermt2ps %%zmm4, %%zmm6, %%zmm0") /* zmm0 = r0' r1' r4' r5' r8' r9' r12' r13' r16' r17' r20' r21' r24' r25' r28' r29' */
109 __ASM_EMIT("vpermt2ps %%zmm5, %%zmm6, %%zmm2") /* zmm2 = i0' i1' i4' i5' i8' i9' i12' i13' i16' i17' i20' i21' i24' i25' i28' i29' */
110 __ASM_EMIT("vpermt2ps %%zmm5, %%zmm7, %%zmm1") /* zmm1 = r2' i3' r6' i7' r10' i11' r14' i15' r18' i19' r22' i23' r26' i27' r30' i31' */
111 __ASM_EMIT("vpermt2ps %%zmm4, %%zmm7, %%zmm3") /* zmm3 = i2' r3' i6' r7' i10' r11' i14' r15' i18' r19' i22' r23' i26' r27' i30' r31' */
112 __ASM_EMIT("vsubps %%zmm1, %%zmm0, %%zmm4") /* zmm4 = r2" r3" r6" r7" r10" r11" r14" r15" r18" r19" r22" r23" r26" r27" r30" r31" = r0'-r2' r1'-i3' r4'-r6' r5'-i7' r8'-r10' r9'-i11' r12'-r14' r13'-i15' r16'-r18' r17'-i19' r20'-r22' r21'-i23' r24'-r26' r25'-i27' r28'-r30' r29'-i31' */
113 __ASM_EMIT("vsubps %%zmm3, %%zmm2, %%zmm5") /* zmm5 = i2" i1" i6" i5" i10" i9" i14" i13" i18" i17" i22" i21" i26" i25" i30" i29" = i0'-i2' i1'-r3' i4'-i6' i5'-r7' i8'-i10' i9'-r11' i12'-i14' i13'-r15' i16'-i18' i17'-r19' i20'-i22' i21'-r23' i24'-i26' i25'-r27' i28'-i30' i29'-r31' */
114 __ASM_EMIT("vaddps %%zmm1, %%zmm0, %%zmm0") /* zmm0 = r0" r1" r4" r5" r8" r9" r12" r13" r16" r17" r20" r21" r24" r25" r28" r29" = r0'+r2' r1'+i3' r4'+r6' r5'+i7' r8'+r10' r9'+i11' r12'+r14' r13'+i15' r16'+r18' r17'+i19' r20'+r22' r21'+i23' r24'+r26' r25'+i27' r28'+r30' r29'+i31' */
115 __ASM_EMIT("vaddps %%zmm3, %%zmm2, %%zmm2") /* zmm2 = i0" i3" i4" i7" i8" i11" i12" i15" i16" i19" i20" i23" i24" i27" i28" i31" = i0'+i2' i1'+r3' i4'+i6' i5'+r7' i8'+i10' i9'+r11' i12'+i14' i13'+r15' i16'+i18' i17'+r19' i20'+i22' i21'+r23' i24'+i26' i25'+r27' i28'+i30' i29'+r31' */
116 /* 3rd-order 16x butterfly */
117 __ASM_EMIT("vmovaps %%zmm0, %%zmm1") /* zmm1 = zmm0 = r0" r1" r4" r5" r8" r9" r12" r13" r16" r17" r20" r21" r24" r25" r28" r29" */
118 __ASM_EMIT("vmovaps %%zmm2, %%zmm3") /* zmm3 = zmm2 = i0" i3" i4" i7" i8" i11" i12" i15" i16" i19" i20" i23" i24" i27" i28" i31" */
119 __ASM_EMIT("vmovaps 0x100 + %[FFT_I], %%zmm6") /* zmm6 = shuffle_a[2] */
120 __ASM_EMIT("vmovaps 0x140 + %[FFT_I], %%zmm7") /* zmm7 = shuffle_b[2] */
121 __ASM_EMIT("vpermt2ps %%zmm4, %%zmm6, %%zmm0") /* zmm0 = a_re = r0" r1" r2" r3" r8" r9" r10" r11" r16" r17" r18" r19" r24" r25" r26" r27" */
122 __ASM_EMIT("vpermt2ps %%zmm5, %%zmm7, %%zmm2") /* zmm2 = a_im = i0" i1" i2" i3" i8" i9" i10" i11" i16" i17" i18" i19" i24" i25" i26" i27" */
123 __ASM_EMIT("vmovaps 0x180 + %[FFT_I], %%zmm6") /* zmm6 = shuffle_a[3] */
124 __ASM_EMIT("vmovaps 0x1c0 + %[FFT_I], %%zmm7") /* zmm7 = shuffle_b[3] */
125 __ASM_EMIT("vpermt2ps %%zmm4, %%zmm6, %%zmm1") /* zmm1 = b_re = r4" r5" r6" r7" r12" r13" r14" r15" r20" r21" r22" r23" r28" r29" r30" r31" */
126 __ASM_EMIT("vpermt2ps %%zmm5, %%zmm7, %%zmm3") /* zmm3 = b_im = i4" i5" i6" i7" i12" i13" i14" i15" i20" i21" i22" i23" i28" i29" i30" i31" */
127 __ASM_EMIT("vmulps 0x40 + %[FFT_A], %%zmm1, %%zmm4") /* zmm4 = x_im * b_re */
128 __ASM_EMIT("vmulps 0x40 + %[FFT_A], %%zmm3, %%zmm5") /* zmm5 = x_im * b_im */
129 __ASM_EMIT("vfmadd231ps 0x00 + %[FFT_A], %%zmm1, %%zmm5") /* zmm5 = c_re = x_re * b_re + x_im * b_im */
130 __ASM_EMIT("vfmsub231ps 0x00 + %[FFT_A], %%zmm3, %%zmm4") /* zmm4 = c_im = x_re * b_im - x_im * b_re */
131 __ASM_EMIT("vsubps %%zmm5, %%zmm0, %%zmm1") /* zmm1 = a_re - c_re = r4` r5` r6` r7` r12` r13` r14` r15` r20` r21` r22` r23` r28` r29` r30` r31` */
132 __ASM_EMIT("vsubps %%zmm4, %%zmm2, %%zmm3") /* zmm3 = a_im - c_im = i4` i5` i6` i7` i12` i13` i14` i15` i20` i21` i22` i23` i28` i29` i30` i31` */
133 __ASM_EMIT("vaddps %%zmm5, %%zmm0, %%zmm0") /* zmm0 = a_re + c_re = r0` r1` r2` r3` r8` r9` r10` r11` r16` r17` r18` r19` r24` r25` r26` r27` */
134 __ASM_EMIT("vaddps %%zmm4, %%zmm2, %%zmm2") /* zmm2 = a_im + c_im = i0` i1` i2` i3` i8` i9` i10` i11` i16` i17` i18` i19` i24` i25` i26` i27` */
135 /* 4th-order 16x butterfly */
136 __ASM_EMIT("vmovaps %%zmm0, %%zmm4") /* zmm4 = zmm0 = r0` r1` r2` r3` r8` r9` r10` r11` r16` r17` r18` r19` r24` r25` r26` r27` */
137 __ASM_EMIT("vmovaps %%zmm2, %%zmm5") /* zmm5 = zmm2 = i0` i1` i2` i3` i8` i9` i10` i11` i16` i17` i18` i19` i24` i25` i26` i27` */
138 __ASM_EMIT("vmovaps 0x200 + %[FFT_I], %%zmm6") /* zmm6 = shuffle_a[4] */
139 __ASM_EMIT("vmovaps 0x240 + %[FFT_I], %%zmm7") /* zmm7 = shuffle_b[4] */
140 __ASM_EMIT("vpermt2ps %%zmm1, %%zmm6, %%zmm0") /* zmm0 = a_re = r0` r1` r2` r3` r4` r5` r6` r7` r16` r17` r18` r19` r20` r21` r22` r23` */
141 __ASM_EMIT("vpermt2ps %%zmm1, %%zmm7, %%zmm4") /* zmm4 = b_re = r8` r9` r10` r11` r12` r13` r14` r15` r24` r25` r26` r27` r28` r29` r30` r31` */
142 __ASM_EMIT("vpermt2ps %%zmm3, %%zmm6, %%zmm2") /* zmm2 = a_im = i0` i1` i2` i3` i4` i5` i6` i7` i16` i17` i18` i19` i20` i21` i22` i23` */
143 __ASM_EMIT("vpermt2ps %%zmm3, %%zmm7, %%zmm5") /* zmm5 = b_im = i8` i9` i10` i11` i12` i13` i14` i15` i24` i25` i26` i27` i28` i29` i30` i31` */
144 __ASM_EMIT("vmulps 0xc0 + %[FFT_A], %%zmm4, %%zmm1") /* zmm1 = x_im * b_re */
145 __ASM_EMIT("vmulps 0xc0 + %[FFT_A], %%zmm5, %%zmm3") /* zmm3 = x_im * b_im */
146 __ASM_EMIT("vfmadd231ps 0x80 + %[FFT_A], %%zmm4, %%zmm3") /* zmm3 = c_re = x_re * b_re + x_im * b_im */
147 __ASM_EMIT("vfmsub231ps 0x80 + %[FFT_A], %%zmm5, %%zmm1") /* zmm1 = c_im = x_re * b_im - x_im * b_re */
148 __ASM_EMIT("vsubps %%zmm3, %%zmm0, %%zmm4") /* zmm4 = a_re - c_re = R8 R9 R10 R11 R12 R13 R14 R15 R24 R25 R26 R27 R28 R29 R30 R31 */
149 __ASM_EMIT("vsubps %%zmm1, %%zmm2, %%zmm5") /* zmm5 = a_im - c_im = I8 I9 I10 I11 I12 I13 I14 I15 I24 I25 I26 I27 I28 I29 I30 I31 */
150 __ASM_EMIT("vaddps %%zmm3, %%zmm0, %%zmm0") /* zmm0 = a_re + c_re = R0 R1 R2 R3 R4 R5 R6 R7 R16 R17 R18 R19 R20 R21 R22 R23 */
151 __ASM_EMIT("vaddps %%zmm1, %%zmm2, %%zmm2") /* zmm2 = a_im + c_im = I0 I1 I2 I3 I4 I5 I6 I7 I16 I17 I18 I19 I20 I21 I22 I23 */
152 /* Store */
153 __ASM_EMIT("vmovups %%ymm0, 0x00(%[dst_re], %[off])")
154 __ASM_EMIT("vmovups %%ymm4, 0x20(%[dst_re], %[off])")
155 __ASM_EMIT("vextractf64x4 $1, %%zmm0, 0x40(%[dst_re], %[off])")
156 __ASM_EMIT("vextractf64x4 $1, %%zmm4, 0x60(%[dst_re], %[off])")
157 __ASM_EMIT("vmovups %%ymm2, 0x00(%[dst_im], %[off])")
158 __ASM_EMIT("vmovups %%ymm5, 0x20(%[dst_im], %[off])")
159 __ASM_EMIT("vextractf64x4 $1, %%zmm2, 0x40(%[dst_im], %[off])")
160 __ASM_EMIT("vextractf64x4 $1, %%zmm5, 0x60(%[dst_im], %[off])")
161 /* Move pointers and repeat */
162 __ASM_EMIT("add $0x80, %[off]")
163 __ASM_EMIT("sub $4, %[items]")
164 __ASM_EMIT("jae 1b")
165 __ASM_EMIT("2:")
166
167 /* Loop 2x 4-element butterflies */
168 __ASM_EMIT("add $2, %[items]")
169 __ASM_EMIT("jl 4f")
170 /* Load data to registers */
171 __ASM_EMIT("vmovups 0x00(%[dst_re], %[off]), %%xmm0") /* xmm0 = r0 r1 r2 r3 */
172 __ASM_EMIT("vmovups 0x10(%[dst_re], %[off]), %%xmm4") /* xmm4 = r4 r5 r6 r7 */
173 __ASM_EMIT("vinsertf128 $1, 0x20(%[dst_re], %[off]), %%ymm0, %%ymm0") /* ymm0 = r0 r1 r2 r3 */
174 __ASM_EMIT("vinsertf128 $1, 0x30(%[dst_re], %[off]), %%ymm4, %%ymm4") /* ymm4 = r4 r5 r6 r7 */
175 __ASM_EMIT("vmovups 0x00(%[dst_im], %[off]), %%xmm2") /* xmm2 = i0 i1 i2 i3 */
176 __ASM_EMIT("vmovups 0x10(%[dst_im], %[off]), %%xmm6") /* xmm6 = i4 i5 i6 i7 */
177 __ASM_EMIT("vinsertf128 $1, 0x20(%[dst_im], %[off]), %%ymm2, %%ymm2") /* ymm2 = i0 i1 i2 i3 */
178 __ASM_EMIT("vinsertf128 $1, 0x30(%[dst_im], %[off]), %%ymm6, %%ymm6") /* ymm6 = i4 i5 i6 i7 */
179 /* 1st-order 8x butterfly */
180 __ASM_EMIT("vhsubps %%ymm4, %%ymm0, %%ymm1") /* ymm1 = r0-r1 r2-r3 r4-r5 r6-r7 = r1' r3' r5' r7' */
181 __ASM_EMIT("vhsubps %%ymm6, %%ymm2, %%ymm3") /* ymm3 = i0-i1 i2-i3 i4-i5 i6-i7 = i1' i3' i5' i7' */
182 __ASM_EMIT("vhaddps %%ymm4, %%ymm0, %%ymm0") /* ymm0 = r0+r1 r2+r3 r4+r5 r6+r7 = r0' r2' r4' r6' */
183 __ASM_EMIT("vhaddps %%ymm6, %%ymm2, %%ymm2") /* ymm2 = i0+i1 i2+i3 i4+i5 i6+i7 = i0' i2' i4' i6' */
184 /* 2nd-order 8x butterfly */
185 __ASM_EMIT("vblendps $0xaa, %%ymm3, %%ymm1, %%ymm4") /* ymm4 = r1' i3' r5' i7' */
186 __ASM_EMIT("vblendps $0xaa, %%ymm1, %%ymm3, %%ymm5") /* ymm5 = i1' r3' i5' r7' */
187 __ASM_EMIT("vhsubps %%ymm4, %%ymm0, %%ymm1") /* ymm1 = r0'-r2' r4'-r6' r1'-i3' r5'-i7' = r2" r6" r3" r7" */
188 __ASM_EMIT("vhsubps %%ymm5, %%ymm2, %%ymm3") /* ymm3 = i0'-i2' i4'-i6' i1'-r3' i5'-r7' = i2" i6" i1" i5" */
189 __ASM_EMIT("vhaddps %%ymm4, %%ymm0, %%ymm0") /* ymm0 = r0'+r2' r4'+r6' r1'+i3' r5'+i7' = r0" r4" r1" r5" */
190 __ASM_EMIT("vhaddps %%ymm5, %%ymm2, %%ymm2") /* ymm2 = i0'+i2' i4'+i6' i1'+r3' i5'+r7' = i0" i4" i3" i7" */
191 __ASM_EMIT("vblendps $0xcc, %%ymm3, %%ymm2, %%ymm4") /* ymm4 = i0" i4" i1" i5" */
192 __ASM_EMIT("vblendps $0xcc, %%ymm2, %%ymm3, %%ymm5") /* ymm5 = i2" i6" i3" i7" */
193 __ASM_EMIT("vshufps $0x88, %%ymm1, %%ymm0, %%ymm2") /* ymm2 = r0" r1" r2" r3" */
194 __ASM_EMIT("vshufps $0xdd, %%ymm1, %%ymm0, %%ymm3") /* ymm3 = r4" r5" r6" r7" */
195 __ASM_EMIT("vshufps $0x88, %%ymm5, %%ymm4, %%ymm6") /* ymm6 = i0" i1" i2" i3" */
196 __ASM_EMIT("vshufps $0xdd, %%ymm5, %%ymm4, %%ymm7") /* ymm7 = i4" i5" i6" i7" */
197 /* 3rd-order 8x butterfly */
198 __ASM_EMIT("vmulps 0x40 + %[FFT_A], %%ymm3, %%ymm4") /* ymm4 = x_im * b_re */
199 __ASM_EMIT("vmulps 0x40 + %[FFT_A], %%ymm7, %%ymm5") /* ymm5 = x_im * b_im */
200 __ASM_EMIT("vfmadd231ps 0x00 + %[FFT_A], %%ymm3, %%ymm5") /* ymm5 = c_re = x_re * b_re + x_im * b_im */
201 __ASM_EMIT("vfmsub231ps 0x00 + %[FFT_A], %%ymm7, %%ymm4") /* ymm4 = c_im = x_re * b_im - x_im * b_re */
202 __ASM_EMIT("vsubps %%ymm5, %%ymm2, %%ymm0") /* ymm0 = a_re - c_re = r8 r9 r10 r11 r12 r13 r14 r15 */
203 __ASM_EMIT("vsubps %%ymm4, %%ymm6, %%ymm1") /* ymm1 = a_im - c_im = i8 i9 i10 i11 i12 i13 i14 i15 */
204 __ASM_EMIT("vaddps %%ymm5, %%ymm2, %%ymm2") /* ymm2 = a_re + c_re = r0 r1 r2 r3 r4 r5 r6 r7 */
205 __ASM_EMIT("vaddps %%ymm4, %%ymm6, %%ymm3") /* ymm3 = a_im + c_im = i0 i1 i2 i3 i4 i5 i6 i7 */
206 /* 4th-order 8x butterfly */
207 __ASM_EMIT("vextractf128 $1, %%ymm2, %%xmm4") /* xmm4 = r4 r5 r6 r7 */
208 __ASM_EMIT("vextractf128 $1, %%ymm3, %%xmm5") /* xmm5 = i4 i5 i6 i7 */
209 __ASM_EMIT("vinsertf128 $1, %%xmm0, %%ymm2, %%ymm2") /* ymm2 = a_re = r0 r1 r2 r3 r8 r9 r10 r11 */
210 __ASM_EMIT("vinsertf128 $1, %%xmm1, %%ymm3, %%ymm3") /* ymm3 = a_im = i0 i1 i2 i3 i8 i9 i10 i11 */
211 __ASM_EMIT("vinsertf128 $0, %%xmm4, %%ymm0, %%ymm0") /* ymm0 = b_re = r5 r6 r7 r12 r13 r14 r15 */
212 __ASM_EMIT("vinsertf128 $0, %%xmm5, %%ymm1, %%ymm1") /* ymm1 = b_im = i5 i6 i7 i12 i13 i14 i15 */
213 __ASM_EMIT("vmulps 0xc0 + %[FFT_A], %%ymm0, %%ymm4") /* ymm4 = x_im * b_re */
214 __ASM_EMIT("vmulps 0xc0 + %[FFT_A], %%ymm1, %%ymm5") /* ymm5 = x_im * b_im */
215 __ASM_EMIT("vfmadd231ps 0x80 + %[FFT_A], %%ymm0, %%ymm5") /* ymm5 = c_re = x_re * b_re + x_im * b_im */
216 __ASM_EMIT("vfmsub231ps 0x80 + %[FFT_A], %%ymm1, %%ymm4") /* ymm4 = c_im = x_re * b_im - x_im * b_re */
217 __ASM_EMIT("vsubps %%ymm5, %%ymm2, %%ymm0") /* ymm0 = a_re - c_re */
218 __ASM_EMIT("vsubps %%ymm4, %%ymm3, %%ymm1") /* ymm1 = a_im - c_im */
219 __ASM_EMIT("vaddps %%ymm5, %%ymm2, %%ymm2") /* ymm2 = a_re + c_re */
220 __ASM_EMIT("vaddps %%ymm4, %%ymm3, %%ymm3") /* ymm3 = a_im + c_im */
221 /* Store */
222 __ASM_EMIT("vmovups %%ymm2, 0x00(%[dst_re], %[off])")
223 __ASM_EMIT("vmovups %%ymm0, 0x20(%[dst_re], %[off])")
224 __ASM_EMIT("vmovups %%ymm3, 0x00(%[dst_im], %[off])")
225 __ASM_EMIT("vmovups %%ymm1, 0x20(%[dst_im], %[off])")
226 /* Move pointers */
227 __ASM_EMIT("add $0x40, %[off]")
228 __ASM_EMIT("sub $2, %[items]")
229 __ASM_EMIT("4:")
230 /* x4 scramble block */
231 __ASM_EMIT("add $1, %[items]")
232 __ASM_EMIT("jl 6f")
233 __ASM_EMIT("vmovups 0x00(%[dst_re], %[off]), %%xmm0") /* xmm0 = r0 r1 r2 r3 */
234 __ASM_EMIT("vmovups 0x10(%[dst_re], %[off]), %%xmm4") /* xmm4 = r4 r5 r6 r7 */
235 __ASM_EMIT("vmovups 0x00(%[dst_im], %[off]), %%xmm2") /* xmm2 = i0 i1 i2 i3 */
236 __ASM_EMIT("vmovups 0x10(%[dst_im], %[off]), %%xmm6") /* xmm6 = i4 i5 i6 i7 */
237 /* 1st-order 8x butterfly */
238 __ASM_EMIT("vhsubps %%xmm4, %%xmm0, %%xmm1") /* xmm1 = r0-r1 r2-r3 r4-r5 r6-r7 = r1' r3' r5' r7' */
239 __ASM_EMIT("vhsubps %%xmm6, %%xmm2, %%xmm3") /* xmm3 = i0-i1 i2-i3 i4-i5 i6-i7 = i1' i3' i5' i7' */
240 __ASM_EMIT("vhaddps %%xmm4, %%xmm0, %%xmm0") /* xmm0 = r0+r1 r2+r3 r4+r5 r6+r7 = r0' r2' r4' r6' */
241 __ASM_EMIT("vhaddps %%xmm6, %%xmm2, %%xmm2") /* xmm2 = i0+i1 i2+i3 i4+i5 i6+i7 = i0' i2' i4' i6' */
242 /* 2nd-order 8x butterfly */
243 __ASM_EMIT("vblendps $0xaa, %%xmm3, %%xmm1, %%xmm4") /* xmm4 = r1' i3' r5' i7' */
244 __ASM_EMIT("vblendps $0xaa, %%xmm1, %%xmm3, %%xmm5") /* xmm5 = i1' r3' i5' r7' */
245 __ASM_EMIT("vhsubps %%xmm4, %%xmm0, %%xmm1") /* xmm1 = r0'-r2' r4'-r6' r1'-i3' r5'-i7' = r2" r6" r3" r7" */
246 __ASM_EMIT("vhsubps %%xmm5, %%xmm2, %%xmm3") /* xmm3 = i0'-i2' i4'-i6' i1'-r3' i5'-r7' = i2" i6" i1" i5" */
247 __ASM_EMIT("vhaddps %%xmm4, %%xmm0, %%xmm0") /* xmm0 = r0'+r2' r4'+r6' r1'+i3' r5'+i7' = r0" r4" r1" r5" */
248 __ASM_EMIT("vhaddps %%xmm5, %%xmm2, %%xmm2") /* xmm2 = i0'+i2' i4'+i6' i1'+r3' i5'+r7' = i0" i4" i3" i7" */
249 __ASM_EMIT("vblendps $0xcc, %%xmm3, %%xmm2, %%xmm4") /* xmm4 = i0" i4" i1" i5" */
250 __ASM_EMIT("vblendps $0xcc, %%xmm2, %%xmm3, %%xmm5") /* xmm5 = i2" i6" i3" i7" */
251 __ASM_EMIT("vshufps $0x88, %%xmm1, %%xmm0, %%xmm2") /* xmm2 = r0" r1" r2" r3" */
252 __ASM_EMIT("vshufps $0xdd, %%xmm1, %%xmm0, %%xmm3") /* xmm3 = r4" r5" r6" r7" */
253 __ASM_EMIT("vshufps $0x88, %%xmm5, %%xmm4, %%xmm6") /* xmm6 = i0" i1" i2" i3" */
254 __ASM_EMIT("vshufps $0xdd, %%xmm5, %%xmm4, %%xmm7") /* xmm7 = i4" i5" i6" i7" */
255 /* 3rd-order 8x butterfly */
256 __ASM_EMIT("vmulps 0x40 + %[FFT_A], %%xmm3, %%xmm4") /* xmm4 = x_im * b_re */
257 __ASM_EMIT("vmulps 0x40 + %[FFT_A], %%xmm7, %%xmm5") /* xmm5 = x_im * b_im */
258 __ASM_EMIT("vfmadd231ps 0x00 + %[FFT_A], %%xmm3, %%xmm5") /* xmm5 = c_re = x_re * b_re + x_im * b_im */
259 __ASM_EMIT("vfmsub231ps 0x00 + %[FFT_A], %%xmm7, %%xmm4") /* xmm4 = c_im = x_re * b_im - x_im * b_re */
260 __ASM_EMIT("vsubps %%xmm5, %%xmm2, %%xmm0") /* xmm0 = a_re - c_re */
261 __ASM_EMIT("vsubps %%xmm4, %%xmm6, %%xmm1") /* xmm1 = a_im - c_im */
262 __ASM_EMIT("vaddps %%xmm5, %%xmm2, %%xmm2") /* xmm2 = a_re + c_re */
263 __ASM_EMIT("vaddps %%xmm4, %%xmm6, %%xmm3") /* xmm3 = a_im + c_im */
264 /* Store */
265 __ASM_EMIT("vmovups %%xmm2, 0x00(%[dst_re], %[off])")
266 __ASM_EMIT("vmovups %%xmm0, 0x10(%[dst_re], %[off])")
267 __ASM_EMIT("vmovups %%xmm3, 0x00(%[dst_im], %[off])")
268 __ASM_EMIT("vmovups %%xmm1, 0x10(%[dst_im], %[off])")
269 __ASM_EMIT("6:")
270
271 : [dst_re] "+r"(dst_re), [dst_im] "+r"(dst_im),
272 [off] "+r" (off), [items] "+r"(items)
273 : [FFT_A] "o" (FFT_A), [FFT_I] "o" (FFT_SCRAMBLE_INDICES)
274 : "cc", "memory",
275 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
276 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
277 );
278 }
279
280 static inline void FFT_SCRAMBLE_SELF_REVERSE_NAME(float *dst_re, float *dst_im, size_t rank)
281 {
282 // Calculate number of items
283 size_t items = (1 << rank) - 1;
284
285 for (size_t i = 1; i < items; ++i)
286 {
287 size_t j = reverse_bits(FFT_TYPE(i), rank); /* Reverse the order of the bits */
288 if (i >= j)
289 continue;
290
291 /* Copy the values from the reversed position */
292 ARCH_X86_ASM
293 (
294 __ASM_EMIT("vmovss (%[dst_re], %[i], 4), %%xmm0")
295 __ASM_EMIT("vmovss (%[dst_im], %[i], 4), %%xmm1")
296 __ASM_EMIT("vmovss (%[dst_re], %[j], 4), %%xmm2")
297 __ASM_EMIT("vmovss (%[dst_im], %[j], 4), %%xmm3")
298 __ASM_EMIT("vmovss %%xmm2, (%[dst_re], %[i], 4)")
299 __ASM_EMIT("vmovss %%xmm3, (%[dst_im], %[i], 4)")
300 __ASM_EMIT("vmovss %%xmm0, (%[dst_re], %[j], 4)")
301 __ASM_EMIT("vmovss %%xmm1, (%[dst_im], %[j], 4)")
302 :
303 : [dst_re] "r" (dst_re), [dst_im] "r" (dst_im),
304 [i] "r"(i), [j] "r"(j)
305 : "memory",
306 "%xmm0", "%xmm1", "%xmm2", "%xmm3"
307 );
308 }
309
310 // Perform butterfly 8x
311 size_t off = 0;
312 items = 1 << (rank - 3);
313
314 // Perform 4-element butterflies
315 ARCH_X86_ASM
316 (
317 /* Loop 4x 4-element butterflies */
318 __ASM_EMIT("sub $4, %[items]")
319 __ASM_EMIT("jb 2f")
320 __ASM_EMIT("1:")
321 /* Load data to registers */
322 __ASM_EMIT("vmovups 0x00(%[dst_re], %[off]), %%zmm0") /* zmm0 = r0 r1 r2 r3 r4 r5 r6 r7 r8 r9 r10 r11 r12 r13 r14 r15 */
323 __ASM_EMIT("vmovups 0x40(%[dst_re], %[off]), %%zmm4") /* zmm4 = r16 r17 r18 r19 r20 r21 r22 r23 r24 r25 r26 r27 r28 r29 r30 r31 */
324 __ASM_EMIT("vmovups 0x00(%[dst_im], %[off]), %%zmm2") /* zmm2 = i0 i1 i2 i3 i4 i5 i6 i7 i8 i9 i10 i11 i12 i13 i14 i15 */
325 __ASM_EMIT("vmovups 0x40(%[dst_im], %[off]), %%zmm5") /* zmm5 = i16 i17 i18 i19 i20 i21 i22 i23 i24 i25 i26 i27 i28 i29 i30 i31 */
326 /* 1st-order 16x butterfly */
327 __ASM_EMIT("vmovaps %%zmm0, %%zmm1") /* zmm1 = zmm0 = r0 r1 r2 r3 r4 r5 r6 r7 r8 r9 r10 r11 r12 r13 r14 r15 */
328 __ASM_EMIT("vmovaps %%zmm2, %%zmm3") /* zmm3 = zmm2 = i0 i1 i2 i3 i4 i5 i6 i7 i8 i9 i10 i11 i12 i13 i14 i15 */
329 __ASM_EMIT("vmovaps 0x00 + %[FFT_I], %%zmm6") /* zmm6 = shuffle_a[0] */
330 __ASM_EMIT("vmovaps 0x40 + %[FFT_I], %%zmm7") /* zmm7 = shuffle_b[0] */
331 __ASM_EMIT("vpermt2ps %%zmm4, %%zmm6, %%zmm0") /* zmm0 = r0 r2 r4 r6 r8 r10 r12 r14 r16 r18 r20 r22 r24 r26 r28 r30 */
332 __ASM_EMIT("vpermt2ps %%zmm4, %%zmm7, %%zmm1") /* zmm1 = r1 r3 r5 r7 r9 r11 r13 r15 r17 r19 r21 r23 r25 r27 r29 r31 */
333 __ASM_EMIT("vpermt2ps %%zmm5, %%zmm6, %%zmm2") /* zmm2 = i0 i2 i4 i6 i8 i10 i12 i14 i16 i18 i20 i22 i24 i26 i28 i30 */
334 __ASM_EMIT("vpermt2ps %%zmm5, %%zmm7, %%zmm3") /* zmm3 = i1 i3 i5 i7 i9 i11 i13 i15 i17 i19 i21 i23 i25 i27 i29 i31 */
335 __ASM_EMIT("vsubps %%zmm1, %%zmm0, %%zmm4") /* zmm4 = r0-r1 r2-r3 r4-r5 r6-r7 ... = r1' r3' r5' r7' r9' r11' r13' r15' r17' r19' r21' r23' r25' r27' r29' r31' */
336 __ASM_EMIT("vsubps %%zmm3, %%zmm2, %%zmm5") /* zmm5 = i0-i1 i2-i3 i4-i5 i6-i7 ... = i1' i3' i5' i7' i9' i11' i13' i15' i17' i19' i21' i23' i25' i27' i29' i31' */
337 __ASM_EMIT("vaddps %%zmm1, %%zmm0, %%zmm0") /* zmm0 = r0+r1 r2+r3 r4+r5 r6+r7 ... = r0' r2' r4' r6' r8' r10' r12' r14' r16' r18' r20' r22' r24' r26' r28' r30' */
338 __ASM_EMIT("vaddps %%zmm3, %%zmm2, %%zmm2") /* zmm2 = i0+i1 i2+i3 i4+i5 i6+i7 ... = i0' i2' i4' i6' i8' i10' i12' i14' i16' i18' i20' i22' i24' i26' i28' i30' */
339 /* 2nd-order 16x butterfly */
340 __ASM_EMIT("vmovaps %%zmm0, %%zmm1") /* zmm1 = zmm0 = r0' r2' r4' r6' r8' r10' r12' r14' r16' r18' r20' r22' r24' r26' r28' r30' */
341 __ASM_EMIT("vmovaps %%zmm2, %%zmm3") /* zmm3 = zmm2 = i0' i2' i4' i6' i8' i10' i12' i14' i16' i18' i20' i22' i24' i26' i28' i30' */
342 __ASM_EMIT("vmovaps 0x80 + %[FFT_I], %%zmm6") /* zmm6 = shuffle_a[1] */
343 __ASM_EMIT("vmovaps 0xc0 + %[FFT_I], %%zmm7") /* zmm7 = shuffle_b[1] */
344 __ASM_EMIT("vpermt2ps %%zmm4, %%zmm6, %%zmm0") /* zmm0 = r0' r1' r4' r5' r8' r9' r12' r13' r16' r17' r20' r21' r24' r25' r28' r29' */
345 __ASM_EMIT("vpermt2ps %%zmm5, %%zmm6, %%zmm2") /* zmm2 = i0' i1' i4' i5' i8' i9' i12' i13' i16' i17' i20' i21' i24' i25' i28' i29' */
346 __ASM_EMIT("vpermt2ps %%zmm5, %%zmm7, %%zmm1") /* zmm1 = r2' i3' r6' i7' r10' i11' r14' i15' r18' i19' r22' i23' r26' i27' r30' i31' */
347 __ASM_EMIT("vpermt2ps %%zmm4, %%zmm7, %%zmm3") /* zmm3 = i2' r3' i6' r7' i10' r11' i14' r15' i18' r19' i22' r23' i26' r27' i30' r31' */
348 __ASM_EMIT("vsubps %%zmm1, %%zmm0, %%zmm4") /* zmm4 = r2" r1" r6" r5" r10" r9" r14" r13" r18" r17" r22" r21" r26" r25" r30" r29" = r0'-r2' r1'-i3' r4'-r6' r5'-i7' r8'-r10' r9'-i11' r12'-r14' r13'-i15' r16'-r18' r17'-i19' r20'-r22' r21'-i23' r24'-r26' r25'-i27' r28'-r30' r29'-i31' */
349 __ASM_EMIT("vsubps %%zmm3, %%zmm2, %%zmm5") /* zmm5 = i2" i3" i6" i7" i10" i11" i14" i15" i18" i19" i22" i23" i26" i27" i30" i31" = i0'-i2' i1'-r3' i4'-i6' i5'-r7' i8'-i10' i9'-r11' i12'-i14' i13'-r15' i16'-i18' i17'-r19' i20'-i22' i21'-r23' i24'-i26' i25'-r27' i28'-i30' i29'-r31' */
350 __ASM_EMIT("vaddps %%zmm1, %%zmm0, %%zmm0") /* zmm0 = r0" r3" r4" r7" r8" r11" r12" r15" r16" r19" r20" r23" r24" r27" r28" r31" = r0'+r2' r1'+i3' r4'+r6' r5'+i7' r8'+r10' r9'+i11' r12'+r14' r13'+i15' r16'+r18' r17'+i19' r20'+r22' r21'+i23' r24'+r26' r25'+i27' r28'+r30' r29'+i31' */
351 __ASM_EMIT("vaddps %%zmm3, %%zmm2, %%zmm2") /* zmm2 = i0" i1" i4" i5" i8" i9" i12" i13" i16" i17" i20" i21" i24" i25" i28" i29" = i0'+i2' i1'+r3' i4'+i6' i5'+r7' i8'+i10' i9'+r11' i12'+i14' i13'+r15' i16'+i18' i17'+r19' i20'+i22' i21'+r23' i24'+i26' i25'+r27' i28'+i30' i29'+r31' */
352 /* 3rd-order 16x butterfly */
353 __ASM_EMIT("vmovaps %%zmm0, %%zmm1") /* zmm1 = zmm0 = r0" r3" r4" r7" r8" r11" r12" r15" r16" r19" r20" r23" r24" r27" r28" r31" */
354 __ASM_EMIT("vmovaps %%zmm2, %%zmm3") /* zmm3 = zmm2 = i0" i1" i4" i5" i8" i9" i12" i13" i16" i17" i20" i21" i24" i25" i28" i29" */
355 __ASM_EMIT("vmovaps 0x140 + %[FFT_I], %%zmm6") /* zmm6 = shuffle_b[2] */
356 __ASM_EMIT("vmovaps 0x100 + %[FFT_I], %%zmm7") /* zmm7 = shuffle_a[2] */
357 __ASM_EMIT("vpermt2ps %%zmm4, %%zmm6, %%zmm0") /* zmm0 = a_re = r0" r1" r2" r3" r8" r9" r10" r11" r16" r17" r18" r19" r24" r25" r26" r27" */
358 __ASM_EMIT("vpermt2ps %%zmm5, %%zmm7, %%zmm2") /* zmm2 = a_im = i0" i1" i2" i3" i8" i9" i10" i11" i16" i17" i18" i19" i24" i25" i26" i27" */
359 __ASM_EMIT("vmovaps 0x1c0 + %[FFT_I], %%zmm6") /* zmm6 = shuffle_b[3] */
360 __ASM_EMIT("vmovaps 0x180 + %[FFT_I], %%zmm7") /* zmm7 = shuffle_a[3] */
361 __ASM_EMIT("vpermt2ps %%zmm4, %%zmm6, %%zmm1") /* zmm1 = b_re = r4" r5" r6" r7" r12" r13" r14" r15" r20" r21" r22" r23" r28" r29" r30" r31" */
362 __ASM_EMIT("vpermt2ps %%zmm5, %%zmm7, %%zmm3") /* zmm3 = b_im = i4" i5" i6" i7" i12" i13" i14" i15" i20" i21" i22" i23" i28" i29" i30" i31" */
363 __ASM_EMIT("vmulps 0x40 + %[FFT_A], %%zmm1, %%zmm4") /* zmm4 = x_im * b_re */
364 __ASM_EMIT("vmulps 0x40 + %[FFT_A], %%zmm3, %%zmm5") /* zmm5 = x_im * b_im */
365 __ASM_EMIT("vfmsub231ps 0x00 + %[FFT_A], %%zmm1, %%zmm5") /* zmm5 = c_re = x_re * b_re - x_im * b_im */
366 __ASM_EMIT("vfmadd231ps 0x00 + %[FFT_A], %%zmm3, %%zmm4") /* zmm4 = c_im = x_re * b_im + x_im * b_re */
367 __ASM_EMIT("vsubps %%zmm5, %%zmm0, %%zmm1") /* zmm1 = a_re - c_re = r4` r5` r6` r7` r12` r13` r14` r15` r20` r21` r22` r23` r28` r29` r30` r31` */
368 __ASM_EMIT("vsubps %%zmm4, %%zmm2, %%zmm3") /* zmm3 = a_im - c_im = i4` i5` i6` i7` i12` i13` i14` i15` i20` i21` i22` i23` i28` i29` i30` i31` */
369 __ASM_EMIT("vaddps %%zmm5, %%zmm0, %%zmm0") /* zmm0 = a_re + c_re = r0` r1` r2` r3` r8` r9` r10` r11` r16` r17` r18` r19` r24` r25` r26` r27` */
370 __ASM_EMIT("vaddps %%zmm4, %%zmm2, %%zmm2") /* zmm2 = a_im + c_im = i0` i1` i2` i3` i8` i9` i10` i11` i16` i17` i18` i19` i24` i25` i26` i27` */
371 /* 4th-order 16x butterfly */
372 __ASM_EMIT("vmovaps %%zmm0, %%zmm4") /* zmm4 = zmm0 = r0` r1` r2` r3` r8` r9` r10` r11` r16` r17` r18` r19` r24` r25` r26` r27` */
373 __ASM_EMIT("vmovaps %%zmm2, %%zmm5") /* zmm5 = zmm2 = i0` i1` i2` i3` i8` i9` i10` i11` i16` i17` i18` i19` i24` i25` i26` i27` */
374 __ASM_EMIT("vmovaps 0x200 + %[FFT_I], %%zmm6") /* zmm6 = shuffle_a[4] */
375 __ASM_EMIT("vmovaps 0x240 + %[FFT_I], %%zmm7") /* zmm7 = shuffle_b[4] */
376 __ASM_EMIT("vpermt2ps %%zmm1, %%zmm6, %%zmm0") /* zmm0 = a_re = r0` r1` r2` r3` r4` r5` r6` r7` r16` r17` r18` r19` r20` r21` r22` r23` */
377 __ASM_EMIT("vpermt2ps %%zmm1, %%zmm7, %%zmm4") /* zmm4 = b_re = r8` r9` r10` r11` r12` r13` r14` r15` r24` r25` r26` r27` r28` r29` r30` r31` */
378 __ASM_EMIT("vpermt2ps %%zmm3, %%zmm6, %%zmm2") /* zmm2 = a_im = i0` i1` i2` i3` i4` i5` i6` i7` i16` i17` i18` i19` i20` i21` i22` i23` */
379 __ASM_EMIT("vpermt2ps %%zmm3, %%zmm7, %%zmm5") /* zmm5 = b_im = i8` i9` i10` i11` i12` i13` i14` i15` i24` i25` i26` i27` i28` i29` i30` i31` */
380 __ASM_EMIT("vmulps 0xc0 + %[FFT_A], %%zmm4, %%zmm1") /* zmm1 = x_im * b_re */
381 __ASM_EMIT("vmulps 0xc0 + %[FFT_A], %%zmm5, %%zmm3") /* zmm3 = x_im * b_im */
382 __ASM_EMIT("vfmsub231ps 0x80 + %[FFT_A], %%zmm4, %%zmm3") /* zmm3 = c_re = x_re * b_re - x_im * b_im */
383 __ASM_EMIT("vfmadd231ps 0x80 + %[FFT_A], %%zmm5, %%zmm1") /* zmm1 = c_im = x_re * b_im + x_im * b_re */
384 __ASM_EMIT("vsubps %%zmm3, %%zmm0, %%zmm4") /* zmm4 = a_re - c_re = R8 R9 R10 R11 R12 R13 R14 R15 R24 R25 R26 R27 R28 R29 R30 R31 */
385 __ASM_EMIT("vsubps %%zmm1, %%zmm2, %%zmm5") /* zmm5 = a_im - c_im = I8 I9 I10 I11 I12 I13 I14 I15 I24 I25 I26 I27 I28 I29 I30 I31 */
386 __ASM_EMIT("vaddps %%zmm3, %%zmm0, %%zmm0") /* zmm0 = a_re + c_re = R0 R1 R2 R3 R4 R5 R6 R7 R16 R17 R18 R19 R20 R21 R22 R23 */
387 __ASM_EMIT("vaddps %%zmm1, %%zmm2, %%zmm2") /* zmm2 = a_im + c_im = I0 I1 I2 I3 I4 I5 I6 I7 I16 I17 I18 I19 I20 I21 I22 I23 */
388 /* Store */
389 __ASM_EMIT("vmovups %%ymm0, 0x00(%[dst_re], %[off])")
390 __ASM_EMIT("vmovups %%ymm4, 0x20(%[dst_re], %[off])")
391 __ASM_EMIT("vextractf64x4 $1, %%zmm0, 0x40(%[dst_re], %[off])")
392 __ASM_EMIT("vextractf64x4 $1, %%zmm4, 0x60(%[dst_re], %[off])")
393 __ASM_EMIT("vmovups %%ymm2, 0x00(%[dst_im], %[off])")
394 __ASM_EMIT("vmovups %%ymm5, 0x20(%[dst_im], %[off])")
395 __ASM_EMIT("vextractf64x4 $1, %%zmm2, 0x40(%[dst_im], %[off])")
396 __ASM_EMIT("vextractf64x4 $1, %%zmm5, 0x60(%[dst_im], %[off])")
397 /* Move pointers and repeat */
398 __ASM_EMIT("add $0x80, %[off]")
399 __ASM_EMIT("sub $4, %[items]")
400 __ASM_EMIT("jae 1b")
401 __ASM_EMIT("2:")
402
403 /* Loop 2x 4-element butterflies */
404 __ASM_EMIT("add $2, %[items]")
405 __ASM_EMIT("jl 4f")
406 /* Load data to registers */
407 __ASM_EMIT("vmovups 0x00(%[dst_re], %[off]), %%xmm0") /* xmm0 = r0 r1 r2 r3 */
408 __ASM_EMIT("vmovups 0x10(%[dst_re], %[off]), %%xmm4") /* xmm4 = r4 r5 r6 r7 */
409 __ASM_EMIT("vinsertf128 $1, 0x20(%[dst_re], %[off]), %%ymm0, %%ymm0") /* ymm0 = r0 r1 r2 r3 */
410 __ASM_EMIT("vinsertf128 $1, 0x30(%[dst_re], %[off]), %%ymm4, %%ymm4") /* ymm4 = r4 r5 r6 r7 */
411 __ASM_EMIT("vmovups 0x00(%[dst_im], %[off]), %%xmm2") /* xmm2 = i0 i1 i2 i3 */
412 __ASM_EMIT("vmovups 0x10(%[dst_im], %[off]), %%xmm6") /* xmm6 = i4 i5 i6 i7 */
413 __ASM_EMIT("vinsertf128 $1, 0x20(%[dst_im], %[off]), %%ymm2, %%ymm2") /* ymm2 = i0 i1 i2 i3 */
414 __ASM_EMIT("vinsertf128 $1, 0x30(%[dst_im], %[off]), %%ymm6, %%ymm6") /* ymm6 = i4 i5 i6 i7 */
415 /* 1st-order 8x butterfly */
416 __ASM_EMIT("vhsubps %%ymm4, %%ymm0, %%ymm1") /* ymm1 = r0-r1 r2-r3 r4-r5 r6-r7 = r1' r3' r5' r7' */
417 __ASM_EMIT("vhsubps %%ymm6, %%ymm2, %%ymm3") /* ymm3 = i0-i1 i2-i3 i4-i5 i6-i7 = i1' i3' i5' i7' */
418 __ASM_EMIT("vhaddps %%ymm4, %%ymm0, %%ymm0") /* ymm0 = r0+r1 r2+r3 r4+r5 r6+r7 = r0' r2' r4' r6' */
419 __ASM_EMIT("vhaddps %%ymm6, %%ymm2, %%ymm2") /* ymm2 = i0+i1 i2+i3 i4+i5 i6+i7 = i0' i2' i4' i6' */
420 /* 2nd-order 8x butterfly */
421 __ASM_EMIT("vblendps $0xaa, %%ymm3, %%ymm1, %%ymm4") /* ymm4 = r1' i3' r5' i7' */
422 __ASM_EMIT("vblendps $0xaa, %%ymm1, %%ymm3, %%ymm5") /* ymm5 = i1' r3' i5' r7' */
423 __ASM_EMIT("vhsubps %%ymm4, %%ymm0, %%ymm1") /* ymm1 = r0'-r2' r4'-r6' r1'-i3' r5'-i7' = r2" r6" r1" r5" */
424 __ASM_EMIT("vhsubps %%ymm5, %%ymm2, %%ymm3") /* ymm3 = i0'-i2' i4'-i6' i1'-r3' i5'-r7' = i2" i6" i3" i7" */
425 __ASM_EMIT("vhaddps %%ymm4, %%ymm0, %%ymm0") /* ymm0 = r0'+r2' r4'+r6' r1'+i3' r5'+i7' = r0" r4" r3" r7" */
426 __ASM_EMIT("vhaddps %%ymm5, %%ymm2, %%ymm2") /* ymm2 = i0'+i2' i4'+i6' i1'+r3' i5'+r7' = i0" i4" i1" i5" */
427 __ASM_EMIT("vblendps $0xcc, %%ymm1, %%ymm0, %%ymm4") /* ymm4 = r0" i4" r1" r5" */
428 __ASM_EMIT("vblendps $0xcc, %%ymm0, %%ymm1, %%ymm5") /* ymm5 = r2" r6" r3" r7" */
429 __ASM_EMIT("vshufps $0x88, %%ymm3, %%ymm2, %%ymm6") /* ymm6 = i0" i1" i2" i3" */
430 __ASM_EMIT("vshufps $0xdd, %%ymm3, %%ymm2, %%ymm7") /* ymm7 = i4" i5" i6" i7" */
431 __ASM_EMIT("vshufps $0x88, %%ymm5, %%ymm4, %%ymm2") /* ymm2 = r0" r1" r2" r3" */
432 __ASM_EMIT("vshufps $0xdd, %%ymm5, %%ymm4, %%ymm3") /* ymm3 = r4" r5" r6" r7" */
433 /* 3rd-order 8x butterfly */
434 __ASM_EMIT("vmulps 0x40 + %[FFT_A], %%ymm3, %%ymm4") /* ymm4 = x_im * b_re */
435 __ASM_EMIT("vmulps 0x40 + %[FFT_A], %%ymm7, %%ymm5") /* ymm5 = x_im * b_im */
436 __ASM_EMIT("vfmsub231ps 0x00 + %[FFT_A], %%ymm3, %%ymm5") /* ymm5 = c_re = x_re * b_re + x_im * b_im */
437 __ASM_EMIT("vfmadd231ps 0x00 + %[FFT_A], %%ymm7, %%ymm4") /* ymm4 = c_im = x_re * b_im - x_im * b_re */
438 __ASM_EMIT("vsubps %%ymm5, %%ymm2, %%ymm0") /* ymm0 = a_re - c_re = r8 r9 r10 r11 r12 r13 r14 r15 */
439 __ASM_EMIT("vsubps %%ymm4, %%ymm6, %%ymm1") /* ymm1 = a_im - c_im = i8 i9 i10 i11 i12 i13 i14 i15 */
440 __ASM_EMIT("vaddps %%ymm5, %%ymm2, %%ymm2") /* ymm2 = a_re + c_re = r0 r1 r2 r3 r4 r5 r6 r7 */
441 __ASM_EMIT("vaddps %%ymm4, %%ymm6, %%ymm3") /* ymm3 = a_im + c_im = i0 i1 i2 i3 i4 i5 i6 i7 */
442 /* 4th-order 8x butterfly */
443 __ASM_EMIT("vextractf128 $1, %%ymm2, %%xmm4") /* xmm4 = r4 r5 r6 r7 */
444 __ASM_EMIT("vextractf128 $1, %%ymm3, %%xmm5") /* xmm5 = i4 i5 i6 i7 */
445 __ASM_EMIT("vinsertf128 $1, %%xmm0, %%ymm2, %%ymm2") /* ymm2 = a_re = r0 r1 r2 r3 r8 r9 r10 r11 */
446 __ASM_EMIT("vinsertf128 $1, %%xmm1, %%ymm3, %%ymm3") /* ymm3 = a_im = i0 i1 i2 i3 i8 i9 i10 i11 */
447 __ASM_EMIT("vinsertf128 $0, %%xmm4, %%ymm0, %%ymm0") /* ymm0 = b_re = r5 r6 r7 r12 r13 r14 r15 */
448 __ASM_EMIT("vinsertf128 $0, %%xmm5, %%ymm1, %%ymm1") /* ymm1 = b_im = i5 i6 i7 i12 i13 i14 i15 */
449 __ASM_EMIT("vmulps 0xc0 + %[FFT_A], %%ymm0, %%ymm4") /* ymm4 = x_im * b_re */
450 __ASM_EMIT("vmulps 0xc0 + %[FFT_A], %%ymm1, %%ymm5") /* ymm5 = x_im * b_im */
451 __ASM_EMIT("vfmsub231ps 0x80 + %[FFT_A], %%ymm0, %%ymm5") /* ymm5 = c_re = x_re * b_re - x_im * b_im */
452 __ASM_EMIT("vfmadd231ps 0x80 + %[FFT_A], %%ymm1, %%ymm4") /* ymm4 = c_im = x_re * b_im + x_im * b_re */
453 __ASM_EMIT("vsubps %%ymm5, %%ymm2, %%ymm0") /* ymm0 = a_re - c_re */
454 __ASM_EMIT("vsubps %%ymm4, %%ymm3, %%ymm1") /* ymm1 = a_im - c_im */
455 __ASM_EMIT("vaddps %%ymm5, %%ymm2, %%ymm2") /* ymm2 = a_re + c_re */
456 __ASM_EMIT("vaddps %%ymm4, %%ymm3, %%ymm3") /* ymm3 = a_im + c_im */
457 /* Store */
458 __ASM_EMIT("vmovups %%ymm2, 0x00(%[dst_re], %[off])")
459 __ASM_EMIT("vmovups %%ymm0, 0x20(%[dst_re], %[off])")
460 __ASM_EMIT("vmovups %%ymm3, 0x00(%[dst_im], %[off])")
461 __ASM_EMIT("vmovups %%ymm1, 0x20(%[dst_im], %[off])")
462 /* Move pointers */
463 __ASM_EMIT("add $0x40, %[off]")
464 __ASM_EMIT("sub $2, %[items]")
465 __ASM_EMIT("4:")
466 /* x4 scramble block */
467 __ASM_EMIT("add $1, %[items]")
468 __ASM_EMIT("jl 6f")
469 __ASM_EMIT("vmovups 0x00(%[dst_re], %[off]), %%xmm0") /* xmm0 = r0 r1 r2 r3 */
470 __ASM_EMIT("vmovups 0x10(%[dst_re], %[off]), %%xmm4") /* xmm4 = r4 r5 r6 r7 */
471 __ASM_EMIT("vmovups 0x00(%[dst_im], %[off]), %%xmm2") /* xmm2 = i0 i1 i2 i3 */
472 __ASM_EMIT("vmovups 0x10(%[dst_im], %[off]), %%xmm6") /* xmm6 = i4 i5 i6 i7 */
473 /* 1st-order 8x butterfly */
474 __ASM_EMIT("vhsubps %%xmm4, %%xmm0, %%xmm1") /* xmm1 = r0-r1 r2-r3 r4-r5 r6-r7 = r1' r3' r5' r7' */
475 __ASM_EMIT("vhsubps %%xmm6, %%xmm2, %%xmm3") /* xmm3 = i0-i1 i2-i3 i4-i5 i6-i7 = i1' i3' i5' i7' */
476 __ASM_EMIT("vhaddps %%xmm4, %%xmm0, %%xmm0") /* xmm0 = r0+r1 r2+r3 r4+r5 r6+r7 = r0' r2' r4' r6' */
477 __ASM_EMIT("vhaddps %%xmm6, %%xmm2, %%xmm2") /* xmm2 = i0+i1 i2+i3 i4+i5 i6+i7 = i0' i2' i4' i6' */
478 /* 2nd-order 8x butterfly */
479 __ASM_EMIT("vblendps $0xaa, %%xmm3, %%xmm1, %%xmm4") /* xmm4 = r1' i3' r5' i7' */
480 __ASM_EMIT("vblendps $0xaa, %%xmm1, %%xmm3, %%xmm5") /* xmm5 = i1' r3' i5' r7' */
481 __ASM_EMIT("vhsubps %%xmm4, %%xmm0, %%xmm1") /* xmm1 = r0'-r2' r4'-r6' r1'-i3' r5'-i7' = r2" r6" r1" r5" */
482 __ASM_EMIT("vhsubps %%xmm5, %%xmm2, %%xmm3") /* xmm3 = i0'-i2' i4'-i6' i1'-r3' i5'-r7' = i2" i6" i3" i7" */
483 __ASM_EMIT("vhaddps %%xmm4, %%xmm0, %%xmm0") /* xmm0 = r0'+r2' r4'+r6' r1'+i3' r5'+i7' = r0" r4" r3" r7" */
484 __ASM_EMIT("vhaddps %%xmm5, %%xmm2, %%xmm2") /* xmm2 = i0'+i2' i4'+i6' i1'+r3' i5'+r7' = i0" i4" i1" i5" */
485 __ASM_EMIT("vblendps $0xcc, %%xmm1, %%xmm0, %%xmm4") /* xmm4 = r0" i4" r1" r5" */
486 __ASM_EMIT("vblendps $0xcc, %%xmm0, %%xmm1, %%xmm5") /* xmm5 = r2" r6" r3" r7" */
487 __ASM_EMIT("vshufps $0x88, %%xmm3, %%xmm2, %%xmm6") /* xmm6 = i0" i1" i2" i3" */
488 __ASM_EMIT("vshufps $0xdd, %%xmm3, %%xmm2, %%xmm7") /* xmm7 = i4" i5" i6" i7" */
489 __ASM_EMIT("vshufps $0x88, %%xmm5, %%xmm4, %%xmm2") /* xmm2 = r0" r1" r2" r3" */
490 __ASM_EMIT("vshufps $0xdd, %%xmm5, %%xmm4, %%xmm3") /* xmm3 = r4" r5" r6" r7" */
491 /* 3rd-order 8x butterfly */
492 __ASM_EMIT("vmulps 0x40 + %[FFT_A], %%xmm3, %%xmm4") /* xmm4 = x_im * b_re */
493 __ASM_EMIT("vmulps 0x40 + %[FFT_A], %%xmm7, %%xmm5") /* xmm5 = x_im * b_im */
494 __ASM_EMIT("vfmsub231ps 0x00 + %[FFT_A], %%xmm3, %%xmm5") /* xmm5 = c_re = x_re * b_re - x_im * b_im */
495 __ASM_EMIT("vfmadd231ps 0x00 + %[FFT_A], %%xmm7, %%xmm4") /* xmm4 = c_im = x_re * b_im + x_im * b_re */
496 __ASM_EMIT("vsubps %%xmm5, %%xmm2, %%xmm0") /* xmm0 = a_re - c_re */
497 __ASM_EMIT("vsubps %%xmm4, %%xmm6, %%xmm1") /* xmm1 = a_im - c_im */
498 __ASM_EMIT("vaddps %%xmm5, %%xmm2, %%xmm2") /* xmm2 = a_re + c_re */
499 __ASM_EMIT("vaddps %%xmm4, %%xmm6, %%xmm3") /* xmm3 = a_im + c_im */
500 /* Store */
501 __ASM_EMIT("vmovups %%xmm2, 0x00(%[dst_re], %[off])")
502 __ASM_EMIT("vmovups %%xmm0, 0x10(%[dst_re], %[off])")
503 __ASM_EMIT("vmovups %%xmm3, 0x00(%[dst_im], %[off])")
504 __ASM_EMIT("vmovups %%xmm1, 0x10(%[dst_im], %[off])")
505 __ASM_EMIT("6:")
506
507 : [dst_re] "+r"(dst_re), [dst_im] "+r"(dst_im),
508 [off] "+r" (off), [items] "+r"(items)
509 : [FFT_A] "o" (FFT_A), [FFT_I] "o" (FFT_SCRAMBLE_INDICES)
510 : "cc", "memory",
511 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
512 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
513 );
514 }
515
516 static inline void FFT_SCRAMBLE_COPY_DIRECT_NAME(float *dst_re, float *dst_im, const float *src_re, const float *src_im, size_t rank)
517 {
518 const size_t regs = 1 << (rank);
519
520 for (size_t i=0; i < regs; ++i)
521 {
522 size_t index = reverse_bits(FFT_TYPE(i), rank);
523
524 ARCH_X86_ASM(
525 /* Load even indices */
526 __ASM_EMIT("vmovaps 0x280 + %[FFT_I], %%zmm7") /* zmm7 = indices */
527 FFT_SCRAMBLE_LOAD4X2("%%xmm0", "%%xmm2") /* xmm0 = r0 r16 r8 r24, xmm2 = i0 i16 i8 i24 */
528 FFT_SCRAMBLE_LOAD4X2("%%xmm4", "%%xmm5") /* xmm4 = r4 r20 r12 r28, xmm5 = i4 i20 i12 i28 */
529 __ASM_EMIT("vinsertf32x4 $01, %%xmm4, %%zmm0, %%zmm0") /* zmm0 = r0 r16 r8 r24 r4 r20 r12 r28 x x x x x x x x */
530 __ASM_EMIT("vinsertf32x4 $01, %%xmm5, %%zmm2, %%zmm2") /* zmm2 = i0 i16 i8 i24 i4 i20 i12 i28 x x x x x x x x */
531 FFT_SCRAMBLE_LOAD4X2("%%xmm4", "%%xmm5") /* xmm4 = r2 r18 r10 r26, xmm5 = i2 i18 i10 i26 */
532 __ASM_EMIT("vinsertf32x4 $02, %%xmm4, %%zmm0, %%zmm0") /* zmm0 = r0 r16 r8 r24 r4 r20 r12 r28 r2 r18 r10 r26 x x x x */
533 __ASM_EMIT("vinsertf32x4 $02, %%xmm5, %%zmm2, %%zmm2") /* zmm2 = i0 i16 i8 i24 i4 i20 i12 i28 i2 i18 i10 i26 x x x x */
534 FFT_SCRAMBLE_LOAD4X2("%%xmm4", "%%xmm5") /* xmm4 = r6 r22 r14 r30, xmm5 = i6 i22 i14 i30 */
535 __ASM_EMIT("vinsertf32x4 $03, %%xmm4, %%zmm0, %%zmm0") /* zmm0 = r0 r16 r8 r24 r4 r20 r12 r28 r2 r18 r10 r26 r6 r22 r14 r30 */
536 __ASM_EMIT("vinsertf32x4 $03, %%xmm5, %%zmm2, %%zmm2") /* zmm2 = i0 i16 i8 i24 i4 i20 i12 i28 i2 i18 i10 i26 i6 i22 i14 i30 */
537 /* Load odd indices */
538 FFT_SCRAMBLE_LOAD4X2("%%xmm1", "%%xmm3") /* xmm1 = r1 r17 r9 r25, xmm3 = i1 i17 i9 i25 */
539 FFT_SCRAMBLE_LOAD4X2("%%xmm4", "%%xmm5") /* xmm4 = r5 r21 r13 r29, xmm5 = i5 i21 i13 i29 */
540 __ASM_EMIT("vinsertf32x4 $01, %%xmm4, %%zmm1, %%zmm1") /* zmm1 = r1 r17 r9 r25 r5 r21 r13 r29 x x x x x x x x */
541 __ASM_EMIT("vinsertf32x4 $01, %%xmm5, %%zmm3, %%zmm3") /* zmm3 = i1 i17 i9 i25 i5 i21 i13 i29 x x x x x x x x */
542 FFT_SCRAMBLE_LOAD4X2("%%xmm4", "%%xmm5") /* xmm4 = r3 r19 r11 r27, xmm5 = i3 i19 i11 i27 */
543 __ASM_EMIT("vinsertf32x4 $02, %%xmm4, %%zmm1, %%zmm1") /* zmm1 = r1 r17 r9 r25 r5 r21 r13 r29 r3 r19 r11 r27 x x x x */
544 __ASM_EMIT("vinsertf32x4 $02, %%xmm5, %%zmm3, %%zmm3") /* zmm3 = i1 i17 i9 i25 i5 i21 i13 i29 i3 i19 i11 i27 x x x x */
545 FFT_SCRAMBLE_LOAD4X2("%%xmm4", "%%xmm5") /* xmm4 = r7 r23 r15 r31, xmm5 = i7 i23 i15 i31 */
546 __ASM_EMIT("vinsertf32x4 $03, %%xmm4, %%zmm1, %%zmm1") /* zmm1 = r1 r17 r9 r25 r5 r21 r13 r29 r3 r19 r11 r27 r7 r23 r15 r31 */
547 __ASM_EMIT("vinsertf32x4 $03, %%xmm5, %%zmm3, %%zmm3") /* zmm3 = i1 i17 i9 i25 i5 i21 i13 i29 i3 i19 i11 i27 i7 i23 i15 i31 */
548 /* Final permute */
549 __ASM_EMIT("vpermps %%zmm0, %%zmm7, %%zmm0") /* zmm0 = r0 r2 r4 r6 r8 r10 r12 r14 r16 r18 r20 r22 r24 r26 r28 r30 */
550 __ASM_EMIT("vpermps %%zmm1, %%zmm7, %%zmm1") /* zmm1 = r1 r3 r5 r7 r9 r11 r13 r15 r17 r19 r21 r23 r25 r27 r29 r31 */
551 __ASM_EMIT("vpermps %%zmm2, %%zmm7, %%zmm2") /* zmm2 = i0 i2 i4 i6 i8 i10 i12 i14 i16 i18 i20 i22 i24 i26 i28 i30 */
552 __ASM_EMIT("vpermps %%zmm3, %%zmm7, %%zmm3") /* zmm3 = i1 i3 i5 i7 i9 i11 i13 i15 i17 i19 i21 i23 i25 i27 i29 i31 */
553
554 /* 1st-order 16x butterfly */
555 /* zmm0 = r0 r2 r4 r6 r8 r10 r12 r14 r16 r18 r20 r22 r24 r26 r28 r30 */
556 /* zmm1 = r1 r3 r5 r7 r9 r11 r13 r15 r17 r19 r21 r23 r25 r27 r29 r31 */
557 /* zmm2 = i0 i2 i4 i6 i8 i10 i12 i14 i16 i18 i20 i22 i24 i26 i28 i30 */
558 /* zmm3 = i1 i3 i5 i7 i9 i11 i13 i15 i17 i19 i21 i23 i25 i27 i29 i31 */
559 __ASM_EMIT("vsubps %%zmm1, %%zmm0, %%zmm4") /* zmm4 = r0-r1 r2-r3 r4-r5 r6-r7 ... = r1' r3' r5' r7' r9' r11' r13' r15' r17' r19' r21' r23' r25' r27' r29' r31' */
560 __ASM_EMIT("vsubps %%zmm3, %%zmm2, %%zmm5") /* zmm5 = i0-i1 i2-i3 i4-i5 i6-i7 ... = i1' i3' i5' i7' i9' i11' i13' i15' i17' i19' i21' i23' i25' i27' i29' i31' */
561 __ASM_EMIT("vaddps %%zmm1, %%zmm0, %%zmm0") /* zmm0 = r0+r1 r2+r3 r4+r5 r6+r7 ... = r0' r2' r4' r6' r8' r10' r12' r14' r16' r18' r20' r22' r24' r26' r28' r30' */
562 __ASM_EMIT("vaddps %%zmm3, %%zmm2, %%zmm2") /* zmm2 = i0+i1 i2+i3 i4+i5 i6+i7 ... = i0' i2' i4' i6' i8' i10' i12' i14' i16' i18' i20' i22' i24' i26' i28' i30' */
563 /* 2nd-order 16x butterfly */
564 __ASM_EMIT("vmovaps %%zmm0, %%zmm1") /* zmm1 = zmm0 = r0' r2' r4' r6' r8' r10' r12' r14' r16' r18' r20' r22' r24' r26' r28' r30' */
565 __ASM_EMIT("vmovaps %%zmm2, %%zmm3") /* zmm3 = zmm2 = i0' i2' i4' i6' i8' i10' i12' i14' i16' i18' i20' i22' i24' i26' i28' i30' */
566 __ASM_EMIT("vmovaps 0x80 + %[FFT_I], %%zmm6") /* zmm6 = shuffle_a[1] */
567 __ASM_EMIT("vmovaps 0xc0 + %[FFT_I], %%zmm7") /* zmm7 = shuffle_b[1] */
568 __ASM_EMIT("vpermt2ps %%zmm4, %%zmm6, %%zmm0") /* zmm0 = r0' r1' r4' r5' r8' r9' r12' r13' r16' r17' r20' r21' r24' r25' r28' r29' */
569 __ASM_EMIT("vpermt2ps %%zmm5, %%zmm6, %%zmm2") /* zmm2 = i0' i1' i4' i5' i8' i9' i12' i13' i16' i17' i20' i21' i24' i25' i28' i29' */
570 __ASM_EMIT("vpermt2ps %%zmm5, %%zmm7, %%zmm1") /* zmm1 = r2' i3' r6' i7' r10' i11' r14' i15' r18' i19' r22' i23' r26' i27' r30' i31' */
571 __ASM_EMIT("vpermt2ps %%zmm4, %%zmm7, %%zmm3") /* zmm3 = i2' r3' i6' r7' i10' r11' i14' r15' i18' r19' i22' r23' i26' r27' i30' r31' */
572 __ASM_EMIT("vsubps %%zmm1, %%zmm0, %%zmm4") /* zmm4 = r2" r3" r6" r7" r10" r11" r14" r15" r18" r19" r22" r23" r26" r27" r30" r31" = r0'-r2' r1'-i3' r4'-r6' r5'-i7' r8'-r10' r9'-i11' r12'-r14' r13'-i15' r16'-r18' r17'-i19' r20'-r22' r21'-i23' r24'-r26' r25'-i27' r28'-r30' r29'-i31' */
573 __ASM_EMIT("vsubps %%zmm3, %%zmm2, %%zmm5") /* zmm5 = i2" i1" i6" i5" i10" i9" i14" i13" i18" i17" i22" i21" i26" i25" i30" i29" = i0'-i2' i1'-r3' i4'-i6' i5'-r7' i8'-i10' i9'-r11' i12'-i14' i13'-r15' i16'-i18' i17'-r19' i20'-i22' i21'-r23' i24'-i26' i25'-r27' i28'-i30' i29'-r31' */
574 __ASM_EMIT("vaddps %%zmm1, %%zmm0, %%zmm0") /* zmm0 = r0" r1" r4" r5" r8" r9" r12" r13" r16" r17" r20" r21" r24" r25" r28" r29" = r0'+r2' r1'+i3' r4'+r6' r5'+i7' r8'+r10' r9'+i11' r12'+r14' r13'+i15' r16'+r18' r17'+i19' r20'+r22' r21'+i23' r24'+r26' r25'+i27' r28'+r30' r29'+i31' */
575 __ASM_EMIT("vaddps %%zmm3, %%zmm2, %%zmm2") /* zmm2 = i0" i3" i4" i7" i8" i11" i12" i15" i16" i19" i20" i23" i24" i27" i28" i31" = i0'+i2' i1'+r3' i4'+i6' i5'+r7' i8'+i10' i9'+r11' i12'+i14' i13'+r15' i16'+i18' i17'+r19' i20'+i22' i21'+r23' i24'+i26' i25'+r27' i28'+i30' i29'+r31' */
576 /* 3rd-order 16x butterfly */
577 __ASM_EMIT("vmovaps %%zmm0, %%zmm1") /* zmm1 = zmm0 = r0" r1" r4" r5" r8" r9" r12" r13" r16" r17" r20" r21" r24" r25" r28" r29" */
578 __ASM_EMIT("vmovaps %%zmm2, %%zmm3") /* zmm3 = zmm2 = i0" i3" i4" i7" i8" i11" i12" i15" i16" i19" i20" i23" i24" i27" i28" i31" */
579 __ASM_EMIT("vmovaps 0x100 + %[FFT_I], %%zmm6") /* zmm6 = shuffle_a[2] */
580 __ASM_EMIT("vmovaps 0x140 + %[FFT_I], %%zmm7") /* zmm7 = shuffle_b[2] */
581 __ASM_EMIT("vpermt2ps %%zmm4, %%zmm6, %%zmm0") /* zmm0 = a_re = r0" r1" r2" r3" r8" r9" r10" r11" r16" r17" r18" r19" r24" r25" r26" r27" */
582 __ASM_EMIT("vpermt2ps %%zmm5, %%zmm7, %%zmm2") /* zmm2 = a_im = i0" i1" i2" i3" i8" i9" i10" i11" i16" i17" i18" i19" i24" i25" i26" i27" */
583 __ASM_EMIT("vmovaps 0x180 + %[FFT_I], %%zmm6") /* zmm6 = shuffle_a[3] */
584 __ASM_EMIT("vmovaps 0x1c0 + %[FFT_I], %%zmm7") /* zmm7 = shuffle_b[3] */
585 __ASM_EMIT("vpermt2ps %%zmm4, %%zmm6, %%zmm1") /* zmm1 = b_re = r4" r5" r6" r7" r12" r13" r14" r15" r20" r21" r22" r23" r28" r29" r30" r31" */
586 __ASM_EMIT("vpermt2ps %%zmm5, %%zmm7, %%zmm3") /* zmm3 = b_im = i4" i5" i6" i7" i12" i13" i14" i15" i20" i21" i22" i23" i28" i29" i30" i31" */
587 __ASM_EMIT("vmulps 0x40 + %[FFT_A], %%zmm1, %%zmm4") /* zmm4 = x_im * b_re */
588 __ASM_EMIT("vmulps 0x40 + %[FFT_A], %%zmm3, %%zmm5") /* zmm5 = x_im * b_im */
589 __ASM_EMIT("vfmadd231ps 0x00 + %[FFT_A], %%zmm1, %%zmm5") /* zmm5 = c_re = x_re * b_re + x_im * b_im */
590 __ASM_EMIT("vfmsub231ps 0x00 + %[FFT_A], %%zmm3, %%zmm4") /* zmm4 = c_im = x_re * b_im - x_im * b_re */
591 __ASM_EMIT("vsubps %%zmm5, %%zmm0, %%zmm1") /* zmm1 = a_re - c_re = r4` r5` r6` r7` r12` r13` r14` r15` r20` r21` r22` r23` r28` r29` r30` r31` */
592 __ASM_EMIT("vsubps %%zmm4, %%zmm2, %%zmm3") /* zmm3 = a_im - c_im = i4` i5` i6` i7` i12` i13` i14` i15` i20` i21` i22` i23` i28` i29` i30` i31` */
593 __ASM_EMIT("vaddps %%zmm5, %%zmm0, %%zmm0") /* zmm0 = a_re + c_re = r0` r1` r2` r3` r8` r9` r10` r11` r16` r17` r18` r19` r24` r25` r26` r27` */
594 __ASM_EMIT("vaddps %%zmm4, %%zmm2, %%zmm2") /* zmm2 = a_im + c_im = i0` i1` i2` i3` i8` i9` i10` i11` i16` i17` i18` i19` i24` i25` i26` i27` */
595 /* 4th-order 16x butterfly */
596 __ASM_EMIT("vmovaps %%zmm0, %%zmm4") /* zmm4 = zmm0 = r0` r1` r2` r3` r8` r9` r10` r11` r16` r17` r18` r19` r24` r25` r26` r27` */
597 __ASM_EMIT("vmovaps %%zmm2, %%zmm5") /* zmm5 = zmm2 = i0` i1` i2` i3` i8` i9` i10` i11` i16` i17` i18` i19` i24` i25` i26` i27` */
598 __ASM_EMIT("vmovaps 0x200 + %[FFT_I], %%zmm6") /* zmm6 = shuffle_a[4] */
599 __ASM_EMIT("vmovaps 0x240 + %[FFT_I], %%zmm7") /* zmm7 = shuffle_b[4] */
600 __ASM_EMIT("vpermt2ps %%zmm1, %%zmm6, %%zmm0") /* zmm0 = a_re = r0` r1` r2` r3` r4` r5` r6` r7` r16` r17` r18` r19` r20` r21` r22` r23` */
601 __ASM_EMIT("vpermt2ps %%zmm1, %%zmm7, %%zmm4") /* zmm4 = b_re = r8` r9` r10` r11` r12` r13` r14` r15` r24` r25` r26` r27` r28` r29` r30` r31` */
602 __ASM_EMIT("vpermt2ps %%zmm3, %%zmm6, %%zmm2") /* zmm2 = a_im = i0` i1` i2` i3` i4` i5` i6` i7` i16` i17` i18` i19` i20` i21` i22` i23` */
603 __ASM_EMIT("vpermt2ps %%zmm3, %%zmm7, %%zmm5") /* zmm5 = b_im = i8` i9` i10` i11` i12` i13` i14` i15` i24` i25` i26` i27` i28` i29` i30` i31` */
604 __ASM_EMIT("vmulps 0xc0 + %[FFT_A], %%zmm4, %%zmm1") /* zmm1 = x_im * b_re */
605 __ASM_EMIT("vmulps 0xc0 + %[FFT_A], %%zmm5, %%zmm3") /* zmm3 = x_im * b_im */
606 __ASM_EMIT("vfmadd231ps 0x80 + %[FFT_A], %%zmm4, %%zmm3") /* zmm3 = c_re = x_re * b_re + x_im * b_im */
607 __ASM_EMIT("vfmsub231ps 0x80 + %[FFT_A], %%zmm5, %%zmm1") /* zmm1 = c_im = x_re * b_im - x_im * b_re */
608 __ASM_EMIT("vsubps %%zmm3, %%zmm0, %%zmm4") /* zmm4 = a_re - c_re = R8 R9 R10 R11 R12 R13 R14 R15 R24 R25 R26 R27 R28 R29 R30 R31 */
609 __ASM_EMIT("vsubps %%zmm1, %%zmm2, %%zmm5") /* zmm5 = a_im - c_im = I8 I9 I10 I11 I12 I13 I14 I15 I24 I25 I26 I27 I28 I29 I30 I31 */
610 __ASM_EMIT("vaddps %%zmm3, %%zmm0, %%zmm0") /* zmm0 = a_re + c_re = R0 R1 R2 R3 R4 R5 R6 R7 R16 R17 R18 R19 R20 R21 R22 R23 */
611 __ASM_EMIT("vaddps %%zmm1, %%zmm2, %%zmm2") /* zmm2 = a_im + c_im = I0 I1 I2 I3 I4 I5 I6 I7 I16 I17 I18 I19 I20 I21 I22 I23 */
612 /* Store */
613 __ASM_EMIT("vmovups %%ymm0, 0x00(%[dst_re])")
614 __ASM_EMIT("vmovups %%ymm4, 0x20(%[dst_re])")
615 __ASM_EMIT("vextractf64x4 $1, %%zmm0, 0x40(%[dst_re])")
616 __ASM_EMIT("vextractf64x4 $1, %%zmm4, 0x60(%[dst_re])")
617 __ASM_EMIT("vmovups %%ymm2, 0x00(%[dst_im])")
618 __ASM_EMIT("vmovups %%ymm5, 0x20(%[dst_im])")
619 __ASM_EMIT("vextractf64x4 $1, %%zmm2, 0x40(%[dst_im])")
620 __ASM_EMIT("vextractf64x4 $1, %%zmm5, 0x60(%[dst_im])")
621 __ASM_EMIT("add $0x80, %[dst_re]")
622 __ASM_EMIT("add $0x80, %[dst_im]")
623
624 : [dst_re] "+r" (dst_re), [dst_im] "+r" (dst_im),
625 [index] "+r"(index)
626 : [src_re] "r" (src_re), [src_im] "r" (src_im),
627 [regs] X86_GREG (regs),
628 [FFT_A] "o" (FFT_A),
629 [FFT_I] "o" (FFT_SCRAMBLE_INDICES)
630 : "cc", "memory",
631 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
632 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
633 );
634 }
635 }
636
637 static inline void FFT_SCRAMBLE_COPY_REVERSE_NAME(float *dst_re, float *dst_im, const float *src_re, const float *src_im, size_t rank)
638 {
639 const size_t regs = 1 << (rank);
640
641 for (size_t i=0; i < regs; ++i)
642 {
643 size_t index = reverse_bits(FFT_TYPE(i), rank);
644
645 ARCH_X86_ASM(
646 /* Load even indices */
647 __ASM_EMIT("vmovaps 0x280 + %[FFT_I], %%zmm7") /* zmm7 = indices */
648 FFT_SCRAMBLE_LOAD4X2("%%xmm0", "%%xmm2") /* xmm0 = r0 r16 r8 r24, xmm2 = i0 i16 i8 i24 */
649 FFT_SCRAMBLE_LOAD4X2("%%xmm4", "%%xmm5") /* xmm4 = r4 r20 r12 r28, xmm5 = i4 i20 i12 i28 */
650 __ASM_EMIT("vinsertf32x4 $01, %%xmm4, %%zmm0, %%zmm0") /* zmm0 = r0 r16 r8 r24 r4 r20 r12 r28 x x x x x x x x */
651 __ASM_EMIT("vinsertf32x4 $01, %%xmm5, %%zmm2, %%zmm2") /* zmm2 = i0 i16 i8 i24 i4 i20 i12 i28 x x x x x x x x */
652 FFT_SCRAMBLE_LOAD4X2("%%xmm4", "%%xmm5") /* xmm4 = r2 r18 r10 r26, xmm5 = i2 i18 i10 i26 */
653 __ASM_EMIT("vinsertf32x4 $02, %%xmm4, %%zmm0, %%zmm0") /* zmm0 = r0 r16 r8 r24 r4 r20 r12 r28 r2 r18 r10 r26 x x x x */
654 __ASM_EMIT("vinsertf32x4 $02, %%xmm5, %%zmm2, %%zmm2") /* zmm2 = i0 i16 i8 i24 i4 i20 i12 i28 i2 i18 i10 i26 x x x x */
655 FFT_SCRAMBLE_LOAD4X2("%%xmm4", "%%xmm5") /* xmm4 = r6 r22 r14 r30, xmm5 = i6 i22 i14 i30 */
656 __ASM_EMIT("vinsertf32x4 $03, %%xmm4, %%zmm0, %%zmm0") /* zmm0 = r0 r16 r8 r24 r4 r20 r12 r28 r2 r18 r10 r26 r6 r22 r14 r30 */
657 __ASM_EMIT("vinsertf32x4 $03, %%xmm5, %%zmm2, %%zmm2") /* zmm2 = i0 i16 i8 i24 i4 i20 i12 i28 i2 i18 i10 i26 i6 i22 i14 i30 */
658 /* Load odd indices */
659 FFT_SCRAMBLE_LOAD4X2("%%xmm1", "%%xmm3") /* xmm1 = r1 r17 r9 r25, xmm3 = i1 i17 i9 i25 */
660 FFT_SCRAMBLE_LOAD4X2("%%xmm4", "%%xmm5") /* xmm4 = r5 r21 r13 r29, xmm5 = i5 i21 i13 i29 */
661 __ASM_EMIT("vinsertf32x4 $01, %%xmm4, %%zmm1, %%zmm1") /* zmm1 = r1 r17 r9 r25 r5 r21 r13 r29 x x x x x x x x */
662 __ASM_EMIT("vinsertf32x4 $01, %%xmm5, %%zmm3, %%zmm3") /* zmm3 = i1 i17 i9 i25 i5 i21 i13 i29 x x x x x x x x */
663 FFT_SCRAMBLE_LOAD4X2("%%xmm4", "%%xmm5") /* xmm4 = r3 r19 r11 r27, xmm5 = i3 i19 i11 i27 */
664 __ASM_EMIT("vinsertf32x4 $02, %%xmm4, %%zmm1, %%zmm1") /* zmm1 = r1 r17 r9 r25 r5 r21 r13 r29 r3 r19 r11 r27 x x x x */
665 __ASM_EMIT("vinsertf32x4 $02, %%xmm5, %%zmm3, %%zmm3") /* zmm3 = i1 i17 i9 i25 i5 i21 i13 i29 i3 i19 i11 i27 x x x x */
666 FFT_SCRAMBLE_LOAD4X2("%%xmm4", "%%xmm5") /* xmm4 = r7 r23 r15 r31, xmm5 = i7 i23 i15 i31 */
667 __ASM_EMIT("vinsertf32x4 $03, %%xmm4, %%zmm1, %%zmm1") /* zmm1 = r1 r17 r9 r25 r5 r21 r13 r29 r3 r19 r11 r27 r7 r23 r15 r31 */
668 __ASM_EMIT("vinsertf32x4 $03, %%xmm5, %%zmm3, %%zmm3") /* zmm3 = i1 i17 i9 i25 i5 i21 i13 i29 i3 i19 i11 i27 i7 i23 i15 i31 */
669 /* Final permute */
670 __ASM_EMIT("vpermps %%zmm0, %%zmm7, %%zmm0") /* zmm0 = r0 r2 r4 r6 r8 r10 r12 r14 r16 r18 r20 r22 r24 r26 r28 r30 */
671 __ASM_EMIT("vpermps %%zmm1, %%zmm7, %%zmm1") /* zmm1 = r1 r3 r5 r7 r9 r11 r13 r15 r17 r19 r21 r23 r25 r27 r29 r31 */
672 __ASM_EMIT("vpermps %%zmm2, %%zmm7, %%zmm2") /* zmm2 = i0 i2 i4 i6 i8 i10 i12 i14 i16 i18 i20 i22 i24 i26 i28 i30 */
673 __ASM_EMIT("vpermps %%zmm3, %%zmm7, %%zmm3") /* zmm3 = i1 i3 i5 i7 i9 i11 i13 i15 i17 i19 i21 i23 i25 i27 i29 i31 */
674
675 /* 1st-order 16x butterfly */
676 /* zmm0 = r0 r2 r4 r6 r8 r10 r12 r14 r16 r18 r20 r22 r24 r26 r28 r30 */
677 /* zmm1 = r1 r3 r5 r7 r9 r11 r13 r15 r17 r19 r21 r23 r25 r27 r29 r31 */
678 /* zmm2 = i0 i2 i4 i6 i8 i10 i12 i14 i16 i18 i20 i22 i24 i26 i28 i30 */
679 /* zmm3 = i1 i3 i5 i7 i9 i11 i13 i15 i17 i19 i21 i23 i25 i27 i29 i31 */
680 __ASM_EMIT("vsubps %%zmm1, %%zmm0, %%zmm4") /* zmm4 = r0-r1 r2-r3 r4-r5 r6-r7 ... = r1' r3' r5' r7' r9' r11' r13' r15' r17' r19' r21' r23' r25' r27' r29' r31' */
681 __ASM_EMIT("vsubps %%zmm3, %%zmm2, %%zmm5") /* zmm5 = i0-i1 i2-i3 i4-i5 i6-i7 ... = i1' i3' i5' i7' i9' i11' i13' i15' i17' i19' i21' i23' i25' i27' i29' i31' */
682 __ASM_EMIT("vaddps %%zmm1, %%zmm0, %%zmm0") /* zmm0 = r0+r1 r2+r3 r4+r5 r6+r7 ... = r0' r2' r4' r6' r8' r10' r12' r14' r16' r18' r20' r22' r24' r26' r28' r30' */
683 __ASM_EMIT("vaddps %%zmm3, %%zmm2, %%zmm2") /* zmm2 = i0+i1 i2+i3 i4+i5 i6+i7 ... = i0' i2' i4' i6' i8' i10' i12' i14' i16' i18' i20' i22' i24' i26' i28' i30' */
684 /* 2nd-order 16x butterfly */
685 __ASM_EMIT("vmovaps %%zmm0, %%zmm1") /* zmm1 = zmm0 = r0' r2' r4' r6' r8' r10' r12' r14' r16' r18' r20' r22' r24' r26' r28' r30' */
686 __ASM_EMIT("vmovaps %%zmm2, %%zmm3") /* zmm3 = zmm2 = i0' i2' i4' i6' i8' i10' i12' i14' i16' i18' i20' i22' i24' i26' i28' i30' */
687 __ASM_EMIT("vmovaps 0x80 + %[FFT_I], %%zmm6") /* zmm6 = shuffle_a[1] */
688 __ASM_EMIT("vmovaps 0xc0 + %[FFT_I], %%zmm7") /* zmm7 = shuffle_b[1] */
689 __ASM_EMIT("vpermt2ps %%zmm4, %%zmm6, %%zmm0") /* zmm0 = r0' r1' r4' r5' r8' r9' r12' r13' r16' r17' r20' r21' r24' r25' r28' r29' */
690 __ASM_EMIT("vpermt2ps %%zmm5, %%zmm6, %%zmm2") /* zmm2 = i0' i1' i4' i5' i8' i9' i12' i13' i16' i17' i20' i21' i24' i25' i28' i29' */
691 __ASM_EMIT("vpermt2ps %%zmm5, %%zmm7, %%zmm1") /* zmm1 = r2' i3' r6' i7' r10' i11' r14' i15' r18' i19' r22' i23' r26' i27' r30' i31' */
692 __ASM_EMIT("vpermt2ps %%zmm4, %%zmm7, %%zmm3") /* zmm3 = i2' r3' i6' r7' i10' r11' i14' r15' i18' r19' i22' r23' i26' r27' i30' r31' */
693 __ASM_EMIT("vsubps %%zmm1, %%zmm0, %%zmm4") /* zmm4 = r2" r1" r6" r5" r10" r9" r14" r13" r18" r17" r22" r21" r26" r25" r30" r29" = r0'-r2' r1'-i3' r4'-r6' r5'-i7' r8'-r10' r9'-i11' r12'-r14' r13'-i15' r16'-r18' r17'-i19' r20'-r22' r21'-i23' r24'-r26' r25'-i27' r28'-r30' r29'-i31' */
694 __ASM_EMIT("vsubps %%zmm3, %%zmm2, %%zmm5") /* zmm5 = i2" i3" i6" i7" i10" i11" i14" i15" i18" i19" i22" i23" i26" i27" i30" i31" = i0'-i2' i1'-r3' i4'-i6' i5'-r7' i8'-i10' i9'-r11' i12'-i14' i13'-r15' i16'-i18' i17'-r19' i20'-i22' i21'-r23' i24'-i26' i25'-r27' i28'-i30' i29'-r31' */
695 __ASM_EMIT("vaddps %%zmm1, %%zmm0, %%zmm0") /* zmm0 = r0" r3" r4" r7" r8" r11" r12" r15" r16" r19" r20" r23" r24" r27" r28" r31" = r0'+r2' r1'+i3' r4'+r6' r5'+i7' r8'+r10' r9'+i11' r12'+r14' r13'+i15' r16'+r18' r17'+i19' r20'+r22' r21'+i23' r24'+r26' r25'+i27' r28'+r30' r29'+i31' */
696 __ASM_EMIT("vaddps %%zmm3, %%zmm2, %%zmm2") /* zmm2 = i0" i1" i4" i5" i8" i9" i12" i13" i16" i17" i20" i21" i24" i25" i28" i29" = i0'+i2' i1'+r3' i4'+i6' i5'+r7' i8'+i10' i9'+r11' i12'+i14' i13'+r15' i16'+i18' i17'+r19' i20'+i22' i21'+r23' i24'+i26' i25'+r27' i28'+i30' i29'+r31' */
697 /* 3rd-order 16x butterfly */
698 __ASM_EMIT("vmovaps %%zmm0, %%zmm1") /* zmm1 = zmm0 = r0" r3" r4" r7" r8" r11" r12" r15" r16" r19" r20" r23" r24" r27" r28" r31" */
699 __ASM_EMIT("vmovaps %%zmm2, %%zmm3") /* zmm3 = zmm2 = i0" i1" i4" i5" i8" i9" i12" i13" i16" i17" i20" i21" i24" i25" i28" i29" */
700 __ASM_EMIT("vmovaps 0x140 + %[FFT_I], %%zmm6") /* zmm6 = shuffle_b[2] */
701 __ASM_EMIT("vmovaps 0x100 + %[FFT_I], %%zmm7") /* zmm7 = shuffle_a[2] */
702 __ASM_EMIT("vpermt2ps %%zmm4, %%zmm6, %%zmm0") /* zmm0 = a_re = r0" r1" r2" r3" r8" r9" r10" r11" r16" r17" r18" r19" r24" r25" r26" r27" */
703 __ASM_EMIT("vpermt2ps %%zmm5, %%zmm7, %%zmm2") /* zmm2 = a_im = i0" i1" i2" i3" i8" i9" i10" i11" i16" i17" i18" i19" i24" i25" i26" i27" */
704 __ASM_EMIT("vmovaps 0x1c0 + %[FFT_I], %%zmm6") /* zmm6 = shuffle_b[3] */
705 __ASM_EMIT("vmovaps 0x180 + %[FFT_I], %%zmm7") /* zmm7 = shuffle_a[3] */
706 __ASM_EMIT("vpermt2ps %%zmm4, %%zmm6, %%zmm1") /* zmm1 = b_re = r4" r5" r6" r7" r12" r13" r14" r15" r20" r21" r22" r23" r28" r29" r30" r31" */
707 __ASM_EMIT("vpermt2ps %%zmm5, %%zmm7, %%zmm3") /* zmm3 = b_im = i4" i5" i6" i7" i12" i13" i14" i15" i20" i21" i22" i23" i28" i29" i30" i31" */
708 __ASM_EMIT("vmulps 0x40 + %[FFT_A], %%zmm1, %%zmm4") /* zmm4 = x_im * b_re */
709 __ASM_EMIT("vmulps 0x40 + %[FFT_A], %%zmm3, %%zmm5") /* zmm5 = x_im * b_im */
710 __ASM_EMIT("vfmsub231ps 0x00 + %[FFT_A], %%zmm1, %%zmm5") /* zmm5 = c_re = x_re * b_re - x_im * b_im */
711 __ASM_EMIT("vfmadd231ps 0x00 + %[FFT_A], %%zmm3, %%zmm4") /* zmm4 = c_im = x_re * b_im + x_im * b_re */
712 __ASM_EMIT("vsubps %%zmm5, %%zmm0, %%zmm1") /* zmm1 = a_re - c_re = r4` r5` r6` r7` r12` r13` r14` r15` r20` r21` r22` r23` r28` r29` r30` r31` */
713 __ASM_EMIT("vsubps %%zmm4, %%zmm2, %%zmm3") /* zmm3 = a_im - c_im = i4` i5` i6` i7` i12` i13` i14` i15` i20` i21` i22` i23` i28` i29` i30` i31` */
714 __ASM_EMIT("vaddps %%zmm5, %%zmm0, %%zmm0") /* zmm0 = a_re + c_re = r0` r1` r2` r3` r8` r9` r10` r11` r16` r17` r18` r19` r24` r25` r26` r27` */
715 __ASM_EMIT("vaddps %%zmm4, %%zmm2, %%zmm2") /* zmm2 = a_im + c_im = i0` i1` i2` i3` i8` i9` i10` i11` i16` i17` i18` i19` i24` i25` i26` i27` */
716 /* 4th-order 16x butterfly */
717 __ASM_EMIT("vmovaps %%zmm0, %%zmm4") /* zmm4 = zmm0 = r0` r1` r2` r3` r8` r9` r10` r11` r16` r17` r18` r19` r24` r25` r26` r27` */
718 __ASM_EMIT("vmovaps %%zmm2, %%zmm5") /* zmm5 = zmm2 = i0` i1` i2` i3` i8` i9` i10` i11` i16` i17` i18` i19` i24` i25` i26` i27` */
719 __ASM_EMIT("vmovaps 0x200 + %[FFT_I], %%zmm6") /* zmm6 = shuffle_a[4] */
720 __ASM_EMIT("vmovaps 0x240 + %[FFT_I], %%zmm7") /* zmm7 = shuffle_b[4] */
721 __ASM_EMIT("vpermt2ps %%zmm1, %%zmm6, %%zmm0") /* zmm0 = a_re = r0` r1` r2` r3` r4` r5` r6` r7` r16` r17` r18` r19` r20` r21` r22` r23` */
722 __ASM_EMIT("vpermt2ps %%zmm1, %%zmm7, %%zmm4") /* zmm4 = b_re = r8` r9` r10` r11` r12` r13` r14` r15` r24` r25` r26` r27` r28` r29` r30` r31` */
723 __ASM_EMIT("vpermt2ps %%zmm3, %%zmm6, %%zmm2") /* zmm2 = a_im = i0` i1` i2` i3` i4` i5` i6` i7` i16` i17` i18` i19` i20` i21` i22` i23` */
724 __ASM_EMIT("vpermt2ps %%zmm3, %%zmm7, %%zmm5") /* zmm5 = b_im = i8` i9` i10` i11` i12` i13` i14` i15` i24` i25` i26` i27` i28` i29` i30` i31` */
725 __ASM_EMIT("vmulps 0xc0 + %[FFT_A], %%zmm4, %%zmm1") /* zmm1 = x_im * b_re */
726 __ASM_EMIT("vmulps 0xc0 + %[FFT_A], %%zmm5, %%zmm3") /* zmm3 = x_im * b_im */
727 __ASM_EMIT("vfmsub231ps 0x80 + %[FFT_A], %%zmm4, %%zmm3") /* zmm3 = c_re = x_re * b_re - x_im * b_im */
728 __ASM_EMIT("vfmadd231ps 0x80 + %[FFT_A], %%zmm5, %%zmm1") /* zmm1 = c_im = x_re * b_im + x_im * b_re */
729 __ASM_EMIT("vsubps %%zmm3, %%zmm0, %%zmm4") /* zmm4 = a_re - c_re = R8 R9 R10 R11 R12 R13 R14 R15 R24 R25 R26 R27 R28 R29 R30 R31 */
730 __ASM_EMIT("vsubps %%zmm1, %%zmm2, %%zmm5") /* zmm5 = a_im - c_im = I8 I9 I10 I11 I12 I13 I14 I15 I24 I25 I26 I27 I28 I29 I30 I31 */
731 __ASM_EMIT("vaddps %%zmm3, %%zmm0, %%zmm0") /* zmm0 = a_re + c_re = R0 R1 R2 R3 R4 R5 R6 R7 R16 R17 R18 R19 R20 R21 R22 R23 */
732 __ASM_EMIT("vaddps %%zmm1, %%zmm2, %%zmm2") /* zmm2 = a_im + c_im = I0 I1 I2 I3 I4 I5 I6 I7 I16 I17 I18 I19 I20 I21 I22 I23 */
733 /* Store */
734 __ASM_EMIT("vmovups %%ymm0, 0x00(%[dst_re])")
735 __ASM_EMIT("vmovups %%ymm4, 0x20(%[dst_re])")
736 __ASM_EMIT("vextractf64x4 $1, %%zmm0, 0x40(%[dst_re])")
737 __ASM_EMIT("vextractf64x4 $1, %%zmm4, 0x60(%[dst_re])")
738 __ASM_EMIT("vmovups %%ymm2, 0x00(%[dst_im])")
739 __ASM_EMIT("vmovups %%ymm5, 0x20(%[dst_im])")
740 __ASM_EMIT("vextractf64x4 $1, %%zmm2, 0x40(%[dst_im])")
741 __ASM_EMIT("vextractf64x4 $1, %%zmm5, 0x60(%[dst_im])")
742 __ASM_EMIT("add $0x80, %[dst_re]")
743 __ASM_EMIT("add $0x80, %[dst_im]")
744
745 : [dst_re] "+r" (dst_re), [dst_im] "+r" (dst_im),
746 [index] "+r"(index)
747 : [src_re] "r" (src_re), [src_im] "r" (src_im),
748 [regs] X86_GREG (regs),
749 [FFT_A] "o" (FFT_A),
750 [FFT_I] "o" (FFT_SCRAMBLE_INDICES)
751 : "cc", "memory",
752 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
753 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
754 );
755 }
756 }
757
758 } /* namespace avx512 */
759 } /* namespace lsp */
760
761 #undef FFT_SCRAMBLE_SELF_DIRECT_NAME
762 #undef FFT_SCRAMBLE_SELF_REVERSE_NAME
763 #undef FFT_SCRAMBLE_COPY_DIRECT_NAME
764 #undef FFT_SCRAMBLE_COPY_REVERSE_NAME
765 #undef FFT_TYPE
766 #undef FFT_GATHER_LOOP
767 #undef FFT_SCRAMBLE_LOAD4X2
768