Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/x86/avx512/fft/p_scramble.h

raw
1 /*
2 * Copyright (C) 2026 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2026 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 18 мар. 2026 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #define FFT_PSCRAMBLE_LOAD4X4(dst, a, b, c, d) \
23 __ASM_EMIT("vmovlps (%[src], %[index], 8), " a ", " a) /* a = R0 I0 x x */ \
24 __ASM_EMIT("add %[regs], %[index]") \
25 __ASM_EMIT("vmovhps (%[src], %[index], 8), " a ", " a) /* a = R0 I0 R1 I1 */ \
26 __ASM_EMIT("add %[regs], %[index]") \
27 __ASM_EMIT("vmovlps (%[src], %[index], 8), " b ", " b) /* a = R2 I2 x x */ \
28 __ASM_EMIT("add %[regs], %[index]") \
29 __ASM_EMIT("vmovhps (%[src], %[index], 8), " b ", " b) /* a = R2 I2 R3 I3 */ \
30 __ASM_EMIT("add %[regs], %[index]") \
31 __ASM_EMIT("vinsertf32x4 $01, " b ", " dst ", " dst) /* dst = R0 I0 R1 I1 R2 I2 R3 I3 x x x x x x x x */ \
32 __ASM_EMIT("vmovlps (%[src], %[index], 8), " c ", " c) /* a = R4 I4 x x */ \
33 __ASM_EMIT("add %[regs], %[index]") \
34 __ASM_EMIT("vmovhps (%[src], %[index], 8), " c ", " c) /* a = R4 I4 R5 I5 */ \
35 __ASM_EMIT("add %[regs], %[index]") \
36 __ASM_EMIT("vinsertf32x4 $02, " c ", " dst ", " dst) /* dst = R0 I0 R1 I1 R2 I2 R3 I3 R4 I4 R5 I5 x x x x */ \
37 __ASM_EMIT("vmovlps (%[src], %[index], 8), " d ", " d) /* a = R6 I6 x x */ \
38 __ASM_EMIT("add %[regs], %[index]") \
39 __ASM_EMIT("vmovhps (%[src], %[index], 8), " d ", " d) /* a = R6 I6 R7 I7 */ \
40 __ASM_EMIT("add %[regs], %[index]") \
41 __ASM_EMIT("vinsertf32x4 $03, " d ", " dst ", " dst) /* dst = R0 I0 R1 I1 R2 I2 R3 I3 R4 I4 R5 I5 R6 I6 R7 I7 */
42
43
44 namespace lsp
45 {
46 namespace avx512
47 {
48 static inline void FFT_PSCRAMBLE_SELF_DIRECT_NAME(float *dst, size_t rank)
49 {
50 // Calculate number of items
51 size_t items = (1 << rank) - 1;
52
53 for (size_t i = 1; i < items; ++i)
54 {
55 size_t j = reverse_bits(FFT_TYPE(i), rank); /* Reverse the order of the bits */
56 if (i >= j)
57 continue;
58
59 /* Copy the values from the reversed position */
60 ARCH_X86_ASM
61 (
62 __ASM_EMIT("vmovlps (%[dst], %[i], 8), %%xmm0, %%xmm0")
63 __ASM_EMIT("vmovlps (%[dst], %[j], 8), %%xmm1, %%xmm1")
64 __ASM_EMIT("vmovlps %%xmm0, (%[dst], %[j], 8)")
65 __ASM_EMIT("vmovlps %%xmm1, (%[dst], %[i], 8)")
66 :
67 : [dst] "r"(dst), [i] "r"(i), [j] "r"(j)
68 : "memory",
69 "%xmm0", "%xmm1"
70 );
71 }
72
73 // Perform butterfly 8x
74 size_t off = 0;
75 items = 1 << (rank - 3);
76
77 // Perform 4-element butterflies
78 ARCH_X86_ASM
79 (
80 /* Loop 4x 4-element butterflies */
81 __ASM_EMIT("sub $4, %[items]")
82 __ASM_EMIT("jb 2f")
83 __ASM_EMIT("kmovw 0x00 + %[MASK], %%k4")
84 __ASM_EMIT("kmovw 0x02 + %[MASK], %%k5")
85 __ASM_EMIT("1:")
86 /* Load data to registers */
87 __ASM_EMIT("vmovups 0x00(%[dst], %[off]), %%zmm0") /* zmm0 = r0 i0 r1 i1 r2 i2 r3 i3 r4 i4 r5 i5 r6 i6 r7 i7 */
88 __ASM_EMIT("vmovups 0x40(%[dst], %[off]), %%zmm1") /* zmm1 = r8 i8 r9 i9 r10 i10 r11 i11 r12 i12 r13 i13 r14 i14 r15 i15 */
89 __ASM_EMIT("vmovups 0x80(%[dst], %[off]), %%zmm4") /* zmm4 = r16 i16 r17 i17 r18 i18 r19 i19 r20 i20 r21 i21 r22 i22 r23 i23 */
90 __ASM_EMIT("vmovups 0xc0(%[dst], %[off]), %%zmm6") /* zmm6 = r24 i24 r25 i25 r26 i26 r27 i27 r28 i28 r29 i29 r30 i30 r31 i31 */
91 __ASM_EMIT("vcompressps %%zmm0, %%zmm2 %{%%k5%}") /* zmm2 = i0 i1 i2 i3 i4 i5 i6 i7 ? ? ? ? ? ? ? ? */
92 __ASM_EMIT("vcompressps %%zmm1, %%zmm3 %{%%k5%}") /* zmm3 = i8 i9 i10 i11 i12 i13 i14 i15 ? ? ? ? ? ? ? ? */
93 __ASM_EMIT("vcompressps %%zmm4, %%zmm5 %{%%k5%}") /* zmm5 = i16 i17 i18 i19 i20 i21 i22 i23 ? ? ? ? ? ? ? ? */
94 __ASM_EMIT("vcompressps %%zmm6, %%zmm7 %{%%k5%}") /* zmm7 = i24 i25 i26 i27 i28 i29 i30 i31 ? ? ? ? ? ? ? ? */
95 __ASM_EMIT("vcompressps %%zmm0, %%zmm0 %{%%k4%}") /* zmm0 = r0 r1 r2 r3 r4 r5 r6 r7 ? ? ? ? ? ? ? ? */
96 __ASM_EMIT("vcompressps %%zmm1, %%zmm1 %{%%k4%}") /* zmm1 = r8 r9 r10 r11 r12 r13 r14 r15 ? ? ? ? ? ? ? ? */
97 __ASM_EMIT("vcompressps %%zmm4, %%zmm4 %{%%k4%}") /* zmm4 = r16 r17 r18 r19 r20 r21 r22 r23 ? ? ? ? ? ? ? ? */
98 __ASM_EMIT("vcompressps %%zmm6, %%zmm6 %{%%k4%}") /* zmm6 = r24 r25 r26 r27 r28 r29 r30 r31 ? ? ? ? ? ? ? ? */
99 __ASM_EMIT("vinsertf64x4 $1, %%ymm3, %%zmm2, %%zmm2") /* zmm2 = i0 i1 i2 i3 i4 i5 i6 i7 i8 i9 i10 i11 i12 i13 i14 i15 */
100 __ASM_EMIT("vinsertf64x4 $1, %%ymm7, %%zmm5, %%zmm5") /* zmm5 = i16 i17 i18 i19 i20 i21 i22 i23 i24 i25 i26 i27 i28 i29 i30 i31 */
101 __ASM_EMIT("vinsertf64x4 $1, %%ymm1, %%zmm0, %%zmm0") /* zmm0 = r0 r1 r2 r3 r4 r5 r6 r7 r8 r9 r10 r11 r12 r13 r14 r15 */
102 __ASM_EMIT("vinsertf64x4 $1, %%ymm6, %%zmm4, %%zmm4") /* zmm4 = r16 r17 r18 r19 r20 r21 r22 r23 r24 r25 r26 r27 r28 r29 r30 r31 */
103 /* 1st-order 16x butterfly */
104 __ASM_EMIT("vmovaps %%zmm0, %%zmm1") /* zmm1 = zmm0 = r0 r1 r2 r3 r4 r5 r6 r7 r8 r9 r10 r11 r12 r13 r14 r15 */
105 __ASM_EMIT("vmovaps %%zmm2, %%zmm3") /* zmm3 = zmm2 = i0 i1 i2 i3 i4 i5 i6 i7 i8 i9 i10 i11 i12 i13 i14 i15 */
106 __ASM_EMIT("vmovaps 0x00 + %[FFT_I], %%zmm6") /* zmm6 = shuffle_a[0] */
107 __ASM_EMIT("vmovaps 0x40 + %[FFT_I], %%zmm7") /* zmm7 = shuffle_b[0] */
108 __ASM_EMIT("vpermt2ps %%zmm4, %%zmm6, %%zmm0") /* zmm0 = r0 r2 r4 r6 r8 r10 r12 r14 r16 r18 r20 r22 r24 r26 r28 r30 */
109 __ASM_EMIT("vpermt2ps %%zmm4, %%zmm7, %%zmm1") /* zmm1 = r1 r3 r5 r7 r9 r11 r13 r15 r17 r19 r21 r23 r25 r27 r29 r31 */
110 __ASM_EMIT("vpermt2ps %%zmm5, %%zmm6, %%zmm2") /* zmm2 = i0 i2 i4 i6 i8 i10 i12 i14 i16 i18 i20 i22 i24 i26 i28 i30 */
111 __ASM_EMIT("vpermt2ps %%zmm5, %%zmm7, %%zmm3") /* zmm3 = i1 i3 i5 i7 i9 i11 i13 i15 i17 i19 i21 i23 i25 i27 i29 i31 */
112 __ASM_EMIT("vsubps %%zmm1, %%zmm0, %%zmm4") /* zmm4 = r0-r1 r2-r3 r4-r5 r6-r7 ... = r1' r3' r5' r7' r9' r11' r13' r15' r17' r19' r21' r23' r25' r27' r29' r31' */
113 __ASM_EMIT("vsubps %%zmm3, %%zmm2, %%zmm5") /* zmm5 = i0-i1 i2-i3 i4-i5 i6-i7 ... = i1' i3' i5' i7' i9' i11' i13' i15' i17' i19' i21' i23' i25' i27' i29' i31' */
114 __ASM_EMIT("vaddps %%zmm1, %%zmm0, %%zmm0") /* zmm0 = r0+r1 r2+r3 r4+r5 r6+r7 ... = r0' r2' r4' r6' r8' r10' r12' r14' r16' r18' r20' r22' r24' r26' r28' r30' */
115 __ASM_EMIT("vaddps %%zmm3, %%zmm2, %%zmm2") /* zmm2 = i0+i1 i2+i3 i4+i5 i6+i7 ... = i0' i2' i4' i6' i8' i10' i12' i14' i16' i18' i20' i22' i24' i26' i28' i30' */
116 /* 2nd-order 16x butterfly */
117 __ASM_EMIT("vmovaps %%zmm0, %%zmm1") /* zmm1 = zmm0 = r0' r2' r4' r6' r8' r10' r12' r14' r16' r18' r20' r22' r24' r26' r28' r30' */
118 __ASM_EMIT("vmovaps %%zmm2, %%zmm3") /* zmm3 = zmm2 = i0' i2' i4' i6' i8' i10' i12' i14' i16' i18' i20' i22' i24' i26' i28' i30' */
119 __ASM_EMIT("vmovaps 0x80 + %[FFT_I], %%zmm6") /* zmm6 = shuffle_a[1] */
120 __ASM_EMIT("vmovaps 0xc0 + %[FFT_I], %%zmm7") /* zmm7 = shuffle_b[1] */
121 __ASM_EMIT("vpermt2ps %%zmm4, %%zmm6, %%zmm0") /* zmm0 = r0' r1' r4' r5' r8' r9' r12' r13' r16' r17' r20' r21' r24' r25' r28' r29' */
122 __ASM_EMIT("vpermt2ps %%zmm5, %%zmm6, %%zmm2") /* zmm2 = i0' i1' i4' i5' i8' i9' i12' i13' i16' i17' i20' i21' i24' i25' i28' i29' */
123 __ASM_EMIT("vpermt2ps %%zmm5, %%zmm7, %%zmm1") /* zmm1 = r2' i3' r6' i7' r10' i11' r14' i15' r18' i19' r22' i23' r26' i27' r30' i31' */
124 __ASM_EMIT("vpermt2ps %%zmm4, %%zmm7, %%zmm3") /* zmm3 = i2' r3' i6' r7' i10' r11' i14' r15' i18' r19' i22' r23' i26' r27' i30' r31' */
125 __ASM_EMIT("vsubps %%zmm1, %%zmm0, %%zmm4") /* zmm4 = r2" r3" r6" r7" r10" r11" r14" r15" r18" r19" r22" r23" r26" r27" r30" r31" = r0'-r2' r1'-i3' r4'-r6' r5'-i7' r8'-r10' r9'-i11' r12'-r14' r13'-i15' r16'-r18' r17'-i19' r20'-r22' r21'-i23' r24'-r26' r25'-i27' r28'-r30' r29'-i31' */
126 __ASM_EMIT("vsubps %%zmm3, %%zmm2, %%zmm5") /* zmm5 = i2" i1" i6" i5" i10" i9" i14" i13" i18" i17" i22" i21" i26" i25" i30" i29" = i0'-i2' i1'-r3' i4'-i6' i5'-r7' i8'-i10' i9'-r11' i12'-i14' i13'-r15' i16'-i18' i17'-r19' i20'-i22' i21'-r23' i24'-i26' i25'-r27' i28'-i30' i29'-r31' */
127 __ASM_EMIT("vaddps %%zmm1, %%zmm0, %%zmm0") /* zmm0 = r0" r1" r4" r5" r8" r9" r12" r13" r16" r17" r20" r21" r24" r25" r28" r29" = r0'+r2' r1'+i3' r4'+r6' r5'+i7' r8'+r10' r9'+i11' r12'+r14' r13'+i15' r16'+r18' r17'+i19' r20'+r22' r21'+i23' r24'+r26' r25'+i27' r28'+r30' r29'+i31' */
128 __ASM_EMIT("vaddps %%zmm3, %%zmm2, %%zmm2") /* zmm2 = i0" i3" i4" i7" i8" i11" i12" i15" i16" i19" i20" i23" i24" i27" i28" i31" = i0'+i2' i1'+r3' i4'+i6' i5'+r7' i8'+i10' i9'+r11' i12'+i14' i13'+r15' i16'+i18' i17'+r19' i20'+i22' i21'+r23' i24'+i26' i25'+r27' i28'+i30' i29'+r31' */
129 /* 3rd-order 16x butterfly */
130 __ASM_EMIT("vmovaps %%zmm0, %%zmm1") /* zmm1 = zmm0 = r0" r1" r4" r5" r8" r9" r12" r13" r16" r17" r20" r21" r24" r25" r28" r29" */
131 __ASM_EMIT("vmovaps %%zmm2, %%zmm3") /* zmm3 = zmm2 = i0" i3" i4" i7" i8" i11" i12" i15" i16" i19" i20" i23" i24" i27" i28" i31" */
132 __ASM_EMIT("vmovaps 0x100 + %[FFT_I], %%zmm6") /* zmm6 = shuffle_a[2] */
133 __ASM_EMIT("vmovaps 0x140 + %[FFT_I], %%zmm7") /* zmm7 = shuffle_b[2] */
134 __ASM_EMIT("vpermt2ps %%zmm4, %%zmm6, %%zmm0") /* zmm0 = a_re = r0" r1" r2" r3" r8" r9" r10" r11" r16" r17" r18" r19" r24" r25" r26" r27" */
135 __ASM_EMIT("vpermt2ps %%zmm5, %%zmm7, %%zmm2") /* zmm2 = a_im = i0" i1" i2" i3" i8" i9" i10" i11" i16" i17" i18" i19" i24" i25" i26" i27" */
136 __ASM_EMIT("vmovaps 0x180 + %[FFT_I], %%zmm6") /* zmm6 = shuffle_a[3] */
137 __ASM_EMIT("vmovaps 0x1c0 + %[FFT_I], %%zmm7") /* zmm7 = shuffle_b[3] */
138 __ASM_EMIT("vpermt2ps %%zmm4, %%zmm6, %%zmm1") /* zmm1 = b_re = r4" r5" r6" r7" r12" r13" r14" r15" r20" r21" r22" r23" r28" r29" r30" r31" */
139 __ASM_EMIT("vpermt2ps %%zmm5, %%zmm7, %%zmm3") /* zmm3 = b_im = i4" i5" i6" i7" i12" i13" i14" i15" i20" i21" i22" i23" i28" i29" i30" i31" */
140 __ASM_EMIT("vmulps 0x40 + %[FFT_A], %%zmm1, %%zmm4") /* zmm4 = x_im * b_re */
141 __ASM_EMIT("vmulps 0x40 + %[FFT_A], %%zmm3, %%zmm5") /* zmm5 = x_im * b_im */
142 __ASM_EMIT("vfmadd231ps 0x00 + %[FFT_A], %%zmm1, %%zmm5") /* zmm5 = c_re = x_re * b_re + x_im * b_im */
143 __ASM_EMIT("vfmsub231ps 0x00 + %[FFT_A], %%zmm3, %%zmm4") /* zmm4 = c_im = x_re * b_im - x_im * b_re */
144 __ASM_EMIT("vsubps %%zmm5, %%zmm0, %%zmm1") /* zmm1 = a_re - c_re = r4` r5` r6` r7` r12` r13` r14` r15` r20` r21` r22` r23` r28` r29` r30` r31` */
145 __ASM_EMIT("vsubps %%zmm4, %%zmm2, %%zmm3") /* zmm3 = a_im - c_im = i4` i5` i6` i7` i12` i13` i14` i15` i20` i21` i22` i23` i28` i29` i30` i31` */
146 __ASM_EMIT("vaddps %%zmm5, %%zmm0, %%zmm0") /* zmm0 = a_re + c_re = r0` r1` r2` r3` r8` r9` r10` r11` r16` r17` r18` r19` r24` r25` r26` r27` */
147 __ASM_EMIT("vaddps %%zmm4, %%zmm2, %%zmm2") /* zmm2 = a_im + c_im = i0` i1` i2` i3` i8` i9` i10` i11` i16` i17` i18` i19` i24` i25` i26` i27` */
148 /* 4th-order 16x butterfly */
149 __ASM_EMIT("vmovaps %%zmm0, %%zmm4") /* zmm4 = zmm0 = r0` r1` r2` r3` r8` r9` r10` r11` r16` r17` r18` r19` r24` r25` r26` r27` */
150 __ASM_EMIT("vmovaps %%zmm2, %%zmm5") /* zmm5 = zmm2 = i0` i1` i2` i3` i8` i9` i10` i11` i16` i17` i18` i19` i24` i25` i26` i27` */
151 __ASM_EMIT("vmovaps 0x200 + %[FFT_I], %%zmm6") /* zmm6 = shuffle_a[4] */
152 __ASM_EMIT("vmovaps 0x240 + %[FFT_I], %%zmm7") /* zmm7 = shuffle_b[4] */
153 __ASM_EMIT("vpermt2ps %%zmm1, %%zmm6, %%zmm0") /* zmm0 = a_re = r0` r1` r2` r3` r4` r5` r6` r7` r16` r17` r18` r19` r20` r21` r22` r23` */
154 __ASM_EMIT("vpermt2ps %%zmm1, %%zmm7, %%zmm4") /* zmm4 = b_re = r8` r9` r10` r11` r12` r13` r14` r15` r24` r25` r26` r27` r28` r29` r30` r31` */
155 __ASM_EMIT("vpermt2ps %%zmm3, %%zmm6, %%zmm2") /* zmm2 = a_im = i0` i1` i2` i3` i4` i5` i6` i7` i16` i17` i18` i19` i20` i21` i22` i23` */
156 __ASM_EMIT("vpermt2ps %%zmm3, %%zmm7, %%zmm5") /* zmm5 = b_im = i8` i9` i10` i11` i12` i13` i14` i15` i24` i25` i26` i27` i28` i29` i30` i31` */
157 __ASM_EMIT("vmulps 0xc0 + %[FFT_A], %%zmm4, %%zmm1") /* zmm1 = x_im * b_re */
158 __ASM_EMIT("vmulps 0xc0 + %[FFT_A], %%zmm5, %%zmm3") /* zmm3 = x_im * b_im */
159 __ASM_EMIT("vfmadd231ps 0x80 + %[FFT_A], %%zmm4, %%zmm3") /* zmm3 = c_re = x_re * b_re + x_im * b_im */
160 __ASM_EMIT("vfmsub231ps 0x80 + %[FFT_A], %%zmm5, %%zmm1") /* zmm1 = c_im = x_re * b_im - x_im * b_re */
161 __ASM_EMIT("vsubps %%zmm3, %%zmm0, %%zmm4") /* zmm4 = a_re - c_re = R8 R9 R10 R11 R12 R13 R14 R15 R24 R25 R26 R27 R28 R29 R30 R31 */
162 __ASM_EMIT("vsubps %%zmm1, %%zmm2, %%zmm5") /* zmm5 = a_im - c_im = I8 I9 I10 I11 I12 I13 I14 I15 I24 I25 I26 I27 I28 I29 I30 I31 */
163 __ASM_EMIT("vaddps %%zmm3, %%zmm0, %%zmm0") /* zmm0 = a_re + c_re = R0 R1 R2 R3 R4 R5 R6 R7 R16 R17 R18 R19 R20 R21 R22 R23 */
164 __ASM_EMIT("vaddps %%zmm1, %%zmm2, %%zmm2") /* zmm2 = a_im + c_im = I0 I1 I2 I3 I4 I5 I6 I7 I16 I17 I18 I19 I20 I21 I22 I23 */
165 /* Store */
166 __ASM_EMIT("vmovups %%ymm0, 0x00(%[dst], %[off])")
167 __ASM_EMIT("vmovups %%ymm4, 0x20(%[dst], %[off])")
168 __ASM_EMIT("vmovups %%ymm2, 0x40(%[dst], %[off])")
169 __ASM_EMIT("vmovups %%ymm5, 0x60(%[dst], %[off])")
170 __ASM_EMIT("vextractf64x4 $1, %%zmm0, 0x80(%[dst], %[off])")
171 __ASM_EMIT("vextractf64x4 $1, %%zmm4, 0xa0(%[dst], %[off])")
172 __ASM_EMIT("vextractf64x4 $1, %%zmm2, 0xc0(%[dst], %[off])")
173 __ASM_EMIT("vextractf64x4 $1, %%zmm5, 0xe0(%[dst], %[off])")
174 /* Move pointers and repeat */
175 __ASM_EMIT("add $0x100, %[off]")
176 __ASM_EMIT("sub $4, %[items]")
177 __ASM_EMIT("jae 1b")
178 __ASM_EMIT("2:")
179
180 /* Loop 2x 4-element butterflies */
181 __ASM_EMIT("add $2, %[items]")
182 __ASM_EMIT("jl 4f")
183 /* Load data to registers */
184 __ASM_EMIT("vmovups 0x00(%[dst]), %%xmm0") /* xmm0 = r0 i0 r1 i1 */
185 __ASM_EMIT("vmovups 0x10(%[dst]), %%xmm1") /* xmm1 = r2 i2 r3 i3 */
186 __ASM_EMIT("vmovups 0x20(%[dst]), %%xmm4") /* xmm4 = r4 i4 r5 i5 */
187 __ASM_EMIT("vmovups 0x30(%[dst]), %%xmm5") /* xmm5 = r6 i6 r7 i7 */
188 __ASM_EMIT("vinsertf128 $1, 0x40(%[dst]), %%ymm0, %%ymm0") /* ymm0 = r0 i0 r1 i1 r8 i8 r9 i9 */
189 __ASM_EMIT("vinsertf128 $1, 0x50(%[dst]), %%ymm1, %%ymm1") /* ymm1 = r2 i2 r3 i3 r10 i10 r11 i11 */
190 __ASM_EMIT("vinsertf128 $1, 0x60(%[dst]), %%ymm4, %%ymm4") /* ymm4 = r4 i4 r5 i5 r12 i12 r13 i13 */
191 __ASM_EMIT("vinsertf128 $1, 0x70(%[dst]), %%ymm5, %%ymm5") /* ymm5 = r6 i6 r7 i7 r14 i14 r15 i15 */
192 __ASM_EMIT("vshufps $0xdd, %%ymm1, %%ymm0, %%ymm2") /* ymm2 = i0 i1 i2 i3 i8 i9 i10 i11 */
193 __ASM_EMIT("vshufps $0x88, %%ymm1, %%ymm0, %%ymm0") /* ymm0 = r0 r1 r2 r3 r8 r9 r10 r11 */
194 __ASM_EMIT("vshufps $0xdd, %%ymm5, %%ymm4, %%ymm6") /* ymm6 = i4 i5 i6 i7 i12 i13 i14 i15 */
195 __ASM_EMIT("vshufps $0x88, %%ymm5, %%ymm4, %%ymm4") /* ymm4 = r4 r5 r6 r7 r12 r13 r14 r15 */
196 /* 1st-order 8x butterfly */
197 __ASM_EMIT("vhsubps %%ymm4, %%ymm0, %%ymm1") /* ymm1 = r0-r1 r2-r3 r4-r5 r6-r7 = r1' r3' r5' r7' */
198 __ASM_EMIT("vhsubps %%ymm6, %%ymm2, %%ymm3") /* ymm3 = i0-i1 i2-i3 i4-i5 i6-i7 = i1' i3' i5' i7' */
199 __ASM_EMIT("vhaddps %%ymm4, %%ymm0, %%ymm0") /* ymm0 = r0+r1 r2+r3 r4+r5 r6+r7 = r0' r2' r4' r6' */
200 __ASM_EMIT("vhaddps %%ymm6, %%ymm2, %%ymm2") /* ymm2 = i0+i1 i2+i3 i4+i5 i6+i7 = i0' i2' i4' i6' */
201 /* 2nd-order 8x butterfly */
202 __ASM_EMIT("vblendps $0xaa, %%ymm3, %%ymm1, %%ymm4") /* ymm4 = r1' i3' r5' i7' */
203 __ASM_EMIT("vblendps $0xaa, %%ymm1, %%ymm3, %%ymm5") /* ymm5 = i1' r3' i5' r7' */
204 __ASM_EMIT("vhsubps %%ymm4, %%ymm0, %%ymm1") /* ymm1 = r0'-r2' r4'-r6' r1'-i3' r5'-i7' = r2" r6" r3" r7" */
205 __ASM_EMIT("vhsubps %%ymm5, %%ymm2, %%ymm3") /* ymm3 = i0'-i2' i4'-i6' i1'-r3' i5'-r7' = i2" i6" i1" i5" */
206 __ASM_EMIT("vhaddps %%ymm4, %%ymm0, %%ymm0") /* ymm0 = r0'+r2' r4'+r6' r1'+i3' r5'+i7' = r0" r4" r1" r5" */
207 __ASM_EMIT("vhaddps %%ymm5, %%ymm2, %%ymm2") /* ymm2 = i0'+i2' i4'+i6' i1'+r3' i5'+r7' = i0" i4" i3" i7" */
208 __ASM_EMIT("vblendps $0xcc, %%ymm3, %%ymm2, %%ymm4") /* ymm4 = i0" i4" i1" i5" */
209 __ASM_EMIT("vblendps $0xcc, %%ymm2, %%ymm3, %%ymm5") /* ymm5 = i2" i6" i3" i7" */
210 __ASM_EMIT("vshufps $0x88, %%ymm1, %%ymm0, %%ymm2") /* ymm2 = r0" r1" r2" r3" */
211 __ASM_EMIT("vshufps $0xdd, %%ymm1, %%ymm0, %%ymm3") /* ymm3 = r4" r5" r6" r7" */
212 __ASM_EMIT("vshufps $0x88, %%ymm5, %%ymm4, %%ymm6") /* ymm6 = i0" i1" i2" i3" */
213 __ASM_EMIT("vshufps $0xdd, %%ymm5, %%ymm4, %%ymm7") /* ymm7 = i4" i5" i6" i7" */
214 /* 3rd-order 8x butterfly */
215 __ASM_EMIT("vmulps 0x40 + %[FFT_A], %%ymm3, %%ymm4") /* ymm4 = x_im * b_re */
216 __ASM_EMIT("vmulps 0x40 + %[FFT_A], %%ymm7, %%ymm5") /* ymm5 = x_im * b_im */
217 __ASM_EMIT("vfmadd231ps 0x00 + %[FFT_A], %%ymm3, %%ymm5") /* ymm5 = c_re = x_re * b_re + x_im * b_im */
218 __ASM_EMIT("vfmsub231ps 0x00 + %[FFT_A], %%ymm7, %%ymm4") /* ymm4 = c_im = x_re * b_im - x_im * b_re */
219 __ASM_EMIT("vsubps %%ymm5, %%ymm2, %%ymm0") /* ymm0 = a_re - c_re = r8 r9 r10 r11 r12 r13 r14 r15 */
220 __ASM_EMIT("vsubps %%ymm4, %%ymm6, %%ymm1") /* ymm1 = a_im - c_im = i8 i9 i10 i11 i12 i13 i14 i15 */
221 __ASM_EMIT("vaddps %%ymm5, %%ymm2, %%ymm2") /* ymm2 = a_re + c_re = r0 r1 r2 r3 r4 r5 r6 r7 */
222 __ASM_EMIT("vaddps %%ymm4, %%ymm6, %%ymm3") /* ymm3 = a_im + c_im = i0 i1 i2 i3 i4 i5 i6 i7 */
223 /* 4th-order 8x butterfly */
224 __ASM_EMIT("vextractf128 $1, %%ymm2, %%xmm4") /* xmm4 = r4 r5 r6 r7 */
225 __ASM_EMIT("vextractf128 $1, %%ymm3, %%xmm5") /* xmm5 = i4 i5 i6 i7 */
226 __ASM_EMIT("vinsertf128 $1, %%xmm0, %%ymm2, %%ymm2") /* ymm2 = a_re = r0 r1 r2 r3 r8 r9 r10 r11 */
227 __ASM_EMIT("vinsertf128 $1, %%xmm1, %%ymm3, %%ymm3") /* ymm3 = a_im = i0 i1 i2 i3 i8 i9 i10 i11 */
228 __ASM_EMIT("vinsertf128 $0, %%xmm4, %%ymm0, %%ymm0") /* ymm0 = b_re = r5 r6 r7 r12 r13 r14 r15 */
229 __ASM_EMIT("vinsertf128 $0, %%xmm5, %%ymm1, %%ymm1") /* ymm1 = b_im = i5 i6 i7 i12 i13 i14 i15 */
230 __ASM_EMIT("vmulps 0xc0 + %[FFT_A], %%ymm0, %%ymm4") /* ymm4 = x_im * b_re */
231 __ASM_EMIT("vmulps 0xc0 + %[FFT_A], %%ymm1, %%ymm5") /* ymm5 = x_im * b_im */
232 __ASM_EMIT("vfmadd231ps 0x80 + %[FFT_A], %%ymm0, %%ymm5") /* ymm5 = c_re = x_re * b_re + x_im * b_im */
233 __ASM_EMIT("vfmsub231ps 0x80 + %[FFT_A], %%ymm1, %%ymm4") /* ymm4 = c_im = x_re * b_im - x_im * b_re */
234 __ASM_EMIT("vsubps %%ymm5, %%ymm2, %%ymm0") /* ymm0 = a_re - c_re */
235 __ASM_EMIT("vsubps %%ymm4, %%ymm3, %%ymm1") /* ymm1 = a_im - c_im */
236 __ASM_EMIT("vaddps %%ymm5, %%ymm2, %%ymm2") /* ymm2 = a_re + c_re */
237 __ASM_EMIT("vaddps %%ymm4, %%ymm3, %%ymm3") /* ymm3 = a_im + c_im */
238 /* Store */
239 __ASM_EMIT("vmovups %%ymm2, 0x00(%[dst], %[off])")
240 __ASM_EMIT("vmovups %%ymm0, 0x20(%[dst], %[off])")
241 __ASM_EMIT("vmovups %%ymm3, 0x40(%[dst], %[off])")
242 __ASM_EMIT("vmovups %%ymm1, 0x60(%[dst], %[off])")
243 /* Move pointers */
244 __ASM_EMIT("add $0x80, %[off]")
245 __ASM_EMIT("sub $2, %[items]")
246 __ASM_EMIT("4:")
247 /* x4 scramble block */
248 __ASM_EMIT("add $1, %[items]")
249 __ASM_EMIT("jl 6f")
250 __ASM_EMIT("vmovups 0x00(%[dst]), %%xmm0") /* xmm0 = r0 i0 r1 i1 */
251 __ASM_EMIT("vmovups 0x10(%[dst]), %%xmm1") /* xmm1 = r2 i2 r3 i3 */
252 __ASM_EMIT("vmovups 0x20(%[dst]), %%xmm4") /* xmm4 = r4 i4 r5 i5 */
253 __ASM_EMIT("vmovups 0x30(%[dst]), %%xmm5") /* xmm5 = r6 i6 r7 i7 */
254 __ASM_EMIT("vshufps $0xdd, %%xmm1, %%xmm0, %%xmm2") /* xmm2 = i0 i1 i2 i3 */
255 __ASM_EMIT("vshufps $0x88, %%xmm1, %%xmm0, %%xmm0") /* xmm0 = r0 r1 r2 r3 */
256 __ASM_EMIT("vshufps $0xdd, %%xmm5, %%xmm4, %%xmm6") /* xmm4 = i4 i5 i6 i7 */
257 __ASM_EMIT("vshufps $0x88, %%xmm5, %%xmm4, %%xmm4") /* xmm6 = r4 r5 r6 r7 */
258 /* 1st-order 8x butterfly */
259 __ASM_EMIT("vhsubps %%xmm4, %%xmm0, %%xmm1") /* xmm1 = r0-r1 r2-r3 r4-r5 r6-r7 = r1' r3' r5' r7' */
260 __ASM_EMIT("vhsubps %%xmm6, %%xmm2, %%xmm3") /* xmm3 = i0-i1 i2-i3 i4-i5 i6-i7 = i1' i3' i5' i7' */
261 __ASM_EMIT("vhaddps %%xmm4, %%xmm0, %%xmm0") /* xmm0 = r0+r1 r2+r3 r4+r5 r6+r7 = r0' r2' r4' r6' */
262 __ASM_EMIT("vhaddps %%xmm6, %%xmm2, %%xmm2") /* xmm2 = i0+i1 i2+i3 i4+i5 i6+i7 = i0' i2' i4' i6' */
263 /* 2nd-order 8x butterfly */
264 __ASM_EMIT("vblendps $0xaa, %%xmm3, %%xmm1, %%xmm4") /* xmm4 = r1' i3' r5' i7' */
265 __ASM_EMIT("vblendps $0xaa, %%xmm1, %%xmm3, %%xmm5") /* xmm5 = i1' r3' i5' r7' */
266 __ASM_EMIT("vhsubps %%xmm4, %%xmm0, %%xmm1") /* xmm1 = r0'-r2' r4'-r6' r1'-i3' r5'-i7' = r2" r6" r3" r7" */
267 __ASM_EMIT("vhsubps %%xmm5, %%xmm2, %%xmm3") /* xmm3 = i0'-i2' i4'-i6' i1'-r3' i5'-r7' = i2" i6" i1" i5" */
268 __ASM_EMIT("vhaddps %%xmm4, %%xmm0, %%xmm0") /* xmm0 = r0'+r2' r4'+r6' r1'+i3' r5'+i7' = r0" r4" r1" r5" */
269 __ASM_EMIT("vhaddps %%xmm5, %%xmm2, %%xmm2") /* xmm2 = i0'+i2' i4'+i6' i1'+r3' i5'+r7' = i0" i4" i3" i7" */
270 __ASM_EMIT("vblendps $0xcc, %%xmm3, %%xmm2, %%xmm4") /* xmm4 = i0" i4" i1" i5" */
271 __ASM_EMIT("vblendps $0xcc, %%xmm2, %%xmm3, %%xmm5") /* xmm5 = i2" i6" i3" i7" */
272 __ASM_EMIT("vshufps $0x88, %%xmm1, %%xmm0, %%xmm2") /* xmm2 = r0" r1" r2" r3" */
273 __ASM_EMIT("vshufps $0xdd, %%xmm1, %%xmm0, %%xmm3") /* xmm3 = r4" r5" r6" r7" */
274 __ASM_EMIT("vshufps $0x88, %%xmm5, %%xmm4, %%xmm6") /* xmm6 = i0" i1" i2" i3" */
275 __ASM_EMIT("vshufps $0xdd, %%xmm5, %%xmm4, %%xmm7") /* xmm7 = i4" i5" i6" i7" */
276 /* 3rd-order 8x butterfly */
277 __ASM_EMIT("vmulps 0x40 + %[FFT_A], %%xmm3, %%xmm4") /* xmm4 = x_im * b_re */
278 __ASM_EMIT("vmulps 0x40 + %[FFT_A], %%xmm7, %%xmm5") /* xmm5 = x_im * b_im */
279 __ASM_EMIT("vfmadd231ps 0x00 + %[FFT_A], %%xmm3, %%xmm5") /* xmm5 = c_re = x_re * b_re + x_im * b_im */
280 __ASM_EMIT("vfmsub231ps 0x00 + %[FFT_A], %%xmm7, %%xmm4") /* xmm4 = c_im = x_re * b_im - x_im * b_re */
281 __ASM_EMIT("vsubps %%xmm5, %%xmm2, %%xmm0") /* xmm0 = a_re - c_re */
282 __ASM_EMIT("vsubps %%xmm4, %%xmm6, %%xmm1") /* xmm1 = a_im - c_im */
283 __ASM_EMIT("vaddps %%xmm5, %%xmm2, %%xmm2") /* xmm2 = a_re + c_re */
284 __ASM_EMIT("vaddps %%xmm4, %%xmm6, %%xmm3") /* xmm3 = a_im + c_im */
285 /* Store */
286 __ASM_EMIT("vmovups %%xmm2, 0x00(%[dst], %[off])")
287 __ASM_EMIT("vmovups %%xmm0, 0x10(%[dst], %[off])")
288 __ASM_EMIT("vmovups %%xmm3, 0x20(%[dst], %[off])")
289 __ASM_EMIT("vmovups %%xmm1, 0x30(%[dst], %[off])")
290 __ASM_EMIT("6:")
291
292 : [dst] "+r" (dst),
293 [off] "+r" (off), [items] "+r" (items)
294 : [FFT_A] "o" (FFT_A), [FFT_I] "o" (FFT_SCRAMBLE_INDICES),
295 [MASK] "o" (FFT_REPACK_MASKS)
296 : "cc", "memory",
297 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
298 "%xmm4", "%xmm5", "%xmm6", "%xmm7",
299 "%k4", "%k5"
300 );
301 }
302
303 static inline void FFT_PSCRAMBLE_SELF_REVERSE_NAME(float *dst, size_t rank)
304 {
305 // Calculate number of items
306 size_t items = (1 << rank) - 1;
307
308 for (size_t i = 1; i < items; ++i)
309 {
310 size_t j = reverse_bits(FFT_TYPE(i), rank); /* Reverse the order of the bits */
311 if (i >= j)
312 continue;
313
314 /* Copy the values from the reversed position */
315 ARCH_X86_ASM
316 (
317 __ASM_EMIT("vmovlps (%[dst], %[i], 8), %%xmm0, %%xmm0")
318 __ASM_EMIT("vmovlps (%[dst], %[j], 8), %%xmm1, %%xmm1")
319 __ASM_EMIT("vmovlps %%xmm0, (%[dst], %[j], 8)")
320 __ASM_EMIT("vmovlps %%xmm1, (%[dst], %[i], 8)")
321 :
322 : [dst] "r"(dst), [i] "r"(i), [j] "r"(j)
323 : "memory",
324 "%xmm0", "%xmm1"
325 );
326 }
327
328 // Perform butterfly 8x
329 size_t off = 0;
330 items = 1 << (rank - 3);
331
332 // Perform 4-element butterflies
333 ARCH_X86_ASM
334 (
335 /* Loop 4x 4-element butterflies */
336 __ASM_EMIT("sub $4, %[items]")
337 __ASM_EMIT("jb 2f")
338 __ASM_EMIT("kmovw 0x00 + %[MASK], %%k4")
339 __ASM_EMIT("kmovw 0x02 + %[MASK], %%k5")
340 __ASM_EMIT("1:")
341 /* Load data to registers */
342 __ASM_EMIT("vmovups 0x00(%[dst], %[off]), %%zmm0") /* zmm0 = r0 i0 r1 i1 r2 i2 r3 i3 r4 i4 r5 i5 r6 i6 r7 i7 */
343 __ASM_EMIT("vmovups 0x40(%[dst], %[off]), %%zmm1") /* zmm1 = r8 i8 r9 i9 r10 i10 r11 i11 r12 i12 r13 i13 r14 i14 r15 i15 */
344 __ASM_EMIT("vmovups 0x80(%[dst], %[off]), %%zmm4") /* zmm4 = r16 i16 r17 i17 r18 i18 r19 i19 r20 i20 r21 i21 r22 i22 r23 i23 */
345 __ASM_EMIT("vmovups 0xc0(%[dst], %[off]), %%zmm6") /* zmm6 = r24 i24 r25 i25 r26 i26 r27 i27 r28 i28 r29 i29 r30 i30 r31 i31 */
346 __ASM_EMIT("vcompressps %%zmm0, %%zmm2 %{%%k5%}") /* zmm2 = i0 i1 i2 i3 i4 i5 i6 i7 ? ? ? ? ? ? ? ? */
347 __ASM_EMIT("vcompressps %%zmm1, %%zmm3 %{%%k5%}") /* zmm3 = i8 i9 i10 i11 i12 i13 i14 i15 ? ? ? ? ? ? ? ? */
348 __ASM_EMIT("vcompressps %%zmm4, %%zmm5 %{%%k5%}") /* zmm5 = i16 i17 i18 i19 i20 i21 i22 i23 ? ? ? ? ? ? ? ? */
349 __ASM_EMIT("vcompressps %%zmm6, %%zmm7 %{%%k5%}") /* zmm7 = i24 i25 i26 i27 i28 i29 i30 i31 ? ? ? ? ? ? ? ? */
350 __ASM_EMIT("vcompressps %%zmm0, %%zmm0 %{%%k4%}") /* zmm0 = r0 r1 r2 r3 r4 r5 r6 r7 ? ? ? ? ? ? ? ? */
351 __ASM_EMIT("vcompressps %%zmm1, %%zmm1 %{%%k4%}") /* zmm1 = r8 r9 r10 r11 r12 r13 r14 r15 ? ? ? ? ? ? ? ? */
352 __ASM_EMIT("vcompressps %%zmm4, %%zmm4 %{%%k4%}") /* zmm4 = r16 r17 r18 r19 r20 r21 r22 r23 ? ? ? ? ? ? ? ? */
353 __ASM_EMIT("vcompressps %%zmm6, %%zmm6 %{%%k4%}") /* zmm6 = r24 r25 r26 r27 r28 r29 r30 r31 ? ? ? ? ? ? ? ? */
354 __ASM_EMIT("vinsertf64x4 $1, %%ymm3, %%zmm2, %%zmm2") /* zmm2 = i0 i1 i2 i3 i4 i5 i6 i7 i8 i9 i10 i11 i12 i13 i14 i15 */
355 __ASM_EMIT("vinsertf64x4 $1, %%ymm7, %%zmm5, %%zmm5") /* zmm5 = i16 i17 i18 i19 i20 i21 i22 i23 i24 i25 i26 i27 i28 i29 i30 i31 */
356 __ASM_EMIT("vinsertf64x4 $1, %%ymm1, %%zmm0, %%zmm0") /* zmm0 = r0 r1 r2 r3 r4 r5 r6 r7 r8 r9 r10 r11 r12 r13 r14 r15 */
357 __ASM_EMIT("vinsertf64x4 $1, %%ymm6, %%zmm4, %%zmm4") /* zmm4 = r16 r17 r18 r19 r20 r21 r22 r23 r24 r25 r26 r27 r28 r29 r30 r31 */
358 /* 1st-order 16x butterfly */
359 __ASM_EMIT("vmovaps %%zmm0, %%zmm1") /* zmm1 = zmm0 = r0 r1 r2 r3 r4 r5 r6 r7 r8 r9 r10 r11 r12 r13 r14 r15 */
360 __ASM_EMIT("vmovaps %%zmm2, %%zmm3") /* zmm3 = zmm2 = i0 i1 i2 i3 i4 i5 i6 i7 i8 i9 i10 i11 i12 i13 i14 i15 */
361 __ASM_EMIT("vmovaps 0x00 + %[FFT_I], %%zmm6") /* zmm6 = shuffle_a[0] */
362 __ASM_EMIT("vmovaps 0x40 + %[FFT_I], %%zmm7") /* zmm7 = shuffle_b[0] */
363 __ASM_EMIT("vpermt2ps %%zmm4, %%zmm6, %%zmm0") /* zmm0 = r0 r2 r4 r6 r8 r10 r12 r14 r16 r18 r20 r22 r24 r26 r28 r30 */
364 __ASM_EMIT("vpermt2ps %%zmm4, %%zmm7, %%zmm1") /* zmm1 = r1 r3 r5 r7 r9 r11 r13 r15 r17 r19 r21 r23 r25 r27 r29 r31 */
365 __ASM_EMIT("vpermt2ps %%zmm5, %%zmm6, %%zmm2") /* zmm2 = i0 i2 i4 i6 i8 i10 i12 i14 i16 i18 i20 i22 i24 i26 i28 i30 */
366 __ASM_EMIT("vpermt2ps %%zmm5, %%zmm7, %%zmm3") /* zmm3 = i1 i3 i5 i7 i9 i11 i13 i15 i17 i19 i21 i23 i25 i27 i29 i31 */
367 __ASM_EMIT("vsubps %%zmm1, %%zmm0, %%zmm4") /* zmm4 = r0-r1 r2-r3 r4-r5 r6-r7 ... = r1' r3' r5' r7' r9' r11' r13' r15' r17' r19' r21' r23' r25' r27' r29' r31' */
368 __ASM_EMIT("vsubps %%zmm3, %%zmm2, %%zmm5") /* zmm5 = i0-i1 i2-i3 i4-i5 i6-i7 ... = i1' i3' i5' i7' i9' i11' i13' i15' i17' i19' i21' i23' i25' i27' i29' i31' */
369 __ASM_EMIT("vaddps %%zmm1, %%zmm0, %%zmm0") /* zmm0 = r0+r1 r2+r3 r4+r5 r6+r7 ... = r0' r2' r4' r6' r8' r10' r12' r14' r16' r18' r20' r22' r24' r26' r28' r30' */
370 __ASM_EMIT("vaddps %%zmm3, %%zmm2, %%zmm2") /* zmm2 = i0+i1 i2+i3 i4+i5 i6+i7 ... = i0' i2' i4' i6' i8' i10' i12' i14' i16' i18' i20' i22' i24' i26' i28' i30' */
371 /* 2nd-order 16x butterfly */
372 __ASM_EMIT("vmovaps %%zmm0, %%zmm1") /* zmm1 = zmm0 = r0' r2' r4' r6' r8' r10' r12' r14' r16' r18' r20' r22' r24' r26' r28' r30' */
373 __ASM_EMIT("vmovaps %%zmm2, %%zmm3") /* zmm3 = zmm2 = i0' i2' i4' i6' i8' i10' i12' i14' i16' i18' i20' i22' i24' i26' i28' i30' */
374 __ASM_EMIT("vmovaps 0x80 + %[FFT_I], %%zmm6") /* zmm6 = shuffle_a[1] */
375 __ASM_EMIT("vmovaps 0xc0 + %[FFT_I], %%zmm7") /* zmm7 = shuffle_b[1] */
376 __ASM_EMIT("vpermt2ps %%zmm4, %%zmm6, %%zmm0") /* zmm0 = r0' r1' r4' r5' r8' r9' r12' r13' r16' r17' r20' r21' r24' r25' r28' r29' */
377 __ASM_EMIT("vpermt2ps %%zmm5, %%zmm6, %%zmm2") /* zmm2 = i0' i1' i4' i5' i8' i9' i12' i13' i16' i17' i20' i21' i24' i25' i28' i29' */
378 __ASM_EMIT("vpermt2ps %%zmm5, %%zmm7, %%zmm1") /* zmm1 = r2' i3' r6' i7' r10' i11' r14' i15' r18' i19' r22' i23' r26' i27' r30' i31' */
379 __ASM_EMIT("vpermt2ps %%zmm4, %%zmm7, %%zmm3") /* zmm3 = i2' r3' i6' r7' i10' r11' i14' r15' i18' r19' i22' r23' i26' r27' i30' r31' */
380 __ASM_EMIT("vsubps %%zmm1, %%zmm0, %%zmm4") /* zmm4 = r2" r1" r6" r5" r10" r9" r14" r13" r18" r17" r22" r21" r26" r25" r30" r29" = r0'-r2' r1'-i3' r4'-r6' r5'-i7' r8'-r10' r9'-i11' r12'-r14' r13'-i15' r16'-r18' r17'-i19' r20'-r22' r21'-i23' r24'-r26' r25'-i27' r28'-r30' r29'-i31' */
381 __ASM_EMIT("vsubps %%zmm3, %%zmm2, %%zmm5") /* zmm5 = i2" i3" i6" i7" i10" i11" i14" i15" i18" i19" i22" i23" i26" i27" i30" i31" = i0'-i2' i1'-r3' i4'-i6' i5'-r7' i8'-i10' i9'-r11' i12'-i14' i13'-r15' i16'-i18' i17'-r19' i20'-i22' i21'-r23' i24'-i26' i25'-r27' i28'-i30' i29'-r31' */
382 __ASM_EMIT("vaddps %%zmm1, %%zmm0, %%zmm0") /* zmm0 = r0" r3" r4" r7" r8" r11" r12" r15" r16" r19" r20" r23" r24" r27" r28" r31" = r0'+r2' r1'+i3' r4'+r6' r5'+i7' r8'+r10' r9'+i11' r12'+r14' r13'+i15' r16'+r18' r17'+i19' r20'+r22' r21'+i23' r24'+r26' r25'+i27' r28'+r30' r29'+i31' */
383 __ASM_EMIT("vaddps %%zmm3, %%zmm2, %%zmm2") /* zmm2 = i0" i1" i4" i5" i8" i9" i12" i13" i16" i17" i20" i21" i24" i25" i28" i29" = i0'+i2' i1'+r3' i4'+i6' i5'+r7' i8'+i10' i9'+r11' i12'+i14' i13'+r15' i16'+i18' i17'+r19' i20'+i22' i21'+r23' i24'+i26' i25'+r27' i28'+i30' i29'+r31' */
384 /* 3rd-order 16x butterfly */
385 __ASM_EMIT("vmovaps %%zmm0, %%zmm1") /* zmm1 = zmm0 = r0" r3" r4" r7" r8" r11" r12" r15" r16" r19" r20" r23" r24" r27" r28" r31" */
386 __ASM_EMIT("vmovaps %%zmm2, %%zmm3") /* zmm3 = zmm2 = i0" i1" i4" i5" i8" i9" i12" i13" i16" i17" i20" i21" i24" i25" i28" i29" */
387 __ASM_EMIT("vmovaps 0x140 + %[FFT_I], %%zmm6") /* zmm6 = shuffle_b[2] */
388 __ASM_EMIT("vmovaps 0x100 + %[FFT_I], %%zmm7") /* zmm7 = shuffle_a[2] */
389 __ASM_EMIT("vpermt2ps %%zmm4, %%zmm6, %%zmm0") /* zmm0 = a_re = r0" r1" r2" r3" r8" r9" r10" r11" r16" r17" r18" r19" r24" r25" r26" r27" */
390 __ASM_EMIT("vpermt2ps %%zmm5, %%zmm7, %%zmm2") /* zmm2 = a_im = i0" i1" i2" i3" i8" i9" i10" i11" i16" i17" i18" i19" i24" i25" i26" i27" */
391 __ASM_EMIT("vmovaps 0x1c0 + %[FFT_I], %%zmm6") /* zmm6 = shuffle_b[3] */
392 __ASM_EMIT("vmovaps 0x180 + %[FFT_I], %%zmm7") /* zmm7 = shuffle_a[3] */
393 __ASM_EMIT("vpermt2ps %%zmm4, %%zmm6, %%zmm1") /* zmm1 = b_re = r4" r5" r6" r7" r12" r13" r14" r15" r20" r21" r22" r23" r28" r29" r30" r31" */
394 __ASM_EMIT("vpermt2ps %%zmm5, %%zmm7, %%zmm3") /* zmm3 = b_im = i4" i5" i6" i7" i12" i13" i14" i15" i20" i21" i22" i23" i28" i29" i30" i31" */
395 __ASM_EMIT("vmulps 0x40 + %[FFT_A], %%zmm1, %%zmm4") /* zmm4 = x_im * b_re */
396 __ASM_EMIT("vmulps 0x40 + %[FFT_A], %%zmm3, %%zmm5") /* zmm5 = x_im * b_im */
397 __ASM_EMIT("vfmsub231ps 0x00 + %[FFT_A], %%zmm1, %%zmm5") /* zmm5 = c_re = x_re * b_re - x_im * b_im */
398 __ASM_EMIT("vfmadd231ps 0x00 + %[FFT_A], %%zmm3, %%zmm4") /* zmm4 = c_im = x_re * b_im + x_im * b_re */
399 __ASM_EMIT("vsubps %%zmm5, %%zmm0, %%zmm1") /* zmm1 = a_re - c_re = r4` r5` r6` r7` r12` r13` r14` r15` r20` r21` r22` r23` r28` r29` r30` r31` */
400 __ASM_EMIT("vsubps %%zmm4, %%zmm2, %%zmm3") /* zmm3 = a_im - c_im = i4` i5` i6` i7` i12` i13` i14` i15` i20` i21` i22` i23` i28` i29` i30` i31` */
401 __ASM_EMIT("vaddps %%zmm5, %%zmm0, %%zmm0") /* zmm0 = a_re + c_re = r0` r1` r2` r3` r8` r9` r10` r11` r16` r17` r18` r19` r24` r25` r26` r27` */
402 __ASM_EMIT("vaddps %%zmm4, %%zmm2, %%zmm2") /* zmm2 = a_im + c_im = i0` i1` i2` i3` i8` i9` i10` i11` i16` i17` i18` i19` i24` i25` i26` i27` */
403 /* 4th-order 16x butterfly */
404 __ASM_EMIT("vmovaps %%zmm0, %%zmm4") /* zmm4 = zmm0 = r0` r1` r2` r3` r8` r9` r10` r11` r16` r17` r18` r19` r24` r25` r26` r27` */
405 __ASM_EMIT("vmovaps %%zmm2, %%zmm5") /* zmm5 = zmm2 = i0` i1` i2` i3` i8` i9` i10` i11` i16` i17` i18` i19` i24` i25` i26` i27` */
406 __ASM_EMIT("vmovaps 0x200 + %[FFT_I], %%zmm6") /* zmm6 = shuffle_a[4] */
407 __ASM_EMIT("vmovaps 0x240 + %[FFT_I], %%zmm7") /* zmm7 = shuffle_b[4] */
408 __ASM_EMIT("vpermt2ps %%zmm1, %%zmm6, %%zmm0") /* zmm0 = a_re = r0` r1` r2` r3` r4` r5` r6` r7` r16` r17` r18` r19` r20` r21` r22` r23` */
409 __ASM_EMIT("vpermt2ps %%zmm1, %%zmm7, %%zmm4") /* zmm4 = b_re = r8` r9` r10` r11` r12` r13` r14` r15` r24` r25` r26` r27` r28` r29` r30` r31` */
410 __ASM_EMIT("vpermt2ps %%zmm3, %%zmm6, %%zmm2") /* zmm2 = a_im = i0` i1` i2` i3` i4` i5` i6` i7` i16` i17` i18` i19` i20` i21` i22` i23` */
411 __ASM_EMIT("vpermt2ps %%zmm3, %%zmm7, %%zmm5") /* zmm5 = b_im = i8` i9` i10` i11` i12` i13` i14` i15` i24` i25` i26` i27` i28` i29` i30` i31` */
412 __ASM_EMIT("vmulps 0xc0 + %[FFT_A], %%zmm4, %%zmm1") /* zmm1 = x_im * b_re */
413 __ASM_EMIT("vmulps 0xc0 + %[FFT_A], %%zmm5, %%zmm3") /* zmm3 = x_im * b_im */
414 __ASM_EMIT("vfmsub231ps 0x80 + %[FFT_A], %%zmm4, %%zmm3") /* zmm3 = c_re = x_re * b_re - x_im * b_im */
415 __ASM_EMIT("vfmadd231ps 0x80 + %[FFT_A], %%zmm5, %%zmm1") /* zmm1 = c_im = x_re * b_im + x_im * b_re */
416 __ASM_EMIT("vsubps %%zmm3, %%zmm0, %%zmm4") /* zmm4 = a_re - c_re = R8 R9 R10 R11 R12 R13 R14 R15 R24 R25 R26 R27 R28 R29 R30 R31 */
417 __ASM_EMIT("vsubps %%zmm1, %%zmm2, %%zmm5") /* zmm5 = a_im - c_im = I8 I9 I10 I11 I12 I13 I14 I15 I24 I25 I26 I27 I28 I29 I30 I31 */
418 __ASM_EMIT("vaddps %%zmm3, %%zmm0, %%zmm0") /* zmm0 = a_re + c_re = R0 R1 R2 R3 R4 R5 R6 R7 R16 R17 R18 R19 R20 R21 R22 R23 */
419 __ASM_EMIT("vaddps %%zmm1, %%zmm2, %%zmm2") /* zmm2 = a_im + c_im = I0 I1 I2 I3 I4 I5 I6 I7 I16 I17 I18 I19 I20 I21 I22 I23 */
420 /* Store */
421 __ASM_EMIT("vmovups %%ymm0, 0x00(%[dst], %[off])")
422 __ASM_EMIT("vmovups %%ymm4, 0x20(%[dst], %[off])")
423 __ASM_EMIT("vmovups %%ymm2, 0x40(%[dst], %[off])")
424 __ASM_EMIT("vmovups %%ymm5, 0x60(%[dst], %[off])")
425 __ASM_EMIT("vextractf64x4 $1, %%zmm0, 0x80(%[dst], %[off])")
426 __ASM_EMIT("vextractf64x4 $1, %%zmm4, 0xa0(%[dst], %[off])")
427 __ASM_EMIT("vextractf64x4 $1, %%zmm2, 0xc0(%[dst], %[off])")
428 __ASM_EMIT("vextractf64x4 $1, %%zmm5, 0xe0(%[dst], %[off])")
429 /* Move pointers and repeat */
430 __ASM_EMIT("add $0x100, %[off]")
431 __ASM_EMIT("sub $4, %[items]")
432 __ASM_EMIT("jae 1b")
433 __ASM_EMIT("2:")
434
435 /* Loop 2x 4-element butterflies */
436 __ASM_EMIT("add $2, %[items]")
437 __ASM_EMIT("jl 4f")
438 /* Load data to registers */
439 __ASM_EMIT("vmovups 0x00(%[dst]), %%xmm0") /* xmm0 = r0 i0 r1 i1 */
440 __ASM_EMIT("vmovups 0x10(%[dst]), %%xmm1") /* xmm1 = r2 i2 r3 i3 */
441 __ASM_EMIT("vmovups 0x20(%[dst]), %%xmm4") /* xmm4 = r4 i4 r5 i5 */
442 __ASM_EMIT("vmovups 0x30(%[dst]), %%xmm5") /* xmm5 = r6 i6 r7 i7 */
443 __ASM_EMIT("vinsertf128 $1, 0x40(%[dst]), %%ymm0, %%ymm0") /* ymm0 = r0 i0 r1 i1 r8 i8 r9 i9 */
444 __ASM_EMIT("vinsertf128 $1, 0x50(%[dst]), %%ymm1, %%ymm1") /* ymm1 = r2 i2 r3 i3 r10 i10 r11 i11 */
445 __ASM_EMIT("vinsertf128 $1, 0x60(%[dst]), %%ymm4, %%ymm4") /* ymm4 = r4 i4 r5 i5 r12 i12 r13 i13 */
446 __ASM_EMIT("vinsertf128 $1, 0x70(%[dst]), %%ymm5, %%ymm5") /* ymm5 = r6 i6 r7 i7 r14 i14 r15 i15 */
447 __ASM_EMIT("vshufps $0xdd, %%ymm1, %%ymm0, %%ymm2") /* ymm2 = i0 i1 i2 i3 i8 i9 i10 i11 */
448 __ASM_EMIT("vshufps $0x88, %%ymm1, %%ymm0, %%ymm0") /* ymm0 = r0 r1 r2 r3 r8 r9 r10 r11 */
449 __ASM_EMIT("vshufps $0xdd, %%ymm5, %%ymm4, %%ymm6") /* ymm6 = i4 i5 i6 i7 i12 i13 i14 i15 */
450 __ASM_EMIT("vshufps $0x88, %%ymm5, %%ymm4, %%ymm4") /* ymm4 = r4 r5 r6 r7 r12 r13 r14 r15 */
451 /* 1st-order 8x butterfly */
452 __ASM_EMIT("vhsubps %%ymm4, %%ymm0, %%ymm1") /* ymm1 = r0-r1 r2-r3 r4-r5 r6-r7 = r1' r3' r5' r7' */
453 __ASM_EMIT("vhsubps %%ymm6, %%ymm2, %%ymm3") /* ymm3 = i0-i1 i2-i3 i4-i5 i6-i7 = i1' i3' i5' i7' */
454 __ASM_EMIT("vhaddps %%ymm4, %%ymm0, %%ymm0") /* ymm0 = r0+r1 r2+r3 r4+r5 r6+r7 = r0' r2' r4' r6' */
455 __ASM_EMIT("vhaddps %%ymm6, %%ymm2, %%ymm2") /* ymm2 = i0+i1 i2+i3 i4+i5 i6+i7 = i0' i2' i4' i6' */
456 /* 2nd-order 8x butterfly */
457 __ASM_EMIT("vblendps $0xaa, %%ymm3, %%ymm1, %%ymm4") /* ymm4 = r1' i3' r5' i7' */
458 __ASM_EMIT("vblendps $0xaa, %%ymm1, %%ymm3, %%ymm5") /* ymm5 = i1' r3' i5' r7' */
459 __ASM_EMIT("vhsubps %%ymm4, %%ymm0, %%ymm1") /* ymm1 = r0'-r2' r4'-r6' r1'-i3' r5'-i7' = r2" r6" r1" r5" */
460 __ASM_EMIT("vhsubps %%ymm5, %%ymm2, %%ymm3") /* ymm3 = i0'-i2' i4'-i6' i1'-r3' i5'-r7' = i2" i6" i3" i7" */
461 __ASM_EMIT("vhaddps %%ymm4, %%ymm0, %%ymm0") /* ymm0 = r0'+r2' r4'+r6' r1'+i3' r5'+i7' = r0" r4" r3" r7" */
462 __ASM_EMIT("vhaddps %%ymm5, %%ymm2, %%ymm2") /* ymm2 = i0'+i2' i4'+i6' i1'+r3' i5'+r7' = i0" i4" i1" i5" */
463 __ASM_EMIT("vblendps $0xcc, %%ymm1, %%ymm0, %%ymm4") /* ymm4 = r0" i4" r1" r5" */
464 __ASM_EMIT("vblendps $0xcc, %%ymm0, %%ymm1, %%ymm5") /* ymm5 = r2" r6" r3" r7" */
465 __ASM_EMIT("vshufps $0x88, %%ymm3, %%ymm2, %%ymm6") /* ymm6 = i0" i1" i2" i3" */
466 __ASM_EMIT("vshufps $0xdd, %%ymm3, %%ymm2, %%ymm7") /* ymm7 = i4" i5" i6" i7" */
467 __ASM_EMIT("vshufps $0x88, %%ymm5, %%ymm4, %%ymm2") /* ymm2 = r0" r1" r2" r3" */
468 __ASM_EMIT("vshufps $0xdd, %%ymm5, %%ymm4, %%ymm3") /* ymm3 = r4" r5" r6" r7" */
469 /* 3rd-order 8x butterfly */
470 __ASM_EMIT("vmulps 0x40 + %[FFT_A], %%ymm3, %%ymm4") /* ymm4 = x_im * b_re */
471 __ASM_EMIT("vmulps 0x40 + %[FFT_A], %%ymm7, %%ymm5") /* ymm5 = x_im * b_im */
472 __ASM_EMIT("vfmsub231ps 0x00 + %[FFT_A], %%ymm3, %%ymm5") /* ymm5 = c_re = x_re * b_re + x_im * b_im */
473 __ASM_EMIT("vfmadd231ps 0x00 + %[FFT_A], %%ymm7, %%ymm4") /* ymm4 = c_im = x_re * b_im - x_im * b_re */
474 __ASM_EMIT("vsubps %%ymm5, %%ymm2, %%ymm0") /* ymm0 = a_re - c_re = r8 r9 r10 r11 r12 r13 r14 r15 */
475 __ASM_EMIT("vsubps %%ymm4, %%ymm6, %%ymm1") /* ymm1 = a_im - c_im = i8 i9 i10 i11 i12 i13 i14 i15 */
476 __ASM_EMIT("vaddps %%ymm5, %%ymm2, %%ymm2") /* ymm2 = a_re + c_re = r0 r1 r2 r3 r4 r5 r6 r7 */
477 __ASM_EMIT("vaddps %%ymm4, %%ymm6, %%ymm3") /* ymm3 = a_im + c_im = i0 i1 i2 i3 i4 i5 i6 i7 */
478 /* 4th-order 8x butterfly */
479 __ASM_EMIT("vextractf128 $1, %%ymm2, %%xmm4") /* xmm4 = r4 r5 r6 r7 */
480 __ASM_EMIT("vextractf128 $1, %%ymm3, %%xmm5") /* xmm5 = i4 i5 i6 i7 */
481 __ASM_EMIT("vinsertf128 $1, %%xmm0, %%ymm2, %%ymm2") /* ymm2 = a_re = r0 r1 r2 r3 r8 r9 r10 r11 */
482 __ASM_EMIT("vinsertf128 $1, %%xmm1, %%ymm3, %%ymm3") /* ymm3 = a_im = i0 i1 i2 i3 i8 i9 i10 i11 */
483 __ASM_EMIT("vinsertf128 $0, %%xmm4, %%ymm0, %%ymm0") /* ymm0 = b_re = r5 r6 r7 r12 r13 r14 r15 */
484 __ASM_EMIT("vinsertf128 $0, %%xmm5, %%ymm1, %%ymm1") /* ymm1 = b_im = i5 i6 i7 i12 i13 i14 i15 */
485 __ASM_EMIT("vmulps 0xc0 + %[FFT_A], %%ymm0, %%ymm4") /* ymm4 = x_im * b_re */
486 __ASM_EMIT("vmulps 0xc0 + %[FFT_A], %%ymm1, %%ymm5") /* ymm5 = x_im * b_im */
487 __ASM_EMIT("vfmsub231ps 0x80 + %[FFT_A], %%ymm0, %%ymm5") /* ymm5 = c_re = x_re * b_re - x_im * b_im */
488 __ASM_EMIT("vfmadd231ps 0x80 + %[FFT_A], %%ymm1, %%ymm4") /* ymm4 = c_im = x_re * b_im + x_im * b_re */
489 __ASM_EMIT("vsubps %%ymm5, %%ymm2, %%ymm0") /* ymm0 = a_re - c_re */
490 __ASM_EMIT("vsubps %%ymm4, %%ymm3, %%ymm1") /* ymm1 = a_im - c_im */
491 __ASM_EMIT("vaddps %%ymm5, %%ymm2, %%ymm2") /* ymm2 = a_re + c_re */
492 __ASM_EMIT("vaddps %%ymm4, %%ymm3, %%ymm3") /* ymm3 = a_im + c_im */
493 /* Store */
494 __ASM_EMIT("vmovups %%ymm2, 0x00(%[dst], %[off])")
495 __ASM_EMIT("vmovups %%ymm0, 0x20(%[dst], %[off])")
496 __ASM_EMIT("vmovups %%ymm3, 0x40(%[dst], %[off])")
497 __ASM_EMIT("vmovups %%ymm1, 0x60(%[dst], %[off])")
498 /* Move pointers */
499 __ASM_EMIT("add $0x80, %[off]")
500 __ASM_EMIT("sub $2, %[items]")
501 __ASM_EMIT("4:")
502 /* x4 scramble block */
503 __ASM_EMIT("add $1, %[items]")
504 __ASM_EMIT("jl 6f")
505 __ASM_EMIT("vmovups 0x00(%[dst]), %%xmm0") /* xmm0 = r0 i0 r1 i1 */
506 __ASM_EMIT("vmovups 0x10(%[dst]), %%xmm1") /* xmm1 = r2 i2 r3 i3 */
507 __ASM_EMIT("vmovups 0x20(%[dst]), %%xmm4") /* xmm4 = r4 i4 r5 i5 */
508 __ASM_EMIT("vmovups 0x30(%[dst]), %%xmm5") /* xmm5 = r6 i6 r7 i7 */
509 __ASM_EMIT("vshufps $0xdd, %%xmm1, %%xmm0, %%xmm2") /* xmm2 = i0 i1 i2 i3 */
510 __ASM_EMIT("vshufps $0x88, %%xmm1, %%xmm0, %%xmm0") /* xmm0 = r0 r1 r2 r3 */
511 __ASM_EMIT("vshufps $0xdd, %%xmm5, %%xmm4, %%xmm6") /* xmm4 = i4 i5 i6 i7 */
512 __ASM_EMIT("vshufps $0x88, %%xmm5, %%xmm4, %%xmm4") /* xmm6 = r4 r5 r6 r7 */
513 /* 1st-order 8x butterfly */
514 __ASM_EMIT("vhsubps %%xmm4, %%xmm0, %%xmm1") /* xmm1 = r0-r1 r2-r3 r4-r5 r6-r7 = r1' r3' r5' r7' */
515 __ASM_EMIT("vhsubps %%xmm6, %%xmm2, %%xmm3") /* xmm3 = i0-i1 i2-i3 i4-i5 i6-i7 = i1' i3' i5' i7' */
516 __ASM_EMIT("vhaddps %%xmm4, %%xmm0, %%xmm0") /* xmm0 = r0+r1 r2+r3 r4+r5 r6+r7 = r0' r2' r4' r6' */
517 __ASM_EMIT("vhaddps %%xmm6, %%xmm2, %%xmm2") /* xmm2 = i0+i1 i2+i3 i4+i5 i6+i7 = i0' i2' i4' i6' */
518 /* 2nd-order 8x butterfly */
519 __ASM_EMIT("vblendps $0xaa, %%xmm3, %%xmm1, %%xmm4") /* xmm4 = r1' i3' r5' i7' */
520 __ASM_EMIT("vblendps $0xaa, %%xmm1, %%xmm3, %%xmm5") /* xmm5 = i1' r3' i5' r7' */
521 __ASM_EMIT("vhsubps %%xmm4, %%xmm0, %%xmm1") /* xmm1 = r0'-r2' r4'-r6' r1'-i3' r5'-i7' = r2" r6" r1" r5" */
522 __ASM_EMIT("vhsubps %%xmm5, %%xmm2, %%xmm3") /* xmm3 = i0'-i2' i4'-i6' i1'-r3' i5'-r7' = i2" i6" i3" i7" */
523 __ASM_EMIT("vhaddps %%xmm4, %%xmm0, %%xmm0") /* xmm0 = r0'+r2' r4'+r6' r1'+i3' r5'+i7' = r0" r4" r3" r7" */
524 __ASM_EMIT("vhaddps %%xmm5, %%xmm2, %%xmm2") /* xmm2 = i0'+i2' i4'+i6' i1'+r3' i5'+r7' = i0" i4" i1" i5" */
525 __ASM_EMIT("vblendps $0xcc, %%xmm1, %%xmm0, %%xmm4") /* xmm4 = r0" i4" r1" r5" */
526 __ASM_EMIT("vblendps $0xcc, %%xmm0, %%xmm1, %%xmm5") /* xmm5 = r2" r6" r3" r7" */
527 __ASM_EMIT("vshufps $0x88, %%xmm3, %%xmm2, %%xmm6") /* xmm6 = i0" i1" i2" i3" */
528 __ASM_EMIT("vshufps $0xdd, %%xmm3, %%xmm2, %%xmm7") /* xmm7 = i4" i5" i6" i7" */
529 __ASM_EMIT("vshufps $0x88, %%xmm5, %%xmm4, %%xmm2") /* xmm2 = r0" r1" r2" r3" */
530 __ASM_EMIT("vshufps $0xdd, %%xmm5, %%xmm4, %%xmm3") /* xmm3 = r4" r5" r6" r7" */
531 /* 3rd-order 8x butterfly */
532 __ASM_EMIT("vmulps 0x40 + %[FFT_A], %%xmm3, %%xmm4") /* xmm4 = x_im * b_re */
533 __ASM_EMIT("vmulps 0x40 + %[FFT_A], %%xmm7, %%xmm5") /* xmm5 = x_im * b_im */
534 __ASM_EMIT("vfmsub231ps 0x00 + %[FFT_A], %%xmm3, %%xmm5") /* xmm5 = c_re = x_re * b_re - x_im * b_im */
535 __ASM_EMIT("vfmadd231ps 0x00 + %[FFT_A], %%xmm7, %%xmm4") /* xmm4 = c_im = x_re * b_im + x_im * b_re */
536 __ASM_EMIT("vsubps %%xmm5, %%xmm2, %%xmm0") /* xmm0 = a_re - c_re */
537 __ASM_EMIT("vsubps %%xmm4, %%xmm6, %%xmm1") /* xmm1 = a_im - c_im */
538 __ASM_EMIT("vaddps %%xmm5, %%xmm2, %%xmm2") /* xmm2 = a_re + c_re */
539 __ASM_EMIT("vaddps %%xmm4, %%xmm6, %%xmm3") /* xmm3 = a_im + c_im */
540 /* Store */
541 __ASM_EMIT("vmovups %%xmm2, 0x00(%[dst], %[off])")
542 __ASM_EMIT("vmovups %%xmm0, 0x10(%[dst], %[off])")
543 __ASM_EMIT("vmovups %%xmm3, 0x20(%[dst], %[off])")
544 __ASM_EMIT("vmovups %%xmm1, 0x30(%[dst], %[off])")
545 __ASM_EMIT("6:")
546
547 : [dst] "+r" (dst),
548 [off] "+r" (off), [items] "+r" (items)
549 : [FFT_A] "o" (FFT_A), [FFT_I] "o" (FFT_SCRAMBLE_INDICES),
550 [MASK] "o" (FFT_REPACK_MASKS)
551 : "cc", "memory",
552 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
553 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
554 );
555 }
556
557 static inline void FFT_PSCRAMBLE_COPY_DIRECT_NAME(float *dst, const float *src, size_t rank)
558 {
559 const size_t regs = 1 << (rank);
560
561 for (size_t i=0; i < regs; ++i)
562 {
563 size_t index = reverse_bits(FFT_TYPE(i), rank);
564
565 ARCH_X86_ASM(
566 __ASM_EMIT("vmovaps 0x280 + %[FFT_I], %%zmm7") /* zmm7 = indices */
567 /* Load even indices */
568 FFT_PSCRAMBLE_LOAD4X4("%%zmm0", "%%xmm0", "%%xmm4", "%%xmm5", "%%xmm6") /* zmm0 = r0 i0 r16 i16 r8 i8 r24 i24 r4 i4 r20 i20 r12 i12 r28 i28 */
569 FFT_PSCRAMBLE_LOAD4X4("%%zmm2", "%%xmm2", "%%xmm4", "%%xmm5", "%%xmm6") /* zmm2 = r2 i2 r18 i8 r10 i10 r26 i26 r6 i6 r22 i22 r14 i14 r30 i30 */
570 /* Load odd indices */
571 FFT_PSCRAMBLE_LOAD4X4("%%zmm1", "%%xmm1", "%%xmm4", "%%xmm5", "%%xmm6") /* zmm1 = r1 i1 r17 i17 r9 i9 r25 i25 r5 i5 r21 i21 r13 i13 r29 i29 */
572 FFT_PSCRAMBLE_LOAD4X4("%%zmm3", "%%xmm3", "%%xmm4", "%%xmm5", "%%xmm6") /* zmm3 = r3 i3 r19 i19 r11 i11 r27 i27 r7 i7 r23 i23 r15 i15 r31 i31 */
573 /* Unpack complex numbers */
574 __ASM_EMIT("kmovw 0x00 + %[MASK], %%k4")
575 __ASM_EMIT("kmovw 0x02 + %[MASK], %%k5")
576 __ASM_EMIT("vcompressps %%zmm0, %%zmm4 %{%%k5%}") /* zmm4 = i0 i16 i8 i24 i4 i20 i12 i28 ? ? ? ? ? ? ? ? */
577 __ASM_EMIT("vcompressps %%zmm2, %%zmm5 %{%%k5%}") /* zmm5 = i2 i8 i10 i26 i6 i22 i14 i30 ? ? ? ? ? ? ? ? */
578 __ASM_EMIT("vcompressps %%zmm0, %%zmm0 %{%%k4%}") /* zmm0 = r0 r16 r8 r24 r4 r20 r12 r28 ? ? ? ? ? ? ? ? */
579 __ASM_EMIT("vcompressps %%zmm2, %%zmm6 %{%%k4%}") /* zmm6 = r2 r8 r10 r26 r6 r22 r14 r30 ? ? ? ? ? ? ? ? */
580 __ASM_EMIT("vinsertf64x4 $1, %%ymm6, %%zmm0, %%zmm0") /* zmm0 = r0 r16 r8 r24 r4 r20 r12 r28 r2 r18 r10 r26 r6 r22 r14 r30 */
581 __ASM_EMIT("vinsertf64x4 $1, %%ymm5, %%zmm4, %%zmm2") /* zmm2 = i0 i16 i8 i24 i4 i20 i12 i28 i2 i18 i10 i26 i6 i22 i14 i30 */
582 __ASM_EMIT("vcompressps %%zmm1, %%zmm4 %{%%k5%}") /* zmm4 = i1 i17 i9 i25 i5 i21 i13 i29 ? ? ? ? ? ? ? ? */
583 __ASM_EMIT("vcompressps %%zmm3, %%zmm5 %{%%k5%}") /* zmm5 = i3 i19 i11 i27 i7 i23 i15 i31 ? ? ? ? ? ? ? ? */
584 __ASM_EMIT("vcompressps %%zmm1, %%zmm1 %{%%k4%}") /* zmm1 = r1 r17 r9 r25 r5 r21 r13 r29 ? ? ? ? ? ? ? ? */
585 __ASM_EMIT("vcompressps %%zmm3, %%zmm6 %{%%k4%}") /* zmm6 = r3 r19 r11 r27 r7 r23 r15 r31 ? ? ? ? ? ? ? ? */
586 __ASM_EMIT("vinsertf64x4 $1, %%ymm6, %%zmm1, %%zmm1") /* zmm1 = r1 r17 r9 r25 r5 r21 r13 r29 r3 r19 r11 r27 r7 r23 r15 r31 */
587 __ASM_EMIT("vinsertf64x4 $1, %%ymm5, %%zmm4, %%zmm3") /* zmm3 = i1 i17 i9 i25 i5 i21 i13 i29 i3 i19 i11 i27 i7 i23 i15 i31 */
588 /* Final permute */
589 __ASM_EMIT("vpermps %%zmm0, %%zmm7, %%zmm0") /* zmm0 = r0 r2 r4 r6 r8 r10 r12 r14 r16 r18 r20 r22 r24 r26 r28 r30 */
590 __ASM_EMIT("vpermps %%zmm1, %%zmm7, %%zmm1") /* zmm1 = r1 r3 r5 r7 r9 r11 r13 r15 r17 r19 r21 r23 r25 r27 r29 r31 */
591 __ASM_EMIT("vpermps %%zmm2, %%zmm7, %%zmm2") /* zmm2 = i0 i2 i4 i6 i8 i10 i12 i14 i16 i18 i20 i22 i24 i26 i28 i30 */
592 __ASM_EMIT("vpermps %%zmm3, %%zmm7, %%zmm3") /* zmm3 = i1 i3 i5 i7 i9 i11 i13 i15 i17 i19 i21 i23 i25 i27 i29 i31 */
593
594 /* 1st-order 16x butterfly */
595 /* zmm0 = r0 r2 r4 r6 r8 r10 r12 r14 r16 r18 r20 r22 r24 r26 r28 r30 */
596 /* zmm1 = r1 r3 r5 r7 r9 r11 r13 r15 r17 r19 r21 r23 r25 r27 r29 r31 */
597 /* zmm2 = i0 i2 i4 i6 i8 i10 i12 i14 i16 i18 i20 i22 i24 i26 i28 i30 */
598 /* zmm3 = i1 i3 i5 i7 i9 i11 i13 i15 i17 i19 i21 i23 i25 i27 i29 i31 */
599 __ASM_EMIT("vsubps %%zmm1, %%zmm0, %%zmm4") /* zmm4 = r0-r1 r2-r3 r4-r5 r6-r7 ... = r1' r3' r5' r7' r9' r11' r13' r15' r17' r19' r21' r23' r25' r27' r29' r31' */
600 __ASM_EMIT("vsubps %%zmm3, %%zmm2, %%zmm5") /* zmm5 = i0-i1 i2-i3 i4-i5 i6-i7 ... = i1' i3' i5' i7' i9' i11' i13' i15' i17' i19' i21' i23' i25' i27' i29' i31' */
601 __ASM_EMIT("vaddps %%zmm1, %%zmm0, %%zmm0") /* zmm0 = r0+r1 r2+r3 r4+r5 r6+r7 ... = r0' r2' r4' r6' r8' r10' r12' r14' r16' r18' r20' r22' r24' r26' r28' r30' */
602 __ASM_EMIT("vaddps %%zmm3, %%zmm2, %%zmm2") /* zmm2 = i0+i1 i2+i3 i4+i5 i6+i7 ... = i0' i2' i4' i6' i8' i10' i12' i14' i16' i18' i20' i22' i24' i26' i28' i30' */
603 /* 2nd-order 16x butterfly */
604 __ASM_EMIT("vmovaps %%zmm0, %%zmm1") /* zmm1 = zmm0 = r0' r2' r4' r6' r8' r10' r12' r14' r16' r18' r20' r22' r24' r26' r28' r30' */
605 __ASM_EMIT("vmovaps %%zmm2, %%zmm3") /* zmm3 = zmm2 = i0' i2' i4' i6' i8' i10' i12' i14' i16' i18' i20' i22' i24' i26' i28' i30' */
606 __ASM_EMIT("vmovaps 0x80 + %[FFT_I], %%zmm6") /* zmm6 = shuffle_a[1] */
607 __ASM_EMIT("vmovaps 0xc0 + %[FFT_I], %%zmm7") /* zmm7 = shuffle_b[1] */
608 __ASM_EMIT("vpermt2ps %%zmm4, %%zmm6, %%zmm0") /* zmm0 = r0' r1' r4' r5' r8' r9' r12' r13' r16' r17' r20' r21' r24' r25' r28' r29' */
609 __ASM_EMIT("vpermt2ps %%zmm5, %%zmm6, %%zmm2") /* zmm2 = i0' i1' i4' i5' i8' i9' i12' i13' i16' i17' i20' i21' i24' i25' i28' i29' */
610 __ASM_EMIT("vpermt2ps %%zmm5, %%zmm7, %%zmm1") /* zmm1 = r2' i3' r6' i7' r10' i11' r14' i15' r18' i19' r22' i23' r26' i27' r30' i31' */
611 __ASM_EMIT("vpermt2ps %%zmm4, %%zmm7, %%zmm3") /* zmm3 = i2' r3' i6' r7' i10' r11' i14' r15' i18' r19' i22' r23' i26' r27' i30' r31' */
612 __ASM_EMIT("vsubps %%zmm1, %%zmm0, %%zmm4") /* zmm4 = r2" r3" r6" r7" r10" r11" r14" r15" r18" r19" r22" r23" r26" r27" r30" r31" = r0'-r2' r1'-i3' r4'-r6' r5'-i7' r8'-r10' r9'-i11' r12'-r14' r13'-i15' r16'-r18' r17'-i19' r20'-r22' r21'-i23' r24'-r26' r25'-i27' r28'-r30' r29'-i31' */
613 __ASM_EMIT("vsubps %%zmm3, %%zmm2, %%zmm5") /* zmm5 = i2" i1" i6" i5" i10" i9" i14" i13" i18" i17" i22" i21" i26" i25" i30" i29" = i0'-i2' i1'-r3' i4'-i6' i5'-r7' i8'-i10' i9'-r11' i12'-i14' i13'-r15' i16'-i18' i17'-r19' i20'-i22' i21'-r23' i24'-i26' i25'-r27' i28'-i30' i29'-r31' */
614 __ASM_EMIT("vaddps %%zmm1, %%zmm0, %%zmm0") /* zmm0 = r0" r1" r4" r5" r8" r9" r12" r13" r16" r17" r20" r21" r24" r25" r28" r29" = r0'+r2' r1'+i3' r4'+r6' r5'+i7' r8'+r10' r9'+i11' r12'+r14' r13'+i15' r16'+r18' r17'+i19' r20'+r22' r21'+i23' r24'+r26' r25'+i27' r28'+r30' r29'+i31' */
615 __ASM_EMIT("vaddps %%zmm3, %%zmm2, %%zmm2") /* zmm2 = i0" i3" i4" i7" i8" i11" i12" i15" i16" i19" i20" i23" i24" i27" i28" i31" = i0'+i2' i1'+r3' i4'+i6' i5'+r7' i8'+i10' i9'+r11' i12'+i14' i13'+r15' i16'+i18' i17'+r19' i20'+i22' i21'+r23' i24'+i26' i25'+r27' i28'+i30' i29'+r31' */
616 /* 3rd-order 16x butterfly */
617 __ASM_EMIT("vmovaps %%zmm0, %%zmm1") /* zmm1 = zmm0 = r0" r1" r4" r5" r8" r9" r12" r13" r16" r17" r20" r21" r24" r25" r28" r29" */
618 __ASM_EMIT("vmovaps %%zmm2, %%zmm3") /* zmm3 = zmm2 = i0" i3" i4" i7" i8" i11" i12" i15" i16" i19" i20" i23" i24" i27" i28" i31" */
619 __ASM_EMIT("vmovaps 0x100 + %[FFT_I], %%zmm6") /* zmm6 = shuffle_a[2] */
620 __ASM_EMIT("vmovaps 0x140 + %[FFT_I], %%zmm7") /* zmm7 = shuffle_b[2] */
621 __ASM_EMIT("vpermt2ps %%zmm4, %%zmm6, %%zmm0") /* zmm0 = a_re = r0" r1" r2" r3" r8" r9" r10" r11" r16" r17" r18" r19" r24" r25" r26" r27" */
622 __ASM_EMIT("vpermt2ps %%zmm5, %%zmm7, %%zmm2") /* zmm2 = a_im = i0" i1" i2" i3" i8" i9" i10" i11" i16" i17" i18" i19" i24" i25" i26" i27" */
623 __ASM_EMIT("vmovaps 0x180 + %[FFT_I], %%zmm6") /* zmm6 = shuffle_a[3] */
624 __ASM_EMIT("vmovaps 0x1c0 + %[FFT_I], %%zmm7") /* zmm7 = shuffle_b[3] */
625 __ASM_EMIT("vpermt2ps %%zmm4, %%zmm6, %%zmm1") /* zmm1 = b_re = r4" r5" r6" r7" r12" r13" r14" r15" r20" r21" r22" r23" r28" r29" r30" r31" */
626 __ASM_EMIT("vpermt2ps %%zmm5, %%zmm7, %%zmm3") /* zmm3 = b_im = i4" i5" i6" i7" i12" i13" i14" i15" i20" i21" i22" i23" i28" i29" i30" i31" */
627 __ASM_EMIT("vmulps 0x40 + %[FFT_A], %%zmm1, %%zmm4") /* zmm4 = x_im * b_re */
628 __ASM_EMIT("vmulps 0x40 + %[FFT_A], %%zmm3, %%zmm5") /* zmm5 = x_im * b_im */
629 __ASM_EMIT("vfmadd231ps 0x00 + %[FFT_A], %%zmm1, %%zmm5") /* zmm5 = c_re = x_re * b_re + x_im * b_im */
630 __ASM_EMIT("vfmsub231ps 0x00 + %[FFT_A], %%zmm3, %%zmm4") /* zmm4 = c_im = x_re * b_im - x_im * b_re */
631 __ASM_EMIT("vsubps %%zmm5, %%zmm0, %%zmm1") /* zmm1 = a_re - c_re = r4` r5` r6` r7` r12` r13` r14` r15` r20` r21` r22` r23` r28` r29` r30` r31` */
632 __ASM_EMIT("vsubps %%zmm4, %%zmm2, %%zmm3") /* zmm3 = a_im - c_im = i4` i5` i6` i7` i12` i13` i14` i15` i20` i21` i22` i23` i28` i29` i30` i31` */
633 __ASM_EMIT("vaddps %%zmm5, %%zmm0, %%zmm0") /* zmm0 = a_re + c_re = r0` r1` r2` r3` r8` r9` r10` r11` r16` r17` r18` r19` r24` r25` r26` r27` */
634 __ASM_EMIT("vaddps %%zmm4, %%zmm2, %%zmm2") /* zmm2 = a_im + c_im = i0` i1` i2` i3` i8` i9` i10` i11` i16` i17` i18` i19` i24` i25` i26` i27` */
635 /* 4th-order 16x butterfly */
636 __ASM_EMIT("vmovaps %%zmm0, %%zmm4") /* zmm4 = zmm0 = r0` r1` r2` r3` r8` r9` r10` r11` r16` r17` r18` r19` r24` r25` r26` r27` */
637 __ASM_EMIT("vmovaps %%zmm2, %%zmm5") /* zmm5 = zmm2 = i0` i1` i2` i3` i8` i9` i10` i11` i16` i17` i18` i19` i24` i25` i26` i27` */
638 __ASM_EMIT("vmovaps 0x200 + %[FFT_I], %%zmm6") /* zmm6 = shuffle_a[4] */
639 __ASM_EMIT("vmovaps 0x240 + %[FFT_I], %%zmm7") /* zmm7 = shuffle_b[4] */
640 __ASM_EMIT("vpermt2ps %%zmm1, %%zmm6, %%zmm0") /* zmm0 = a_re = r0` r1` r2` r3` r4` r5` r6` r7` r16` r17` r18` r19` r20` r21` r22` r23` */
641 __ASM_EMIT("vpermt2ps %%zmm1, %%zmm7, %%zmm4") /* zmm4 = b_re = r8` r9` r10` r11` r12` r13` r14` r15` r24` r25` r26` r27` r28` r29` r30` r31` */
642 __ASM_EMIT("vpermt2ps %%zmm3, %%zmm6, %%zmm2") /* zmm2 = a_im = i0` i1` i2` i3` i4` i5` i6` i7` i16` i17` i18` i19` i20` i21` i22` i23` */
643 __ASM_EMIT("vpermt2ps %%zmm3, %%zmm7, %%zmm5") /* zmm5 = b_im = i8` i9` i10` i11` i12` i13` i14` i15` i24` i25` i26` i27` i28` i29` i30` i31` */
644 __ASM_EMIT("vmulps 0xc0 + %[FFT_A], %%zmm4, %%zmm1") /* zmm1 = x_im * b_re */
645 __ASM_EMIT("vmulps 0xc0 + %[FFT_A], %%zmm5, %%zmm3") /* zmm3 = x_im * b_im */
646 __ASM_EMIT("vfmadd231ps 0x80 + %[FFT_A], %%zmm4, %%zmm3") /* zmm3 = c_re = x_re * b_re + x_im * b_im */
647 __ASM_EMIT("vfmsub231ps 0x80 + %[FFT_A], %%zmm5, %%zmm1") /* zmm1 = c_im = x_re * b_im - x_im * b_re */
648 __ASM_EMIT("vsubps %%zmm3, %%zmm0, %%zmm4") /* zmm4 = a_re - c_re = R8 R9 R10 R11 R12 R13 R14 R15 R24 R25 R26 R27 R28 R29 R30 R31 */
649 __ASM_EMIT("vsubps %%zmm1, %%zmm2, %%zmm5") /* zmm5 = a_im - c_im = I8 I9 I10 I11 I12 I13 I14 I15 I24 I25 I26 I27 I28 I29 I30 I31 */
650 __ASM_EMIT("vaddps %%zmm3, %%zmm0, %%zmm0") /* zmm0 = a_re + c_re = R0 R1 R2 R3 R4 R5 R6 R7 R16 R17 R18 R19 R20 R21 R22 R23 */
651 __ASM_EMIT("vaddps %%zmm1, %%zmm2, %%zmm2") /* zmm2 = a_im + c_im = I0 I1 I2 I3 I4 I5 I6 I7 I16 I17 I18 I19 I20 I21 I22 I23 */
652 /* Store */
653 __ASM_EMIT("vmovups %%ymm0, 0x00(%[dst])")
654 __ASM_EMIT("vmovups %%ymm4, 0x20(%[dst])")
655 __ASM_EMIT("vmovups %%ymm2, 0x40(%[dst])")
656 __ASM_EMIT("vmovups %%ymm5, 0x60(%[dst])")
657 __ASM_EMIT("vextractf64x4 $1, %%zmm0, 0x80(%[dst])")
658 __ASM_EMIT("vextractf64x4 $1, %%zmm4, 0xa0(%[dst])")
659 __ASM_EMIT("vextractf64x4 $1, %%zmm2, 0xc0(%[dst])")
660 __ASM_EMIT("vextractf64x4 $1, %%zmm5, 0xe0(%[dst])")
661 __ASM_EMIT("add $0x100, %[dst]")
662
663 : [dst] "+r" (dst),
664 [index] "+r"(index)
665 : [src] "r" (src),
666 [regs] "r" (regs),
667 [FFT_A] "o" (FFT_A),
668 [FFT_I] "o" (FFT_SCRAMBLE_INDICES),
669 [MASK] "o" (FFT_REPACK_MASKS)
670 : "cc", "memory",
671 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
672 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
673 );
674 }
675 }
676
677 static inline void FFT_PSCRAMBLE_COPY_REVERSE_NAME(float *dst, const float *src, size_t rank)
678 {
679 const size_t regs = 1 << (rank);
680
681 for (size_t i=0; i < regs; ++i)
682 {
683 size_t index = reverse_bits(FFT_TYPE(i), rank);
684
685 ARCH_X86_ASM(
686 __ASM_EMIT("vmovaps 0x280 + %[FFT_I], %%zmm7") /* zmm7 = indices */
687 /* Load even indices */
688 FFT_PSCRAMBLE_LOAD4X4("%%zmm0", "%%xmm0", "%%xmm4", "%%xmm5", "%%xmm6") /* zmm0 = r0 i0 r16 i16 r8 i8 r24 i24 r4 i4 r20 i20 r12 i12 r28 i28 */
689 FFT_PSCRAMBLE_LOAD4X4("%%zmm2", "%%xmm2", "%%xmm4", "%%xmm5", "%%xmm6") /* zmm2 = r2 i2 r18 i8 r10 i10 r26 i26 r6 i6 r22 i22 r14 i14 r30 i30 */
690 /* Load odd indices */
691 FFT_PSCRAMBLE_LOAD4X4("%%zmm1", "%%xmm1", "%%xmm4", "%%xmm5", "%%xmm6") /* zmm1 = r1 i1 r17 i17 r9 i9 r25 i25 r5 i5 r21 i21 r13 i13 r29 i29 */
692 FFT_PSCRAMBLE_LOAD4X4("%%zmm3", "%%xmm3", "%%xmm4", "%%xmm5", "%%xmm6") /* zmm3 = r3 i3 r19 i19 r11 i11 r27 i27 r7 i7 r23 i23 r15 i15 r31 i31 */
693 /* Unpack complex numbers */
694 __ASM_EMIT("kmovw 0x00 + %[MASK], %%k4")
695 __ASM_EMIT("kmovw 0x02 + %[MASK], %%k5")
696 __ASM_EMIT("vcompressps %%zmm0, %%zmm4 %{%%k5%}") /* zmm4 = i0 i16 i8 i24 i4 i20 i12 i28 ? ? ? ? ? ? ? ? */
697 __ASM_EMIT("vcompressps %%zmm2, %%zmm5 %{%%k5%}") /* zmm5 = i2 i8 i10 i26 i6 i22 i14 i30 ? ? ? ? ? ? ? ? */
698 __ASM_EMIT("vcompressps %%zmm0, %%zmm0 %{%%k4%}") /* zmm0 = r0 r16 r8 r24 r4 r20 r12 r28 ? ? ? ? ? ? ? ? */
699 __ASM_EMIT("vcompressps %%zmm2, %%zmm6 %{%%k4%}") /* zmm6 = r2 r8 r10 r26 r6 r22 r14 r30 ? ? ? ? ? ? ? ? */
700 __ASM_EMIT("vinsertf64x4 $1, %%ymm6, %%zmm0, %%zmm0") /* zmm0 = r0 r16 r8 r24 r4 r20 r12 r28 r2 r18 r10 r26 r6 r22 r14 r30 */
701 __ASM_EMIT("vinsertf64x4 $1, %%ymm5, %%zmm4, %%zmm2") /* zmm2 = i0 i16 i8 i24 i4 i20 i12 i28 i2 i18 i10 i26 i6 i22 i14 i30 */
702 __ASM_EMIT("vcompressps %%zmm1, %%zmm4 %{%%k5%}") /* zmm4 = i1 i17 i9 i25 i5 i21 i13 i29 ? ? ? ? ? ? ? ? */
703 __ASM_EMIT("vcompressps %%zmm3, %%zmm5 %{%%k5%}") /* zmm5 = i3 i19 i11 i27 i7 i23 i15 i31 ? ? ? ? ? ? ? ? */
704 __ASM_EMIT("vcompressps %%zmm1, %%zmm1 %{%%k4%}") /* zmm1 = r1 r17 r9 r25 r5 r21 r13 r29 ? ? ? ? ? ? ? ? */
705 __ASM_EMIT("vcompressps %%zmm3, %%zmm6 %{%%k4%}") /* zmm6 = r3 r19 r11 r27 r7 r23 r15 r31 ? ? ? ? ? ? ? ? */
706 __ASM_EMIT("vinsertf64x4 $1, %%ymm6, %%zmm1, %%zmm1") /* zmm1 = r1 r17 r9 r25 r5 r21 r13 r29 r3 r19 r11 r27 r7 r23 r15 r31 */
707 __ASM_EMIT("vinsertf64x4 $1, %%ymm5, %%zmm4, %%zmm3") /* zmm3 = i1 i17 i9 i25 i5 i21 i13 i29 i3 i19 i11 i27 i7 i23 i15 i31 */
708 /* Final permute */
709 __ASM_EMIT("vpermps %%zmm0, %%zmm7, %%zmm0") /* zmm0 = r0 r2 r4 r6 r8 r10 r12 r14 r16 r18 r20 r22 r24 r26 r28 r30 */
710 __ASM_EMIT("vpermps %%zmm1, %%zmm7, %%zmm1") /* zmm1 = r1 r3 r5 r7 r9 r11 r13 r15 r17 r19 r21 r23 r25 r27 r29 r31 */
711 __ASM_EMIT("vpermps %%zmm2, %%zmm7, %%zmm2") /* zmm2 = i0 i2 i4 i6 i8 i10 i12 i14 i16 i18 i20 i22 i24 i26 i28 i30 */
712 __ASM_EMIT("vpermps %%zmm3, %%zmm7, %%zmm3") /* zmm3 = i1 i3 i5 i7 i9 i11 i13 i15 i17 i19 i21 i23 i25 i27 i29 i31 */
713
714 /* 1st-order 16x butterfly */
715 /* zmm0 = r0 r2 r4 r6 r8 r10 r12 r14 r16 r18 r20 r22 r24 r26 r28 r30 */
716 /* zmm1 = r1 r3 r5 r7 r9 r11 r13 r15 r17 r19 r21 r23 r25 r27 r29 r31 */
717 /* zmm2 = i0 i2 i4 i6 i8 i10 i12 i14 i16 i18 i20 i22 i24 i26 i28 i30 */
718 /* zmm3 = i1 i3 i5 i7 i9 i11 i13 i15 i17 i19 i21 i23 i25 i27 i29 i31 */
719 __ASM_EMIT("vsubps %%zmm1, %%zmm0, %%zmm4") /* zmm4 = r0-r1 r2-r3 r4-r5 r6-r7 ... = r1' r3' r5' r7' r9' r11' r13' r15' r17' r19' r21' r23' r25' r27' r29' r31' */
720 __ASM_EMIT("vsubps %%zmm3, %%zmm2, %%zmm5") /* zmm5 = i0-i1 i2-i3 i4-i5 i6-i7 ... = i1' i3' i5' i7' i9' i11' i13' i15' i17' i19' i21' i23' i25' i27' i29' i31' */
721 __ASM_EMIT("vaddps %%zmm1, %%zmm0, %%zmm0") /* zmm0 = r0+r1 r2+r3 r4+r5 r6+r7 ... = r0' r2' r4' r6' r8' r10' r12' r14' r16' r18' r20' r22' r24' r26' r28' r30' */
722 __ASM_EMIT("vaddps %%zmm3, %%zmm2, %%zmm2") /* zmm2 = i0+i1 i2+i3 i4+i5 i6+i7 ... = i0' i2' i4' i6' i8' i10' i12' i14' i16' i18' i20' i22' i24' i26' i28' i30' */
723 /* 2nd-order 16x butterfly */
724 __ASM_EMIT("vmovaps %%zmm0, %%zmm1") /* zmm1 = zmm0 = r0' r2' r4' r6' r8' r10' r12' r14' r16' r18' r20' r22' r24' r26' r28' r30' */
725 __ASM_EMIT("vmovaps %%zmm2, %%zmm3") /* zmm3 = zmm2 = i0' i2' i4' i6' i8' i10' i12' i14' i16' i18' i20' i22' i24' i26' i28' i30' */
726 __ASM_EMIT("vmovaps 0x80 + %[FFT_I], %%zmm6") /* zmm6 = shuffle_a[1] */
727 __ASM_EMIT("vmovaps 0xc0 + %[FFT_I], %%zmm7") /* zmm7 = shuffle_b[1] */
728 __ASM_EMIT("vpermt2ps %%zmm4, %%zmm6, %%zmm0") /* zmm0 = r0' r1' r4' r5' r8' r9' r12' r13' r16' r17' r20' r21' r24' r25' r28' r29' */
729 __ASM_EMIT("vpermt2ps %%zmm5, %%zmm6, %%zmm2") /* zmm2 = i0' i1' i4' i5' i8' i9' i12' i13' i16' i17' i20' i21' i24' i25' i28' i29' */
730 __ASM_EMIT("vpermt2ps %%zmm5, %%zmm7, %%zmm1") /* zmm1 = r2' i3' r6' i7' r10' i11' r14' i15' r18' i19' r22' i23' r26' i27' r30' i31' */
731 __ASM_EMIT("vpermt2ps %%zmm4, %%zmm7, %%zmm3") /* zmm3 = i2' r3' i6' r7' i10' r11' i14' r15' i18' r19' i22' r23' i26' r27' i30' r31' */
732 __ASM_EMIT("vsubps %%zmm1, %%zmm0, %%zmm4") /* zmm4 = r2" r1" r6" r5" r10" r9" r14" r13" r18" r17" r22" r21" r26" r25" r30" r29" = r0'-r2' r1'-i3' r4'-r6' r5'-i7' r8'-r10' r9'-i11' r12'-r14' r13'-i15' r16'-r18' r17'-i19' r20'-r22' r21'-i23' r24'-r26' r25'-i27' r28'-r30' r29'-i31' */
733 __ASM_EMIT("vsubps %%zmm3, %%zmm2, %%zmm5") /* zmm5 = i2" i3" i6" i7" i10" i11" i14" i15" i18" i19" i22" i23" i26" i27" i30" i31" = i0'-i2' i1'-r3' i4'-i6' i5'-r7' i8'-i10' i9'-r11' i12'-i14' i13'-r15' i16'-i18' i17'-r19' i20'-i22' i21'-r23' i24'-i26' i25'-r27' i28'-i30' i29'-r31' */
734 __ASM_EMIT("vaddps %%zmm1, %%zmm0, %%zmm0") /* zmm0 = r0" r3" r4" r7" r8" r11" r12" r15" r16" r19" r20" r23" r24" r27" r28" r31" = r0'+r2' r1'+i3' r4'+r6' r5'+i7' r8'+r10' r9'+i11' r12'+r14' r13'+i15' r16'+r18' r17'+i19' r20'+r22' r21'+i23' r24'+r26' r25'+i27' r28'+r30' r29'+i31' */
735 __ASM_EMIT("vaddps %%zmm3, %%zmm2, %%zmm2") /* zmm2 = i0" i1" i4" i5" i8" i9" i12" i13" i16" i17" i20" i21" i24" i25" i28" i29" = i0'+i2' i1'+r3' i4'+i6' i5'+r7' i8'+i10' i9'+r11' i12'+i14' i13'+r15' i16'+i18' i17'+r19' i20'+i22' i21'+r23' i24'+i26' i25'+r27' i28'+i30' i29'+r31' */
736 /* 3rd-order 16x butterfly */
737 __ASM_EMIT("vmovaps %%zmm0, %%zmm1") /* zmm1 = zmm0 = r0" r3" r4" r7" r8" r11" r12" r15" r16" r19" r20" r23" r24" r27" r28" r31" */
738 __ASM_EMIT("vmovaps %%zmm2, %%zmm3") /* zmm3 = zmm2 = i0" i1" i4" i5" i8" i9" i12" i13" i16" i17" i20" i21" i24" i25" i28" i29" */
739 __ASM_EMIT("vmovaps 0x140 + %[FFT_I], %%zmm6") /* zmm6 = shuffle_b[2] */
740 __ASM_EMIT("vmovaps 0x100 + %[FFT_I], %%zmm7") /* zmm7 = shuffle_a[2] */
741 __ASM_EMIT("vpermt2ps %%zmm4, %%zmm6, %%zmm0") /* zmm0 = a_re = r0" r1" r2" r3" r8" r9" r10" r11" r16" r17" r18" r19" r24" r25" r26" r27" */
742 __ASM_EMIT("vpermt2ps %%zmm5, %%zmm7, %%zmm2") /* zmm2 = a_im = i0" i1" i2" i3" i8" i9" i10" i11" i16" i17" i18" i19" i24" i25" i26" i27" */
743 __ASM_EMIT("vmovaps 0x1c0 + %[FFT_I], %%zmm6") /* zmm6 = shuffle_b[3] */
744 __ASM_EMIT("vmovaps 0x180 + %[FFT_I], %%zmm7") /* zmm7 = shuffle_a[3] */
745 __ASM_EMIT("vpermt2ps %%zmm4, %%zmm6, %%zmm1") /* zmm1 = b_re = r4" r5" r6" r7" r12" r13" r14" r15" r20" r21" r22" r23" r28" r29" r30" r31" */
746 __ASM_EMIT("vpermt2ps %%zmm5, %%zmm7, %%zmm3") /* zmm3 = b_im = i4" i5" i6" i7" i12" i13" i14" i15" i20" i21" i22" i23" i28" i29" i30" i31" */
747 __ASM_EMIT("vmulps 0x40 + %[FFT_A], %%zmm1, %%zmm4") /* zmm4 = x_im * b_re */
748 __ASM_EMIT("vmulps 0x40 + %[FFT_A], %%zmm3, %%zmm5") /* zmm5 = x_im * b_im */
749 __ASM_EMIT("vfmsub231ps 0x00 + %[FFT_A], %%zmm1, %%zmm5") /* zmm5 = c_re = x_re * b_re - x_im * b_im */
750 __ASM_EMIT("vfmadd231ps 0x00 + %[FFT_A], %%zmm3, %%zmm4") /* zmm4 = c_im = x_re * b_im + x_im * b_re */
751 __ASM_EMIT("vsubps %%zmm5, %%zmm0, %%zmm1") /* zmm1 = a_re - c_re = r4` r5` r6` r7` r12` r13` r14` r15` r20` r21` r22` r23` r28` r29` r30` r31` */
752 __ASM_EMIT("vsubps %%zmm4, %%zmm2, %%zmm3") /* zmm3 = a_im - c_im = i4` i5` i6` i7` i12` i13` i14` i15` i20` i21` i22` i23` i28` i29` i30` i31` */
753 __ASM_EMIT("vaddps %%zmm5, %%zmm0, %%zmm0") /* zmm0 = a_re + c_re = r0` r1` r2` r3` r8` r9` r10` r11` r16` r17` r18` r19` r24` r25` r26` r27` */
754 __ASM_EMIT("vaddps %%zmm4, %%zmm2, %%zmm2") /* zmm2 = a_im + c_im = i0` i1` i2` i3` i8` i9` i10` i11` i16` i17` i18` i19` i24` i25` i26` i27` */
755 /* 4th-order 16x butterfly */
756 __ASM_EMIT("vmovaps %%zmm0, %%zmm4") /* zmm4 = zmm0 = r0` r1` r2` r3` r8` r9` r10` r11` r16` r17` r18` r19` r24` r25` r26` r27` */
757 __ASM_EMIT("vmovaps %%zmm2, %%zmm5") /* zmm5 = zmm2 = i0` i1` i2` i3` i8` i9` i10` i11` i16` i17` i18` i19` i24` i25` i26` i27` */
758 __ASM_EMIT("vmovaps 0x200 + %[FFT_I], %%zmm6") /* zmm6 = shuffle_a[4] */
759 __ASM_EMIT("vmovaps 0x240 + %[FFT_I], %%zmm7") /* zmm7 = shuffle_b[4] */
760 __ASM_EMIT("vpermt2ps %%zmm1, %%zmm6, %%zmm0") /* zmm0 = a_re = r0` r1` r2` r3` r4` r5` r6` r7` r16` r17` r18` r19` r20` r21` r22` r23` */
761 __ASM_EMIT("vpermt2ps %%zmm1, %%zmm7, %%zmm4") /* zmm4 = b_re = r8` r9` r10` r11` r12` r13` r14` r15` r24` r25` r26` r27` r28` r29` r30` r31` */
762 __ASM_EMIT("vpermt2ps %%zmm3, %%zmm6, %%zmm2") /* zmm2 = a_im = i0` i1` i2` i3` i4` i5` i6` i7` i16` i17` i18` i19` i20` i21` i22` i23` */
763 __ASM_EMIT("vpermt2ps %%zmm3, %%zmm7, %%zmm5") /* zmm5 = b_im = i8` i9` i10` i11` i12` i13` i14` i15` i24` i25` i26` i27` i28` i29` i30` i31` */
764 __ASM_EMIT("vmulps 0xc0 + %[FFT_A], %%zmm4, %%zmm1") /* zmm1 = x_im * b_re */
765 __ASM_EMIT("vmulps 0xc0 + %[FFT_A], %%zmm5, %%zmm3") /* zmm3 = x_im * b_im */
766 __ASM_EMIT("vfmsub231ps 0x80 + %[FFT_A], %%zmm4, %%zmm3") /* zmm3 = c_re = x_re * b_re - x_im * b_im */
767 __ASM_EMIT("vfmadd231ps 0x80 + %[FFT_A], %%zmm5, %%zmm1") /* zmm1 = c_im = x_re * b_im + x_im * b_re */
768 __ASM_EMIT("vsubps %%zmm3, %%zmm0, %%zmm4") /* zmm4 = a_re - c_re = R8 R9 R10 R11 R12 R13 R14 R15 R24 R25 R26 R27 R28 R29 R30 R31 */
769 __ASM_EMIT("vsubps %%zmm1, %%zmm2, %%zmm5") /* zmm5 = a_im - c_im = I8 I9 I10 I11 I12 I13 I14 I15 I24 I25 I26 I27 I28 I29 I30 I31 */
770 __ASM_EMIT("vaddps %%zmm3, %%zmm0, %%zmm0") /* zmm0 = a_re + c_re = R0 R1 R2 R3 R4 R5 R6 R7 R16 R17 R18 R19 R20 R21 R22 R23 */
771 __ASM_EMIT("vaddps %%zmm1, %%zmm2, %%zmm2") /* zmm2 = a_im + c_im = I0 I1 I2 I3 I4 I5 I6 I7 I16 I17 I18 I19 I20 I21 I22 I23 */
772 /* Store */
773 __ASM_EMIT("vmovups %%ymm0, 0x00(%[dst])")
774 __ASM_EMIT("vmovups %%ymm4, 0x20(%[dst])")
775 __ASM_EMIT("vmovups %%ymm2, 0x40(%[dst])")
776 __ASM_EMIT("vmovups %%ymm5, 0x60(%[dst])")
777 __ASM_EMIT("vextractf64x4 $1, %%zmm0, 0x80(%[dst])")
778 __ASM_EMIT("vextractf64x4 $1, %%zmm4, 0xa0(%[dst])")
779 __ASM_EMIT("vextractf64x4 $1, %%zmm2, 0xc0(%[dst])")
780 __ASM_EMIT("vextractf64x4 $1, %%zmm5, 0xe0(%[dst])")
781 __ASM_EMIT("add $0x100, %[dst]")
782
783 : [dst] "+r" (dst),
784 [index] "+r"(index)
785 : [src] "r" (src),
786 [regs] "r" (regs),
787 [FFT_A] "o" (FFT_A),
788 [FFT_I] "o" (FFT_SCRAMBLE_INDICES),
789 [MASK] "o" (FFT_REPACK_MASKS)
790 : "cc", "memory",
791 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
792 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
793 );
794 }
795 }
796
797 } /* namespace avx512 */
798 } /* namespace lsp */
799
800 #undef FFT_PSCRAMBLE_SELF_DIRECT_NAME
801 #undef FFT_PSCRAMBLE_SELF_REVERSE_NAME
802 #undef FFT_PSCRAMBLE_COPY_DIRECT_NAME
803 #undef FFT_PSCRAMBLE_COPY_REVERSE_NAME
804 #undef FFT_TYPE
805 #undef FFT_PSCRAMBLE_LOAD4X4