Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/x86/sse/fft/p_switch.h

raw
1 /*
2 * Copyright (C) 2020 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2020 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 31 мар. 2020 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_X86_SSE_IMPL
23 #error "This header should not be included directly"
24 #endif /* PRIVATE_DSP_ARCH_X86_SSE_IMPL */
25
26 #define _SSE_FFT_NAME(id) id
27 #define _SSE_CALL_NAME(id, bits) id ## bits
28
29 #define SSE_FFT_NAME(id) _SSE_FFT_NAME(id)
30 #define SSE_CALL_NAME(id, bits) _SSE_CALL_NAME(id, bits)(dst, src, rank)
31
32 namespace lsp
33 {
34 namespace sse
35 {
36 static inline void SSE_FFT_NAME(FFT_SCRAMBLE_DIRECT_NAME)(float *dst, const float *src, size_t rank)
37 {
38 // Scramble the order of samples
39 if (dst == src)
40 {
41 if (rank <= 8)
42 SSE_CALL_NAME(FFT_SCRAMBLE_SELF_DIRECT_NAME, 8);
43 else //if (rank <= 16)
44 SSE_CALL_NAME(FFT_SCRAMBLE_SELF_DIRECT_NAME, 16);
45 }
46 else
47 {
48 rank -= 3;
49
50 if (rank <= 8)
51 SSE_CALL_NAME(FFT_SCRAMBLE_COPY_DIRECT_NAME, 8);
52 else //if (rank <= 16)
53 SSE_CALL_NAME(FFT_SCRAMBLE_COPY_DIRECT_NAME, 16);
54 }
55 }
56
57 static inline void SSE_FFT_NAME(FFT_SCRAMBLE_REVERSE_NAME)(float *dst, const float *src, size_t rank)
58 {
59 // Scramble the order of samples
60 if (dst == src)
61 {
62 if (rank <= 8)
63 SSE_CALL_NAME(FFT_SCRAMBLE_SELF_REVERSE_NAME, 8);
64 else //if (rank <= 16)
65 SSE_CALL_NAME(FFT_SCRAMBLE_SELF_REVERSE_NAME, 16);
66 }
67 else
68 {
69 rank -= 3;
70
71 if (rank <= 8)
72 SSE_CALL_NAME(FFT_SCRAMBLE_COPY_REVERSE_NAME, 8);
73 else //if (rank <= 16)
74 SSE_CALL_NAME(FFT_SCRAMBLE_COPY_REVERSE_NAME, 16);
75 }
76 }
77
78 static inline void SSE_FFT_NAME(FFT_REPACK)(float *dst, size_t rank)
79 {
80 size_t blocks = 1 << (rank-3);
81
82 // Perform 4-element butterflies
83 ARCH_X86_ASM
84 (
85 __ASM_EMIT(".align 16")
86 __ASM_EMIT("1:")
87
88 /* Load data */
89 __ASM_EMIT("movups 0x00(%[dst]), %%xmm0") /* xmm0 = r0 r1 r2 r3 */
90 __ASM_EMIT("movups 0x10(%[dst]), %%xmm1") /* xmm1 = i0 i1 i2 i3 */
91 __ASM_EMIT("movups 0x20(%[dst]), %%xmm4") /* xmm4 = r4 r5 r6 r7 */
92 __ASM_EMIT("movups 0x30(%[dst]), %%xmm5") /* xmm5 = i4 i5 i6 i7 */
93
94 /* 1st step: repack pairs */
95 __ASM_EMIT("movaps %%xmm0, %%xmm2") /* xmm2 = r0 r1 r2 r3 */
96 __ASM_EMIT("movaps %%xmm4, %%xmm6") /* xmm6 = r4 r5 r6 r7 */
97 __ASM_EMIT("unpcklps %%xmm1, %%xmm0") /* xmm0 = r0 i0 r1 i1 */
98 __ASM_EMIT("unpcklps %%xmm5, %%xmm4") /* xmm4 = r4 i4 r5 i5 */
99 __ASM_EMIT("unpckhps %%xmm1, %%xmm2") /* xmm2 = r2 i2 r3 i3 */
100 __ASM_EMIT("unpckhps %%xmm5, %%xmm6") /* xmm6 = r6 i6 r7 i7 */
101
102 /* Store data */
103 __ASM_EMIT("movups %%xmm0, 0x00(%[dst])")
104 __ASM_EMIT("movups %%xmm2, 0x10(%[dst])")
105 __ASM_EMIT("movups %%xmm4, 0x20(%[dst])")
106 __ASM_EMIT("movups %%xmm6, 0x30(%[dst])")
107
108 /* Move pointers and repeat cycle */
109 __ASM_EMIT("add $0x40, %[dst]")
110 __ASM_EMIT("dec %[blocks]")
111 __ASM_EMIT("jnz 1b")
112
113 : [dst] "+r"(dst), [blocks] "+r"(blocks)
114 :
115 : "cc", "memory",
116 "%xmm0", "%xmm1", "%xmm2",
117 "%xmm4", "%xmm5", "%xmm6"
118 );
119 }
120
121 static inline void SSE_FFT_NAME(FFT_REPACK_NORMALIZE)(float *dst, size_t rank)
122 {
123 size_t blocks = 1 << (rank-3);
124 float k = 0.125f/blocks;
125
126 // Perform 4-element butterflies
127 ARCH_X86_ASM
128 (
129 __ASM_EMIT("shufps $0x00, %%xmm0, %%xmm0") /* xmm0 = k k k k */
130 __ASM_EMIT("movaps %%xmm0, %%xmm1") /* xmm1 = k k k k */
131 __ASM_EMIT(".align 16")
132 __ASM_EMIT("1:")
133
134 /* Load data */
135 __ASM_EMIT("movups 0x00(%[dst]), %%xmm2") /* xmm2 = r0 r1 r2 r3 */
136 __ASM_EMIT("movups 0x10(%[dst]), %%xmm3") /* xmm3 = i0 i1 i2 i3 */
137 __ASM_EMIT("movups 0x20(%[dst]), %%xmm5") /* xmm5 = r4 r5 r6 r7 */
138 __ASM_EMIT("movups 0x30(%[dst]), %%xmm6") /* xmm6 = i4 i5 i6 i7 */
139
140 /* 1st step: repack pairs */
141 __ASM_EMIT("movaps %%xmm2, %%xmm4") /* xmm4 = r0 r1 r2 r3 */
142 __ASM_EMIT("movaps %%xmm5, %%xmm7") /* xmm7 = r4 r5 r6 r7 */
143 __ASM_EMIT("unpcklps %%xmm3, %%xmm2") /* xmm2 = r0 i0 r1 i1 */
144 __ASM_EMIT("unpcklps %%xmm6, %%xmm5") /* xmm5 = r4 i4 r5 i5 */
145 __ASM_EMIT("unpckhps %%xmm3, %%xmm4") /* xmm4 = r2 i2 r3 i3 */
146 __ASM_EMIT("unpckhps %%xmm6, %%xmm7") /* xmm7 = r6 i6 r7 i7 */
147
148 /* 3rd step: apply normalization */
149 __ASM_EMIT("mulps %%xmm0, %%xmm2")
150 __ASM_EMIT("mulps %%xmm1, %%xmm4")
151 __ASM_EMIT("mulps %%xmm0, %%xmm5")
152 __ASM_EMIT("mulps %%xmm1, %%xmm7")
153
154 /* Store data */
155 __ASM_EMIT("movups %%xmm2, 0x00(%[dst])")
156 __ASM_EMIT("movups %%xmm4, 0x10(%[dst])")
157 __ASM_EMIT("movups %%xmm5, 0x20(%[dst])")
158 __ASM_EMIT("movups %%xmm7, 0x30(%[dst])")
159
160 /* Move pointers and repeat cycle */
161 __ASM_EMIT("add $0x40, %[dst]")
162 __ASM_EMIT("dec %[blocks]")
163 __ASM_EMIT("jnz 1b")
164
165 : [dst] "+r"(dst), [blocks] "+r"(blocks), "+Yz"(k)
166 :
167 : "cc", "memory",
168 "%xmm1", "%xmm2", "%xmm3",
169 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
170 );
171 }
172 }
173 }
174
175 #undef SSE_FFT_NAME
176 #undef SSE_CALL_NAME
177 #undef _SSE_FFT_NAME
178 #undef _SSE_CALL_NAME
179
180 #undef FFT_SCRAMBLE_SELF_DIRECT_NAME
181 #undef FFT_SCRAMBLE_COPY_DIRECT_NAME
182 #undef FFT_SCRAMBLE_SELF_REVERSE_NAME
183 #undef FFT_SCRAMBLE_COPY_REVERSE_NAME
184 #undef LS_RE
185
186 #undef FFT_SCRAMBLE_DIRECT_NAME
187 #undef FFT_SCRAMBLE_REVERSE_NAME
188 #undef FFT_REPACK
189 #undef FFT_REPACK_NORMALIZED
190 #undef FFT_MODE
191