Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/x86/sse/fastconv/irestore.h

raw
1 /*
2 * Copyright (C) 2020 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2020 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 31 мар. 2020 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_X86_SSE_FASTCONV_IRESTORE_H_
23 #define PRIVATE_DSP_ARCH_X86_SSE_FASTCONV_IRESTORE_H_
24
25 #ifndef PRIVATE_DSP_ARCH_X86_SSE_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_X86_SSE_IMPL */
28
29 namespace lsp
30 {
31 namespace sse
32 {
33 static inline void fastconv_restore_internal(float *dst, float *tmp, size_t rank)
34 {
35 // Prepare for butterflies
36 size_t last = size_t(1) << rank;
37 size_t items = last << 1;
38 size_t n = 8;
39 size_t bs = n << 1;
40
41 const float *wk = XFFT_W;
42 const float *ak = XFFT_A;
43
44 // Iterate butterflies
45 while (n < last)
46 {
47 for (size_t p=0; p<items; p += bs)
48 {
49 // Set initial values of pointers
50 float *a = &tmp[p];
51 float *b = &a[n];
52 size_t k = n;
53
54 ARCH_X86_ASM
55 (
56 __ASM_EMIT("movaps 0x00(%[ak]), %%xmm6") /* xmm6 = rak[i] */
57 __ASM_EMIT("movaps 0x10(%[ak]), %%xmm7") /* xmm7 = iak[i] */
58 :
59 : [ak] "r"(ak)
60 : "%xmm6", "%xmm7"
61 );
62
63 ARCH_X86_ASM
64 (
65 // __ASM_EMIT(".align 16")
66 __ASM_EMIT("1:")
67
68 __ASM_EMIT("movups 0x00(%[a]), %%xmm0") /* xmm0 = ra[i] */
69 __ASM_EMIT("movups 0x10(%[a]), %%xmm1") /* xmm1 = ia[i] */
70 __ASM_EMIT("movups 0x00(%[b]), %%xmm2") /* xmm2 = rb[i] */
71 __ASM_EMIT("movups 0x10(%[b]), %%xmm3") /* xmm3 = ib[i] */
72
73 __ASM_EMIT("movaps %%xmm2, %%xmm4") /* xmm4 = rb[i] */
74 __ASM_EMIT("movaps %%xmm3, %%xmm5") /* xmm5 = ib[i] */
75 __ASM_EMIT("mulps %%xmm6, %%xmm2") /* xmm2 = rak[i]*rb[i] */
76 __ASM_EMIT("mulps %%xmm7, %%xmm4") /* xmm4 = iak[i]*rb[i] */
77 __ASM_EMIT("mulps %%xmm6, %%xmm3") /* xmm3 = rak[i]*ib[i] */
78 __ASM_EMIT("mulps %%xmm7, %%xmm5") /* xmm5 = iak[i]*ib[i] */
79 __ASM_EMIT("addps %%xmm4, %%xmm3") /* xmm3 = rak[i]*ib[i] + iak[i]*rb[i] = ic[i] */
80 __ASM_EMIT("subps %%xmm5, %%xmm2") /* xmm2 = rak[i]*rb[i] - iak[i]*ib[i] = rc[i] */
81 __ASM_EMIT("movaps %%xmm0, %%xmm4") /* xmm4 = ra[i] */
82 __ASM_EMIT("movaps %%xmm1, %%xmm5") /* xmm5 = ia[i] */
83 __ASM_EMIT("subps %%xmm2, %%xmm0") /* xmm0 = ra[i] - rc[i] */
84 __ASM_EMIT("subps %%xmm3, %%xmm1") /* xmm1 = ia[i] - ic[i] */
85 __ASM_EMIT("addps %%xmm4, %%xmm2") /* xmm2 = ra[i] + rc[i] */
86 __ASM_EMIT("addps %%xmm5, %%xmm3") /* xmm3 = ia[i] + ic[i] */
87
88 __ASM_EMIT("movups %%xmm2, 0x00(%[a])")
89 __ASM_EMIT("movups %%xmm3, 0x10(%[a])")
90 __ASM_EMIT("movups %%xmm0, 0x00(%[b])")
91 __ASM_EMIT("movups %%xmm1, 0x10(%[b])")
92
93 __ASM_EMIT("add $0x20, %[a]")
94 __ASM_EMIT("add $0x20, %[b]")
95 __ASM_EMIT("sub $8, %[k]")
96 __ASM_EMIT("jz 2f")
97
98 /* Rotate angle */
99 __ASM_EMIT("movaps 0x00(%[wk]), %%xmm0") /* xmm0 = rw */
100 __ASM_EMIT("movaps 0x10(%[wk]), %%xmm1") /* xmm1 = iw */
101 __ASM_EMIT("movaps %%xmm0, %%xmm2") /* xmm2 = rw */
102 __ASM_EMIT("movaps %%xmm1, %%xmm3") /* xmm3 = iw */
103 __ASM_EMIT("mulps %%xmm6, %%xmm3") /* xmm3 = ra * iw */
104 __ASM_EMIT("mulps %%xmm7, %%xmm1") /* xmm1 = ia * iw */
105 __ASM_EMIT("mulps %%xmm0, %%xmm6") /* xmm6 = ra * rw */
106 __ASM_EMIT("mulps %%xmm2, %%xmm7") /* xmm7 = ia * rw */
107 __ASM_EMIT("subps %%xmm1, %%xmm6") /* xmm6 = ra * rw - ia * iw */
108 __ASM_EMIT("addps %%xmm3, %%xmm7") /* xmm7 = ra * iw + ia * rw */
109 __ASM_EMIT("jmp 1b")
110
111 __ASM_EMIT("2:")
112
113 : [a] "+r"(a), [b] "+r"(b), [k] "+r" (k)
114 : [wk] "r"(wk)
115 : "cc", "memory",
116 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
117 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
118 );
119 }
120
121 wk += 8;
122 ak += 8;
123 n <<= 1;
124 bs <<= 1;
125 }
126
127 if (n < items)
128 {
129 // ONE LARGE CYCLE
130 // Set initial values of pointers
131 float kn = 1.0f / last;
132 size_t k = n;
133
134 ARCH_X86_ASM
135 (
136 __ASM_EMIT("movaps 0x00(%[ak]), %%xmm6") /* xmm6 = rak[i] */
137 __ASM_EMIT("movaps 0x10(%[ak]), %%xmm7") /* xmm7 = iak[i] */
138 __ASM_EMIT("shufps $0x00, %%xmm0, %%xmm0") /* xmm0 = kn */
139 __ASM_EMIT("movaps %%xmm0, %%xmm1") /* xmm1 = kn */
140 : "+Yz" (kn)
141 : [ak] "r" (ak)
142 : "%xmm1", "%xmm6", "%xmm7"
143 );
144
145 ARCH_X86_ASM
146 (
147 // __ASM_EMIT(".align 16")
148 __ASM_EMIT("1:")
149
150 __ASM_EMIT("movups 0x00(%[tmp]), %%xmm4") /* xmm4 = ra[i] */
151 __ASM_EMIT("movups 0x00(%[tmp],%[n],4), %%xmm2") /* xmm2 = rb[i] */
152 __ASM_EMIT("movups 0x10(%[tmp],%[n],4), %%xmm3") /* xmm3 = ib[i] */
153
154 __ASM_EMIT("mulps %%xmm6, %%xmm2") /* xmm2 = rak[i]*rb[i] */
155 __ASM_EMIT("mulps %%xmm7, %%xmm3") /* xmm3 = iak[i]*ib[i] */
156 __ASM_EMIT("movaps %%xmm4, %%xmm5") /* xmm5 = ra[i] */
157 __ASM_EMIT("subps %%xmm3, %%xmm2") /* xmm2 = rak[i]*rb[i] - iak[i]*ib[i] = rc[i] */
158 __ASM_EMIT("addps %%xmm2, %%xmm4") /* xmm4 = ra[i] + rc[i] */
159 __ASM_EMIT("subps %%xmm2, %%xmm5") /* xmm5 = ra[i] - rc[i] */
160 __ASM_EMIT("mulps %%xmm0, %%xmm4") /* xmm4 = kn*(ra[i] + rc[i]) */
161 __ASM_EMIT("mulps %%xmm1, %%xmm5") /* xmm5 = kn*(ra[i] - rc[i]) */
162
163 __ASM_EMIT("movups 0x00(%[dst]), %%xmm2")
164 __ASM_EMIT("movups 0x00(%[dst], %[n], 2), %%xmm3")
165 __ASM_EMIT("addps %%xmm4, %%xmm2")
166 __ASM_EMIT("addps %%xmm5, %%xmm3")
167 __ASM_EMIT("movups %%xmm2, 0x00(%[dst])")
168 __ASM_EMIT("movups %%xmm3, 0x00(%[dst],%[n],2)")
169
170 __ASM_EMIT("add $0x20, %[tmp]")
171 __ASM_EMIT("add $0x10, %[dst]")
172 __ASM_EMIT("sub $8, %[k]")
173 __ASM_EMIT("jz 2f")
174
175 /* Rotate angle */
176 __ASM_EMIT("movaps 0x00(%[wk]), %%xmm4") /* xmm4 = rw */
177 __ASM_EMIT("movaps 0x10(%[wk]), %%xmm5") /* xmm5 = iw */
178 __ASM_EMIT("movaps %%xmm4, %%xmm2") /* xmm2 = rw */
179 __ASM_EMIT("movaps %%xmm5, %%xmm3") /* xmm3 = iw */
180 __ASM_EMIT("mulps %%xmm6, %%xmm3") /* xmm3 = ra * iw */
181 __ASM_EMIT("mulps %%xmm7, %%xmm5") /* xmm5 = ia * iw */
182 __ASM_EMIT("mulps %%xmm4, %%xmm6") /* xmm6 = ra * rw */
183 __ASM_EMIT("mulps %%xmm2, %%xmm7") /* xmm7 = ia * rw */
184 __ASM_EMIT("subps %%xmm5, %%xmm6") /* xmm6 = ra * rw - ia * iw */
185 __ASM_EMIT("addps %%xmm3, %%xmm7") /* xmm7 = ra * iw + ia * rw */
186 __ASM_EMIT("jmp 1b")
187
188 __ASM_EMIT("2:")
189
190 : [tmp] "+r"(tmp), [dst] "+r"(dst), [n] "+r"(n), [k] "+r" (k)
191 : [wk] "r"(wk)
192 : "cc", "memory",
193 "%xmm0", "%xmm5", "%xmm2", "%xmm3",
194 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
195 );
196 }
197 else
198 {
199 // Add real result to the target (ignore complex result)
200 float kn = 1.0f / last;
201
202 // Unpack 4x split complex to 4x real
203 ARCH_X86_ASM
204 (
205 __ASM_EMIT("shufps $0x00, %%xmm0, %%xmm0") /* xmm0 = kn */
206 __ASM_EMIT("movups 0x00(%[tmp]), %%xmm1") /* xmm0 = s[i] */
207 __ASM_EMIT("movups 0x00(%[dst]), %%xmm2")
208 __ASM_EMIT("mulps %%xmm0, %%xmm1")
209 __ASM_EMIT("addps %%xmm1, %%xmm2")
210 __ASM_EMIT("movups %%xmm2, 0x00(%[dst])")
211 : "+Yz" (kn)
212 : [tmp] "r" (tmp), [dst] "r" (dst)
213 : "cc", "memory",
214 "%xmm1", "%xmm2"
215 );
216 }
217 }
218 }
219 }
220
221 #endif /* PRIVATE_DSP_ARCH_X86_SSE_FASTCONV_IRESTORE_H_ */