Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/aarch64/asimd/fastconv/restore.h

raw
1 /*
2 * Copyright (C) 2020 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2020 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 31 мар. 2020 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_AARCH64_ASIMD_FASTCONV_RESTORE_H_
23 #define PRIVATE_DSP_ARCH_AARCH64_ASIMD_FASTCONV_RESTORE_H_
24
25 #ifndef PRIVATE_DSP_ARCH_AARCH64_ASIMD_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_AARCH64_ASIMD_IMPL */
28
29 namespace lsp
30 {
31 namespace asimd
32 {
33 void fastconv_restore(float *dst, float *tmp, size_t rank)
34 {
35 size_t items = 1 << rank; // number of complex numbers per half of block (16 for rank=4)
36 size_t n;
37
38 IF_ARCH_AARCH64(
39 float *a, *b;
40 size_t k, p;
41 );
42
43 ARCH_AARCH64_ASM(
44 __ASM_EMIT("mov %[a], %[tmp]")
45 __ASM_EMIT("subs %[n], %[items], #8")
46 __ASM_EMIT("ldp q16, q17, [%[MASK]]")
47 __ASM_EMIT("b.lo 2f")
48
49 __ASM_EMIT("1:")
50 __ASM_EMIT("ld4 {v0.4s-v3.4s}, [%[a]]")
51 // v0 = r0 i0 r4 i4
52 // v1 = r1 i1 r5 i5
53 // v2 = r2 i2 r6 i6
54 // v3 = r3 i3 r7 i7
55 __ASM_EMIT("fsub v6.4s, v0.4s, v1.4s") // v6 = r0-r1 i0-i1 r4-r5 i4-i5 = r1' i1' r5' i5'
56 __ASM_EMIT("fsub v7.4s, v2.4s, v3.4s") // v7 = r2-r3 i2-i3 r6-r7 i6-i7 = r3' i3' r7' i7'
57 __ASM_EMIT("fadd v4.4s, v0.4s, v1.4s") // v4 = r0+r1 i0+i1 r4+r5 i4+i5 = r0' i0' r4' i4'
58 __ASM_EMIT("fadd v5.4s, v2.4s, v3.4s") // v5 = r2+r3 i2+i3 r6+r7 i6+i7 = r2' i2' r6' i6'
59 __ASM_EMIT("rev64 v7.4s, v7.4s") // v7 = i3' r3' i7' r7'
60 // v4 = r0' i0' r4' i4'
61 // v5 = r2' i2' r6' i6'
62 // v6 = r1' i1' r5' i5'
63 // v7 = i3' r3' i7' r7'
64 __ASM_EMIT("fadd v0.4s, v4.4s, v5.4s") // v0 = r0'+r2' i0'+i2' r4'+r6' i4'+i6' = r0" i0" r4" i4"
65 __ASM_EMIT("fsub v1.4s, v6.4s, v7.4s") // v2 = r1'-i3' i1'-r3' r5'-i7' i5'-r7' = r1" i3" r5" i7"
66 __ASM_EMIT("fsub v2.4s, v4.4s, v5.4s") // v1 = r0'-r2' i0'-i2' r4'-r6' i4'-i6' = r2" i2" r6" i6"
67 __ASM_EMIT("fadd v3.4s, v6.4s, v7.4s") // v3 = r1'+i3' i1'+r3' r5'+i7' i5'+r7' = r3" i1" r7" i5"
68 __ASM_EMIT("mov v6.16b, v1.16b") // v6 = r1" i3" r5" i7"
69 __ASM_EMIT("mov v7.16b, v3.16b") // v7 = r3" i1" r7" i5"
70 __ASM_EMIT("bit v1.16b, v7.16b, v16.16b") // v1 = r1" i1" r5" i5"
71 __ASM_EMIT("bit v3.16b, v6.16b, v17.16b") // v3 = r3" i3" r7" i7"
72 __ASM_EMIT("st4 {v0.4s-v3.4s}, [%[a]]")
73 __ASM_EMIT("subs %[n], %[n], #8") // n -= 8
74 __ASM_EMIT("add %[a], %[a], #0x40")
75 __ASM_EMIT("b.hs 1b")
76 __ASM_EMIT("2:")
77
78 : [a] "=&r" (a), [n] "=&r" (n)
79 : [tmp] "r" (tmp), [items] "r" (items),
80 [MASK] "r" (&fastconv_swp_mask[0])
81 : "cc", "memory",
82 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
83 "v16", "v17"
84 );
85
86 n = 8;
87 const float *xfft_a = XFFT_A;
88
89 // Perform butterflies without need of vector rotation
90 if (n < items)
91 {
92 ARCH_AARCH64_ASM(
93 __ASM_EMIT("mov %[a], %[tmp]")
94 __ASM_EMIT("subs %[k], %[items], #16")
95 __ASM_EMIT("ldp q16, q17, [%[XFFT_A], #0x00]") // v16 = wr1, v17 = wr2
96 __ASM_EMIT("ldp q18, q19, [%[XFFT_A], #0x20]") // v18 = wi1, v19 = wi2
97 __ASM_EMIT("b.lo 2f")
98 // 8x butterflies
99 // Calculate complex c = w * b
100 __ASM_EMIT("1:")
101 // Calc cr and ci
102 __ASM_EMIT("ldp q0, q1, [%[a], #0x00]") // v0 = ar1, v1 = ai1
103 __ASM_EMIT("ldp q2, q3, [%[a], #0x20]") // v2 = ar2, v3 = ai2
104 __ASM_EMIT("ldp q4, q5, [%[a], #0x40]") // v4 = ar3, v5 = ai3
105 __ASM_EMIT("ldp q6, q7, [%[a], #0x60]") // v6 = ar4, v7 = ai4
106 __ASM_EMIT("fmul v20.4s, v16.4s, v2.4s") // v20 = wr1 * br1
107 __ASM_EMIT("fmul v21.4s, v17.4s, v6.4s") // v21 = wr2 * br2
108 __ASM_EMIT("fmul v22.4s, v16.4s, v3.4s") // v22 = wr1 * bi1
109 __ASM_EMIT("fmul v23.4s, v17.4s, v7.4s") // v23 = wr2 * bi2
110 __ASM_EMIT("fmls v20.4s, v18.4s, v3.4s") // v20 = wr1 * br1 - wi1 * bi1 = cr1
111 __ASM_EMIT("fmls v21.4s, v19.4s, v7.4s") // v21 = wr2 * br2 - wi2 * bi2 = cr2
112 __ASM_EMIT("fmla v22.4s, v18.4s, v2.4s") // v22 = wr1 * bi1 + wi1 * br1 = ci1
113 __ASM_EMIT("fmla v23.4s, v19.4s, v6.4s") // v23 = wr1 * bi1 + wi2 * br2 = ci2
114 // Apply butterfly
115 __ASM_EMIT("fsub v2.4s, v0.4s, v20.4s") // v2 = ar1 - cr1
116 __ASM_EMIT("fsub v3.4s, v1.4s, v22.4s") // v3 = ai1 - ci1
117 __ASM_EMIT("fsub v6.4s, v4.4s, v21.4s") // v6 = ar2 - cr2
118 __ASM_EMIT("fsub v7.4s, v5.4s, v23.4s") // v7 = ai2 - ci2
119 __ASM_EMIT("fadd v0.4s, v0.4s, v20.4s") // v0 = ar1 + cr1
120 __ASM_EMIT("fadd v1.4s, v1.4s, v22.4s") // v1 = ai1 + ci1
121 __ASM_EMIT("fadd v4.4s, v4.4s, v21.4s") // v4 = ar2 + cr2
122 __ASM_EMIT("fadd v5.4s, v5.4s, v23.4s") // v5 = ai2 + ci2
123 __ASM_EMIT("stp q0, q1, [%[a], #0x00]")
124 __ASM_EMIT("stp q2, q3, [%[a], #0x20]")
125 __ASM_EMIT("stp q4, q5, [%[a], #0x40]")
126 __ASM_EMIT("stp q6, q7, [%[a], #0x60]")
127 __ASM_EMIT("subs %[k], %[k], #16")
128 __ASM_EMIT("add %[a], %[a], #0x80")
129 __ASM_EMIT("b.ge 1b")
130
131 __ASM_EMIT("2:")
132 __ASM_EMIT("adds %[k], %[k], #8")
133 __ASM_EMIT("b.lo 4f")
134 // 4x butterflies
135 __ASM_EMIT("ldp q0, q1, [%[a], #0x00]") // v0 = ar1, v1 = ai1
136 __ASM_EMIT("ldp q2, q3, [%[a], #0x20]") // v2 = ar2, v3 = ai2
137 __ASM_EMIT("fmul v20.4s, v16.4s, v2.4s") // v20 = wr1 * br1
138 __ASM_EMIT("fmul v22.4s, v16.4s, v3.4s") // v22 = wr1 * bi1
139 __ASM_EMIT("fmls v20.4s, v18.4s, v3.4s") // v20 = wr1 * br1 - wi1 * bi1 = cr1
140 __ASM_EMIT("fmla v22.4s, v18.4s, v2.4s") // v22 = wr1 * bi1 + wi1 * br1 = ci1
141 // Apply butterfly
142 __ASM_EMIT("fsub v2.4s, v0.4s, v20.4s") // v2 = ar1 - cr1
143 __ASM_EMIT("fsub v3.4s, v1.4s, v22.4s") // v3 = ai1 - ci1
144 __ASM_EMIT("fadd v0.4s, v0.4s, v20.4s") // v0 = ar1 + cr1
145 __ASM_EMIT("fadd v1.4s, v1.4s, v22.4s") // v1 = ai1 + ci1
146 __ASM_EMIT("stp q0, q1, [%[a], #0x00]")
147 __ASM_EMIT("stp q2, q3, [%[a], #0x20]")
148 __ASM_EMIT("4:")
149 __ASM_EMIT("add %[XFFT_A], %[XFFT_A], #0x40")
150
151 : [a] "=&r" (a), [k] "=&r" (k),
152 [XFFT_A] "+r" (xfft_a)
153 : [tmp] "r" (tmp), [items] "r" (items)
154 : "cc", "memory",
155 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
156 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"
157 );
158
159 n <<= 1;
160 }
161
162 const float *xfft_dw = &XFFT_DW[8];
163
164 // Perform loops except last one
165 while (n < items)
166 {
167 ARCH_AARCH64_ASM(
168 __ASM_EMIT("mov %[k], %[items]")
169 __ASM_EMIT("mov %[a], %[tmp]")
170 __ASM_EMIT("1:")
171 // Initialize sub-loop
172 __ASM_EMIT("mov %[p], %[n]")
173 __ASM_EMIT("ldp q16, q17, [%[XFFT_A], #0x00]") // v16 = wr1, v17 = wr2
174 __ASM_EMIT("ldp q18, q19, [%[XFFT_A], #0x20]") // v18 = wi1, v19 = wi2
175 __ASM_EMIT("ldp q24, q25, [%[XFFT_DW]]") // v24 = dr, v25 = di
176 __ASM_EMIT("add %[b], %[a], %[n], LSL #2") // b = &a[n*4]
177 // 8x butterflies
178 __ASM_EMIT("3:")
179 // Calc cr and ci
180 __ASM_EMIT("ldp q0, q1, [%[a], #0x00]") // v0 = ar1, v1 = ai1
181 __ASM_EMIT("ldp q2, q3, [%[a], #0x20]") // v2 = ar2, v3 = ai2
182 __ASM_EMIT("ldp q4, q5, [%[b], #0x00]") // v4 = br1, v5 = bi1
183 __ASM_EMIT("ldp q6, q7, [%[b], #0x20]") // v6 = br2, v7 = bi2
184 __ASM_EMIT("fmul v20.4s, v16.4s, v4.4s") // v20 = wr1 * br1
185 __ASM_EMIT("fmul v21.4s, v17.4s, v6.4s") // v21 = wr2 * br2
186 __ASM_EMIT("fmul v22.4s, v16.4s, v5.4s") // v22 = wr1 * bi1
187 __ASM_EMIT("fmul v23.4s, v17.4s, v7.4s") // v23 = wr2 * bi2
188 __ASM_EMIT("fmls v20.4s, v18.4s, v5.4s") // v20 = wr1 * br1 - wi1 * bi1 = cr1
189 __ASM_EMIT("fmls v21.4s, v19.4s, v7.4s") // v21 = wr2 * br2 - wi2 * bi2 = cr2
190 __ASM_EMIT("fmla v22.4s, v18.4s, v4.4s") // v22 = wr1 * bi1 + wi1 * br1 = ci1
191 __ASM_EMIT("fmla v23.4s, v19.4s, v6.4s") // v23 = wr1 * bi1 + wi2 * br2 = ci2
192 // Apply butterfly
193 __ASM_EMIT("fsub v4.4s, v0.4s, v20.4s") // v4 = ar1 - cr1
194 __ASM_EMIT("fsub v5.4s, v1.4s, v22.4s") // v5 = ai1 - ci1
195 __ASM_EMIT("fsub v6.4s, v2.4s, v21.4s") // v6 = ar2 - cr2
196 __ASM_EMIT("fsub v7.4s, v3.4s, v23.4s") // v7 = ai2 - ci2
197 __ASM_EMIT("fadd v0.4s, v0.4s, v20.4s") // v0 = ar1 + cr1
198 __ASM_EMIT("fadd v1.4s, v1.4s, v22.4s") // v1 = ai1 + ci1
199 __ASM_EMIT("fadd v2.4s, v2.4s, v21.4s") // v2 = ar2 + cr2
200 __ASM_EMIT("fadd v3.4s, v3.4s, v23.4s") // v3 = ai2 + ci2
201 __ASM_EMIT("stp q0, q1, [%[a], #0x00]")
202 __ASM_EMIT("stp q2, q3, [%[a], #0x20]")
203 __ASM_EMIT("stp q4, q5, [%[b], #0x00]")
204 __ASM_EMIT("stp q6, q7, [%[b], #0x20]")
205 __ASM_EMIT("subs %[p], %[p], #16")
206 __ASM_EMIT("add %[a], %[a], #0x40")
207 __ASM_EMIT("add %[b], %[b], #0x40")
208 __ASM_EMIT("b.eq 4f")
209 // Prepare next loop: rotate angle
210 __ASM_EMIT("fmul v20.4s, v16.4s, v25.4s") // v20 = wr1 * di
211 __ASM_EMIT("fmul v21.4s, v17.4s, v25.4s") // v21 = wr2 * di
212 __ASM_EMIT("fmul v22.4s, v18.4s, v25.4s") // v22 = wi1 * di
213 __ASM_EMIT("fmul v23.4s, v19.4s, v25.4s") // v23 = wi2 * di
214 __ASM_EMIT("fmul v16.4s, v16.4s, v24.4s") // v16 = wr1 * dr
215 __ASM_EMIT("fmul v17.4s, v17.4s, v24.4s") // v17 = wr2 * dr
216 __ASM_EMIT("fmul v18.4s, v18.4s, v24.4s") // v18 = wi1 * dr
217 __ASM_EMIT("fmul v19.4s, v19.4s, v24.4s") // v19 = wi2 * dr
218 __ASM_EMIT("fsub v16.4s, v16.4s, v22.4s") // v16 = wr1*dr - wi1*di
219 __ASM_EMIT("fsub v17.4s, v17.4s, v23.4s") // v17 = wr2*dr - wi2*di
220 __ASM_EMIT("fadd v18.4s, v18.4s, v20.4s") // v18 = wi1*dr + wr1*di
221 __ASM_EMIT("fadd v19.4s, v19.4s, v21.4s") // v19 = wi2*dr + wr2*di
222 __ASM_EMIT("b 3b")
223 __ASM_EMIT("4:")
224 __ASM_EMIT("subs %[k], %[k], %[n]")
225 __ASM_EMIT("mov %[a], %[b]")
226 __ASM_EMIT("b.ne 1b")
227
228 : [a] "=&r" (a), [b] "=&r" (b),
229 [p] "=&r" (p), [k] "=&r" (k)
230 : [tmp] "r" (tmp), [items] "r" (items), [n] "r" (n),
231 [XFFT_A] "r" (xfft_a), [XFFT_DW] "r" (xfft_dw)
232 : "cc", "memory",
233 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
234 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23",
235 "v24", "v25"
236 );
237
238 n <<= 1;
239 xfft_a += 16;
240 xfft_dw += 8;
241 }
242
243 // Last one large loop
244 IF_ARCH_AARCH64(float fftk = 1.0f / items);
245
246 ARCH_AARCH64_ASM(
247 // Initialize sub-loop
248 __ASM_EMIT("add %[b], %[a], %[n], LSL #2") // b = &a[n*4]
249 __ASM_EMIT("ld1r {v23.4s}, [%[fftk]]") // v23 = 1 / items
250 __ASM_EMIT("add %[db], %[da], %[n], LSL #1") // db = &da[n*2]
251 __ASM_EMIT("ldp q16, q17, [%[XFFT_A], #0x00]") // v16 = wr1, v17 = wr2
252 __ASM_EMIT("ldp q18, q19, [%[XFFT_A], #0x20]") // v18 = wi1, v19 = wi2
253 __ASM_EMIT("ldp q24, q25, [%[XFFT_DW]]") // v24 = dr, v25 = di
254 __ASM_EMIT("mov v22.16b, v23.16b") // v22 = 1 / items
255 // 8x butterflies
256 __ASM_EMIT("1:")
257 __ASM_EMIT("ldp q0, q1, [%[a], #0x00]") // v0 = ar1, v1 = ai1
258 __ASM_EMIT("ldp q2, q3, [%[a], #0x20]") // v2 = ar2, v3 = ai2
259 __ASM_EMIT("ldp q4, q5, [%[b], #0x00]") // v4 = br1, v5 = bi1
260 __ASM_EMIT("ldp q6, q7, [%[b], #0x20]") // v6 = br2, v7 = bi2
261 __ASM_EMIT("fmul v20.4s, v16.4s, v4.4s") // v20 = wr1 * br1
262 __ASM_EMIT("fmul v21.4s, v17.4s, v6.4s") // v21 = wr2 * br2
263 __ASM_EMIT("fmls v20.4s, v18.4s, v5.4s") // v20 = wr1 * br1 - wi1 * bi1 = cr1
264 __ASM_EMIT("fmls v21.4s, v19.4s, v7.4s") // v21 = wr2 * br2 - wi2 * bi2 = cr2
265 // Apply butterfly, we do not need imaginary part
266 __ASM_EMIT("fadd v4.4s, v0.4s, v20.4s") // v4 = ar1 + cr1
267 __ASM_EMIT("fadd v5.4s, v2.4s, v21.4s") // v5 = ar2 + cr2
268 __ASM_EMIT("fsub v6.4s, v0.4s, v20.4s") // v6 = ar1 - cr1
269 __ASM_EMIT("fsub v7.4s, v2.4s, v21.4s") // v7 = ar2 - cr2
270 __ASM_EMIT("fmul v4.4s, v4.4s, v22.4s") // v4 = (ar1 + cr1) / items
271 __ASM_EMIT("fmul v5.4s, v5.4s, v23.4s") // v5 = (ar2 + cr2) / items
272 __ASM_EMIT("fmul v6.4s, v6.4s, v22.4s") // v6 = (ar1 - cr1) / items
273 __ASM_EMIT("fmul v7.4s, v7.4s, v23.4s") // v7 = (ar2 - cr2) / items
274 __ASM_EMIT("stp q4, q5, [%[da]]")
275 __ASM_EMIT("stp q6, q7, [%[db]]")
276 __ASM_EMIT("subs %[items], %[items], #16")
277 __ASM_EMIT("add %[a], %[a], #0x40")
278 __ASM_EMIT("add %[b], %[b], #0x40")
279 __ASM_EMIT("add %[da], %[da], #0x20")
280 __ASM_EMIT("add %[db], %[db], #0x20")
281 __ASM_EMIT("b.ls 2f")
282 // Prepare next loop: rotate angle
283 __ASM_EMIT("fmul v2.4s, v16.4s, v25.4s") // v2 = wr1 * di
284 __ASM_EMIT("fmul v3.4s, v17.4s, v25.4s") // v3 = wr2 * di
285 __ASM_EMIT("fmul v4.4s, v18.4s, v25.4s") // v4 = wi1 * di
286 __ASM_EMIT("fmul v5.4s, v19.4s, v25.4s") // v5 = wi2 * di
287 __ASM_EMIT("fmul v16.4s, v16.4s, v24.4s") // v16 = wr1 * dr
288 __ASM_EMIT("fmul v17.4s, v17.4s, v24.4s") // v17 = wr2 * dr
289 __ASM_EMIT("fmul v18.4s, v18.4s, v24.4s") // v18 = wi1 * dr
290 __ASM_EMIT("fmul v19.4s, v19.4s, v24.4s") // v19 = wi2 * dr
291 __ASM_EMIT("fsub v16.4s, v16.4s, v4.4s") // v16 = wr1*dr - wi1*di
292 __ASM_EMIT("fsub v17.4s, v17.4s, v5.4s") // v17 = wr2*dr - wi2*di
293 __ASM_EMIT("fadd v18.4s, v18.4s, v2.4s") // v18 = wi1*dr + wr1*di
294 __ASM_EMIT("fadd v19.4s, v19.4s, v3.4s") // v19 = wi2*dr + wr2*di
295 __ASM_EMIT("b 1b")
296 __ASM_EMIT("2:")
297
298 : [da] "+r" (dst), [db] "=&r" (a),
299 [a] "+r" (tmp), [b] "=&r" (b),
300 [items] "+r" (items)
301 : [n] "r" (n), [fftk] "r" (&fftk),
302 [XFFT_A] "r" (xfft_a), [XFFT_DW] "r" (xfft_dw)
303 : "cc", "memory",
304 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
305 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23",
306 "v24", "v25"
307 );
308 }
309 }
310 }
311
312 #endif /* PRIVATE_DSP_ARCH_AARCH64_ASIMD_FASTCONV_RESTORE_H_ */