Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/aarch64/asimd/fft/pbutterfly.h

raw
1 /*
2 * Copyright (C) 2020 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2020 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 31 мар. 2020 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_AARCH64_ASIMD_FFT_PBUTTERFLY_H_
23 #define PRIVATE_DSP_ARCH_AARCH64_ASIMD_FFT_PBUTTERFLY_H_
24
25 #ifndef PRIVATE_DSP_ARCH_AARCH64_ASIMD_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_AARCH64_ASIMD_IMPL */
28
29 namespace lsp
30 {
31 namespace asimd
32 {
33 #define PBUTTERFLY_RANK3(op1, op2) \
34 /* 8x butterflies */ \
35 __ASM_EMIT("ldp q28, q29, [%[XFFT_A], #0x00]") /* q28 = wr1, q29 = wr2 */ \
36 __ASM_EMIT("ldp q30, q31, [%[XFFT_A], #0x20]") /* q30 = wi1, q31 = wi2 */ \
37 __ASM_EMIT("subs %[blocks], %[blocks], #2") \
38 __ASM_EMIT("b.lo 2f") \
39 /* Calculate complex c = w * b */ \
40 __ASM_EMIT("1:") \
41 __ASM_EMIT("ldp q0, q4, [%[dst], #0x00]") /* v0 = ar1, v4 = ai1 */ \
42 __ASM_EMIT("ldp q1, q5, [%[dst], #0x20]") /* v1 = br1, v5 = bi1 */ \
43 __ASM_EMIT("ldp q2, q6, [%[dst], #0x40]") /* v2 = ai2, v6 = ai2 */ \
44 __ASM_EMIT("ldp q3, q7, [%[dst], #0x60]") /* v3 = bi2, v7 = bi2 */ \
45 /* Calc cr and ci */ \
46 __ASM_EMIT("fmul v16.4s, v28.4s, v1.4s") /* v16 = wr1 * br1 */ \
47 __ASM_EMIT("fmul v17.4s, v29.4s, v3.4s") /* v17 = wr2 * br2 */ \
48 __ASM_EMIT("fmul v18.4s, v28.4s, v5.4s") /* v18 = wr1 * bi1 */ \
49 __ASM_EMIT("fmul v19.4s, v29.4s, v7.4s") /* v19 = wr2 * bi2 */ \
50 __ASM_EMIT(op1 " v16.4s, v30.4s, v5.4s") /* v16 = wr1 * br1 +- wi1 * bi1 = cr1 */ \
51 __ASM_EMIT(op1 " v17.4s, v31.4s, v7.4s") /* v17 = wr2 * br2 +- wi2 * bi2 = cr2 */ \
52 __ASM_EMIT(op2 " v18.4s, v30.4s, v1.4s") /* v18 = wr1 * bi1 -+ wi1 * br1 = ci1 */ \
53 __ASM_EMIT(op2 " v19.4s, v31.4s, v3.4s") /* v19 = wr1 * bi1 -+ wi2 * br2 = ci2 */ \
54 /* Apply butterfly */ \
55 __ASM_EMIT("fsub v1.4s, v0.4s, v16.4s") /* v1 = ar1 - cr1 */ \
56 __ASM_EMIT("fsub v3.4s, v2.4s, v17.4s") /* v3 = ar2 - cr2 */ \
57 __ASM_EMIT("fsub v5.4s, v4.4s, v18.4s") /* v5 = ai1 - ci1 */ \
58 __ASM_EMIT("fsub v7.4s, v6.4s, v19.4s") /* v7 = ai2 - ci2 */ \
59 __ASM_EMIT("fadd v0.4s, v0.4s, v16.4s") /* v0 = ar1 + cr1 */ \
60 __ASM_EMIT("fadd v2.4s, v2.4s, v17.4s") /* v2 = ar2 + cr2 */ \
61 __ASM_EMIT("fadd v4.4s, v4.4s, v18.4s") /* v4 = ai1 + ci1 */ \
62 __ASM_EMIT("fadd v6.4s, v6.4s, v19.4s") /* v6 = ai2 + ci2 */ \
63 __ASM_EMIT("stp q0, q4, [%[dst], #0x00]") \
64 __ASM_EMIT("stp q1, q5, [%[dst], #0x20]") \
65 __ASM_EMIT("stp q2, q6, [%[dst], #0x40]") \
66 __ASM_EMIT("stp q3, q7, [%[dst], #0x60]") \
67 __ASM_EMIT("subs %[blocks], %[blocks], #2") \
68 __ASM_EMIT("add %[dst], %[dst], #0x80") \
69 __ASM_EMIT("b.ge 1b") \
70 __ASM_EMIT("2:") \
71 /* 4x butterflies */ \
72 __ASM_EMIT("adds %[blocks], %[blocks], #1") \
73 __ASM_EMIT("b.lo 4f") \
74 __ASM_EMIT("ldp q0, q4, [%[dst], #0x00]") /* v0 = ar1, v4 = ai1 */ \
75 __ASM_EMIT("ldp q1, q5, [%[dst], #0x20]") /* v1 = br1, v5 = bi1 */ \
76 /* Calc cr and ci */ \
77 __ASM_EMIT("fmul v16.4s, v28.4s, v1.4s") /* v16 = wr1 * br1 */ \
78 __ASM_EMIT("fmul v18.4s, v28.4s, v5.4s") /* v18 = wr1 * bi1 */ \
79 __ASM_EMIT(op1 " v16.4s, v30.4s, v5.4s") /* v16 = wr1 * br1 +- wi1 * bi1 = cr1 */ \
80 __ASM_EMIT(op2 " v18.4s, v30.4s, v1.4s") /* v18 = wr1 * bi1 -+ wi1 * br1 = ci1 */ \
81 /* Apply butterfly */ \
82 __ASM_EMIT("fsub v1.4s, v0.4s, v16.4s") /* v1 = ar1 - cr1 */ \
83 __ASM_EMIT("fsub v5.4s, v4.4s, v18.4s") /* v5 = ai1 - ci1 */ \
84 __ASM_EMIT("fadd v0.4s, v0.4s, v16.4s") /* v0 = ar1 + cr1 */ \
85 __ASM_EMIT("fadd v4.4s, v4.4s, v18.4s") /* v4 = ai1 + ci1 */ \
86 __ASM_EMIT("stp q0, q4, [%[dst], #0x00]") \
87 __ASM_EMIT("stp q1, q5, [%[dst], #0x20]") \
88 __ASM_EMIT("4:")
89
90 void packed_direct_butterfly_rank3(float *dst, size_t blocks)
91 {
92 ARCH_AARCH64_ASM(
93 PBUTTERFLY_RANK3("fmla", "fmls")
94 : [dst] "+r" (dst), [blocks] "+r" (blocks)
95 : [XFFT_A] "r" (&XFFT_A[0])
96 : "cc", "memory",
97 "v0", "v1", "v2", "v3",
98 "v4", "v5", "v6", "v7",
99 "v16", "v17", "v18", "v19",
100 "v28", "v29", "v30", "v31"
101 );
102 }
103
104 void packed_reverse_butterfly_rank3(float *dst, size_t blocks)
105 {
106 ARCH_AARCH64_ASM(
107 PBUTTERFLY_RANK3("fmls", "fmla")
108 : [dst] "+r" (dst),
109 [blocks] "+r" (blocks)
110 : [XFFT_A] "r" (&XFFT_A[0])
111 : "cc", "memory",
112 "v0", "v1", "v2", "v3",
113 "v4", "v5", "v6", "v7",
114 "v16", "v17", "v18", "v19",
115 "v28", "v29", "v30", "v31"
116 );
117 }
118
119 #undef PBUTTERFLY_RANK3
120
121 #define BUTTERFLY_RANK4(op1, op2) \
122 __ASM_EMIT("1:") \
123 /* Initialize sub-loop */ \
124 __ASM_EMIT("mov %[pairs], #1") /* pairs = 1 */ \
125 __ASM_EMIT("ldp q28, q29, [%[XFFT_A], #0x00]") /* q28 = wr1, q29 = wr2 */ \
126 __ASM_EMIT("ldp q30, q31, [%[XFFT_A], #0x20]") /* q30 = wi1, q31 = wi2 */ \
127 __ASM_EMIT("ldp q24, q25, [%[XFFT_W], #0x00]") /* q24 = dr, q25 = di */ \
128 __ASM_EMIT("lsl %[pairs], %[pairs], %[rank]") /* pairs = 1 << rank */ \
129 __ASM_EMIT("add %[b], %[a], %[pairs], LSL #5") /* b = &a[pairs*8] */ \
130 /* 8x butterflies */ \
131 /* Calculate complex c = w * b */ \
132 __ASM_EMIT("3:") \
133 __ASM_EMIT("ldp q0, q2, [%[a], #0x00]") /* v0 = ar1, v2 = ai1 */ \
134 __ASM_EMIT("ldp q1, q3, [%[a], #0x20]") /* v1 = ar2, v3 = ai2 */ \
135 __ASM_EMIT("ldp q4, q6, [%[b], #0x00]") /* v4 = br1, v6 = bi1 */ \
136 __ASM_EMIT("ldp q5, q7, [%[b], #0x20]") /* v5 = br2, v7 = bi2 */ \
137 /* Calc cr and ci */ \
138 __ASM_EMIT("fmul v16.4s, v28.4s, v4.4s") /* v16 = wr1 * br1 */ \
139 __ASM_EMIT("fmul v17.4s, v29.4s, v5.4s") /* v17 = wr2 * br2 */ \
140 __ASM_EMIT("fmul v18.4s, v28.4s, v6.4s") /* v18 = wr1 * bi1 */ \
141 __ASM_EMIT("fmul v19.4s, v29.4s, v7.4s") /* v19 = wr2 * bi2 */ \
142 __ASM_EMIT(op1 " v16.4s, v30.4s, v6.4s") /* v16 = wr1 * br1 +- wi1 * bi1 = cr1 */ \
143 __ASM_EMIT(op1 " v17.4s, v31.4s, v7.4s") /* v17 = wr2 * br2 +- wi2 * bi2 = cr2 */ \
144 __ASM_EMIT(op2 " v18.4s, v30.4s, v4.4s") /* v18 = wr1 * bi1 -+ wi1 * br1 = ci1 */ \
145 __ASM_EMIT(op2 " v19.4s, v31.4s, v5.4s") /* v19 = wr1 * bi1 -+ wi2 * br2 = ci2 */ \
146 /* Apply butterfly */ \
147 __ASM_EMIT("fsub v4.4s, v0.4s, v16.4s") /* v4 = ar1 - cr1 */ \
148 __ASM_EMIT("fsub v5.4s, v1.4s, v17.4s") /* v5 = ar2 - cr2 */ \
149 __ASM_EMIT("fsub v6.4s, v2.4s, v18.4s") /* v6 = ai1 - ci1 */ \
150 __ASM_EMIT("fsub v7.4s, v3.4s, v19.4s") /* v7 = ai2 - ci2 */ \
151 __ASM_EMIT("fadd v0.4s, v0.4s, v16.4s") /* v0 = ar1 + cr1 */ \
152 __ASM_EMIT("fadd v1.4s, v1.4s, v17.4s") /* v1 = ar2 + cr2 */ \
153 __ASM_EMIT("fadd v2.4s, v2.4s, v18.4s") /* v2 = ai1 + ci1 */ \
154 __ASM_EMIT("fadd v3.4s, v3.4s, v19.4s") /* v3 = ai2 + ci2 */ \
155 __ASM_EMIT("stp q0, q2, [%[a], #0x00]") \
156 __ASM_EMIT("stp q1, q3, [%[a], #0x20]") \
157 __ASM_EMIT("stp q4, q6, [%[b], #0x00]") \
158 __ASM_EMIT("stp q5, q7, [%[b], #0x20]") \
159 __ASM_EMIT("subs %[pairs], %[pairs], #2") \
160 __ASM_EMIT("add %[a], %[a], #0x40") \
161 __ASM_EMIT("add %[b], %[b], #0x40") \
162 __ASM_EMIT("b.le 4f") \
163 /* Prepare next loop: rotate angle */ \
164 __ASM_EMIT("fmul v16.4s, v28.4s, v25.4s") /* v16 = wr1 * di */ \
165 __ASM_EMIT("fmul v17.4s, v29.4s, v25.4s") /* v17 = wr2 * di */ \
166 __ASM_EMIT("fmul v18.4s, v30.4s, v25.4s") /* v18 = wi1 * di */ \
167 __ASM_EMIT("fmul v19.4s, v31.4s, v25.4s") /* v19 = wi2 * di */ \
168 __ASM_EMIT("fmul v28.4s, v28.4s, v24.4s") /* v28 = wr1 * dr */ \
169 __ASM_EMIT("fmul v29.4s, v29.4s, v24.4s") /* v29 = wr2 * dr */ \
170 __ASM_EMIT("fmul v30.4s, v30.4s, v24.4s") /* v30 = wi1 * dr */ \
171 __ASM_EMIT("fmul v31.4s, v31.4s, v24.4s") /* v31 = wi2 * dr */ \
172 __ASM_EMIT("fsub v28.4s, v28.4s, v18.4s") /* v28 = wr1*dr - wi1*di */ \
173 __ASM_EMIT("fsub v29.4s, v29.4s, v19.4s") /* v29 = wr2*dr - wi2*di */ \
174 __ASM_EMIT("fadd v30.4s, v30.4s, v16.4s") /* v30 = wi1*dr + wr1*di */ \
175 __ASM_EMIT("fadd v31.4s, v31.4s, v17.4s") /* v31 = wi2*dr + wr2*di */ \
176 __ASM_EMIT("b 3b") \
177 __ASM_EMIT("4:") \
178 __ASM_EMIT("subs %[blocks], %[blocks], #1") \
179 __ASM_EMIT("mov %[a], %[b]") \
180 __ASM_EMIT("b.gt 1b")
181
182 void packed_direct_butterfly_rank4p(float *dst, size_t rank, size_t blocks)
183 {
184 IF_ARCH_AARCH64(
185 rank -= 3;
186 const float *xfft_a = &XFFT_A[rank << 4];
187 const float *xfft_dw = &XFFT_DW[rank << 3];
188 float *b;
189 size_t pairs;
190 )
191
192 ARCH_AARCH64_ASM(
193 BUTTERFLY_RANK4("fmla", "fmls")
194 : [a] "+r" (dst), [b] "=&r" (b),
195 [blocks] "+r" (blocks), [pairs] "=&r" (pairs)
196 : [XFFT_A] "r" (xfft_a), [XFFT_W] "r" (xfft_dw),
197 [rank] "r" (rank)
198 : "cc", "memory",
199 "v0", "v1", "v2", "v3",
200 "v4", "v5", "v6", "v7",
201 "v16", "v17", "v18", "v19",
202 "v24", "v25",
203 "v28", "v29", "v30", "v31"
204 );
205 }
206
207 void packed_reverse_butterfly_rank4p(float *dst, size_t rank, size_t blocks)
208 {
209 IF_ARCH_AARCH64(
210 rank -= 3;
211 const float *xfft_a = &XFFT_A[rank << 4];
212 const float *xfft_dw = &XFFT_DW[rank << 3];
213 float *b;
214 size_t pairs;
215 )
216
217 ARCH_AARCH64_ASM(
218 BUTTERFLY_RANK4("fmls", "fmla")
219 : [a] "+r" (dst), [b] "=&r" (b),
220 [blocks] "+r" (blocks), [pairs] "=&r" (pairs)
221 : [XFFT_A] "r" (xfft_a), [XFFT_W] "r" (xfft_dw),
222 [rank] "r" (rank)
223 : "cc", "memory",
224 "v0", "v1", "v2", "v3",
225 "v4", "v5", "v6", "v7",
226 "v16", "v17", "v18", "v19",
227 "v24", "v25",
228 "v28", "v29", "v30", "v31"
229 );
230 }
231
232 #undef PBUTTERFLY_RANK4
233 }
234 }
235
236 #endif /* PRIVATE_DSP_ARCH_AARCH64_ASIMD_FFT_PBUTTERFLY_H_ */