Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/arm/neon-d32/fft/scramble.h

raw
1 /*
2 * Copyright (C) 2020 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2020 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 31 мар. 2020 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_ARM_NEON_D32_FFT_SCRAMBLE_H_
23 #define PRIVATE_DSP_ARCH_ARM_NEON_D32_FFT_SCRAMBLE_H_
24
25 #ifndef PRIVATE_DSP_ARCH_ARM_NEON_D32_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_ARM_NEON_D32_IMPL */
28
29 #include <lsp-plug.in/common/bits.h>
30
31 namespace lsp
32 {
33 namespace neon_d32
34 {
35 void scramble_direct(float *dst_re, float *dst_im, const float *src_re, const float *src_im, size_t rank)
36 {
37 if ((dst_re == src_re) || (dst_im == src_im))
38 {
39 IF_ARCH_ARM(
40 size_t count = 1 << rank;
41 size_t i, j, rrank = rank;
42 float *d_re, *d_im;
43 );
44
45 // Self algorithm
46 ARCH_ARM_ASM(
47 // Do bit-reverse shuffle
48 __ASM_EMIT("rsb %[rrank], %[rrank], #32") // rrank = 32 - rank
49 __ASM_EMIT("mov %[i], #1") // i = 1
50
51 __ASM_EMIT("1:")
52 #if defined(ARCH_ARM6)
53 __ASM_EMIT("push {%[dst_re]}")
54 __ASM_EMIT("push {%[dst_im]}")
55 LSP_ARMV6_MV_RBIT32("%[j]", "%[i]", "%[dst_re]", "%[dst_im]", "%[masks]") // j = reverse_bits(i)
56 __ASM_EMIT("pop {%[dst_im]}")
57 __ASM_EMIT("pop {%[dst_re]}")
58 #else
59 __ASM_EMIT("rbit %[j], %[i]") // j = reverse_bits(i)
60 #endif
61 __ASM_EMIT("add %[src_re], #4")
62 __ASM_EMIT("lsr %[j], %[rrank]") // j = reverse_bits(i) >> rank
63 __ASM_EMIT("add %[src_im], #4")
64 __ASM_EMIT("cmp %[i], %[j]") // i <=> j
65 __ASM_EMIT("bhs 2f") // if (i >= j) continue
66 __ASM_EMIT("add %[d_re], %[dst_re], %[j], LSL #2") // d_re = &dst_re[j]
67 __ASM_EMIT("vldm %[src_re], {s0}") // s0 = *src_re
68 __ASM_EMIT("add %[d_im], %[dst_im], %[j], LSL #2") // d_im = &dst_im[j]
69 __ASM_EMIT("vldm %[src_im], {s1}") // s1 = *src_im
70 __ASM_EMIT("vldm %[d_re], {s2}") // s2 = *td_re
71 __ASM_EMIT("vldm %[d_im], {s3}") // s3 = *td_im
72 __ASM_EMIT("vstm %[src_re], {s2}") // *(src_re++) = s2
73 __ASM_EMIT("vstm %[src_im], {s3}") // *(src_im++) = s3
74 __ASM_EMIT("vstm %[d_re], {s0}") // *d_re = s0
75 __ASM_EMIT("vstm %[d_im], {s1}") // *d_im = s1
76 __ASM_EMIT("2:")
77 __ASM_EMIT("add %[i], #1") // i++
78 __ASM_EMIT("cmp %[i], %[count]") // i <=> count
79 __ASM_EMIT("blo 1b")
80
81 // Perform x8 butterflies
82 __ASM_EMIT("3:")
83 __ASM_EMIT("vld2.32 {q0-q1}, [%[dst_re]]") // q0 = r0 r2 r4 r6, q1 = r1 r3 r5 r7
84 __ASM_EMIT("vld2.32 {q2-q3}, [%[dst_im]]") // q2 = i0 i2 i4 i6, q3 = i1 i3 i5 i7
85 __ASM_EMIT("vadd.f32 q4, q0, q1") // q4 = r0+r1 r2+r3 r4+r5 r6+r7 = r0' r2' r4' r6'
86 __ASM_EMIT("vadd.f32 q5, q2, q3") // q5 = i0+i1 i2+i3 i4+i5 i6+i7 = i0' i2' i4' i6'
87 __ASM_EMIT("vsub.f32 q0, q0, q1") // q0 = r0-r1 r2-r3 r4-r5 r6-r7 = r1' r3' r5' r7'
88 __ASM_EMIT("vsub.f32 q1, q2, q3") // q1 = i0-i1 i2-i3 i4-i5 i6-i7 = i1' i3' i5' i7'
89
90 // q4 = r0' r2' r4' r6'
91 // q0 = r1' r3' r5' r7'
92 // q5 = i0' i2' i4' i6'
93 // q1 = i1' i3' i5' i7'
94 __ASM_EMIT("vuzp.32 q4, q0") // q4 = r0' r4' r1' r5', q0 = r2' r6' r3' r7'
95 __ASM_EMIT("vuzp.32 q5, q1") // q5 = i0' i4' i1' i5', q1 = i2' i6' i3' i7'
96 __ASM_EMIT("vswp d1, d3") // q0 = r2' r6' i3' i7', q1 = i2' i6' r3' r7'
97 __ASM_EMIT("vadd.f32 q2, q4, q0") // q2 = r0'+r2' r4'+r6' r1'+i3' r5'+i7' = r0" r4" r1" r5"
98 __ASM_EMIT("vsub.f32 q3, q4, q0") // q3 = r0'-r2' r4'-r6' r1'-i3' r5'-i7' = r2" r6" r3" r7"
99 __ASM_EMIT("vadd.f32 q0, q5, q1") // q0 = i0'+i2' i4'+i6' i1'+r3' i5'+r7' = i0" i4" i3" i7"
100 __ASM_EMIT("vsub.f32 q1, q5, q1") // q1 = i0'-i2' i4'-i6' i1'-r3' i5'-r7' = i2" i6" i1" i5"
101
102 // q0 = i0" i4" i3" i7"
103 // q1 = i2" i6" i1" i5"
104 // q2 = r0" r4" r1" r5"
105 // q3 = r2" r6" r3" r7"
106 __ASM_EMIT("vswp d1, d3") // q0 = i0" i4" i1" i5", q1 = i2" i6" i3" i7"
107 __ASM_EMIT("vuzp.32 q2, q3") // q2 = r0" r1" r2" r3", q3 = r4" r5" r6" r7"
108 __ASM_EMIT("vuzp.32 q0, q1") // q0 = i0" i1" i2" i3", q1 = i4" i5" i6" i7"
109
110 __ASM_EMIT("vst1.32 {q2-q3}, [%[dst_re]]!")
111 __ASM_EMIT("subs %[count], #8") // i <=> count
112 __ASM_EMIT("vst1.32 {q0-q1}, [%[dst_im]]!")
113 __ASM_EMIT("bne 3b")
114
115
116 : [src_re] "+r" (src_re), [src_im] "+r" (src_im),
117 [dst_re] "+r" (dst_re), [dst_im] "+r" (dst_im),
118 [d_re] "=&r" (d_re), [d_im] "=&r" (d_im),
119 [rrank] "+r" (rrank), [i] "=&r" (i), [j] "=&r" (j),
120 [count] "+r" (count)
121 : IF_ARCH_ARM6([masks] "r" (lsp_rb_masks))
122 : "cc", "memory",
123 "q0", "q1", "q2", "q3", "q4", "q5"
124 );
125 }
126 else
127 {
128 IF_ARCH_ARM(
129 size_t i, j, rrank = rank - 3;
130 size_t regs = 1 << rrank;
131 float *s_re, *s_im;
132 );
133
134 ARCH_ARM_ASM(
135 __ASM_EMIT("eor %[i], %[i]") // i = 0
136 __ASM_EMIT("rsb %[rrank], %[rrank], #32") // rrank = 32 - rank
137
138 __ASM_EMIT("1:")
139 #if defined(ARCH_ARM6)
140 __ASM_EMIT("push {%[dst_re]}")
141 __ASM_EMIT("push {%[dst_im]}")
142 LSP_ARMV6_MV_RBIT32("%[j]", "%[i]", "%[dst_re]", "%[dst_im]", "%[masks]") // j = reverse_bits(i)
143 __ASM_EMIT("pop {%[dst_im]}")
144 __ASM_EMIT("pop {%[dst_re]}")
145 #else
146 __ASM_EMIT("rbit %[j], %[i]") // j = reverse_bits(i)
147 #endif
148 __ASM_EMIT("lsr %[j], %[rrank]") // j = reverse_bits(i) >> rank
149
150 __ASM_EMIT("add %[s_re], %[src_re], %[j], LSL #2") // s_re = &src_re[i]
151 __ASM_EMIT("add %[s_im], %[src_im], %[j], LSL #2") // s_re = &src_im[i]
152 __ASM_EMIT("vldm %[s_re], {s0}") // q0 = r0 ? ? ?
153 __ASM_EMIT("vldm %[s_im], {s8}") // q2 = i0 ? ? ?
154 __ASM_EMIT("add %[s_re], %[s_re], %[regs], LSL #2") // s_re = &src_re[i + regs]
155 __ASM_EMIT("add %[s_im], %[s_im], %[regs], LSL #2") // s_re = &src_im[i + regs]
156 __ASM_EMIT("vldm %[s_re], {s2}") // q0 = r0 ? r4 ?
157 __ASM_EMIT("vldm %[s_im], {s10}") // q2 = i0 ? i4 ?
158 __ASM_EMIT("add %[s_re], %[s_re], %[regs], LSL #2") // s_re = &src_re[i + regs*2]
159 __ASM_EMIT("add %[s_im], %[s_im], %[regs], LSL #2") // s_re = &src_im[i + regs*2]
160 __ASM_EMIT("vldm %[s_re], {s1}") // q0 = r0 r2 r4 ?
161 __ASM_EMIT("vldm %[s_im], {s9}") // q2 = i0 i2 i4 ?
162 __ASM_EMIT("add %[s_re], %[s_re], %[regs], LSL #2") // s_re = &src_re[i + regs*3]
163 __ASM_EMIT("add %[s_im], %[s_im], %[regs], LSL #2") // s_re = &src_im[i + regs*3]
164 __ASM_EMIT("vldm %[s_re], {s3}") // q0 = r0 r2 r4 r6
165 __ASM_EMIT("vldm %[s_im], {s11}") // q2 = i0 i2 i4 i6
166
167 __ASM_EMIT("add %[s_re], %[s_re], %[regs], LSL #2") // s_re = &src_re[i + regs*4]
168 __ASM_EMIT("add %[s_im], %[s_im], %[regs], LSL #2") // s_re = &src_im[i + regs*4]
169 __ASM_EMIT("vldm %[s_re], {s4}") // q1 = r1 ? ? ?
170 __ASM_EMIT("vldm %[s_im], {s12}") // q3 = i1 ? ? ?
171 __ASM_EMIT("add %[s_re], %[s_re], %[regs], LSL #2") // s_re = &src_re[i + regs*5]
172 __ASM_EMIT("add %[s_im], %[s_im], %[regs], LSL #2") // s_re = &src_im[i + regs*5]
173 __ASM_EMIT("vldm %[s_re], {s6}") // q1 = r1 ? r5 ?
174 __ASM_EMIT("vldm %[s_im], {s14}") // q3 = i1 ? i5 ?
175 __ASM_EMIT("add %[s_re], %[s_re], %[regs], LSL #2") // s_re = &src_re[i + regs*6]
176 __ASM_EMIT("add %[s_im], %[s_im], %[regs], LSL #2") // s_re = &src_im[i + regs*6]
177 __ASM_EMIT("vldm %[s_re], {s5}") // q1 = r1 r3 r5 ?
178 __ASM_EMIT("vldm %[s_im], {s13}") // q3 = i1 i3 i5 ?
179 __ASM_EMIT("add %[s_re], %[s_re], %[regs], LSL #2") // s_re = &src_re[i + regs*7]
180 __ASM_EMIT("add %[s_im], %[s_im], %[regs], LSL #2") // s_re = &src_im[i + regs*7]
181 __ASM_EMIT("vldm %[s_re], {s7}") // q1 = r1 r3 r5 r7
182 __ASM_EMIT("vldm %[s_im], {s15}") // q3 = i1 i3 i5 i7
183
184 // q0 = r0 r2 r4 r6
185 // q1 = r1 r3 r5 r7
186 // q2 = i0 i2 i4 i6
187 // q3 = i1 i3 i5 i7
188 __ASM_EMIT("add %[i], #1")
189
190 // Perform x8 butterflies
191 __ASM_EMIT("vadd.f32 q4, q0, q1") // q4 = r0+r1 r2+r3 r4+r5 r6+r7 = r0' r2' r4' r6'
192 __ASM_EMIT("vadd.f32 q5, q2, q3") // q5 = i0+i1 i2+i3 i4+i5 i6+i7 = i0' i2' i4' i6'
193 __ASM_EMIT("vsub.f32 q0, q0, q1") // q0 = r0-r1 r2-r3 r4-r5 r6-r7 = r1' r3' r5' r7'
194 __ASM_EMIT("vsub.f32 q1, q2, q3") // q1 = i0-i1 i2-i3 i4-i5 i6-i7 = i1' i3' i5' i7'
195
196 // q4 = r0' r2' r4' r6'
197 // q0 = r1' r3' r5' r7'
198 // q5 = i0' i2' i4' i6'
199 // q1 = i1' i3' i5' i7'
200 __ASM_EMIT("vuzp.32 q4, q0") // q4 = r0' r4' r1' r5', q0 = r2' r6' r3' r7'
201 __ASM_EMIT("vuzp.32 q5, q1") // q5 = i0' i4' i1' i5', q1 = i2' i6' i3' i7'
202 __ASM_EMIT("vswp d1, d3") // q0 = r2' r6' i3' i7', q1 = i2' i6' r3' r7'
203 __ASM_EMIT("vadd.f32 q2, q4, q0") // q2 = r0'+r2' r4'+r6' r1'+i3' r5'+i7' = r0" r4" r1" r5"
204 __ASM_EMIT("vsub.f32 q3, q4, q0") // q3 = r0'-r2' r4'-r6' r1'-i3' r5'-i7' = r2" r6" r3" r7"
205 __ASM_EMIT("vadd.f32 q0, q5, q1") // q0 = i0'+i2' i4'+i6' i1'+r3' i5'+r7' = i0" i4" i3" i7"
206 __ASM_EMIT("vsub.f32 q1, q5, q1") // q1 = i0'-i2' i4'-i6' i1'-r3' i5'-r7' = i2" i6" i1" i5"
207
208 // q0 = i0" i4" i3" i7"
209 // q1 = i2" i6" i1" i5"
210 // q2 = r0" r4" r1" r5"
211 // q3 = r2" r6" r3" r7"
212 __ASM_EMIT("vswp d1, d3") // q0 = i0" i4" i1" i5", q1 = i2" i6" i3" i7"
213 __ASM_EMIT("vuzp.32 q2, q3") // q2 = r0" r1" r2" r3", q3 = r4" r5" r6" r7"
214 __ASM_EMIT("vuzp.32 q0, q1") // q0 = i0" i1" i2" i3", q1 = i4" i5" i6" i7"
215
216 __ASM_EMIT("vst1.32 {q2-q3}, [%[dst_re]]!")
217 __ASM_EMIT("cmp %[i], %[regs]")
218 __ASM_EMIT("vst1.32 {q0-q1}, [%[dst_im]]!")
219 __ASM_EMIT("blo 1b")
220
221 : [dst_re] "+r" (dst_re), [dst_im] "+r" (dst_im),
222 [s_re] "=&r" (s_re), [s_im] "=&r" (s_im),
223 [rrank] "+r" (rrank), [i] "=&r" (i), [j] "=&r" (j)
224 : [src_re] "r" (src_re), [src_im] "r" (src_im),
225 [regs] "r" (regs)
226 IF_ARCH_ARM6(, [masks] "r" (lsp_rb_masks))
227 : "cc", "memory",
228 "q0", "q1", "q2", "q3", "q4", "q5"
229 );
230 }
231 }
232
233 void scramble_reverse(float *dst_re, float *dst_im, const float *src_re, const float *src_im, size_t rank)
234 {
235 if ((dst_re == src_re) || (dst_im == src_im))
236 {
237 IF_ARCH_ARM(
238 size_t count = 1 << rank;
239 size_t i, j, rrank = rank;
240 float *d_re, *d_im;
241 );
242
243 // Self algorithm
244 ARCH_ARM_ASM(
245 // Do bit-reverse shuffle
246 __ASM_EMIT("rsb %[rrank], %[rrank], #32") // rrank = 32 - rank
247 __ASM_EMIT("mov %[i], #1") // i = 1
248
249 __ASM_EMIT("1:")
250 #if defined(ARCH_ARM6)
251 __ASM_EMIT("push {%[dst_re]}")
252 __ASM_EMIT("push {%[dst_im]}")
253 LSP_ARMV6_MV_RBIT32("%[j]", "%[i]", "%[dst_re]", "%[dst_im]", "%[masks]") // j = reverse_bits(i)
254 __ASM_EMIT("pop {%[dst_im]}")
255 __ASM_EMIT("pop {%[dst_re]}")
256 #else
257 __ASM_EMIT("rbit %[j], %[i]") // j = reverse_bits(i)
258 #endif
259 __ASM_EMIT("add %[src_re], #4")
260 __ASM_EMIT("lsr %[j], %[rrank]") // j = reverse_bits(i) >> rank
261 __ASM_EMIT("add %[src_im], #4")
262 __ASM_EMIT("cmp %[i], %[j]") // i <=> j
263 __ASM_EMIT("bhs 2f") // if (i >= j) continue
264 __ASM_EMIT("add %[d_re], %[dst_re], %[j], LSL #2") // d_re = &dst_re[j]
265 __ASM_EMIT("vldm %[src_re], {s0}") // s0 = *src_re
266 __ASM_EMIT("add %[d_im], %[dst_im], %[j], LSL #2") // d_im = &dst_im[j]
267 __ASM_EMIT("vldm %[src_im], {s1}") // s1 = *src_im
268 __ASM_EMIT("vldm %[d_re], {s2}") // s2 = *td_re
269 __ASM_EMIT("vldm %[d_im], {s3}") // s3 = *td_im
270 __ASM_EMIT("vstm %[src_re], {s2}") // *(src_re++) = s2
271 __ASM_EMIT("vstm %[src_im], {s3}") // *(src_im++) = s3
272 __ASM_EMIT("vstm %[d_re], {s0}") // *d_re = s0
273 __ASM_EMIT("vstm %[d_im], {s1}") // *d_im = s1
274 __ASM_EMIT("2:")
275 __ASM_EMIT("add %[i], #1") // i++
276 __ASM_EMIT("cmp %[i], %[count]") // i <=> count
277 __ASM_EMIT("blo 1b")
278
279 __ASM_EMIT("eor %[i], %[i]")
280
281 // Perform x8 butterflies
282 __ASM_EMIT("3:")
283 __ASM_EMIT("vld2.32 {q0-q1}, [%[dst_re]]") // q0 = r0 r2 r4 r6, q1 = r1 r3 r5 r7
284 __ASM_EMIT("vld2.32 {q2-q3}, [%[dst_im]]") // q2 = i0 i2 i4 i6, q3 = i1 i3 i5 i7
285 __ASM_EMIT("vadd.f32 q4, q0, q1") // q4 = r0+r1 r2+r3 r4+r5 r6+r7 = r0' r2' r4' r6'
286 __ASM_EMIT("vadd.f32 q5, q2, q3") // q5 = i0+i1 i2+i3 i4+i5 i6+i7 = i0' i2' i4' i6'
287 __ASM_EMIT("vsub.f32 q0, q0, q1") // q0 = r0-r1 r2-r3 r4-r5 r6-r7 = r1' r3' r5' r7'
288 __ASM_EMIT("vsub.f32 q1, q2, q3") // q1 = i0-i1 i2-i3 i4-i5 i6-i7 = i1' i3' i5' i7'
289
290 // q4 = r0' r2' r4' r6'
291 // q0 = r1' r3' r5' r7'
292 // q5 = i0' i2' i4' i6'
293 // q1 = i1' i3' i5' i7'
294 __ASM_EMIT("vuzp.32 q4, q0") // q4 = r0' r4' r1' r5', q0 = r2' r6' r3' r7'
295 __ASM_EMIT("vuzp.32 q5, q1") // q5 = i0' i4' i1' i5', q1 = i2' i6' i3' i7'
296 __ASM_EMIT("vswp d1, d3") // q0 = r2' r6' i3' i7', q1 = i2' i6' r3' r7'
297 __ASM_EMIT("vadd.f32 q2, q4, q0") // q2 = r0'+r2' r4'+r6' r1'+i3' r5'+i7' = r0" r4" r3" r7"
298 __ASM_EMIT("vsub.f32 q3, q4, q0") // q3 = r0'-r2' r4'-r6' r1'-i3' r5'-i7' = r2" r6" r1" r5"
299 __ASM_EMIT("vadd.f32 q0, q5, q1") // q0 = i0'+i2' i4'+i6' i1'+r3' i5'+r7' = i0" i4" i1" i5"
300 __ASM_EMIT("vsub.f32 q1, q5, q1") // q1 = i0'-i2' i4'-i6' i1'-r3' i5'-r7' = i2" i6" i3" i7"
301
302 // q0 = i0" i4" i1" i5"
303 // q1 = i2" i6" i3" i7"
304 // q2 = r0" r4" r3" r7"
305 // q3 = r2" r6" r1" r5"
306 __ASM_EMIT("vswp d5, d7") // q2 = r0" r4" r1" r5", q3 = r2" r6" r3" r7"
307 __ASM_EMIT("vuzp.32 q2, q3") // q2 = r0" r1" r2" r3", q3 = r4" r5" r6" r7"
308 __ASM_EMIT("vuzp.32 q0, q1") // q0 = i0" i1" i2" i3", q1 = i4" i5" i6" i7"
309
310 __ASM_EMIT("vst1.32 {q2-q3}, [%[dst_re]]!")
311 __ASM_EMIT("subs %[count], #8") // i <=> count
312 __ASM_EMIT("vst1.32 {q0-q1}, [%[dst_im]]!")
313 __ASM_EMIT("bne 3b")
314
315
316 : [src_re] "+r" (src_re), [src_im] "+r" (src_im),
317 [dst_re] "+r" (dst_re), [dst_im] "+r" (dst_im),
318 [d_re] "=&r" (d_re), [d_im] "=&r" (d_im),
319 [rrank] "+r" (rrank), [i] "=&r" (i), [j] "=&r" (j),
320 [count] "+r" (count)
321 : IF_ARCH_ARM6([masks] "r" (lsp_rb_masks))
322 : "cc", "memory",
323 "q0", "q1", "q2", "q3", "q4", "q5"
324 );
325 }
326 else
327 {
328 IF_ARCH_ARM(
329 size_t i, j, rrank = rank - 3;
330 size_t regs = 1 << rrank;
331 float *s_re, *s_im;
332 );
333
334 ARCH_ARM_ASM(
335 __ASM_EMIT("eor %[i], %[i]") // i = 0
336 __ASM_EMIT("rsb %[rrank], %[rrank], #32") // rrank = 32 - rank
337
338 __ASM_EMIT("1:")
339 #if defined(ARCH_ARM6)
340 __ASM_EMIT("push {%[dst_re]}")
341 __ASM_EMIT("push {%[dst_im]}")
342 LSP_ARMV6_MV_RBIT32("%[j]", "%[i]", "%[dst_re]", "%[dst_im]", "%[masks]") // j = reverse_bits(i)
343 __ASM_EMIT("pop {%[dst_im]}")
344 __ASM_EMIT("pop {%[dst_re]}")
345 #else
346 __ASM_EMIT("rbit %[j], %[i]") // j = reverse_bits(i)
347 #endif
348 __ASM_EMIT("lsr %[j], %[rrank]") // j = reverse_bits(i) >> rank
349
350 __ASM_EMIT("add %[s_re], %[src_re], %[j], LSL #2") // s_re = &src_re[i]
351 __ASM_EMIT("add %[s_im], %[src_im], %[j], LSL #2") // s_re = &src_im[i]
352 __ASM_EMIT("vldm %[s_re], {s0}") // q0 = r0 ? ? ?
353 __ASM_EMIT("vldm %[s_im], {s8}") // q2 = i0 ? ? ?
354 __ASM_EMIT("add %[s_re], %[s_re], %[regs], LSL #2") // s_re = &src_re[i + regs]
355 __ASM_EMIT("add %[s_im], %[s_im], %[regs], LSL #2") // s_re = &src_im[i + regs]
356 __ASM_EMIT("vldm %[s_re], {s2}") // q0 = r0 ? r4 ?
357 __ASM_EMIT("vldm %[s_im], {s10}") // q2 = i0 ? i4 ?
358 __ASM_EMIT("add %[s_re], %[s_re], %[regs], LSL #2") // s_re = &src_re[i + regs*2]
359 __ASM_EMIT("add %[s_im], %[s_im], %[regs], LSL #2") // s_re = &src_im[i + regs*2]
360 __ASM_EMIT("vldm %[s_re], {s1}") // q0 = r0 r2 r4 ?
361 __ASM_EMIT("vldm %[s_im], {s9}") // q2 = i0 i2 i4 ?
362 __ASM_EMIT("add %[s_re], %[s_re], %[regs], LSL #2") // s_re = &src_re[i + regs*3]
363 __ASM_EMIT("add %[s_im], %[s_im], %[regs], LSL #2") // s_re = &src_im[i + regs*3]
364 __ASM_EMIT("vldm %[s_re], {s3}") // q0 = r0 r2 r4 r6
365 __ASM_EMIT("vldm %[s_im], {s11}") // q2 = i0 i2 i4 i6
366
367 __ASM_EMIT("add %[s_re], %[s_re], %[regs], LSL #2") // s_re = &src_re[i + regs*4]
368 __ASM_EMIT("add %[s_im], %[s_im], %[regs], LSL #2") // s_re = &src_im[i + regs*4]
369 __ASM_EMIT("vldm %[s_re], {s4}") // q1 = r1 ? ? ?
370 __ASM_EMIT("vldm %[s_im], {s12}") // q3 = i1 ? ? ?
371 __ASM_EMIT("add %[s_re], %[s_re], %[regs], LSL #2") // s_re = &src_re[i + regs*5]
372 __ASM_EMIT("add %[s_im], %[s_im], %[regs], LSL #2") // s_re = &src_im[i + regs*5]
373 __ASM_EMIT("vldm %[s_re], {s6}") // q1 = r1 ? r5 ?
374 __ASM_EMIT("vldm %[s_im], {s14}") // q3 = i1 ? i5 ?
375 __ASM_EMIT("add %[s_re], %[s_re], %[regs], LSL #2") // s_re = &src_re[i + regs*6]
376 __ASM_EMIT("add %[s_im], %[s_im], %[regs], LSL #2") // s_re = &src_im[i + regs*6]
377 __ASM_EMIT("vldm %[s_re], {s5}") // q1 = r1 r3 r5 ?
378 __ASM_EMIT("vldm %[s_im], {s13}") // q3 = i1 i3 i5 ?
379 __ASM_EMIT("add %[s_re], %[s_re], %[regs], LSL #2") // s_re = &src_re[i + regs*7]
380 __ASM_EMIT("add %[s_im], %[s_im], %[regs], LSL #2") // s_re = &src_im[i + regs*7]
381 __ASM_EMIT("vldm %[s_re], {s7}") // q1 = r1 r3 r5 r7
382 __ASM_EMIT("vldm %[s_im], {s15}") // q3 = i1 i3 i5 i7
383
384 // q0 = r0 r2 r4 r6
385 // q1 = r1 r3 r5 r7
386 // q2 = i0 i2 i4 i6
387 // q3 = i1 i3 i5 i7
388 __ASM_EMIT("add %[i], #1")
389
390 // Perform x8 butterflies
391 __ASM_EMIT("vadd.f32 q4, q0, q1") // q4 = r0+r1 r2+r3 r4+r5 r6+r7 = r0' r2' r4' r6'
392 __ASM_EMIT("vadd.f32 q5, q2, q3") // q5 = i0+i1 i2+i3 i4+i5 i6+i7 = i0' i2' i4' i6'
393 __ASM_EMIT("vsub.f32 q0, q0, q1") // q0 = r0-r1 r2-r3 r4-r5 r6-r7 = r1' r3' r5' r7'
394 __ASM_EMIT("vsub.f32 q1, q2, q3") // q1 = i0-i1 i2-i3 i4-i5 i6-i7 = i1' i3' i5' i7'
395
396 // q4 = r0' r2' r4' r6'
397 // q0 = r1' r3' r5' r7'
398 // q5 = i0' i2' i4' i6'
399 // q1 = i1' i3' i5' i7'
400 __ASM_EMIT("vuzp.32 q4, q0") // q4 = r0' r4' r1' r5', q0 = r2' r6' r3' r7'
401 __ASM_EMIT("vuzp.32 q5, q1") // q5 = i0' i4' i1' i5', q1 = i2' i6' i3' i7'
402 __ASM_EMIT("vswp d1, d3") // q0 = r2' r6' i3' i7', q1 = i2' i6' r3' r7'
403 __ASM_EMIT("vadd.f32 q2, q4, q0") // q2 = r0'+r2' r4'+r6' r1'+i3' r5'+i7' = r0" r4" r3" r7"
404 __ASM_EMIT("vsub.f32 q3, q4, q0") // q3 = r0'-r2' r4'-r6' r1'-i3' r5'-i7' = r2" r6" r1" r5"
405 __ASM_EMIT("vadd.f32 q0, q5, q1") // q0 = i0'+i2' i4'+i6' i1'+r3' i5'+r7' = i0" i4" i1" i5"
406 __ASM_EMIT("vsub.f32 q1, q5, q1") // q1 = i0'-i2' i4'-i6' i1'-r3' i5'-r7' = i2" i6" i3" i7"
407
408 // q0 = i0" i4" i1" i5"
409 // q1 = i2" i6" i3" i7"
410 // q2 = r0" r4" r3" r7"
411 // q3 = r2" r6" r1" r5"
412 __ASM_EMIT("vswp d5, d7") // q2 = r0" r4" r1" r5", q3 = r2" r6" r3" r7"
413 __ASM_EMIT("vuzp.32 q2, q3") // q2 = r0" r1" r2" r3", q3 = r4" r5" r6" r7"
414 __ASM_EMIT("vuzp.32 q0, q1") // q0 = i0" i1" i2" i3", q1 = i4" i5" i6" i7"
415
416 __ASM_EMIT("vst1.32 {q2-q3}, [%[dst_re]]!")
417 __ASM_EMIT("cmp %[i], %[regs]")
418 __ASM_EMIT("vst1.32 {q0-q1}, [%[dst_im]]!")
419 __ASM_EMIT("blo 1b")
420
421 : [dst_re] "+r" (dst_re), [dst_im] "+r" (dst_im),
422 [s_re] "=&r" (s_re), [s_im] "=&r" (s_im),
423 [rrank] "+r" (rrank), [i] "=&r" (i), [j] "=&r" (j)
424 : [src_re] "r" (src_re), [src_im] "r" (src_im),
425 [regs] "r" (regs)
426 IF_ARCH_ARM6(, [masks] "r" (lsp_rb_masks))
427 : "cc", "memory",
428 "q0", "q1", "q2", "q3", "q4", "q5"
429 );
430 }
431 }
432 }
433 }
434
435 #endif /* PRIVATE_DSP_ARCH_ARM_NEON_D32_FFT_SCRAMBLE_H_ */