Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/aarch64/asimd/fft/pscramble.h

raw
1 /*
2 * Copyright (C) 2020 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2020 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 31 мар. 2020 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_AARCH64_ASIMD_FFT_PSCRAMBLE_H_
23 #define PRIVATE_DSP_ARCH_AARCH64_ASIMD_FFT_PSCRAMBLE_H_
24
25 #ifndef PRIVATE_DSP_ARCH_AARCH64_ASIMD_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_AARCH64_ASIMD_IMPL */
28
29 namespace lsp
30 {
31 namespace asimd
32 {
33 void packed_scramble_self_direct(float *dst, size_t rank)
34 {
35 IF_ARCH_AARCH64(
36 size_t count = 1 << rank;
37 size_t i, j, rrank = 64 - rank;
38 float *dp, *src = dst;
39 );
40
41 // Self algorithm
42 ARCH_AARCH64_ASM(
43 // Do bit-reverse shuffle
44 __ASM_EMIT("mov %[i], #1") // i = 1
45 __ASM_EMIT("1:")
46 __ASM_EMIT("rbit %[j], %[i]") // j = reverse_bits(i)
47 __ASM_EMIT("add %[src], %[src], #8")
48 __ASM_EMIT("lsr %[j], %[j], %[rrank]") // j = reverse_bits(i) >> rank
49 __ASM_EMIT("cmp %[i], %[j]") // i <=> j
50 __ASM_EMIT("b.hs 2f") // if (i >= j) continue
51 __ASM_EMIT("add %[dp], %[dst], %[j], LSL #3") // d_re = &dst[j]
52 __ASM_EMIT("ldr d0, [%[src]]") // d0 = *src
53 __ASM_EMIT("ldr d1, [%[dp]]") // d1 = *dp
54 __ASM_EMIT("str d0, [%[dp]]") // *dp = d0
55 __ASM_EMIT("str d1, [%[src]]") // *src = d1
56 __ASM_EMIT("2:")
57 __ASM_EMIT("add %[i], %[i], #1") // i++
58 __ASM_EMIT("cmp %[i], %[count]") // i <=> count
59 __ASM_EMIT("b.lo 1b")
60 // Perform x8 butterflies
61 __ASM_EMIT("3:")
62 __ASM_EMIT("ld4 {v0.4s-v3.4s}, [%[dst]]")
63 // v0 = r0 r2 r4 r6
64 // v1 = i0 i2 i4 i6
65 // v2 = r1 r3 r5 r7
66 // v3 = i1 i3 i5 i7
67 __ASM_EMIT("fadd v6.4s, v0.4s, v2.4s") // v6 = r0+r1 r2+r3 r4+r5 r6+r7 = r0' r2' r4' r6'
68 __ASM_EMIT("fadd v7.4s, v1.4s, v3.4s") // v7 = i0+i1 i2+i3 i4+i5 i6+i7 = i0' i2' i4' i6'
69 __ASM_EMIT("fsub v0.4s, v0.4s, v2.4s") // v0 = r0-r1 r2-r3 r4-r5 r6-r7 = r1' r3' r5' r7'
70 __ASM_EMIT("fsub v1.4s, v1.4s, v3.4s") // v1 = i0-i1 i2-i3 i4-i5 i6-i7 = i1' i3' i5' i7'
71 // v6 = r0' r2' r4' r6'
72 // v0 = r1' r3' r5' r7'
73 // v7 = i0' i2' i4' i6'
74 // v1 = i1' i3' i5' i7'
75 __ASM_EMIT("uzp1 v4.4s, v6.4s, v0.4s") // v4 = r0' r4' r1' r5'
76 __ASM_EMIT("uzp2 v0.4s, v6.4s, v0.4s") // v0 = r2' r6' r3' r7'
77 __ASM_EMIT("uzp1 v5.4s, v7.4s, v1.4s") // v5 = i0' i4' i1' i5'
78 __ASM_EMIT("uzp2 v1.4s, v7.4s, v1.4s") // v1 = i2' i6' i3' i7'
79 __ASM_EMIT("mov v2.16b, v0.16b") // v2 = r2' r6' r3' r7'
80 __ASM_EMIT("ins v0.d[1], v1.d[1]") // v0 = r2' r6' i3' i7'
81 __ASM_EMIT("ins v1.d[1], v2.d[1]") // v1 = i2' i6' r3' r7'
82 __ASM_EMIT("fadd v2.4s, v4.4s, v0.4s") // v2 = r0'+r2' r4'+r6' r1'+i3' r5'+i7' = r0" r4" r1" r5"
83 __ASM_EMIT("fsub v3.4s, v4.4s, v0.4s") // v3 = r0'-r2' r4'-r6' r1'-i3' r5'-i7' = r2" r6" r3" r7"
84 __ASM_EMIT("fadd v0.4s, v5.4s, v1.4s") // v0 = i0'+i2' i4'+i6' i1'+r3' i5'+r7' = i0" i4" i3" i7"
85 __ASM_EMIT("fsub v1.4s, v5.4s, v1.4s") // v1 = i0'-i2' i4'-i6' i1'-r3' i5'-r7' = i2" i6" i1" i5"
86 // v0 = i0" i4" i3" i7"
87 // v1 = i2" i6" i1" i5"
88 // v2 = r0" r4" r1" r5"
89 // v3 = r2" r6" r3" r7"
90 __ASM_EMIT("mov v4.16b, v0.16b") // v4 = i0" i4" i3" i7"
91 __ASM_EMIT("ins v0.d[1], v1.d[1]") // v0 = i0" i4" i1" i5"
92 __ASM_EMIT("ins v1.d[1], v4.d[1]") // v1 = i2" i6" i3" i7"
93 __ASM_EMIT("uzp1 v4.4s, v2.4s, v3.4s") // v4 = r0" r1" r2" r3"
94 __ASM_EMIT("uzp2 v5.4s, v2.4s, v3.4s") // v5 = r4" r5" r6" r7"
95 __ASM_EMIT("uzp1 v6.4s, v0.4s, v1.4s") // v6 = i0" i1" i2" i3"
96 __ASM_EMIT("uzp2 v7.4s, v0.4s, v1.4s") // v7 = i4" i5" i6" i7"
97 // Store
98 __ASM_EMIT("stp q4, q6, [%[dst], #0x00]")
99 __ASM_EMIT("stp q5, q7, [%[dst], #0x20]")
100 __ASM_EMIT("subs %[count], %[count], #8") // i <=> count
101 __ASM_EMIT("add %[dst], %[dst], #0x40")
102 __ASM_EMIT("b.ne 3b")
103
104 : [src] "+r" (src), [dst] "+r" (dst), [dp] "=&r" (dp),
105 [rrank] "+r" (rrank), [i] "=&r" (i), [j] "=&r" (j),
106 [count] "+r" (count)
107 :
108 : "cc", "memory",
109 "v0", "v1", "v2", "v3",
110 "v4", "v5", "v6", "v7"
111 );
112 }
113
114 void packed_scramble_copy_direct(float *dst, const float *src, size_t rank)
115 {
116 IF_ARCH_AARCH64(
117 size_t i, j;
118 float *sr1, *sr2;
119 size_t regs = 1 << (rank - 3), rrank = 64 - (rank - 3);
120 );
121
122 ARCH_AARCH64_ASM(
123 __ASM_EMIT("eor %[i], %[i], %[i]") // i = 0
124 __ASM_EMIT("1:")
125 __ASM_EMIT("rbit %[j], %[i]") // j = reverse_bits(i)
126 __ASM_EMIT("lsr %[j], %[j], %[rrank]") // j = reverse_bits(i) >> rank
127
128 __ASM_EMIT("add %[sr1], %[src], %[j], LSL #3") // sr1 = &src[j]
129 __ASM_EMIT("add %[sr2], %[sr1], %[regs], LSL #3") // sr2 = &src[j + regs]
130 __ASM_EMIT("ld2 {v0.s, v1.s}[0], [%[sr1]]") // v0 = r0 ? ? ?, v1 = i0 ? ? ?
131 __ASM_EMIT("ld2 {v0.s, v1.s}[2], [%[sr2]]") // v0 = r0 ? r4 ?, v1 = i0 ? i4 ?
132 __ASM_EMIT("add %[sr1], %[sr1], %[regs], LSL #4") // sr1 = &src[j + regs*2]
133 __ASM_EMIT("add %[sr2], %[sr2], %[regs], LSL #4") // sr2 = &src[j + regs*3]
134 __ASM_EMIT("ld2 {v0.s, v1.s}[1], [%[sr1]]") // v0 = r0 r2 r4 ?, v1 = i0 i2 i4 ?
135 __ASM_EMIT("ld2 {v0.s, v1.s}[3], [%[sr2]]") // v0 = r0 r2 r4 r6, v1 = i0 i2 i4 i6
136
137 __ASM_EMIT("add %[sr1], %[sr1], %[regs], LSL #4") // sr1 = &src[j + regs*4]
138 __ASM_EMIT("add %[sr2], %[sr2], %[regs], LSL #4") // sr2 = &src[j + regs*5]
139 __ASM_EMIT("ld2 {v2.s, v3.s}[0], [%[sr1]]") // v2 = r1 ? ? ?, v3 = i1 ? ? ?
140 __ASM_EMIT("ld2 {v2.s, v3.s}[2], [%[sr2]]") // v2 = r1 ? r5 ?, v3 = i1 ? i5 ?
141 __ASM_EMIT("add %[sr1], %[sr1], %[regs], LSL #4") // sr1 = &src[j + regs*6]
142 __ASM_EMIT("add %[sr2], %[sr2], %[regs], LSL #4") // sr2 = &src[j + regs*7]
143 __ASM_EMIT("ld2 {v2.s, v3.s}[1], [%[sr1]]") // v2 = r1 r3 r5 ?, v3 = i1 i3 i5 ?
144 __ASM_EMIT("ld2 {v2.s, v3.s}[3], [%[sr2]]") // v2 = r1 r3 r5 r7, v3 = i1 i3 i5 i7
145 // v0 = r0 r2 r4 r6
146 // v1 = i0 i2 i4 i6
147 // v2 = r1 r3 r5 r7
148 // v3 = i1 i3 i5 i7
149 __ASM_EMIT("fadd v6.4s, v0.4s, v2.4s") // v6 = r0+r1 r2+r3 r4+r5 r6+r7 = r0' r2' r4' r6'
150 __ASM_EMIT("fadd v7.4s, v1.4s, v3.4s") // v7 = i0+i1 i2+i3 i4+i5 i6+i7 = i0' i2' i4' i6'
151 __ASM_EMIT("fsub v0.4s, v0.4s, v2.4s") // v0 = r0-r1 r2-r3 r4-r5 r6-r7 = r1' r3' r5' r7'
152 __ASM_EMIT("fsub v1.4s, v1.4s, v3.4s") // v1 = i0-i1 i2-i3 i4-i5 i6-i7 = i1' i3' i5' i7'
153 // v6 = r0' r2' r4' r6'
154 // v0 = r1' r3' r5' r7'
155 // v7 = i0' i2' i4' i6'
156 // v1 = i1' i3' i5' i7'
157 __ASM_EMIT("uzp1 v4.4s, v6.4s, v0.4s") // v4 = r0' r4' r1' r5'
158 __ASM_EMIT("uzp2 v0.4s, v6.4s, v0.4s") // v0 = r2' r6' r3' r7'
159 __ASM_EMIT("uzp1 v5.4s, v7.4s, v1.4s") // v5 = i0' i4' i1' i5'
160 __ASM_EMIT("uzp2 v1.4s, v7.4s, v1.4s") // v1 = i2' i6' i3' i7'
161 __ASM_EMIT("mov v2.16b, v0.16b") // v2 = r2' r6' r3' r7'
162 __ASM_EMIT("ins v0.d[1], v1.d[1]") // v0 = r2' r6' i3' i7'
163 __ASM_EMIT("ins v1.d[1], v2.d[1]") // v1 = i2' i6' r3' r7'
164 __ASM_EMIT("fadd v2.4s, v4.4s, v0.4s") // v2 = r0'+r2' r4'+r6' r1'+i3' r5'+i7' = r0" r4" r1" r5"
165 __ASM_EMIT("fsub v3.4s, v4.4s, v0.4s") // v3 = r0'-r2' r4'-r6' r1'-i3' r5'-i7' = r2" r6" r3" r7"
166 __ASM_EMIT("fadd v0.4s, v5.4s, v1.4s") // v0 = i0'+i2' i4'+i6' i1'+r3' i5'+r7' = i0" i4" i3" i7"
167 __ASM_EMIT("fsub v1.4s, v5.4s, v1.4s") // v1 = i0'-i2' i4'-i6' i1'-r3' i5'-r7' = i2" i6" i1" i5"
168 // v0 = i0" i4" i3" i7"
169 // v1 = i2" i6" i1" i5"
170 // v2 = r0" r4" r1" r5"
171 // v3 = r2" r6" r3" r7"
172 __ASM_EMIT("mov v4.16b, v0.16b") // v4 = i0" i4" i3" i7"
173 __ASM_EMIT("ins v0.d[1], v1.d[1]") // v0 = i0" i4" i1" i5"
174 __ASM_EMIT("ins v1.d[1], v4.d[1]") // v1 = i2" i6" i3" i7"
175 __ASM_EMIT("uzp1 v4.4s, v2.4s, v3.4s") // v4 = r0" r1" r2" r3"
176 __ASM_EMIT("uzp2 v5.4s, v2.4s, v3.4s") // v5 = r4" r5" r6" r7"
177 __ASM_EMIT("uzp1 v6.4s, v0.4s, v1.4s") // v6 = i0" i1" i2" i3"
178 __ASM_EMIT("uzp2 v7.4s, v0.4s, v1.4s") // v7 = i4" i5" i6" i7"
179 // Store
180 __ASM_EMIT("add %[i], %[i], #1")
181 __ASM_EMIT("stp q4, q6, [%[dst], #0x00]")
182 __ASM_EMIT("stp q5, q7, [%[dst], #0x20]")
183 __ASM_EMIT("cmp %[i], %[regs]")
184 __ASM_EMIT("add %[dst], %[dst], #0x40")
185 __ASM_EMIT("b.lo 1b")
186
187 : [dst] "+r" (dst), [sr1] "=&r" (sr1), [sr2] "=&r" (sr2),
188 [rrank] "+r" (rrank), [i] "=&r" (i), [j] "=&r" (j)
189 : [src] "r" (src), [regs] "r" (regs)
190 : "cc", "memory",
191 "v0", "v1", "v2", "v3",
192 "v4", "v5", "v6", "v7"
193 );
194 }
195
196 void packed_scramble_self_reverse(float *dst, size_t rank)
197 {
198 IF_ARCH_AARCH64(
199 size_t count = 1 << rank;
200 size_t i, j, rrank = 64 - rank;
201 float *dp, *src = dst;
202 );
203
204 // Self algorithm
205 ARCH_AARCH64_ASM(
206 // Do bit-reverse shuffle
207 __ASM_EMIT("mov %[i], #1") // i = 1
208 __ASM_EMIT("1:")
209 __ASM_EMIT("rbit %[j], %[i]") // j = reverse_bits(i)
210 __ASM_EMIT("add %[src], %[src], #8")
211 __ASM_EMIT("lsr %[j], %[j], %[rrank]") // j = reverse_bits(i) >> rank
212 __ASM_EMIT("cmp %[i], %[j]") // i <=> j
213 __ASM_EMIT("b.hs 2f") // if (i >= j) continue
214 __ASM_EMIT("add %[dp], %[dst], %[j], LSL #3") // d_re = &dst[j]
215 __ASM_EMIT("ldr d0, [%[src]]") // d0 = *src
216 __ASM_EMIT("ldr d1, [%[dp]]") // d1 = *dp
217 __ASM_EMIT("str d0, [%[dp]]") // *dp = d0
218 __ASM_EMIT("str d1, [%[src]]") // *src = d1
219 __ASM_EMIT("2:")
220 __ASM_EMIT("add %[i], %[i], #1") // i++
221 __ASM_EMIT("cmp %[i], %[count]") // i <=> count
222 __ASM_EMIT("b.lo 1b")
223 // Perform x8 butterflies
224 __ASM_EMIT("3:")
225 __ASM_EMIT("ld4 {v0.4s-v3.4s}, [%[dst]]")
226 // v0 = r0 r2 r4 r6
227 // v1 = i0 i2 i4 i6
228 // v2 = r1 r3 r5 r7
229 // v3 = i1 i3 i5 i7
230 __ASM_EMIT("fadd v6.4s, v0.4s, v2.4s") // v6 = r0+r1 r2+r3 r4+r5 r6+r7 = r0' r2' r4' r6'
231 __ASM_EMIT("fadd v7.4s, v1.4s, v3.4s") // v7 = i0+i1 i2+i3 i4+i5 i6+i7 = i0' i2' i4' i6'
232 __ASM_EMIT("fsub v0.4s, v0.4s, v2.4s") // v0 = r0-r1 r2-r3 r4-r5 r6-r7 = r1' r3' r5' r7'
233 __ASM_EMIT("fsub v1.4s, v1.4s, v3.4s") // v1 = i0-i1 i2-i3 i4-i5 i6-i7 = i1' i3' i5' i7'
234 // v6 = r0' r2' r4' r6'
235 // v0 = r1' r3' r5' r7'
236 // v7 = i0' i2' i4' i6'
237 // v1 = i1' i3' i5' i7'
238 __ASM_EMIT("uzp1 v4.4s, v6.4s, v0.4s") // v4 = r0' r4' r1' r5'
239 __ASM_EMIT("uzp2 v0.4s, v6.4s, v0.4s") // v0 = r2' r6' r3' r7'
240 __ASM_EMIT("uzp1 v5.4s, v7.4s, v1.4s") // v5 = i0' i4' i1' i5'
241 __ASM_EMIT("uzp2 v1.4s, v7.4s, v1.4s") // v1 = i2' i6' i3' i7'
242 __ASM_EMIT("mov v2.16b, v0.16b") // v2 = r2' r6' r3' r7'
243 __ASM_EMIT("ins v0.d[1], v1.d[1]") // v0 = r2' r6' i3' i7'
244 __ASM_EMIT("ins v1.d[1], v2.d[1]") // v1 = i2' i6' r3' r7'
245 __ASM_EMIT("fadd v2.4s, v4.4s, v0.4s") // v2 = r0'+r2' r4'+r6' r1'+i3' r5'+i7' = r0" r4" r3" r7"
246 __ASM_EMIT("fsub v3.4s, v4.4s, v0.4s") // v3 = r0'-r2' r4'-r6' r1'-i3' r5'-i7' = r2" r6" r1" r5"
247 __ASM_EMIT("fadd v0.4s, v5.4s, v1.4s") // v0 = i0'+i2' i4'+i6' i1'+r3' i5'+r7' = i0" i4" i1" i5"
248 __ASM_EMIT("fsub v1.4s, v5.4s, v1.4s") // v1 = i0'-i2' i4'-i6' i1'-r3' i5'-r7' = i2" i6" i3" i7"
249 // v0 = i0" i4" i1" i5"
250 // v1 = i2" i6" i3" i7"
251 // v2 = r0" r4" r3" r7"
252 // v3 = r2" r6" r1" r5"
253 __ASM_EMIT("mov v4.16b, v2.16b") // v4 = r0" r4" r3" r7"
254 __ASM_EMIT("ins v2.d[1], v3.d[1]") // v2 = r0" r4" r1" r5"
255 __ASM_EMIT("ins v3.d[1], v4.d[1]") // v3 = r2" r6" r3" r7"
256 __ASM_EMIT("uzp1 v4.4s, v2.4s, v3.4s") // v4 = r0" r1" r2" r3"
257 __ASM_EMIT("uzp2 v5.4s, v2.4s, v3.4s") // v5 = r4" r5" r6" r7"
258 __ASM_EMIT("uzp1 v6.4s, v0.4s, v1.4s") // v6 = i0" i1" i2" i3"
259 __ASM_EMIT("uzp2 v7.4s, v0.4s, v1.4s") // v7 = i4" i5" i6" i7"
260 // Store
261 __ASM_EMIT("stp q4, q6, [%[dst], #0x00]")
262 __ASM_EMIT("stp q5, q7, [%[dst], #0x20]")
263 __ASM_EMIT("subs %[count], %[count], #8") // i <=> count
264 __ASM_EMIT("add %[dst], %[dst], #0x40")
265 __ASM_EMIT("b.ne 3b")
266
267 : [src] "+r" (src), [dst] "+r" (dst), [dp] "=&r" (dp),
268 [rrank] "+r" (rrank), [i] "=&r" (i), [j] "=&r" (j),
269 [count] "+r" (count)
270 :
271 : "cc", "memory",
272 "v0", "v1", "v2", "v3",
273 "v4", "v5", "v6", "v7"
274 );
275 }
276
277 void packed_scramble_copy_reverse(float *dst, const float *src, size_t rank)
278 {
279 IF_ARCH_AARCH64(
280 size_t i, j;
281 float *sr1, *sr2;
282 size_t regs = 1 << (rank - 3), rrank = 64 - (rank - 3);
283 );
284
285 ARCH_AARCH64_ASM(
286 __ASM_EMIT("eor %[i], %[i], %[i]") // i = 0
287 __ASM_EMIT("1:")
288 __ASM_EMIT("rbit %[j], %[i]") // j = reverse_bits(i)
289 __ASM_EMIT("lsr %[j], %[j], %[rrank]") // j = reverse_bits(i) >> rank
290
291 __ASM_EMIT("add %[sr1], %[src], %[j], LSL #3") // sr1 = &src[j]
292 __ASM_EMIT("add %[sr2], %[sr1], %[regs], LSL #3") // sr2 = &src[j + regs]
293 __ASM_EMIT("ld2 {v0.s, v1.s}[0], [%[sr1]]") // v0 = r0 ? ? ?, v1 = i0 ? ? ?
294 __ASM_EMIT("ld2 {v0.s, v1.s}[2], [%[sr2]]") // v0 = r0 ? r4 ?, v1 = i0 ? i4 ?
295 __ASM_EMIT("add %[sr1], %[sr1], %[regs], LSL #4") // sr1 = &src[j + regs*2]
296 __ASM_EMIT("add %[sr2], %[sr2], %[regs], LSL #4") // sr2 = &src[j + regs*3]
297 __ASM_EMIT("ld2 {v0.s, v1.s}[1], [%[sr1]]") // v0 = r0 r2 r4 ?, v1 = i0 i2 i4 ?
298 __ASM_EMIT("ld2 {v0.s, v1.s}[3], [%[sr2]]") // v0 = r0 r2 r4 r6, v1 = i0 i2 i4 i6
299
300 __ASM_EMIT("add %[sr1], %[sr1], %[regs], LSL #4") // sr1 = &src[j + regs*4]
301 __ASM_EMIT("add %[sr2], %[sr2], %[regs], LSL #4") // sr2 = &src[j + regs*5]
302 __ASM_EMIT("ld2 {v2.s, v3.s}[0], [%[sr1]]") // v2 = r1 ? ? ?, v3 = i1 ? ? ?
303 __ASM_EMIT("ld2 {v2.s, v3.s}[2], [%[sr2]]") // v2 = r1 ? r5 ?, v3 = i1 ? i5 ?
304 __ASM_EMIT("add %[sr1], %[sr1], %[regs], LSL #4") // sr1 = &src[j + regs*6]
305 __ASM_EMIT("add %[sr2], %[sr2], %[regs], LSL #4") // sr2 = &src[j + regs*7]
306 __ASM_EMIT("ld2 {v2.s, v3.s}[1], [%[sr1]]") // v2 = r1 r3 r5 ?, v3 = i1 i3 i5 ?
307 __ASM_EMIT("ld2 {v2.s, v3.s}[3], [%[sr2]]") // v2 = r1 r3 r5 r7, v3 = i1 i3 i5 i7
308 // v0 = r0 r2 r4 r6
309 // v1 = i0 i2 i4 i6
310 // v2 = r1 r3 r5 r7
311 // v3 = i1 i3 i5 i7
312 __ASM_EMIT("fadd v6.4s, v0.4s, v2.4s") // v6 = r0+r1 r2+r3 r4+r5 r6+r7 = r0' r2' r4' r6'
313 __ASM_EMIT("fadd v7.4s, v1.4s, v3.4s") // v7 = i0+i1 i2+i3 i4+i5 i6+i7 = i0' i2' i4' i6'
314 __ASM_EMIT("fsub v0.4s, v0.4s, v2.4s") // v0 = r0-r1 r2-r3 r4-r5 r6-r7 = r1' r3' r5' r7'
315 __ASM_EMIT("fsub v1.4s, v1.4s, v3.4s") // v1 = i0-i1 i2-i3 i4-i5 i6-i7 = i1' i3' i5' i7'
316 // v6 = r0' r2' r4' r6'
317 // v0 = r1' r3' r5' r7'
318 // v7 = i0' i2' i4' i6'
319 // v1 = i1' i3' i5' i7'
320 __ASM_EMIT("uzp1 v4.4s, v6.4s, v0.4s") // v4 = r0' r4' r1' r5'
321 __ASM_EMIT("uzp2 v0.4s, v6.4s, v0.4s") // v0 = r2' r6' r3' r7'
322 __ASM_EMIT("uzp1 v5.4s, v7.4s, v1.4s") // v5 = i0' i4' i1' i5'
323 __ASM_EMIT("uzp2 v1.4s, v7.4s, v1.4s") // v1 = i2' i6' i3' i7'
324 __ASM_EMIT("mov v2.16b, v0.16b") // v2 = r2' r6' r3' r7'
325 __ASM_EMIT("ins v0.d[1], v1.d[1]") // v0 = r2' r6' i3' i7'
326 __ASM_EMIT("ins v1.d[1], v2.d[1]") // v1 = i2' i6' r3' r7'
327 __ASM_EMIT("fadd v2.4s, v4.4s, v0.4s") // v2 = r0'+r2' r4'+r6' r1'+i3' r5'+i7' = r0" r4" r3" r7"
328 __ASM_EMIT("fsub v3.4s, v4.4s, v0.4s") // v3 = r0'-r2' r4'-r6' r1'-i3' r5'-i7' = r2" r6" r1" r5"
329 __ASM_EMIT("fadd v0.4s, v5.4s, v1.4s") // v0 = i0'+i2' i4'+i6' i1'+r3' i5'+r7' = i0" i4" i1" i5"
330 __ASM_EMIT("fsub v1.4s, v5.4s, v1.4s") // v1 = i0'-i2' i4'-i6' i1'-r3' i5'-r7' = i2" i6" i3" i7"
331 // v0 = i0" i4" i1" i5"
332 // v1 = i2" i6" i3" i7"
333 // v2 = r0" r4" r3" r7"
334 // v3 = r2" r6" r1" r5"
335 __ASM_EMIT("mov v4.16b, v2.16b") // v4 = r0" r4" r3" r7"
336 __ASM_EMIT("ins v2.d[1], v3.d[1]") // v2 = r0" r4" r1" r5"
337 __ASM_EMIT("ins v3.d[1], v4.d[1]") // v3 = r2" r6" r3" r7"
338 __ASM_EMIT("uzp1 v4.4s, v2.4s, v3.4s") // v4 = r0" r1" r2" r3"
339 __ASM_EMIT("uzp2 v5.4s, v2.4s, v3.4s") // v5 = r4" r5" r6" r7"
340 __ASM_EMIT("uzp1 v6.4s, v0.4s, v1.4s") // v6 = i0" i1" i2" i3"
341 __ASM_EMIT("uzp2 v7.4s, v0.4s, v1.4s") // v7 = i4" i5" i6" i7"
342 // Store
343 __ASM_EMIT("add %[i], %[i], #1")
344 __ASM_EMIT("stp q4, q6, [%[dst], #0x00]")
345 __ASM_EMIT("stp q5, q7, [%[dst], #0x20]")
346 __ASM_EMIT("cmp %[i], %[regs]")
347 __ASM_EMIT("add %[dst], %[dst], #0x40")
348 __ASM_EMIT("b.lo 1b")
349
350 : [dst] "+r" (dst), [sr1] "=&r" (sr1), [sr2] "=&r" (sr2),
351 [rrank] "+r" (rrank), [i] "=&r" (i), [j] "=&r" (j)
352 : [src] "r" (src), [regs] "r" (regs)
353 : "cc", "memory",
354 "v0", "v1", "v2", "v3",
355 "v4", "v5", "v6", "v7"
356 );
357 }
358
359 void packed_unscramble_direct(float *dst, size_t rank)
360 {
361 IF_ARCH_AARCH64(
362 size_t count = 1 << rank;
363 );
364
365 ARCH_AARCH64_ASM(
366 // 16x blocks
367 __ASM_EMIT("subs %[count], %[count], #16")
368 __ASM_EMIT("b.lo 2f")
369 __ASM_EMIT("1:")
370 __ASM_EMIT("ldp q0, q16, [%[dst], #0x00]")
371 __ASM_EMIT("ldp q2, q17, [%[dst], #0x20]")
372 __ASM_EMIT("ldp q4, q18, [%[dst], #0x40]")
373 __ASM_EMIT("ldp q6, q19, [%[dst], #0x60]")
374 __ASM_EMIT("zip2 v1.4s, v0.4s, v16.4s")
375 __ASM_EMIT("zip1 v0.4s, v0.4s, v16.4s")
376 __ASM_EMIT("zip2 v3.4s, v2.4s, v17.4s")
377 __ASM_EMIT("zip1 v2.4s, v2.4s, v17.4s")
378 __ASM_EMIT("zip2 v5.4s, v4.4s, v18.4s")
379 __ASM_EMIT("zip1 v4.4s, v4.4s, v18.4s")
380 __ASM_EMIT("zip2 v7.4s, v6.4s, v19.4s")
381 __ASM_EMIT("zip1 v6.4s, v6.4s, v19.4s")
382 __ASM_EMIT("stp q0, q1, [%[dst], #0x00]")
383 __ASM_EMIT("stp q2, q3, [%[dst], #0x20]")
384 __ASM_EMIT("stp q4, q5, [%[dst], #0x40]")
385 __ASM_EMIT("stp q6, q7, [%[dst], #0x60]")
386 __ASM_EMIT("subs %[count], %[count], #16")
387 __ASM_EMIT("add %[dst], %[dst], #0x80")
388 __ASM_EMIT("b.hs 1b")
389 __ASM_EMIT("2:")
390 // 8x block
391 __ASM_EMIT("adds %[count], %[count], #8")
392 __ASM_EMIT("b.lt 4f")
393 __ASM_EMIT("ldp q0, q16, [%[dst], #0x00]")
394 __ASM_EMIT("ldp q2, q17, [%[dst], #0x20]")
395 __ASM_EMIT("zip2 v1.4s, v0.4s, v16.4s")
396 __ASM_EMIT("zip1 v0.4s, v0.4s, v16.4s")
397 __ASM_EMIT("zip2 v3.4s, v2.4s, v17.4s")
398 __ASM_EMIT("zip1 v2.4s, v2.4s, v17.4s")
399 __ASM_EMIT("stp q0, q1, [%[dst], #0x00]")
400 __ASM_EMIT("stp q2, q3, [%[dst], #0x20]")
401 __ASM_EMIT("4:")
402 : [dst] "+r" (dst),
403 [count] "+r" (count)
404 :
405 : "cc", "memory",
406 "v0", "v1", "v2", "v3",
407 "v4", "v5", "v6", "v7",
408 "v16", "v17", "v18", "v19"
409 );
410 }
411
412 void packed_unscramble_reverse(float *dst, size_t rank)
413 {
414 IF_ARCH_AARCH64(
415 size_t count = 1 << rank;
416 float k = 1.0f/(1 << rank);
417 );
418
419 ARCH_AARCH64_ASM(
420 __ASM_EMIT("dup v31.4s, %[k].s[0]") // v31 = k
421 // 16x blocks
422 __ASM_EMIT("subs %[count], %[count], #16")
423 __ASM_EMIT("mov v30.16b, v31.16b")
424 __ASM_EMIT("b.lo 2f")
425 __ASM_EMIT("1:")
426 __ASM_EMIT("ldp q0, q16, [%[dst], #0x00]")
427 __ASM_EMIT("ldp q2, q17, [%[dst], #0x20]")
428 __ASM_EMIT("ldp q4, q18, [%[dst], #0x40]")
429 __ASM_EMIT("ldp q6, q19, [%[dst], #0x60]")
430 __ASM_EMIT("zip2 v1.4s, v0.4s, v16.4s")
431 __ASM_EMIT("zip1 v0.4s, v0.4s, v16.4s")
432 __ASM_EMIT("fmul v1.4s, v1.4s, v30.4s")
433 __ASM_EMIT("fmul v0.4s, v0.4s, v31.4s")
434 __ASM_EMIT("zip2 v3.4s, v2.4s, v17.4s")
435 __ASM_EMIT("zip1 v2.4s, v2.4s, v17.4s")
436 __ASM_EMIT("fmul v3.4s, v3.4s, v30.4s")
437 __ASM_EMIT("fmul v2.4s, v2.4s, v31.4s")
438 __ASM_EMIT("zip2 v5.4s, v4.4s, v18.4s")
439 __ASM_EMIT("zip1 v4.4s, v4.4s, v18.4s")
440 __ASM_EMIT("fmul v5.4s, v5.4s, v30.4s")
441 __ASM_EMIT("fmul v4.4s, v4.4s, v31.4s")
442 __ASM_EMIT("zip2 v7.4s, v6.4s, v19.4s")
443 __ASM_EMIT("zip1 v6.4s, v6.4s, v19.4s")
444 __ASM_EMIT("fmul v7.4s, v7.4s, v30.4s")
445 __ASM_EMIT("fmul v6.4s, v6.4s, v31.4s")
446 __ASM_EMIT("stp q0, q1, [%[dst], #0x00]")
447 __ASM_EMIT("stp q2, q3, [%[dst], #0x20]")
448 __ASM_EMIT("stp q4, q5, [%[dst], #0x40]")
449 __ASM_EMIT("stp q6, q7, [%[dst], #0x60]")
450 __ASM_EMIT("subs %[count], %[count], #16")
451 __ASM_EMIT("add %[dst], %[dst], #0x80")
452 __ASM_EMIT("b.hs 1b")
453 __ASM_EMIT("2:")
454 // 8x block
455 __ASM_EMIT("adds %[count], %[count], #8")
456 __ASM_EMIT("b.lt 4f")
457 __ASM_EMIT("ldp q0, q16, [%[dst], #0x00]")
458 __ASM_EMIT("ldp q2, q17, [%[dst], #0x20]")
459 __ASM_EMIT("zip2 v1.4s, v0.4s, v16.4s")
460 __ASM_EMIT("zip1 v0.4s, v0.4s, v16.4s")
461 __ASM_EMIT("fmul v1.4s, v1.4s, v30.4s")
462 __ASM_EMIT("fmul v0.4s, v0.4s, v31.4s")
463 __ASM_EMIT("zip2 v3.4s, v2.4s, v17.4s")
464 __ASM_EMIT("zip1 v2.4s, v2.4s, v17.4s")
465 __ASM_EMIT("fmul v3.4s, v3.4s, v30.4s")
466 __ASM_EMIT("fmul v2.4s, v2.4s, v31.4s")
467 __ASM_EMIT("stp q0, q1, [%[dst], #0x00]")
468 __ASM_EMIT("stp q2, q3, [%[dst], #0x20]")
469 __ASM_EMIT("4:")
470 : [dst] "+r" (dst),
471 [count] "+r" (count),
472 [k] "+w" (k)
473 :
474 : "cc", "memory",
475 "v0", "v1", "v2", "v3",
476 "v4", "v5", "v6", "v7",
477 "v16", "v17", "v18", "v19",
478 "v30", "v31"
479 );
480 }
481 }
482 }
483
484 #endif /* PRIVATE_DSP_ARCH_AARCH64_ASIMD_FFT_PSCRAMBLE_H_ */