Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/aarch64/asimd/dynamics/gate.h

raw
1 /*
2 * Copyright (C) 2023 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2023 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 20 окт. 2023 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_AARCH64_ASIMD_DYNAMICS_GATE_H_
23 #define PRIVATE_DSP_ARCH_AARCH64_ASIMD_DYNAMICS_GATE_H_
24
25 #ifndef PRIVATE_DSP_ARCH_AARCH64_ASIMD_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_AARCH64_ASIMD_IMPL */
28
29 #include <private/dsp/arch/aarch64/asimd/pmath/exp.h>
30 #include <private/dsp/arch/aarch64/asimd/pmath/log.h>
31
32 namespace lsp
33 {
34 namespace asimd
35 {
36 #define PROCESS_KNEE_SINGLE_X8 \
37 /* in: v0 = lx0, v1 = lx1 */ \
38 __ASM_EMIT("add %[off], %[gate], #0x10") \
39 __ASM_EMIT("ld4r {v8.4s, v9.4s, v10.4s, v11.4s}, [%[off]]" ) /* v8=herm[0], v9=herm[1], v10=herm[2], v11=herm[3] */ \
40 __ASM_EMIT("fmul v2.4s, v0.4s, v8.4s") /* v2 = herm[0]*lx0 */ \
41 __ASM_EMIT("fmul v3.4s, v1.4s, v8.4s") \
42 __ASM_EMIT("fadd v2.4s, v2.4s, v9.4s") /* v2 = herm[0]*lx0+herm[1] */ \
43 __ASM_EMIT("fadd v3.4s, v3.4s, v9.4s") \
44 __ASM_EMIT("fmul v2.4s, v2.4s, v0.4s") /* v2 = (herm[0]*lx0+herm[1])*lx0 */ \
45 __ASM_EMIT("fmul v3.4s, v3.4s, v1.4s") \
46 __ASM_EMIT("fadd v2.4s, v2.4s, v10.4s") /* v2 = (herm[0]*lx0+herm[1])*lx0+herm[2] */ \
47 __ASM_EMIT("fadd v3.4s, v3.4s, v10.4s") \
48 __ASM_EMIT("fmul v2.4s, v2.4s, v0.4s") /* v2 = ((herm[0]*lx0+herm[1])*lx0+herm[2])*lx0 */ \
49 __ASM_EMIT("fmul v3.4s, v3.4s, v1.4s") \
50 __ASM_EMIT("fadd v0.4s, v2.4s, v11.4s") /* v0 = TV = ((herm[0]*lx0+herm[1])*lx0+herm[2])*lx0+herm[3] */ \
51 __ASM_EMIT("fadd v1.4s, v3.4s, v11.4s") \
52 EXP_CORE_X8 /* v0 = KV = expf(TV) */ \
53 __ASM_EMIT("ldp q6, q7, [%[mem], #0x00]") /* v6 = x0 */ \
54 __ASM_EMIT("ld4r {v2.4s, v3.4s, v4.4s, v5.4s}, [%[gate]]" ) /* v2=start, v3=end, v4=gain_start, v5=gain_end */ \
55 __ASM_EMIT("fcmgt v8.4s, v6.4s, v2.4s") /* v8 = [x0 > start] */ \
56 __ASM_EMIT("fcmgt v9.4s, v7.4s, v2.4s") \
57 __ASM_EMIT("fcmge v6.4s, v6.4s, v3.4s") /* v6 = [x0 >= end] */ \
58 __ASM_EMIT("fcmge v7.4s, v7.4s, v3.4s") \
59 __ASM_EMIT("bif v0.16b, v4.16b, v8.16b") /* v0 = [x0 <= start] ? gain_start : KV */ \
60 __ASM_EMIT("bif v1.16b, v4.16b, v9.16b") \
61 __ASM_EMIT("bit v0.16b, v5.16b, v6.16b") /* v0 = [x0 >= end] ? gain_end : [x0 <= start] ? gain_start : KV */ \
62 __ASM_EMIT("bit v1.16b, v5.16b, v7.16b") \
63 /* out: v0 = g0, v1 = g1 */
64
65 #define PROCESS_KNEE_SINGLE_X4 \
66 /* in: v0 = lx0 */ \
67 __ASM_EMIT("add %[off], %[gate], #0x10") \
68 __ASM_EMIT("ld4r {v8.4s, v9.4s, v10.4s, v11.4s}, [%[off]]" ) /* v8=herm[0], v9=herm[1], v10=herm[2], v11=herm[3] */ \
69 __ASM_EMIT("fmul v2.4s, v0.4s, v8.4s") /* v2 = herm[0]*lx0 */ \
70 __ASM_EMIT("fadd v2.4s, v2.4s, v9.4s") /* v2 = herm[0]*lx0+herm[1] */ \
71 __ASM_EMIT("fmul v2.4s, v2.4s, v0.4s") /* v2 = (herm[0]*lx0+herm[1])*lx0 */ \
72 __ASM_EMIT("fadd v2.4s, v2.4s, v10.4s") /* v2 = (herm[0]*lx0+herm[1])*lx0+herm[2] */ \
73 __ASM_EMIT("fmul v2.4s, v2.4s, v0.4s") /* v2 = ((herm[0]*lx0+herm[1])*lx0+herm[2])*lx0 */ \
74 __ASM_EMIT("fadd v0.4s, v2.4s, v11.4s") /* v0 = TV = ((herm[0]*lx0+herm[1])*lx0+herm[2])*lx0+herm[3] */ \
75 EXP_CORE_X4 /* v0 = KV = expf(TV) */ \
76 __ASM_EMIT("ldp q6, q7, [%[mem], #0x00]") /* v6 = x0 */ \
77 __ASM_EMIT("ld4r {v2.4s, v3.4s, v4.4s, v5.4s}, [%[gate]]" ) /* v2=start, v3=end, v4=gain_start, v5=gain_end */ \
78 __ASM_EMIT("fcmgt v8.4s, v6.4s, v2.4s") /* v8 = [x0 > start] */ \
79 __ASM_EMIT("fcmge v6.4s, v6.4s, v3.4s") /* v6 = [x0 >= end] */ \
80 __ASM_EMIT("bif v0.16b, v4.16b, v8.16b") /* v0 = [x0 <= start] ? gain_start : KV */ \
81 __ASM_EMIT("bit v0.16b, v5.16b, v6.16b") /* v0 = [x0 >= end] ? gain_end : [x0 <= start] ? gain_start : KV */ \
82 /* out: v0 = g0 */
83
84 #define PROCESS_GATE_FULL_X8 \
85 /* in: v0 = x0, v1 = x1 */ \
86 __ASM_EMIT("ld2r {v12.4s, v13.4s}, [%[gate]]") /* v12 = begin, v13 = end */ \
87 __ASM_EMIT("fabs v0.4s, v0.4s") /* v0 = fabsf(x0) */ \
88 __ASM_EMIT("fabs v1.4s, v1.4s") \
89 __ASM_EMIT("fcmgt v2.4s, v0.4s, v12.4s") /* v2 = [fabs(x0) > begin] */ \
90 __ASM_EMIT("fcmgt v3.4s, v1.4s, v12.4s") \
91 __ASM_EMIT("fcmgt v4.4s, v13.4s, v0.4s") /* v4 = [fabs(x0) < end] */ \
92 __ASM_EMIT("fcmgt v5.4s, v13.4s, v1.4s") \
93 __ASM_EMIT("and v6.16b, v2.16b, v4.16b") /* v6 = [fabs(x0) > begin] && [fabs(x0) < end] */ \
94 __ASM_EMIT("and v7.16b, v3.16b, v5.16b") \
95 __ASM_EMIT("orr v6.16b, v6.16b, v7.16b") \
96 __ASM_EMIT("ext v7.16b, v6.16b, v6.16b, #8") \
97 __ASM_EMIT("orr v6.16b, v6.16b, v7.16b") \
98 __ASM_EMIT("ext v7.16b, v6.16b, v6.16b, #4") \
99 __ASM_EMIT("orr v6.16b, v6.16b, v7.16b") \
100 __ASM_EMIT("mov %w[off], v6.s[0]") \
101 __ASM_EMIT("cmp %w[off], #0") \
102 __ASM_EMIT("b.ne 100f") \
103 __ASM_EMIT("add %[off], %[gate], #0x08") \
104 __ASM_EMIT("ld1r {v0.4s}, [%[off]]") /* v0 = gain_begin */ \
105 __ASM_EMIT("add %[off], %[gate], #0x0c") \
106 __ASM_EMIT("ld1r {v4.4s}, [%[off]]") /* v4 = gain_end */ \
107 __ASM_EMIT("mov v1.16b, v0.16b") \
108 __ASM_EMIT("bit v0.16b, v4.16b, v2.16b") /* v0 = [fabs(x0) > start] ? gain_end : gain_start */ \
109 __ASM_EMIT("bit v1.16b, v4.16b, v3.16b") \
110 __ASM_EMIT("b 200f") \
111 __ASM_EMIT("100:") \
112 __ASM_EMIT("stp q0, q1, [%[mem], #0x00]") /* mem[0x00] = fabfs(x0) */ \
113 LOGE_CORE_X8 /* v0= lx0 = logf(fabsf(x0)) */ \
114 PROCESS_KNEE_SINGLE_X8 /* apply knee 0 */ \
115 __ASM_EMIT("200:") \
116 /* out: v0 = G0, v1= G1 */
117
118 #define PROCESS_GATE_FULL_X4 \
119 /* in: v0 = x0 */ \
120 __ASM_EMIT("fabs v0.4s, v0.4s") /* v0 = fabsf(x0) */ \
121 __ASM_EMIT("str q0, [%[mem], #0x00]") /* mem[0x00] = fabfs(x0) */ \
122 LOGE_CORE_X4 /* v0= lx0 = logf(fabsf(x0)) */ \
123 PROCESS_KNEE_SINGLE_X4 /* apply knee 0 */ \
124 /* out: v0 = G0 */
125
126 void gate_x1_gain(float *dst, const float *src, const dsp::gate_knee_t *c, size_t count)
127 {
128 IF_ARCH_AARCH64(
129 float mem[8] __lsp_aligned16;
130 size_t off;
131 );
132
133 ARCH_AARCH64_ASM(
134 // x8 blocks
135 __ASM_EMIT("subs %[count], %[count], #8")
136 __ASM_EMIT("b.lo 2f")
137 __ASM_EMIT("1:")
138 __ASM_EMIT("ldp q0, q1, [%[src], #0x00]")
139 PROCESS_GATE_FULL_X8
140 __ASM_EMIT("subs %[count], %[count], #8")
141 __ASM_EMIT("stp q0, q1, [%[dst], #0x00]")
142 __ASM_EMIT("add %[src], %[src], 0x20")
143 __ASM_EMIT("add %[dst], %[dst], 0x20")
144 __ASM_EMIT("b.hs 1b")
145 __ASM_EMIT("2:")
146 // x4 block
147 __ASM_EMIT("adds %[count], %[count], #4")
148 __ASM_EMIT("b.lt 4f")
149 __ASM_EMIT("ldr q0, [%[src], #0x00]")
150 PROCESS_GATE_FULL_X4
151 __ASM_EMIT("sub %[count], %[count], #4")
152 __ASM_EMIT("str q0, [%[dst], #0x00]")
153 __ASM_EMIT("add %[src], %[src], 0x10")
154 __ASM_EMIT("add %[dst], %[dst], 0x10")
155 __ASM_EMIT("4:")
156 // Tail: 1x-3x block
157 __ASM_EMIT("adds %[count], %[count], #4")
158 __ASM_EMIT("b.ls 12f")
159 __ASM_EMIT("tst %[count], #1")
160 __ASM_EMIT("b.eq 6f")
161 __ASM_EMIT("ld1 {v0.s}[0], [%[src]]")
162 __ASM_EMIT("add %[src], %[src], #0x04")
163 __ASM_EMIT("6:")
164 __ASM_EMIT("tst %[count], #2")
165 __ASM_EMIT("b.eq 8f")
166 __ASM_EMIT("ld1 {v0.d}[1], [%[src]]")
167 __ASM_EMIT("8:")
168 PROCESS_GATE_FULL_X4
169 __ASM_EMIT("tst %[count], #1")
170 __ASM_EMIT("b.eq 10f")
171 __ASM_EMIT("st1 {v0.s}[0], [%[dst]]")
172 __ASM_EMIT("add %[dst], %[dst], #0x04")
173 __ASM_EMIT("10:")
174 __ASM_EMIT("tst %[count], #2")
175 __ASM_EMIT("b.eq 12f")
176 __ASM_EMIT("st1 {v0.d}[1], [%[dst]]")
177 __ASM_EMIT("12:")
178
179 : [dst] "+r" (dst), [src] "+r" (src),
180 [count] "+r" (count),
181 [off] "=&r" (off)
182 : [gate] "r" (c),
183 [L2C] "r" (&LOG2_CONST[0]),
184 [LOGC] "r" (&LOGE_C[0]),
185 [mem] "r" (&mem[0]),
186 [E2C] "r" (&EXP2_CONST[0]),
187 [LOG2E] "r" (&EXP_LOG2E[0])
188 : "cc", "memory",
189 "v0", "v1", "v2", "v3",
190 "v4", "v5", "v6", "v7",
191 "v8", "v9", "v10", "v11",
192 "v12", "v13", "v14", "v15",
193 "v16", "v17", "v18", "v19",
194 "v20", "v21", "v22", "v23",
195 "v24", "v25", "v26", "v27",
196 "v28", "v29", "v30", "v31"
197 );
198 }
199
200 void gate_x1_curve(float *dst, const float *src, const dsp::gate_knee_t *c, size_t count)
201 {
202 IF_ARCH_AARCH64(
203 float mem[8] __lsp_aligned16;
204 size_t off;
205 );
206
207 ARCH_AARCH64_ASM(
208 // x8 blocks
209 __ASM_EMIT("subs %[count], %[count], #8")
210 __ASM_EMIT("b.lo 2f")
211 __ASM_EMIT("1:")
212 __ASM_EMIT("ldp q0, q1, [%[src], #0x00]")
213 PROCESS_GATE_FULL_X8
214 __ASM_EMIT("ldp q2, q3, [%[src], #0x00]")
215 __ASM_EMIT("subs %[count], %[count], #8")
216 __ASM_EMIT("fmul v0.4s, v0.4s, v2.4s")
217 __ASM_EMIT("fmul v1.4s, v1.4s, v3.4s")
218 __ASM_EMIT("stp q0, q1, [%[dst], #0x00]")
219 __ASM_EMIT("add %[src], %[src], 0x20")
220 __ASM_EMIT("add %[dst], %[dst], 0x20")
221 __ASM_EMIT("b.hs 1b")
222 __ASM_EMIT("2:")
223 // x4 block
224 __ASM_EMIT("adds %[count], %[count], #4")
225 __ASM_EMIT("b.lt 4f")
226 __ASM_EMIT("ldr q0, [%[src], #0x00]")
227 PROCESS_GATE_FULL_X4
228 __ASM_EMIT("ldr q2, [%[src], #0x00]")
229 __ASM_EMIT("sub %[count], %[count], #4")
230 __ASM_EMIT("fmul v0.4s, v0.4s, v2.4s")
231 __ASM_EMIT("str q0, [%[dst], #0x00]")
232 __ASM_EMIT("add %[src], %[src], 0x10")
233 __ASM_EMIT("add %[dst], %[dst], 0x10")
234 __ASM_EMIT("4:")
235 // Tail: 1x-3x block
236 __ASM_EMIT("adds %[count], %[count], #4")
237 __ASM_EMIT("b.ls 12f")
238 __ASM_EMIT("tst %[count], #1")
239 __ASM_EMIT("b.eq 6f")
240 __ASM_EMIT("ld1 {v0.s}[0], [%[src]]")
241 __ASM_EMIT("add %[src], %[src], #0x04")
242 __ASM_EMIT("6:")
243 __ASM_EMIT("tst %[count], #2")
244 __ASM_EMIT("b.eq 8f")
245 __ASM_EMIT("ld1 {v0.d}[1], [%[src]]")
246 __ASM_EMIT("8:")
247 __ASM_EMIT("mov v1.16b, v0.16b")
248 PROCESS_GATE_FULL_X4
249 __ASM_EMIT("fmul v0.4s, v0.4s, v1.4s")
250 __ASM_EMIT("tst %[count], #1")
251 __ASM_EMIT("b.eq 10f")
252 __ASM_EMIT("st1 {v0.s}[0], [%[dst]]")
253 __ASM_EMIT("add %[dst], %[dst], #0x04")
254 __ASM_EMIT("10:")
255 __ASM_EMIT("tst %[count], #2")
256 __ASM_EMIT("b.eq 12f")
257 __ASM_EMIT("st1 {v0.d}[1], [%[dst]]")
258 __ASM_EMIT("12:")
259
260 : [dst] "+r" (dst), [src] "+r" (src),
261 [count] "+r" (count),
262 [off] "=&r" (off)
263 : [gate] "r" (c),
264 [L2C] "r" (&LOG2_CONST[0]),
265 [LOGC] "r" (&LOGE_C[0]),
266 [mem] "r" (&mem[0]),
267 [E2C] "r" (&EXP2_CONST[0]),
268 [LOG2E] "r" (&EXP_LOG2E[0])
269 : "cc", "memory",
270 "v0", "v1", "v2", "v3",
271 "v4", "v5", "v6", "v7",
272 "v8", "v9", "v10", "v11",
273 "v12", "v13", "v14", "v15",
274 "v16", "v17", "v18", "v19",
275 "v20", "v21", "v22", "v23",
276 "v24", "v25", "v26", "v27",
277 "v28", "v29", "v30", "v31"
278 );
279 }
280
281 #undef KNEE_LOAD
282 #undef PROCESS_KNEE_SINGLE_X4
283 #undef PROCESS_KNEE_SINGLE_X8
284 #undef PROCESS_GATE_FULL_X8
285 #undef PROCESS_GATE_FULL_X4
286
287 } /* namespace asimd */
288 } /* namespace lsp */
289
290
291
292
293 #endif /* PRIVATE_DSP_ARCH_AARCH64_ASIMD_DYNAMICS_GATE_H_ */