Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/aarch64/asimd/dynamics/compressor.h

raw
1 /*
2 * Copyright (C) 2023 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2023 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 8 окт. 2023 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_AARCH64_ASIMD_DYNAMICS_COMPRESSOR_H_
23 #define PRIVATE_DSP_ARCH_AARCH64_ASIMD_DYNAMICS_COMPRESSOR_H_
24
25 #ifndef PRIVATE_DSP_ARCH_AARCH64_ASIMD_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_AARCH64_ASIMD_IMPL */
28
29 #include <private/dsp/arch/aarch64/asimd/pmath/exp.h>
30 #include <private/dsp/arch/aarch64/asimd/pmath/log.h>
31
32 namespace lsp
33 {
34 namespace asimd
35 {
36 #define KNEE_LOAD \
37 __ASM_EMIT("add %[off], %[comp], #0x10") \
38 __ASM_EMIT("ld4r {v12.4s, v13.4s, v14.4s, v15.4s}, [%[comp]]") /* v12=start, v13=end, v14=gain, v15=herm[0] */ \
39 __ASM_EMIT("ld4r {v28.4s, v29.4s, v30.4s, v31.4s}, [%[off]]" ) /* v28=herm[1], v29=herm[2], v30=tilt[0], v31=tilt[1] */ \
40
41 #define PROCESS_KNEE_SINGLE_X8 \
42 /* in: v0 = lx0, v1 = lx1 */ \
43 KNEE_LOAD \
44 __ASM_EMIT("ldp q6, q7, [%[mem], #0x00]") /* v6 = x0 */ \
45 __ASM_EMIT("fmul v2.4s, v0.4s, v15.4s") /* v2 = herm[0]*lx0 */ \
46 __ASM_EMIT("fmul v3.4s, v1.4s, v15.4s") \
47 __ASM_EMIT("fadd v2.4s, v2.4s, v28.4s") /* v2 = herm[0]*lx0+herm[1] */ \
48 __ASM_EMIT("fadd v3.4s, v3.4s, v28.4s") \
49 __ASM_EMIT("fmul v2.4s, v2.4s, v0.4s") /* v2 = (herm[0]*lx0+herm[1])*lx0 */ \
50 __ASM_EMIT("fmul v3.4s, v3.4s, v1.4s") \
51 __ASM_EMIT("fmul v4.4s, v0.4s, v30.4s") /* v4 = tilt[0]*lx0 */ \
52 __ASM_EMIT("fmul v5.4s, v1.4s, v30.4s") \
53 __ASM_EMIT("fcmge v6.4s, v6.4s, v13.4s") /* v6 = [x0 >= end] */ \
54 __ASM_EMIT("fcmge v7.4s, v7.4s, v13.4s") \
55 __ASM_EMIT("fadd v0.4s, v4.4s, v31.4s") /* v0 = TV = tilt[0]*lx0+tilt[1] */ \
56 __ASM_EMIT("fadd v1.4s, v5.4s, v31.4s") \
57 __ASM_EMIT("fadd v2.4s, v2.4s, v29.4s") /* v2 = KV = (herm[0]*lx0+herm[1])*lx0+herm[2] */ \
58 __ASM_EMIT("fadd v3.4s, v3.4s, v29.4s") \
59 __ASM_EMIT("bif v0.16b, v2.16b, v6.16b") /* v0 = [x0 >= end] ? TV : KV */ \
60 __ASM_EMIT("bif v1.16b, v3.16b, v7.16b") \
61 EXP_CORE_X8 /* v0 = EV = expf([x0 >= end] ? TV : KV) */ \
62 __ASM_EMIT("ldp q6, q7, [%[mem], #0x00]") /* v6 = x0 */ \
63 __ASM_EMIT("fcmge v6.4s, v12.4s, v6.4s") /* v6 = [x0 <= start] */ \
64 __ASM_EMIT("fcmge v7.4s, v12.4s, v7.4s") \
65 __ASM_EMIT("bit v0.16b, v14.16b, v6.16b") /* q0 = G = [x0 <= start] ? gain : EV */ \
66 __ASM_EMIT("bit v1.16b, v14.16b, v7.16b") \
67 /* out: v0 = g0, v1 = g1 */
68
69 #define PROCESS_KNEE_SINGLE_X4 \
70 /* in: v0 = lx0 */ \
71 KNEE_LOAD \
72 __ASM_EMIT("ldr q6, [%[mem], #0x00]") /* v6 = x0 */ \
73 __ASM_EMIT("fmul v2.4s, v0.4s, v15.4s") /* v2 = herm[0]*lx0 */ \
74 __ASM_EMIT("fadd v2.4s, v2.4s, v28.4s") /* v2 = herm[0]*lx0+herm[1] */ \
75 __ASM_EMIT("fmul v2.4s, v2.4s, v0.4s") /* v2 = (herm[0]*lx0+herm[1])*lx0 */ \
76 __ASM_EMIT("fmul v4.4s, v0.4s, v30.4s") /* v4 = tilt[0]*lx0 */ \
77 __ASM_EMIT("fcmge v6.4s, v6.4s, v13.4s") /* v6 = [x0 >= end] */ \
78 __ASM_EMIT("fadd v0.4s, v4.4s, v31.4s") /* v0 = TV = tilt[0]*lx0+tilt[1] */ \
79 __ASM_EMIT("fadd v2.4s, v2.4s, v29.4s") /* v2 = KV = (herm[0]*lx0+herm[1])*lx0+herm[2] */ \
80 __ASM_EMIT("bif v0.16b, v2.16b, v6.16b") /* v0 = [x0 >= end] ? TV : KV */ \
81 EXP_CORE_X4 /* v0 = EV = expf([x0 >= end] ? TV : KV) */ \
82 __ASM_EMIT("ldr q6, [%[mem], #0x00]") /* v6 = x0 */ \
83 __ASM_EMIT("fcmge v6.4s, v12.4s, v6.4s") /* v6 = [x0 <= start] */ \
84 __ASM_EMIT("bit v0.16b, v14.16b, v6.16b") /* q0 = G = [x0 <= start] ? gain : EV */ \
85 /* out: v0 = g0 */
86
87 #define PROCESS_COMP_FULL_X8 \
88 /* in: v0 = x0, v1 = x1 */ \
89 __ASM_EMIT("add %[off], %[comp], #0x20") \
90 __ASM_EMIT("ld1r {v12.4s}, [%[comp]]") /* v12 = start[0] */ \
91 __ASM_EMIT("ld1r {v13.4s}, [%[off]]") /* v13 = start[1] */ \
92 __ASM_EMIT("fabs v0.4s, v0.4s") /* v0 = fabsf(x0) */ \
93 __ASM_EMIT("fabs v1.4s, v1.4s") \
94 __ASM_EMIT("fcmgt v2.4s, v0.4s, v12.4s") /* v2 = [fabs(x0) > start[0]] */ \
95 __ASM_EMIT("fcmgt v3.4s, v1.4s, v12.4s") \
96 __ASM_EMIT("fcmgt v4.4s, v0.4s, v13.4s") /* v4 = [fabs(x0) > start[1]] */ \
97 __ASM_EMIT("fcmgt v5.4s, v1.4s, v13.4s") \
98 __ASM_EMIT("orr v2.16b, v2.16b, v3.16b") \
99 __ASM_EMIT("orr v4.16b, v4.16b, v5.16b") \
100 __ASM_EMIT("orr v2.16b, v2.16b, v4.16b") \
101 __ASM_EMIT("ext v4.16b, v2.16b, v2.16b, #8") \
102 __ASM_EMIT("orr v2.16b, v2.16b, v4.16b") \
103 __ASM_EMIT("ext v4.16b, v2.16b, v2.16b, #4") \
104 __ASM_EMIT("orr v2.16b, v2.16b, v4.16b") \
105 __ASM_EMIT("mov %w[off], v2.s[0]") \
106 __ASM_EMIT("cmp %w[off], #0") \
107 __ASM_EMIT("b.ne 100f") \
108 __ASM_EMIT("add %[off], %[comp], #0x08") \
109 __ASM_EMIT("ld1r {v12.4s}, [%[off]]") /* v12 = g1 */ \
110 __ASM_EMIT("add %[off], %[comp], #0x28") \
111 __ASM_EMIT("ld1r {v13.4s}, [%[off]]") /* v13 = g2 */ \
112 __ASM_EMIT("fmul v0.4s, v12.4s, v13.4s") /* v0 = g1*g2 */ \
113 __ASM_EMIT("fmul v1.4s, v12.4s, v13.4s") \
114 __ASM_EMIT("b 200f") \
115 __ASM_EMIT("100:") \
116 __ASM_EMIT("stp q0, q1, [%[mem], #0x00]") /* mem[0x00] = fabfs(x0) */ \
117 LOGE_CORE_X8 /* v0= lx0 = logf(fabsf(x0)) */ \
118 __ASM_EMIT("stp q0, q1, [%[mem], #0x20]") /* mem[0x20] = lx0 */ \
119 PROCESS_KNEE_SINGLE_X8 /* apply knee 0 */ \
120 __ASM_EMIT("stp q0, q1, [%[mem], #0x40]") /* mem[0x40] = g0 */ \
121 __ASM_EMIT("ldp q0, q1, [%[mem], #0x20]") /* q0 = lx0 */ \
122 __ASM_EMIT("add %[comp], %[comp], #0x20") \
123 PROCESS_KNEE_SINGLE_X8 /* apply knee 1 */ \
124 __ASM_EMIT("ldp q2, q3, [%[mem], #0x40]") /* v2 = g0 */ \
125 __ASM_EMIT("sub %[comp], %[comp], #0x20") \
126 __ASM_EMIT("fmul v0.4s, v0.4s, v2.4s") /* v0 = G = */ \
127 __ASM_EMIT("fmul v1.4s, v1.4s, v3.4s") \
128 __ASM_EMIT("200:") \
129 /* out: v0 = G0, v1= G1 */
130
131 #define PROCESS_COMP_FULL_X4 \
132 /* in: q0 = x0, q1 = x1 */ \
133 __ASM_EMIT("fabs v0.4s, v0.4s") /* v0 = fabsf(x0) */ \
134 __ASM_EMIT("str q0, [%[mem], #0x00]") /* mem[0x00] = fabfs(x0) */ \
135 LOGE_CORE_X4 /* v0= lx0 = logf(fabsf(x0)) */ \
136 __ASM_EMIT("str q0, [%[mem], #0x20]") /* mem[0x20] = lx0 */ \
137 PROCESS_KNEE_SINGLE_X4 /* apply knee 0 */ \
138 __ASM_EMIT("str q0, [%[mem], #0x40]") /* mem[0x40] = g0 */ \
139 __ASM_EMIT("ldr q0, [%[mem], #0x20]") /* q0 = lx0 */ \
140 __ASM_EMIT("add %[comp], %[comp], #0x20") \
141 PROCESS_KNEE_SINGLE_X4 /* apply knee 1 */ \
142 __ASM_EMIT("ldr q2, [%[mem], #0x40]") /* v2 = g0 */ \
143 __ASM_EMIT("sub %[comp], %[comp], #0x20") \
144 __ASM_EMIT("fmul v0.4s, v0.4s, v2.4s") /* v0 = G = */ \
145 /* out: q0 = G0, q1= G1 */
146
147 void compressor_x2_gain(float *dst, const float *src, const dsp::compressor_x2_t *c, size_t count)
148 {
149 IF_ARCH_AARCH64(
150 float mem[24] __lsp_aligned16;
151 size_t off;
152 );
153
154 ARCH_AARCH64_ASM(
155 // x8 blocks
156 __ASM_EMIT("subs %[count], %[count], #8")
157 __ASM_EMIT("b.lo 2f")
158 __ASM_EMIT("1:")
159 __ASM_EMIT("ldp q0, q1, [%[src], #0x00]")
160 PROCESS_COMP_FULL_X8
161 __ASM_EMIT("subs %[count], %[count], #8")
162 __ASM_EMIT("stp q0, q1, [%[dst], #0x00]")
163 __ASM_EMIT("add %[src], %[src], 0x20")
164 __ASM_EMIT("add %[dst], %[dst], 0x20")
165 __ASM_EMIT("b.hs 1b")
166 __ASM_EMIT("2:")
167 // x4 block
168 __ASM_EMIT("adds %[count], %[count], #4")
169 __ASM_EMIT("b.lt 4f")
170 __ASM_EMIT("ldr q0, [%[src], #0x00]")
171 PROCESS_COMP_FULL_X4
172 __ASM_EMIT("sub %[count], %[count], #4")
173 __ASM_EMIT("str q0, [%[dst], #0x00]")
174 __ASM_EMIT("add %[src], %[src], 0x10")
175 __ASM_EMIT("add %[dst], %[dst], 0x10")
176 __ASM_EMIT("4:")
177 // Tail: 1x-3x block
178 __ASM_EMIT("adds %[count], %[count], #4")
179 __ASM_EMIT("b.ls 12f")
180 __ASM_EMIT("tst %[count], #1")
181 __ASM_EMIT("b.eq 6f")
182 __ASM_EMIT("ld1 {v0.s}[0], [%[src]]")
183 __ASM_EMIT("add %[src], %[src], #0x04")
184 __ASM_EMIT("6:")
185 __ASM_EMIT("tst %[count], #2")
186 __ASM_EMIT("b.eq 8f")
187 __ASM_EMIT("ld1 {v0.d}[1], [%[src]]")
188 __ASM_EMIT("8:")
189 PROCESS_COMP_FULL_X4
190 __ASM_EMIT("tst %[count], #1")
191 __ASM_EMIT("b.eq 10f")
192 __ASM_EMIT("st1 {v0.s}[0], [%[dst]]")
193 __ASM_EMIT("add %[dst], %[dst], #0x04")
194 __ASM_EMIT("10:")
195 __ASM_EMIT("tst %[count], #2")
196 __ASM_EMIT("b.eq 12f")
197 __ASM_EMIT("st1 {v0.d}[1], [%[dst]]")
198 __ASM_EMIT("12:")
199
200 : [dst] "+r" (dst), [src] "+r" (src),
201 [count] "+r" (count),
202 [off] "=&r" (off)
203 : [comp] "r" (c),
204 [L2C] "r" (&LOG2_CONST[0]),
205 [LOGC] "r" (&LOGE_C[0]),
206 [mem] "r" (&mem[0]),
207 [E2C] "r" (&EXP2_CONST[0]),
208 [LOG2E] "r" (&EXP_LOG2E[0])
209 : "cc", "memory",
210 "v0", "v1", "v2", "v3",
211 "v4", "v5", "v6", "v7",
212 "v8", "v9", "v10", "v11",
213 "v12", "v13", "v14", "v15",
214 "v16", "v17", "v18", "v19",
215 "v20", "v21", "v22", "v23",
216 "v24", "v25", "v26", "v27",
217 "v28", "v29", "v30", "v31"
218 );
219 }
220
221 void compressor_x2_curve(float *dst, const float *src, const dsp::compressor_x2_t *c, size_t count)
222 {
223 IF_ARCH_AARCH64(
224 float mem[24] __lsp_aligned16;
225 size_t off;
226 );
227
228 ARCH_AARCH64_ASM(
229 // x8 blocks
230 __ASM_EMIT("subs %[count], %[count], #8")
231 __ASM_EMIT("b.lo 2f")
232 __ASM_EMIT("1:")
233 __ASM_EMIT("ldp q0, q1, [%[src], #0x00]")
234 PROCESS_COMP_FULL_X8
235 __ASM_EMIT("ldp q2, q3, [%[src], #0x00]")
236 __ASM_EMIT("subs %[count], %[count], #8")
237 __ASM_EMIT("fmul v0.4s, v0.4s, v2.4s")
238 __ASM_EMIT("fmul v1.4s, v1.4s, v3.4s")
239 __ASM_EMIT("stp q0, q1, [%[dst], #0x00]")
240 __ASM_EMIT("add %[src], %[src], 0x20")
241 __ASM_EMIT("add %[dst], %[dst], 0x20")
242 __ASM_EMIT("b.hs 1b")
243 __ASM_EMIT("2:")
244 // x4 block
245 __ASM_EMIT("adds %[count], %[count], #4")
246 __ASM_EMIT("b.lt 4f")
247 __ASM_EMIT("ldr q0, [%[src], #0x00]")
248 PROCESS_COMP_FULL_X4
249 __ASM_EMIT("ldr q2, [%[src], #0x00]")
250 __ASM_EMIT("sub %[count], %[count], #4")
251 __ASM_EMIT("fmul v0.4s, v0.4s, v2.4s")
252 __ASM_EMIT("str q0, [%[dst], #0x00]")
253 __ASM_EMIT("add %[src], %[src], 0x10")
254 __ASM_EMIT("add %[dst], %[dst], 0x10")
255 __ASM_EMIT("4:")
256 // Tail: 1x-3x block
257 __ASM_EMIT("adds %[count], %[count], #4")
258 __ASM_EMIT("b.ls 12f")
259 __ASM_EMIT("tst %[count], #1")
260 __ASM_EMIT("b.eq 6f")
261 __ASM_EMIT("ld1 {v0.s}[0], [%[src]]")
262 __ASM_EMIT("add %[src], %[src], #0x04")
263 __ASM_EMIT("6:")
264 __ASM_EMIT("tst %[count], #2")
265 __ASM_EMIT("b.eq 8f")
266 __ASM_EMIT("ld1 {v0.d}[1], [%[src]]")
267 __ASM_EMIT("8:")
268 __ASM_EMIT("mov v1.16b, v0.16b")
269 PROCESS_COMP_FULL_X4
270 __ASM_EMIT("fmul v0.4s, v0.4s, v1.4s")
271 __ASM_EMIT("tst %[count], #1")
272 __ASM_EMIT("b.eq 10f")
273 __ASM_EMIT("st1 {v0.s}[0], [%[dst]]")
274 __ASM_EMIT("add %[dst], %[dst], #0x04")
275 __ASM_EMIT("10:")
276 __ASM_EMIT("tst %[count], #2")
277 __ASM_EMIT("b.eq 12f")
278 __ASM_EMIT("st1 {v0.d}[1], [%[dst]]")
279 __ASM_EMIT("12:")
280
281 : [dst] "+r" (dst), [src] "+r" (src),
282 [count] "+r" (count),
283 [off] "=&r" (off)
284 : [comp] "r" (c),
285 [L2C] "r" (&LOG2_CONST[0]),
286 [LOGC] "r" (&LOGE_C[0]),
287 [mem] "r" (&mem[0]),
288 [E2C] "r" (&EXP2_CONST[0]),
289 [LOG2E] "r" (&EXP_LOG2E[0])
290 : "cc", "memory",
291 "v0", "v1", "v2", "v3",
292 "v4", "v5", "v6", "v7",
293 "v8", "v9", "v10", "v11",
294 "v12", "v13", "v14", "v15",
295 "v16", "v17", "v18", "v19",
296 "v20", "v21", "v22", "v23",
297 "v24", "v25", "v26", "v27",
298 "v28", "v29", "v30", "v31"
299 );
300 }
301
302 #undef KNEE_LOAD
303 #undef PROCESS_KNEE_SINGLE_X4
304 #undef PROCESS_KNEE_SINGLE_X8
305 #undef PROCESS_COMP_FULL_X8
306 #undef PROCESS_COMP_FULL_X4
307
308 } /* namespace asimd */
309 } /* namespace lsp */
310
311
312 #endif /* PRIVATE_DSP_ARCH_AARCH64_ASIMD_DYNAMICS_COMPRESSOR_H_ */