Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/aarch64/asimd/dynamics/expander.h

raw
1 /*
2 * Copyright (C) 2023 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2023 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 4 нояб. 2023 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef INCLUDE_PRIVATE_DSP_ARCH_AARCH64_ASIMD_DYNAMICS_EXPANDER_H_
23 #define INCLUDE_PRIVATE_DSP_ARCH_AARCH64_ASIMD_DYNAMICS_EXPANDER_H_
24
25 #ifndef PRIVATE_DSP_ARCH_AARCH64_ASIMD_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_AARCH64_ASIMD_IMPL */
28
29 #include <private/dsp/arch/aarch64/asimd/pmath/exp.h>
30 #include <private/dsp/arch/aarch64/asimd/pmath/log.h>
31
32 namespace lsp
33 {
34 namespace asimd
35 {
36 IF_ARCH_AARCH64(
37 static const float expander_const[] __lsp_aligned16 =
38 {
39 LSP_DSP_VEC4(1.0f)
40 };
41 )
42
43 #define PROCESS_UKNEE_SINGLE_X8 \
44 /* in: v0 = lx0, v1 = lx1 */ \
45 __ASM_EMIT("add %[off], %[exp], #0x10") \
46 __ASM_EMIT("ldp q6, q7, [%[mem], #0x00]") /* v6 = x0, v7 = x1 */ \
47 __ASM_EMIT("ld4r {v12.4s, v13.4s, v14.4s, v15.4s}, [%[exp]]") /* v12=start, v13=end, v14=threshold, v15=herm[0] */ \
48 __ASM_EMIT("ld4r {v28.4s, v29.4s, v30.4s, v31.4s}, [%[off]]" ) /* v28=herm[1], v29=herm[2], v30=tilt[0], v31=tilt[1] */ \
49 __ASM_EMIT("fmul v2.4s, v0.4s, v15.4s") /* v2 = herm[0]*lx0 */ \
50 __ASM_EMIT("fmul v3.4s, v1.4s, v15.4s") \
51 __ASM_EMIT("fadd v2.4s, v2.4s, v28.4s") /* v2 = herm[0]*lx0+herm[1] */ \
52 __ASM_EMIT("fadd v3.4s, v3.4s, v28.4s") \
53 __ASM_EMIT("fmul v2.4s, v2.4s, v0.4s") /* v2 = (herm[0]*lx0+herm[1])*lx0 */ \
54 __ASM_EMIT("fmul v3.4s, v3.4s, v1.4s") \
55 __ASM_EMIT("fmul v4.4s, v0.4s, v30.4s") /* v4 = tilt[0]*lx0 */ \
56 __ASM_EMIT("fmul v5.4s, v1.4s, v30.4s") \
57 __ASM_EMIT("fcmge v6.4s, v6.4s, v13.4s") /* v6 = [x0 >= end] */ \
58 __ASM_EMIT("fcmge v7.4s, v7.4s, v13.4s") \
59 __ASM_EMIT("fadd v0.4s, v4.4s, v31.4s") /* v0 = TV = tilt[0]*lx0+tilt[1] */ \
60 __ASM_EMIT("fadd v1.4s, v5.4s, v31.4s") \
61 __ASM_EMIT("fadd v2.4s, v2.4s, v29.4s") /* v2 = KV = (herm[0]*lx0+herm[1])*lx0+herm[2] */ \
62 __ASM_EMIT("fadd v3.4s, v3.4s, v29.4s") \
63 __ASM_EMIT("bif v0.16b, v2.16b, v6.16b") /* v0 = [x0 >= end] ? TV : KV */ \
64 __ASM_EMIT("bif v1.16b, v3.16b, v7.16b") \
65 EXP_CORE_X8 /* v0 = EV = expf([x0 >= end] ? TV : KV) */ \
66 __ASM_EMIT("ldp q2, q3, [%[mem], #0x00]") /* v2 = x0, v3 = x1 */ \
67 __ASM_EMIT("ld1r {v12.4s}, [%[exp]]") /* v12= start */ \
68 __ASM_EMIT("ldr q10, [%[X2C], #0x00]") /* v10= 1.0 */ \
69 __ASM_EMIT("fcmge v4.4s, v12.4s, v2.4s") /* v4 = [x0 <= start] */ \
70 __ASM_EMIT("fcmge v5.4s, v12.4s, v3.4s") \
71 __ASM_EMIT("bit v0.16b, v10.16b, v4.16b") /* v0 = G = [x0 <= start] ? 1.0 : EV */ \
72 __ASM_EMIT("bit v1.16b, v10.16b, v5.16b") \
73 /* out: v0 = g0, v1 = g1 */
74
75 #define PROCESS_UKNEE_SINGLE_X4 \
76 /* in: v0 = lx0 */ \
77 __ASM_EMIT("add %[off], %[exp], #0x10") \
78 __ASM_EMIT("ldr q6, [%[mem], #0x00]") /* v6 = x0 */ \
79 __ASM_EMIT("ld4r {v12.4s, v13.4s, v14.4s, v15.4s}, [%[exp]]") /* v12=start, v13=end, v14=threshold, v15=herm[0] */ \
80 __ASM_EMIT("ld4r {v28.4s, v29.4s, v30.4s, v31.4s}, [%[off]]" ) /* v28=herm[1], v29=herm[2], v30=tilt[0], v31=tilt[1] */ \
81 __ASM_EMIT("fmul v2.4s, v0.4s, v15.4s") /* v2 = herm[0]*lx0 */ \
82 __ASM_EMIT("fadd v2.4s, v2.4s, v28.4s") /* v2 = herm[0]*lx0+herm[1] */ \
83 __ASM_EMIT("fmul v2.4s, v2.4s, v0.4s") /* v2 = (herm[0]*lx0+herm[1])*lx0 */ \
84 __ASM_EMIT("fmul v4.4s, v0.4s, v30.4s") /* v4 = tilt[0]*lx0 */ \
85 __ASM_EMIT("fcmge v6.4s, v6.4s, v13.4s") /* v6 = [x0 >= end] */ \
86 __ASM_EMIT("fadd v0.4s, v4.4s, v31.4s") /* v0 = TV = tilt[0]*lx0+tilt[1] */ \
87 __ASM_EMIT("fadd v2.4s, v2.4s, v29.4s") /* v2 = KV = (herm[0]*lx0+herm[1])*lx0+herm[2] */ \
88 __ASM_EMIT("bif v0.16b, v2.16b, v6.16b") /* v0 = [x0 >= end] ? TV : KV */ \
89 EXP_CORE_X4 /* v0 = EV = expf([x0 >= end] ? TV : KV) */ \
90 __ASM_EMIT("ldr q2, [%[mem], #0x00]") /* v2 = x0 */ \
91 __ASM_EMIT("ld1r {v12.4s}, [%[exp]]") /* v12= start */ \
92 __ASM_EMIT("ldr q10, [%[X2C], #0x00]") /* v10= 1.0 */ \
93 __ASM_EMIT("fcmge v4.4s, v12.4s, v2.4s") /* v4 = [x0 <= start] */ \
94 __ASM_EMIT("bit v0.16b, v10.16b, v4.16b") /* v0 = G = [x0 <= start] ? 1.0 : EV */ \
95 /* out: v0 = g0 */
96
97 #define PROCESS_UEXP_FULL_X8 \
98 /* in: v0 = x0, v1 = x1 */ \
99 __ASM_EMIT("ld3r {v12.4s, v13.4s, v14.4s}, [%[exp]]")/* v12 = start, v13 = end, v14 = threshold */ \
100 __ASM_EMIT("fabs v0.4s, v0.4s") /* v0 = fabsf(x0) */ \
101 __ASM_EMIT("fabs v1.4s, v1.4s") \
102 __ASM_EMIT("fmin v0.4s, v0.4s, v14.4s") /* v0 = min(fabsf(x0), threshold) */ \
103 __ASM_EMIT("fmin v1.4s, v1.4s, v14.4s") \
104 __ASM_EMIT("fcmgt v2.4s, v0.4s, v12.4s") /* v2 = [fabs(x0) > start] */ \
105 __ASM_EMIT("fcmgt v3.4s, v1.4s, v12.4s") \
106 __ASM_EMIT("orr v2.16b, v2.16b, v3.16b") \
107 __ASM_EMIT("ext v4.16b, v2.16b, v2.16b, #8") \
108 __ASM_EMIT("orr v2.16b, v2.16b, v4.16b") \
109 __ASM_EMIT("ext v4.16b, v2.16b, v2.16b, #4") \
110 __ASM_EMIT("orr v2.16b, v2.16b, v4.16b") \
111 __ASM_EMIT("mov %w[off], v2.s[0]") \
112 __ASM_EMIT("cmp %w[off], #0") \
113 __ASM_EMIT("b.ne 100f") \
114 __ASM_EMIT("ld1r {v0.4s}, [%[X2C]]") /* v0 = g1 */ \
115 __ASM_EMIT("mov v1.16b, v0.16b") \
116 __ASM_EMIT("b 200f") \
117 __ASM_EMIT("100:") \
118 __ASM_EMIT("stp q0, q1, [%[mem], #0x00]") /* mem[0x00] = fabfs(x0) */ \
119 LOGE_CORE_X8 /* v0= lx0 = logf(fabsf(x0)) */ \
120 PROCESS_UKNEE_SINGLE_X8 /* apply knee 1 */ \
121 __ASM_EMIT("200:") \
122 /* out: v0 = G0, v1= G1 */
123
124 #define PROCESS_UEXP_FULL_X4 \
125 /* in: v0 = x0 */ \
126 __ASM_EMIT("ld3r {v12.4s, v13.4s, v14.4s}, [%[exp]]")/* v12 = start, v13 = end, v14 = threshold */ \
127 __ASM_EMIT("fabs v0.4s, v0.4s") /* v0 = fabsf(x0) */ \
128 __ASM_EMIT("fmin v0.4s, v0.4s, v14.4s") /* v0 = min(fabsf(x0), threshold) */ \
129 __ASM_EMIT("str q0, [%[mem], #0x00]") /* mem[0x00] = fabfs(x0) */ \
130 LOGE_CORE_X4 /* v0= lx0 = logf(fabsf(x0)) */ \
131 PROCESS_UKNEE_SINGLE_X4 /* apply knee 1 */ \
132 /* out: v0 = G0 */
133
134 void uexpander_x1_gain(float *dst, const float *src, const dsp::expander_knee_t *c, size_t count)
135 {
136 IF_ARCH_AARCH64(
137 float mem[8] __lsp_aligned16;
138 size_t off;
139 );
140
141 ARCH_AARCH64_ASM(
142 // x8 blocks
143 __ASM_EMIT("subs %[count], %[count], #8")
144 __ASM_EMIT("b.lo 2f")
145 __ASM_EMIT("1:")
146 __ASM_EMIT("ldp q0, q1, [%[src], #0x00]")
147 PROCESS_UEXP_FULL_X8
148 __ASM_EMIT("subs %[count], %[count], #8")
149 __ASM_EMIT("stp q0, q1, [%[dst], #0x00]")
150 __ASM_EMIT("add %[src], %[src], 0x20")
151 __ASM_EMIT("add %[dst], %[dst], 0x20")
152 __ASM_EMIT("b.hs 1b")
153 __ASM_EMIT("2:")
154 // x4 block
155 __ASM_EMIT("adds %[count], %[count], #4")
156 __ASM_EMIT("b.lt 4f")
157 __ASM_EMIT("ldr q0, [%[src], #0x00]")
158 PROCESS_UEXP_FULL_X4
159 __ASM_EMIT("sub %[count], %[count], #4")
160 __ASM_EMIT("str q0, [%[dst], #0x00]")
161 __ASM_EMIT("add %[src], %[src], 0x10")
162 __ASM_EMIT("add %[dst], %[dst], 0x10")
163 __ASM_EMIT("4:")
164 // Tail: 1x-3x block
165 __ASM_EMIT("adds %[count], %[count], #4")
166 __ASM_EMIT("b.ls 12f")
167 __ASM_EMIT("tst %[count], #1")
168 __ASM_EMIT("b.eq 6f")
169 __ASM_EMIT("ld1 {v0.s}[0], [%[src]]")
170 __ASM_EMIT("add %[src], %[src], #0x04")
171 __ASM_EMIT("6:")
172 __ASM_EMIT("tst %[count], #2")
173 __ASM_EMIT("b.eq 8f")
174 __ASM_EMIT("ld1 {v0.d}[1], [%[src]]")
175 __ASM_EMIT("8:")
176 PROCESS_UEXP_FULL_X4
177 __ASM_EMIT("tst %[count], #1")
178 __ASM_EMIT("b.eq 10f")
179 __ASM_EMIT("st1 {v0.s}[0], [%[dst]]")
180 __ASM_EMIT("add %[dst], %[dst], #0x04")
181 __ASM_EMIT("10:")
182 __ASM_EMIT("tst %[count], #2")
183 __ASM_EMIT("b.eq 12f")
184 __ASM_EMIT("st1 {v0.d}[1], [%[dst]]")
185 __ASM_EMIT("12:")
186
187 : [dst] "+r" (dst), [src] "+r" (src),
188 [count] "+r" (count),
189 [off] "=&r" (off)
190 : [exp] "r" (c),
191 [L2C] "r" (&LOG2_CONST[0]),
192 [LOGC] "r" (&LOGE_C[0]),
193 [mem] "r" (&mem[0]),
194 [X2C] "r" (&expander_const[0]),
195 [E2C] "r" (&EXP2_CONST[0]),
196 [LOG2E] "r" (&EXP_LOG2E[0])
197 : "cc", "memory",
198 "v0", "v1", "v2", "v3",
199 "v4", "v5", "v6", "v7",
200 "v8", "v9", "v10", "v11",
201 "v12", "v13", "v14", "v15",
202 "v16", "v17", "v18", "v19",
203 "v20", "v21", "v22", "v23",
204 "v24", "v25", "v26", "v27",
205 "v28", "v29", "v30", "v31"
206 );
207 }
208
209 void uexpander_x1_curve(float *dst, const float *src, const dsp::expander_knee_t *c, size_t count)
210 {
211 IF_ARCH_AARCH64(
212 float mem[8] __lsp_aligned16;
213 size_t off;
214 );
215
216 ARCH_AARCH64_ASM(
217 // x8 blocks
218 __ASM_EMIT("subs %[count], %[count], #8")
219 __ASM_EMIT("b.lo 2f")
220 __ASM_EMIT("1:")
221 __ASM_EMIT("ldp q0, q1, [%[src], #0x00]")
222 PROCESS_UEXP_FULL_X8
223 __ASM_EMIT("ldp q2, q3, [%[src], #0x00]")
224 __ASM_EMIT("subs %[count], %[count], #8")
225 __ASM_EMIT("fmul v0.4s, v0.4s, v2.4s")
226 __ASM_EMIT("fmul v1.4s, v1.4s, v3.4s")
227 __ASM_EMIT("stp q0, q1, [%[dst], #0x00]")
228 __ASM_EMIT("add %[src], %[src], 0x20")
229 __ASM_EMIT("add %[dst], %[dst], 0x20")
230 __ASM_EMIT("b.hs 1b")
231 __ASM_EMIT("2:")
232 // x4 block
233 __ASM_EMIT("adds %[count], %[count], #4")
234 __ASM_EMIT("b.lt 4f")
235 __ASM_EMIT("ldr q0, [%[src], #0x00]")
236 PROCESS_UEXP_FULL_X4
237 __ASM_EMIT("ldr q2, [%[src], #0x00]")
238 __ASM_EMIT("sub %[count], %[count], #4")
239 __ASM_EMIT("fmul v0.4s, v0.4s, v2.4s")
240 __ASM_EMIT("str q0, [%[dst], #0x00]")
241 __ASM_EMIT("add %[src], %[src], 0x10")
242 __ASM_EMIT("add %[dst], %[dst], 0x10")
243 __ASM_EMIT("4:")
244 // Tail: 1x-3x block
245 __ASM_EMIT("adds %[count], %[count], #4")
246 __ASM_EMIT("b.ls 12f")
247 __ASM_EMIT("tst %[count], #1")
248 __ASM_EMIT("b.eq 6f")
249 __ASM_EMIT("ld1 {v0.s}[0], [%[src]]")
250 __ASM_EMIT("add %[src], %[src], #0x04")
251 __ASM_EMIT("6:")
252 __ASM_EMIT("tst %[count], #2")
253 __ASM_EMIT("b.eq 8f")
254 __ASM_EMIT("ld1 {v0.d}[1], [%[src]]")
255 __ASM_EMIT("8:")
256 __ASM_EMIT("mov v1.16b, v0.16b")
257 PROCESS_UEXP_FULL_X4
258 __ASM_EMIT("fmul v0.4s, v0.4s, v1.4s")
259 __ASM_EMIT("tst %[count], #1")
260 __ASM_EMIT("b.eq 10f")
261 __ASM_EMIT("st1 {v0.s}[0], [%[dst]]")
262 __ASM_EMIT("add %[dst], %[dst], #0x04")
263 __ASM_EMIT("10:")
264 __ASM_EMIT("tst %[count], #2")
265 __ASM_EMIT("b.eq 12f")
266 __ASM_EMIT("st1 {v0.d}[1], [%[dst]]")
267 __ASM_EMIT("12:")
268
269 : [dst] "+r" (dst), [src] "+r" (src),
270 [count] "+r" (count),
271 [off] "=&r" (off)
272 : [exp] "r" (c),
273 [L2C] "r" (&LOG2_CONST[0]),
274 [LOGC] "r" (&LOGE_C[0]),
275 [mem] "r" (&mem[0]),
276 [X2C] "r" (&expander_const[0]),
277 [E2C] "r" (&EXP2_CONST[0]),
278 [LOG2E] "r" (&EXP_LOG2E[0])
279 : "cc", "memory",
280 "v0", "v1", "v2", "v3",
281 "v4", "v5", "v6", "v7",
282 "v8", "v9", "v10", "v11",
283 "v12", "v13", "v14", "v15",
284 "v16", "v17", "v18", "v19",
285 "v20", "v21", "v22", "v23",
286 "v24", "v25", "v26", "v27",
287 "v28", "v29", "v30", "v31"
288 );
289 }
290
291 #undef PROCESS_UKNEE_SINGLE_X4
292 #undef PROCESS_UKNEE_SINGLE_X8
293 #undef PROCESS_UEXP_FULL_X8
294 #undef PROCESS_UEXP_FULL_X4
295
296 #define PROCESS_DKNEE_SINGLE_X8 \
297 /* in: v0 = lx0, v1 = lx1 */ \
298 __ASM_EMIT("add %[off], %[exp], #0x10") \
299 __ASM_EMIT("ldp q6, q7, [%[mem], #0x00]") /* v6 = x0, v7 = x1 */ \
300 __ASM_EMIT("ld4r {v12.4s, v13.4s, v14.4s, v15.4s}, [%[exp]]") /* v12=start, v13=end, v14=threshold, v15=herm[0] */ \
301 __ASM_EMIT("ld4r {v28.4s, v29.4s, v30.4s, v31.4s}, [%[off]]" ) /* v28=herm[1], v29=herm[2], v30=tilt[0], v31=tilt[1] */ \
302 __ASM_EMIT("fmul v2.4s, v0.4s, v15.4s") /* v2 = herm[0]*lx0 */ \
303 __ASM_EMIT("fmul v3.4s, v1.4s, v15.4s") \
304 __ASM_EMIT("fadd v2.4s, v2.4s, v28.4s") /* v2 = herm[0]*lx0+herm[1] */ \
305 __ASM_EMIT("fadd v3.4s, v3.4s, v28.4s") \
306 __ASM_EMIT("fmul v2.4s, v2.4s, v0.4s") /* v2 = (herm[0]*lx0+herm[1])*lx0 */ \
307 __ASM_EMIT("fmul v3.4s, v3.4s, v1.4s") \
308 __ASM_EMIT("fmul v4.4s, v0.4s, v30.4s") /* v4 = tilt[0]*lx0 */ \
309 __ASM_EMIT("fmul v5.4s, v1.4s, v30.4s") \
310 __ASM_EMIT("fcmge v6.4s, v12.4s, v6.4s") /* v6 = [x0 <= start] */ \
311 __ASM_EMIT("fcmge v7.4s, v12.4s, v7.4s") \
312 __ASM_EMIT("fadd v0.4s, v4.4s, v31.4s") /* v0 = TV = tilt[0]*lx0+tilt[1] */ \
313 __ASM_EMIT("fadd v1.4s, v5.4s, v31.4s") \
314 __ASM_EMIT("fadd v2.4s, v2.4s, v29.4s") /* v2 = KV = (herm[0]*lx0+herm[1])*lx0+herm[2] */ \
315 __ASM_EMIT("fadd v3.4s, v3.4s, v29.4s") \
316 __ASM_EMIT("bif v0.16b, v2.16b, v6.16b") /* v0 = [x0 <= start] ? TV : KV */ \
317 __ASM_EMIT("bif v1.16b, v3.16b, v7.16b") \
318 EXP_CORE_X8 /* v0 = EV = expf([x0 >= end] ? TV : KV) */ \
319 __ASM_EMIT("ldp q2, q3, [%[mem], #0x00]") /* v2 = x0, v3 = x1 */ \
320 __ASM_EMIT("ld3r {v12.4s, v13.4s, v14.4s}, [%[exp]]")/* v12= start, v13 = end, v14=threshold */ \
321 __ASM_EMIT("ldr q10, [%[X2C], #0x00]") /* v10= 1.0 */ \
322 __ASM_EMIT("fcmge v4.4s, v2.4s, v13.4s") /* v4 = [x0 >= end] */ \
323 __ASM_EMIT("fcmge v5.4s, v3.4s, v13.4s") \
324 __ASM_EMIT("fcmge v6.4s, v2.4s, v14.4s") /* v6 = [x0 >= threshold] */ \
325 __ASM_EMIT("fcmge v7.4s, v3.4s, v14.4s") \
326 __ASM_EMIT("bit v0.16b, v10.16b, v4.16b") /* v0 = [x0 >= end] ? 1.0 : EV */ \
327 __ASM_EMIT("bit v1.16b, v10.16b, v5.16b") \
328 __ASM_EMIT("and v0.16b, v0.16b, v6.16b") /* v0 = G = [x0 < threshold] ? 0.0 : [x0 >= end] ? 1.0 : EV */ \
329 __ASM_EMIT("and v1.16b, v1.16b, v7.16b") \
330 /* out: v0 = g0, v1 = g1 */
331
332 #define PROCESS_DKNEE_SINGLE_X4 \
333 /* in: v0 = lx0 */ \
334 __ASM_EMIT("add %[off], %[exp], #0x10") \
335 __ASM_EMIT("ldr q6, [%[mem], #0x00]") /* v6 = x0 */ \
336 __ASM_EMIT("ld4r {v12.4s, v13.4s, v14.4s, v15.4s}, [%[exp]]") /* v12=start, v13=end, v14=threshold, v15=herm[0] */ \
337 __ASM_EMIT("ld4r {v28.4s, v29.4s, v30.4s, v31.4s}, [%[off]]" ) /* v28=herm[1], v29=herm[2], v30=tilt[0], v31=tilt[1] */ \
338 __ASM_EMIT("fmul v2.4s, v0.4s, v15.4s") /* v2 = herm[0]*lx0 */ \
339 __ASM_EMIT("fadd v2.4s, v2.4s, v28.4s") /* v2 = herm[0]*lx0+herm[1] */ \
340 __ASM_EMIT("fmul v2.4s, v2.4s, v0.4s") /* v2 = (herm[0]*lx0+herm[1])*lx0 */ \
341 __ASM_EMIT("fmul v4.4s, v0.4s, v30.4s") /* v4 = tilt[0]*lx0 */ \
342 __ASM_EMIT("fcmge v6.4s, v12.4s, v6.4s") /* v6 = [x0 <= start] */ \
343 __ASM_EMIT("fadd v0.4s, v4.4s, v31.4s") /* v0 = TV = tilt[0]*lx0+tilt[1] */ \
344 __ASM_EMIT("fadd v2.4s, v2.4s, v29.4s") /* v2 = KV = (herm[0]*lx0+herm[1])*lx0+herm[2] */ \
345 __ASM_EMIT("bif v0.16b, v2.16b, v6.16b") /* v0 = [x0 <= start] ? TV : KV */ \
346 EXP_CORE_X4 /* v0 = EV = expf([x0 >= end] ? TV : KV) */ \
347 __ASM_EMIT("ldr q2, [%[mem], #0x00]") /* v2 = x0 */ \
348 __ASM_EMIT("ld3r {v12.4s, v13.4s, v14.4s}, [%[exp]]")/* v12= start, v13 = end, v14=threshold */ \
349 __ASM_EMIT("ldr q10, [%[X2C], #0x00]") /* v10= 1.0 */ \
350 __ASM_EMIT("fcmge v4.4s, v2.4s, v13.4s") /* v4 = [x0 >= end] */ \
351 __ASM_EMIT("fcmge v6.4s, v2.4s, v14.4s") /* v6 = [x0 >= threshold] */ \
352 __ASM_EMIT("bit v0.16b, v10.16b, v4.16b") /* v0 = [x0 >= end] ? 1.0 : EV */ \
353 __ASM_EMIT("and v0.16b, v0.16b, v6.16b") /* v0 = G = [x0 < threshold] ? 0.0 : [x0 >= end] ? 1.0 : EV */ \
354 /* out: v0 = g0 */
355
356 #define PROCESS_DEXP_FULL_X8 \
357 /* in: v0 = x0, v1 = x1 */ \
358 __ASM_EMIT("ld2r {v12.4s, v13.4s}, [%[exp]]") /* v12 = start, v13 = end */ \
359 __ASM_EMIT("fabs v0.4s, v0.4s") /* v0 = fabsf(x0) */ \
360 __ASM_EMIT("fabs v1.4s, v1.4s") \
361 __ASM_EMIT("fcmgt v2.4s, v13.4s, v0.4s") /* v2 = [fabs(x0) < end] */ \
362 __ASM_EMIT("fcmgt v3.4s, v13.4s, v1.4s") \
363 __ASM_EMIT("orr v2.16b, v2.16b, v3.16b") \
364 __ASM_EMIT("ext v4.16b, v2.16b, v2.16b, #8") \
365 __ASM_EMIT("orr v2.16b, v2.16b, v4.16b") \
366 __ASM_EMIT("ext v4.16b, v2.16b, v2.16b, #4") \
367 __ASM_EMIT("orr v2.16b, v2.16b, v4.16b") \
368 __ASM_EMIT("mov %w[off], v2.s[0]") \
369 __ASM_EMIT("cmp %w[off], #0") \
370 __ASM_EMIT("b.ne 100f") \
371 __ASM_EMIT("ld1r {v0.4s}, [%[X2C]]") /* v0 = g1 */ \
372 __ASM_EMIT("mov v1.16b, v0.16b") \
373 __ASM_EMIT("b 200f") \
374 __ASM_EMIT("100:") \
375 __ASM_EMIT("stp q0, q1, [%[mem], #0x00]") /* mem[0x00] = fabfs(x0) */ \
376 LOGE_CORE_X8 /* v0= lx0 = logf(fabsf(x0)) */ \
377 PROCESS_DKNEE_SINGLE_X8 /* apply knee 1 */ \
378 __ASM_EMIT("200:") \
379 /* out: v0 = G0, v1= G1 */
380
381 #define PROCESS_DEXP_FULL_X4 \
382 /* in: v0 = x0 */ \
383 __ASM_EMIT("fabs v0.4s, v0.4s") /* v0 = fabsf(x0) */ \
384 __ASM_EMIT("str q0, [%[mem], #0x00]") /* mem[0x00] = fabfs(x0) */ \
385 LOGE_CORE_X4 /* v0= lx0 = logf(fabsf(x0)) */ \
386 PROCESS_DKNEE_SINGLE_X4 /* apply knee 1 */ \
387 /* out: v0 = G0 */
388
389 void dexpander_x1_gain(float *dst, const float *src, const dsp::expander_knee_t *c, size_t count)
390 {
391 IF_ARCH_AARCH64(
392 float mem[8] __lsp_aligned16;
393 size_t off;
394 );
395
396 ARCH_AARCH64_ASM(
397 // x8 blocks
398 __ASM_EMIT("subs %[count], %[count], #8")
399 __ASM_EMIT("b.lo 2f")
400 __ASM_EMIT("1:")
401 __ASM_EMIT("ldp q0, q1, [%[src], #0x00]")
402 PROCESS_DEXP_FULL_X8
403 __ASM_EMIT("subs %[count], %[count], #8")
404 __ASM_EMIT("stp q0, q1, [%[dst], #0x00]")
405 __ASM_EMIT("add %[src], %[src], 0x20")
406 __ASM_EMIT("add %[dst], %[dst], 0x20")
407 __ASM_EMIT("b.hs 1b")
408 __ASM_EMIT("2:")
409 // x4 block
410 __ASM_EMIT("adds %[count], %[count], #4")
411 __ASM_EMIT("b.lt 4f")
412 __ASM_EMIT("ldr q0, [%[src], #0x00]")
413 PROCESS_DEXP_FULL_X4
414 __ASM_EMIT("sub %[count], %[count], #4")
415 __ASM_EMIT("str q0, [%[dst], #0x00]")
416 __ASM_EMIT("add %[src], %[src], 0x10")
417 __ASM_EMIT("add %[dst], %[dst], 0x10")
418 __ASM_EMIT("4:")
419 // Tail: 1x-3x block
420 __ASM_EMIT("adds %[count], %[count], #4")
421 __ASM_EMIT("b.ls 12f")
422 __ASM_EMIT("tst %[count], #1")
423 __ASM_EMIT("b.eq 6f")
424 __ASM_EMIT("ld1 {v0.s}[0], [%[src]]")
425 __ASM_EMIT("add %[src], %[src], #0x04")
426 __ASM_EMIT("6:")
427 __ASM_EMIT("tst %[count], #2")
428 __ASM_EMIT("b.eq 8f")
429 __ASM_EMIT("ld1 {v0.d}[1], [%[src]]")
430 __ASM_EMIT("8:")
431 PROCESS_DEXP_FULL_X4
432 __ASM_EMIT("tst %[count], #1")
433 __ASM_EMIT("b.eq 10f")
434 __ASM_EMIT("st1 {v0.s}[0], [%[dst]]")
435 __ASM_EMIT("add %[dst], %[dst], #0x04")
436 __ASM_EMIT("10:")
437 __ASM_EMIT("tst %[count], #2")
438 __ASM_EMIT("b.eq 12f")
439 __ASM_EMIT("st1 {v0.d}[1], [%[dst]]")
440 __ASM_EMIT("12:")
441
442 : [dst] "+r" (dst), [src] "+r" (src),
443 [count] "+r" (count),
444 [off] "=&r" (off)
445 : [exp] "r" (c),
446 [L2C] "r" (&LOG2_CONST[0]),
447 [LOGC] "r" (&LOGE_C[0]),
448 [mem] "r" (&mem[0]),
449 [X2C] "r" (&expander_const[0]),
450 [E2C] "r" (&EXP2_CONST[0]),
451 [LOG2E] "r" (&EXP_LOG2E[0])
452 : "cc", "memory",
453 "v0", "v1", "v2", "v3",
454 "v4", "v5", "v6", "v7",
455 "v8", "v9", "v10", "v11",
456 "v12", "v13", "v14", "v15",
457 "v16", "v17", "v18", "v19",
458 "v20", "v21", "v22", "v23",
459 "v24", "v25", "v26", "v27",
460 "v28", "v29", "v30", "v31"
461 );
462 }
463
464 void dexpander_x1_curve(float *dst, const float *src, const dsp::expander_knee_t *c, size_t count)
465 {
466 IF_ARCH_AARCH64(
467 float mem[8] __lsp_aligned16;
468 size_t off;
469 );
470
471 ARCH_AARCH64_ASM(
472 // x8 blocks
473 __ASM_EMIT("subs %[count], %[count], #8")
474 __ASM_EMIT("b.lo 2f")
475 __ASM_EMIT("1:")
476 __ASM_EMIT("ldp q0, q1, [%[src], #0x00]")
477 PROCESS_DEXP_FULL_X8
478 __ASM_EMIT("ldp q2, q3, [%[src], #0x00]")
479 __ASM_EMIT("subs %[count], %[count], #8")
480 __ASM_EMIT("fmul v0.4s, v0.4s, v2.4s")
481 __ASM_EMIT("fmul v1.4s, v1.4s, v3.4s")
482 __ASM_EMIT("stp q0, q1, [%[dst], #0x00]")
483 __ASM_EMIT("add %[src], %[src], 0x20")
484 __ASM_EMIT("add %[dst], %[dst], 0x20")
485 __ASM_EMIT("b.hs 1b")
486 __ASM_EMIT("2:")
487 // x4 block
488 __ASM_EMIT("adds %[count], %[count], #4")
489 __ASM_EMIT("b.lt 4f")
490 __ASM_EMIT("ldr q0, [%[src], #0x00]")
491 PROCESS_DEXP_FULL_X4
492 __ASM_EMIT("ldr q2, [%[src], #0x00]")
493 __ASM_EMIT("sub %[count], %[count], #4")
494 __ASM_EMIT("fmul v0.4s, v0.4s, v2.4s")
495 __ASM_EMIT("str q0, [%[dst], #0x00]")
496 __ASM_EMIT("add %[src], %[src], 0x10")
497 __ASM_EMIT("add %[dst], %[dst], 0x10")
498 __ASM_EMIT("4:")
499 // Tail: 1x-3x block
500 __ASM_EMIT("adds %[count], %[count], #4")
501 __ASM_EMIT("b.ls 12f")
502 __ASM_EMIT("tst %[count], #1")
503 __ASM_EMIT("b.eq 6f")
504 __ASM_EMIT("ld1 {v0.s}[0], [%[src]]")
505 __ASM_EMIT("add %[src], %[src], #0x04")
506 __ASM_EMIT("6:")
507 __ASM_EMIT("tst %[count], #2")
508 __ASM_EMIT("b.eq 8f")
509 __ASM_EMIT("ld1 {v0.d}[1], [%[src]]")
510 __ASM_EMIT("8:")
511 __ASM_EMIT("mov v1.16b, v0.16b")
512 PROCESS_DEXP_FULL_X4
513 __ASM_EMIT("fmul v0.4s, v0.4s, v1.4s")
514 __ASM_EMIT("tst %[count], #1")
515 __ASM_EMIT("b.eq 10f")
516 __ASM_EMIT("st1 {v0.s}[0], [%[dst]]")
517 __ASM_EMIT("add %[dst], %[dst], #0x04")
518 __ASM_EMIT("10:")
519 __ASM_EMIT("tst %[count], #2")
520 __ASM_EMIT("b.eq 12f")
521 __ASM_EMIT("st1 {v0.d}[1], [%[dst]]")
522 __ASM_EMIT("12:")
523
524 : [dst] "+r" (dst), [src] "+r" (src),
525 [count] "+r" (count),
526 [off] "=&r" (off)
527 : [exp] "r" (c),
528 [L2C] "r" (&LOG2_CONST[0]),
529 [LOGC] "r" (&LOGE_C[0]),
530 [mem] "r" (&mem[0]),
531 [X2C] "r" (&expander_const[0]),
532 [E2C] "r" (&EXP2_CONST[0]),
533 [LOG2E] "r" (&EXP_LOG2E[0])
534 : "cc", "memory",
535 "v0", "v1", "v2", "v3",
536 "v4", "v5", "v6", "v7",
537 "v8", "v9", "v10", "v11",
538 "v12", "v13", "v14", "v15",
539 "v16", "v17", "v18", "v19",
540 "v20", "v21", "v22", "v23",
541 "v24", "v25", "v26", "v27",
542 "v28", "v29", "v30", "v31"
543 );
544 }
545
546 #undef PROCESS_UKNEE_SINGLE_X4
547 #undef PROCESS_UKNEE_SINGLE_X8
548 #undef PROCESS_DEXP_FULL_X8
549 #undef PROCESS_DEXP_FULL_X4
550
551
552 } /* namespace asimd */
553 } /* namespace lsp */
554
555
556
557 #endif /* PRIVATE_DSP_ARCH_AARCH64_ASIMD_DYNAMICS_EXPANDER_H_ */