Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/x86/sse2/dynamics/gate.h

raw
1 /*
2 * Copyright (C) 2023 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2023 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 19 окт. 2023 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_X86_SSE2_DYNAMICS_GATE_H_
23 #define PRIVATE_DSP_ARCH_X86_SSE2_DYNAMICS_GATE_H_
24
25 #ifndef PRIVATE_DSP_ARCH_X86_SSE2_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_X86_SSE2_IMPL */
28
29 #include <private/dsp/arch/x86/sse2/pmath/exp.h>
30 #include <private/dsp/arch/x86/sse2/pmath/log.h>
31
32 namespace lsp
33 {
34 namespace sse2
35 {
36 #pragma pack(push, 1)
37 typedef struct gate_knee_t
38 {
39 float start[4]; // +0x00
40 float end[4]; // +0x10
41 float gain_start[4]; // +0x20
42 float gain_end[4]; // +0x30
43 float herm[16]; // +0x40
44 } gate_knee_t;
45 #pragma pack(pop)
46
47 #define UNPACK_GATE_KNEE(DST, SRC) \
48 __ASM_EMIT("movss 0x00(%[" SRC "]), %%xmm0") \
49 __ASM_EMIT("movss 0x04(%[" SRC "]), %%xmm1") \
50 __ASM_EMIT("movss 0x08(%[" SRC "]), %%xmm2") \
51 __ASM_EMIT("movss 0x0c(%[" SRC "]), %%xmm3") \
52 __ASM_EMIT("movss 0x10(%[" SRC "]), %%xmm4") \
53 __ASM_EMIT("movss 0x14(%[" SRC "]), %%xmm5") \
54 __ASM_EMIT("movss 0x18(%[" SRC "]), %%xmm6") \
55 __ASM_EMIT("movss 0x1c(%[" SRC "]), %%xmm7") \
56 __ASM_EMIT("shufps $0x00, %%xmm0, %%xmm0") \
57 __ASM_EMIT("shufps $0x00, %%xmm1, %%xmm1") \
58 __ASM_EMIT("shufps $0x00, %%xmm2, %%xmm2") \
59 __ASM_EMIT("shufps $0x00, %%xmm3, %%xmm3") \
60 __ASM_EMIT("shufps $0x00, %%xmm4, %%xmm4") \
61 __ASM_EMIT("shufps $0x00, %%xmm5, %%xmm5") \
62 __ASM_EMIT("shufps $0x00, %%xmm6, %%xmm6") \
63 __ASM_EMIT("shufps $0x00, %%xmm7, %%xmm7") \
64 __ASM_EMIT("movaps %%xmm0, 0x00 + %[" DST "]") \
65 __ASM_EMIT("movaps %%xmm1, 0x10 + %[" DST "]") \
66 __ASM_EMIT("movaps %%xmm2, 0x20 + %[" DST "]") \
67 __ASM_EMIT("movaps %%xmm3, 0x30 + %[" DST "]") \
68 __ASM_EMIT("movaps %%xmm4, 0x40 + %[" DST "]") \
69 __ASM_EMIT("movaps %%xmm5, 0x50 + %[" DST "]") \
70 __ASM_EMIT("movaps %%xmm6, 0x60 + %[" DST "]") \
71 __ASM_EMIT("movaps %%xmm7, 0x70 + %[" DST "]")
72
73 #define PROCESS_KNEE_SINGLE_X8 \
74 /* in: xmm0 = lx0, xmm4 = lx1 */ \
75 __ASM_EMIT("movaps 0x40 + %[knee], %%xmm1") /* xmm1 = herm[0] */ \
76 __ASM_EMIT("movaps 0x40 + %[knee], %%xmm5") \
77 __ASM_EMIT("mulps %%xmm0, %%xmm1") /* xmm1 = herm[0]*lx0 */ \
78 __ASM_EMIT("mulps %%xmm4, %%xmm5") \
79 __ASM_EMIT("addps 0x50 + %[knee], %%xmm1") /* xmm1 = herm[0]*lx0+herm[1] */ \
80 __ASM_EMIT("addps 0x50 + %[knee], %%xmm5") \
81 __ASM_EMIT("mulps %%xmm0, %%xmm1") /* xmm1 = (herm[0]*lx0+herm[1])*lx0 */ \
82 __ASM_EMIT("mulps %%xmm4, %%xmm5") \
83 __ASM_EMIT("addps 0x60 + %[knee], %%xmm1") /* xmm1 = (herm[0]*lx0+herm[1])*lx0+herm[2] */ \
84 __ASM_EMIT("addps 0x60 + %[knee], %%xmm5") \
85 __ASM_EMIT("mulps %%xmm0, %%xmm1") /* xmm1 = ((herm[0]*lx0+herm[1])*lx0+herm[2])*lx0 */ \
86 __ASM_EMIT("mulps %%xmm4, %%xmm5") \
87 __ASM_EMIT("addps 0x70 + %[knee], %%xmm1") /* xmm1 = KV = ((herm[0]*lx0+herm[1])*lx0+herm[2])*lx0+herm[3] */ \
88 __ASM_EMIT("addps 0x70 + %[knee], %%xmm5") \
89 __ASM_EMIT("movaps %%xmm1, %%xmm0") /* xmm0 = KV */ \
90 __ASM_EMIT("movaps %%xmm5, %%xmm4") \
91 EXP_CORE_X8 /* xmm0 = expf(KV) */ \
92 __ASM_EMIT("movaps 0x00 + %[mem], %%xmm1") /* xmm1 = x0 */ \
93 __ASM_EMIT("movaps 0x10 + %[mem], %%xmm5") \
94 __ASM_EMIT("movaps %%xmm1, %%xmm2") /* xmm2 = x0 */ \
95 __ASM_EMIT("movaps %%xmm5, %%xmm6") \
96 __ASM_EMIT("cmpps $6, 0x00 + %[knee], %%xmm1") /* xmm0 = [x0 > start] */ \
97 __ASM_EMIT("cmpps $6, 0x00 + %[knee], %%xmm5") \
98 __ASM_EMIT("cmpps $1, 0x10 + %[knee], %%xmm2") /* xmm2 = [x0 < end] */ \
99 __ASM_EMIT("cmpps $1, 0x10 + %[knee], %%xmm6") \
100 __ASM_EMIT("andps %%xmm1, %%xmm0") /* xmm0 = [x0 > start] & expf(KV) */ \
101 __ASM_EMIT("andps %%xmm5, %%xmm4") \
102 __ASM_EMIT("andps %%xmm2, %%xmm0") /* xmm0 = ([x0 > start] && [x0 < end]) & expf(KV) */ \
103 __ASM_EMIT("andps %%xmm6, %%xmm4") \
104 __ASM_EMIT("andnps 0x20 + %[knee], %%xmm1") /* xmm1 = [x0 <= start] & gain_start */ \
105 __ASM_EMIT("andnps 0x20 + %[knee], %%xmm5") \
106 __ASM_EMIT("andnps 0x30 + %[knee], %%xmm2") /* xmm2 = [x0 >= end] & gain_end */ \
107 __ASM_EMIT("andnps 0x30 + %[knee], %%xmm6") \
108 __ASM_EMIT("orps %%xmm1, %%xmm0") /* xmm0 = ([x0 <= start]) ? gain_start : ([x0 < end]) ? expf(KV) : 0 */ \
109 __ASM_EMIT("orps %%xmm5, %%xmm4") \
110 __ASM_EMIT("orps %%xmm2, %%xmm0") /* xmm0 = ([x0 <= start]) ? gain_start : ([x0 < end]) ? expf(KV) : gain_end */ \
111 __ASM_EMIT("orps %%xmm6, %%xmm4") \
112 /* out: xmm0 = g0, xmm4 = g1 */
113
114 #define PROCESS_KNEE_SINGLE_X4 \
115 /* in: xmm0 = lx0 */ \
116 __ASM_EMIT("movaps 0x40 + %[knee], %%xmm1") /* xmm1 = herm[0] */ \
117 __ASM_EMIT("mulps %%xmm0, %%xmm1") /* xmm1 = herm[0]*lx0 */ \
118 __ASM_EMIT("addps 0x50 + %[knee], %%xmm1") /* xmm1 = herm[0]*lx0+herm[1] */ \
119 __ASM_EMIT("mulps %%xmm0, %%xmm1") /* xmm1 = (herm[0]*lx0+herm[1])*lx0 */ \
120 __ASM_EMIT("addps 0x60 + %[knee], %%xmm1") /* xmm1 = (herm[0]*lx0+herm[1])*lx0+herm[2] */ \
121 __ASM_EMIT("mulps %%xmm0, %%xmm1") /* xmm1 = ((herm[0]*lx0+herm[1])*lx0+herm[2])*lx0 */ \
122 __ASM_EMIT("addps 0x70 + %[knee], %%xmm1") /* xmm1 = KV = ((herm[0]*lx0+herm[1])*lx0+herm[2])*lx0+herm[3] */ \
123 __ASM_EMIT("movaps %%xmm1, %%xmm0") /* xmm0 = KV */ \
124 EXP_CORE_X4 /* xmm0 = expf(KV) */ \
125 __ASM_EMIT("movaps 0x00 + %[mem], %%xmm1") /* xmm1 = x0 */ \
126 __ASM_EMIT("movaps %%xmm1, %%xmm2") /* xmm2 = x0 */ \
127 __ASM_EMIT("cmpps $6, 0x00 + %[knee], %%xmm1") /* xmm0 = [x0 > start] */ \
128 __ASM_EMIT("cmpps $1, 0x10 + %[knee], %%xmm2") /* xmm2 = [x0 < end] */ \
129 __ASM_EMIT("andps %%xmm1, %%xmm0") /* xmm0 = [x0 > start] & expf(KV) */ \
130 __ASM_EMIT("andps %%xmm2, %%xmm0") /* xmm0 = ([x0 > start] && [x0 < end]) & expf(KV) */ \
131 __ASM_EMIT("andnps 0x20 + %[knee], %%xmm1") /* xmm1 = [x0 <= start] & gain_start */ \
132 __ASM_EMIT("andnps 0x30 + %[knee], %%xmm2") /* xmm2 = [x0 >= end] & gain_end */ \
133 __ASM_EMIT("orps %%xmm1, %%xmm0") /* xmm0 = ([x0 <= start]) ? gain_start : ([x0 < end]) ? expf(KV) : 0 */ \
134 __ASM_EMIT("orps %%xmm2, %%xmm0") /* xmm0 = ([x0 <= start]) ? gain_start : ([x0 < end]) ? expf(KV) : gain_end */ \
135 /* out: xmm0 = g0 */
136
137 #define PROCESS_GATE_FULL_X8 \
138 /* in: xmm0 = x0, xmm4 = x1 */ \
139 __ASM_EMIT("andps 0x00 + %[G2C], %%xmm0") /* xmm0 = fabsf(x0) */ \
140 __ASM_EMIT("andps 0x00 + %[G2C], %%xmm4") \
141 __ASM_EMIT("movaps %%xmm0, %%xmm1") \
142 __ASM_EMIT("movaps %%xmm4, %%xmm5") \
143 __ASM_EMIT("movaps %%xmm0, %%xmm2") \
144 __ASM_EMIT("movaps %%xmm4, %%xmm6") \
145 __ASM_EMIT("cmpps $6, 0x00 + %[knee], %%xmm1") /* xmm1 = [x0 > start] */ \
146 __ASM_EMIT("cmpps $6, 0x00 + %[knee], %%xmm5") \
147 __ASM_EMIT("cmpps $1, 0x10 + %[knee], %%xmm2") /* xmm2 = [x0 < end] */ \
148 __ASM_EMIT("cmpps $1, 0x10 + %[knee], %%xmm6") \
149 __ASM_EMIT("movaps %%xmm1, %%xmm3") /* xmm3 = [x0 > start] */ \
150 __ASM_EMIT("movaps %%xmm5, %%xmm7") \
151 __ASM_EMIT("andps %%xmm2, %%xmm3") /* xmm3 = [x0 > start] && [x0 < end] */ \
152 __ASM_EMIT("andps %%xmm6, %%xmm7") \
153 __ASM_EMIT("orps %%xmm7, %%xmm3") \
154 __ASM_EMIT("movmskps %%xmm3, %[mask]") \
155 __ASM_EMIT("test %[mask], %[mask]") \
156 __ASM_EMIT("jnz 100f") \
157 /* Case when we do not need logarithm because there is no sample at the knee */ \
158 __ASM_EMIT("andnps 0x20 + %[knee], %%xmm1") /* xmm1 = [x0 <= start] & gain_start */ \
159 __ASM_EMIT("andnps 0x20 + %[knee], %%xmm5") \
160 __ASM_EMIT("andnps 0x30 + %[knee], %%xmm2") /* xmm2 = [x0 >= end] & gain_end */ \
161 __ASM_EMIT("andnps 0x30 + %[knee], %%xmm6") \
162 __ASM_EMIT("movaps %%xmm1, %%xmm0") /* xmm0 = [x0 <= start] & gain_start */ \
163 __ASM_EMIT("movaps %%xmm5, %%xmm4") \
164 __ASM_EMIT("orps %%xmm2, %%xmm0") /* xmm0 = [x0 <= start] & gain_start | [x0 >= end] & gain_end */ \
165 __ASM_EMIT("orps %%xmm6, %%xmm4") \
166 __ASM_EMIT("jmp 200f") \
167 __ASM_EMIT("100:") \
168 __ASM_EMIT("movaps %%xmm0, 0x00 + %[mem]") /* store fabsf(x0) */ \
169 __ASM_EMIT("movaps %%xmm4, 0x10 + %[mem]") \
170 LOGE_CORE_X8 /* xmm0 = lx0 = logf(fabsf(x0)) */ \
171 PROCESS_KNEE_SINGLE_X8 /* apply knee 0 */ \
172 __ASM_EMIT("200:") \
173 /* out: xmm0 = g0, xmm4 = g1 */
174
175 #define PROCESS_GATE_FULL_X4 \
176 /* in: xmm0 = x0 */ \
177 __ASM_EMIT("andps 0x00 + %[G2C], %%xmm0") /* xmm0 = fabsf(x0) */ \
178 __ASM_EMIT("movaps %%xmm0, 0x00 + %[mem]") /* store fabsf(x0) */ \
179 LOGE_CORE_X4 /* xmm0 = lx0 = logf(fabsf(x0)) */ \
180 PROCESS_KNEE_SINGLE_X4 /* apply knee 0 */ \
181 /* out: xmm0 = g */
182
183 static const uint32_t gate_const[] __lsp_aligned16 =
184 {
185 LSP_DSP_VEC4(0x7fffffff)
186 };
187
188 void gate_x1_gain(float *dst, const float *src, const dsp::gate_knee_t *c, size_t count)
189 {
190 IF_ARCH_X86(
191 gate_knee_t knee __lsp_aligned16;
192 float mem[8] __lsp_aligned16;
193 float stub[4] __lsp_aligned16;
194 size_t mask;
195 );
196
197 ARCH_X86_ASM
198 (
199 // Prepare stuff
200 UNPACK_GATE_KNEE("knee", "gate")
201
202 // 8x blocks
203 __ASM_EMIT("sub $8, %[count]")
204 __ASM_EMIT("jb 2f")
205 __ASM_EMIT("1:")
206 __ASM_EMIT("movups 0x00(%[src]), %%xmm0")
207 __ASM_EMIT("movups 0x10(%[src]), %%xmm4")
208 PROCESS_GATE_FULL_X8
209 __ASM_EMIT("movups %%xmm0, 0x00(%[dst])")
210 __ASM_EMIT("movups %%xmm4, 0x10(%[dst])")
211 __ASM_EMIT("add $0x20, %[src]")
212 __ASM_EMIT("add $0x20, %[dst]")
213 __ASM_EMIT("sub $8, %[count]")
214 __ASM_EMIT("jae 1b")
215 __ASM_EMIT("2:")
216 // 4x blocks
217 __ASM_EMIT("add $4, %[count]")
218 __ASM_EMIT("jl 4f")
219 __ASM_EMIT("movups 0x00(%[src]), %%xmm0")
220 PROCESS_GATE_FULL_X4
221 __ASM_EMIT("movups %%xmm0, 0x00(%[dst])")
222 __ASM_EMIT("sub $4, %[count]")
223 __ASM_EMIT("add $0x10, %[src]")
224 __ASM_EMIT("add $0x10, %[dst]")
225 __ASM_EMIT("4:")
226 // Tail: 1x-3x block
227 __ASM_EMIT("add $4, %[count]")
228 __ASM_EMIT("jle 12f")
229 __ASM_EMIT("test $1, %[count]")
230 __ASM_EMIT("jz 6f")
231 __ASM_EMIT("movss 0x00(%[src]), %%xmm0")
232 __ASM_EMIT("add $4, %[src]")
233 __ASM_EMIT("6:")
234 __ASM_EMIT("test $2, %[count]")
235 __ASM_EMIT("jz 8f")
236 __ASM_EMIT("movhps 0x00(%[src]), %%xmm0")
237 __ASM_EMIT("8:")
238 PROCESS_GATE_FULL_X4
239 __ASM_EMIT("test $1, %[count]")
240 __ASM_EMIT("jz 10f")
241 __ASM_EMIT("movss %%xmm0, 0x00(%[dst])")
242 __ASM_EMIT("add $4, %[dst]")
243 __ASM_EMIT("10:")
244 __ASM_EMIT("test $2, %[count]")
245 __ASM_EMIT("jz 12f")
246 __ASM_EMIT("movhps %%xmm0, 0x00(%[dst])")
247 __ASM_EMIT("12:")
248
249 : [dst] "+r" (dst), [src] "+r" (src),
250 [count] "+r" (count),
251 [mask] "=&r" (mask)
252 : [gate] "r" (c),
253 [knee] "o" (knee),
254 [mem] "o" (mem),
255 [stub] "o" (stub),
256 [G2C] "o" (gate_const),
257 [L2C] "o" (LOG2_CONST),
258 [LOGC] "o" (LOGE_C),
259 [E2C] "o" (EXP2_CONST),
260 [LOG2E] "m" (EXP_LOG2E)
261 : "cc", "memory",
262 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
263 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
264 );
265 }
266
267 void gate_x1_curve(float *dst, const float *src, const dsp::gate_knee_t *c, size_t count)
268 {
269 IF_ARCH_X86(
270 gate_knee_t knee __lsp_aligned16;
271 float mem[8] __lsp_aligned16;
272 float stub[4] __lsp_aligned16;
273 size_t mask;
274 );
275
276 ARCH_X86_ASM
277 (
278 // Prepare stuff
279 UNPACK_GATE_KNEE("knee", "gate")
280
281 // 8x blocks
282 __ASM_EMIT("sub $8, %[count]")
283 __ASM_EMIT("jb 2f")
284 __ASM_EMIT("1:")
285 __ASM_EMIT("movups 0x00(%[src]), %%xmm0")
286 __ASM_EMIT("movups 0x10(%[src]), %%xmm4")
287 PROCESS_GATE_FULL_X8
288 __ASM_EMIT("movups 0x00(%[src]), %%xmm1")
289 __ASM_EMIT("movups 0x10(%[src]), %%xmm5")
290 __ASM_EMIT("mulps %%xmm1, %%xmm0")
291 __ASM_EMIT("mulps %%xmm5, %%xmm4")
292 __ASM_EMIT("movups %%xmm0, 0x00(%[dst])")
293 __ASM_EMIT("movups %%xmm4, 0x10(%[dst])")
294 __ASM_EMIT("add $0x20, %[src]")
295 __ASM_EMIT("add $0x20, %[dst]")
296 __ASM_EMIT("sub $8, %[count]")
297 __ASM_EMIT("jae 1b")
298 __ASM_EMIT("2:")
299 // 4x blocks
300 __ASM_EMIT("add $4, %[count]")
301 __ASM_EMIT("jl 4f")
302 __ASM_EMIT("movups 0x00(%[src]), %%xmm0")
303 PROCESS_GATE_FULL_X4
304 __ASM_EMIT("movups 0x00(%[src]), %%xmm1")
305 __ASM_EMIT("mulps %%xmm1, %%xmm0")
306 __ASM_EMIT("movups %%xmm0, 0x00(%[dst])")
307 __ASM_EMIT("sub $4, %[count]")
308 __ASM_EMIT("add $0x10, %[src]")
309 __ASM_EMIT("add $0x10, %[dst]")
310 __ASM_EMIT("4:")
311 // Tail: 1x-3x block
312 __ASM_EMIT("add $4, %[count]")
313 __ASM_EMIT("jle 12f")
314 __ASM_EMIT("test $1, %[count]")
315 __ASM_EMIT("jz 6f")
316 __ASM_EMIT("movss 0x00(%[src]), %%xmm0")
317 __ASM_EMIT("add $4, %[src]")
318 __ASM_EMIT("6:")
319 __ASM_EMIT("test $2, %[count]")
320 __ASM_EMIT("jz 8f")
321 __ASM_EMIT("movhps 0x00(%[src]), %%xmm0")
322 __ASM_EMIT("8:")
323 __ASM_EMIT("movaps %%xmm0, %%xmm4")
324 PROCESS_GATE_FULL_X4
325 __ASM_EMIT("mulps %%xmm4, %%xmm0")
326 __ASM_EMIT("test $1, %[count]")
327 __ASM_EMIT("jz 10f")
328 __ASM_EMIT("movss %%xmm0, 0x00(%[dst])")
329 __ASM_EMIT("add $4, %[dst]")
330 __ASM_EMIT("10:")
331 __ASM_EMIT("test $2, %[count]")
332 __ASM_EMIT("jz 12f")
333 __ASM_EMIT("movhps %%xmm0, 0x00(%[dst])")
334 __ASM_EMIT("12:")
335
336 : [dst] "+r" (dst), [src] "+r" (src),
337 [count] "+r" (count),
338 [mask] "=&r" (mask)
339 : [gate] "r" (c),
340 [knee] "o" (knee),
341 [mem] "o" (mem),
342 [stub] "o" (stub),
343 [G2C] "o" (gate_const),
344 [L2C] "o" (LOG2_CONST),
345 [LOGC] "o" (LOGE_C),
346 [E2C] "o" (EXP2_CONST),
347 [LOG2E] "m" (EXP_LOG2E)
348 : "cc", "memory",
349 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
350 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
351 );
352 }
353
354 #undef PROCESS_KNEE_SINGLE_X4
355 #undef PROCESS_KNEE_SINGLE_X8
356 #undef PROCESS_GATE_FULL_X4
357 #undef PROCESS_GATE_FULL_X8
358 #undef UNPACK_GATE_KNEE
359
360 } /* namespace sse2 */
361 } /* namespace lsp */
362
363
364 #endif /* PRIVATE_DSP_ARCH_X86_SSE2_DYNAMICS_GATE_H_ */