Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/x86/avx512/dynamics/compressor.h

raw
1 /*
2 * Copyright (C) 2023 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2023 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 30 окт. 2023 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_X86_AVX512_DYNAMICS_COMPRESSOR_H_
23 #define PRIVATE_DSP_ARCH_X86_AVX512_DYNAMICS_COMPRESSOR_H_
24
25 #ifndef PRIVATE_DSP_ARCH_X86_AVX512_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_X86_AVX512_IMPL */
28
29 #include <private/dsp/arch/x86/avx512/pmath/exp.h>
30 #include <private/dsp/arch/x86/avx512/pmath/log.h>
31
32 namespace lsp
33 {
34 namespace avx512
35 {
36 #pragma pack(push, 1)
37 typedef struct comp_knee_t
38 {
39 float start[16]; // +0x000
40 float end[16]; // +0x040
41 float gain[16]; // +0x080
42 float herm[48]; // +0x0c0
43 float tilt[32]; // +0x180
44 } comp_knee_t;
45 #pragma pack(pop)
46
47 static const uint32_t compressor_const[] __lsp_aligned64 =
48 {
49 LSP_DSP_VEC16(0x7fffffff)
50 };
51
52 #define UNPACK_COMP_KNEE(DST, DOFF, SRC, SOFF) \
53 __ASM_EMIT("vbroadcastss " SOFF " + 0x00(%[" SRC "]), %%zmm0") \
54 __ASM_EMIT("vbroadcastss " SOFF " + 0x04(%[" SRC "]), %%zmm1") \
55 __ASM_EMIT("vbroadcastss " SOFF " + 0x08(%[" SRC "]), %%zmm2") \
56 __ASM_EMIT("vbroadcastss " SOFF " + 0x0c(%[" SRC "]), %%zmm3") \
57 __ASM_EMIT("vbroadcastss " SOFF " + 0x10(%[" SRC "]), %%zmm4") \
58 __ASM_EMIT("vbroadcastss " SOFF " + 0x14(%[" SRC "]), %%zmm5") \
59 __ASM_EMIT("vbroadcastss " SOFF " + 0x18(%[" SRC "]), %%zmm6") \
60 __ASM_EMIT("vbroadcastss " SOFF " + 0x1c(%[" SRC "]), %%zmm7") \
61 __ASM_EMIT("vmovaps %%zmm0, " DOFF " + 0x000 + %[" DST "]") \
62 __ASM_EMIT("vmovaps %%zmm1, " DOFF " + 0x040 + %[" DST "]") \
63 __ASM_EMIT("vmovaps %%zmm2, " DOFF " + 0x080 + %[" DST "]") \
64 __ASM_EMIT("vmovaps %%zmm3, " DOFF " + 0x0c0 + %[" DST "]") \
65 __ASM_EMIT("vmovaps %%zmm4, " DOFF " + 0x100 + %[" DST "]") \
66 __ASM_EMIT("vmovaps %%zmm5, " DOFF " + 0x140 + %[" DST "]") \
67 __ASM_EMIT("vmovaps %%zmm6, " DOFF " + 0x180 + %[" DST "]") \
68 __ASM_EMIT("vmovaps %%zmm7, " DOFF " + 0x1c0 + %[" DST "]")
69
70 #define PROCESS_KNEE_SINGLE_X32(OFF) \
71 /* in: zmm0 = lx0, zmm4 = lx1 */ \
72 __ASM_EMIT("vmulps " OFF " + 0x0c0 + %[knee], %%zmm0, %%zmm1") /* zmm1 = herm[0]*lx0 */ \
73 __ASM_EMIT("vmulps " OFF " + 0x0c0 + %[knee], %%zmm4, %%zmm5") \
74 __ASM_EMIT("vmulps " OFF " + 0x180 + %[knee], %%zmm0, %%zmm2") /* zmm2 = tilt[0]*lx0 */ \
75 __ASM_EMIT("vmulps " OFF " + 0x180 + %[knee], %%zmm4, %%zmm6") \
76 __ASM_EMIT("vmovaps 0x000 + %[mem], %%zmm3") /* zmm3 = x0 */ \
77 __ASM_EMIT("vmovaps 0x040 + %[mem], %%zmm7") \
78 __ASM_EMIT("vaddps " OFF " + 0x100 + %[knee], %%zmm1, %%zmm1") /* zmm1 = herm[0]*lx0+herm[1] */ \
79 __ASM_EMIT("vaddps " OFF " + 0x100 + %[knee], %%zmm5, %%zmm5") \
80 __ASM_EMIT("vaddps " OFF " + 0x1c0 + %[knee], %%zmm2, %%zmm2") /* zmm2 = TV = tilt[0]*lx0+tilt[1] */ \
81 __ASM_EMIT("vaddps " OFF " + 0x1c0 + %[knee], %%zmm6, %%zmm6") \
82 __ASM_EMIT("vcmpps $5, " OFF " + 0x040 + %[knee], %%zmm3, %%k4") /* k4 = [x0 >= end] */ \
83 __ASM_EMIT("vcmpps $5, " OFF " + 0x040 + %[knee], %%zmm7, %%k6") \
84 __ASM_EMIT("vfmadd213ps " OFF " + 0x140 + %[knee], %%zmm1, %%zmm0") /* zmm0 = KV = (herm[0]*lx0+herm[1])*lx0+herm[2] */ \
85 __ASM_EMIT("vfmadd213ps " OFF " + 0x140 + %[knee], %%zmm5, %%zmm4") \
86 __ASM_EMIT("vmovaps %%zmm2, %%zmm0 %{%%k4%}") /* zmm0 = [x0 >= end] ? TV : KV */ \
87 __ASM_EMIT("vmovaps %%zmm6, %%zmm4 %{%%k6%}") \
88 EXP_CORE_X32 /* zmm0 = EV = expf([x0 >= end] ? TV : KV) */ \
89 __ASM_EMIT("vmovaps 0x000 + %[mem], %%zmm1") /* zmm1 = x0 */ \
90 __ASM_EMIT("vmovaps 0x040 + %[mem], %%zmm5") \
91 __ASM_EMIT("vcmpps $2, " OFF " + 0x000 + %[knee], %%zmm1, %%k4") /* k4 = [x0 <= start] */ \
92 __ASM_EMIT("vcmpps $2, " OFF " + 0x000 + %[knee], %%zmm5, %%k6") \
93 __ASM_EMIT("vmovaps " OFF " + 0x80 + %[knee], %%zmm0 %{%%k4%}") /* zmm0 = [x0 <= start] ? GV : EV */ \
94 __ASM_EMIT("vmovaps " OFF " + 0x80 + %[knee], %%zmm4 %{%%k6%}") \
95 /* out: zmm0 = g0, zmm4 = g1 */
96
97 #define PROCESS_KNEE_SINGLE_X16(OFF) \
98 /* in: zmm0 = lx0 */ \
99 __ASM_EMIT("vmulps " OFF " + 0x0c0 + %[knee], %%zmm0, %%zmm1") /* zmm1 = herm[0]*lx0 */ \
100 __ASM_EMIT("vmulps " OFF " + 0x180 + %[knee], %%zmm0, %%zmm2") /* zmm2 = tilt[0]*lx0 */ \
101 __ASM_EMIT("vmovaps 0x000 + %[mem], %%zmm3") /* zmm3 = x0 */ \
102 __ASM_EMIT("vaddps " OFF " + 0x100 + %[knee], %%zmm1, %%zmm1") /* zmm1 = herm[0]*lx0+herm[1] */ \
103 __ASM_EMIT("vaddps " OFF " + 0x1c0 + %[knee], %%zmm2, %%zmm2") /* zmm2 = TV = tilt[0]*lx0+tilt[1] */ \
104 __ASM_EMIT("vcmpps $5, " OFF " + 0x040 + %[knee], %%zmm3, %%k4") /* k4 = [x0 >= end] */ \
105 __ASM_EMIT("vfmadd213ps " OFF " + 0x140 + %[knee], %%zmm1, %%zmm0") /* zmm0 = KV = (herm[0]*lx0+herm[1])*lx0+herm[2] */ \
106 __ASM_EMIT("vmovaps %%zmm2, %%zmm0 %{%%k4%}") /* zmm0 = [x0 >= end] ? TV : KV */ \
107 EXP_CORE_X16 /* zmm0 = EV = expf([x0 >= end] ? TV : KV) */ \
108 __ASM_EMIT("vmovaps 0x000 + %[mem], %%zmm1") /* zmm1 = x0 */ \
109 __ASM_EMIT("vcmpps $2, " OFF " + 0x000 + %[knee], %%zmm1, %%k4") /* k4 = [x0 <= start] */ \
110 __ASM_EMIT("vmovaps " OFF " + 0x80 + %[knee], %%zmm0 %{%%k4%}") /* zmm0 = [x0 <= start] ? GV : EV */ \
111 /* out: zmm0 = g0, zmm4 = g1 */
112
113 #define PROCESS_KNEE_SINGLE_X8(OFF) \
114 /* in: ymm0 = lx0 */ \
115 __ASM_EMIT("vmulps " OFF " + 0x0c0 + %[knee], %%ymm0, %%ymm1") /* ymm1 = herm[0]*lx0 */ \
116 __ASM_EMIT("vmulps " OFF " + 0x180 + %[knee], %%ymm0, %%ymm2") /* ymm2 = tilt[0]*lx0 */ \
117 __ASM_EMIT("vmovaps 0x000 + %[mem], %%ymm3") /* ymm3 = x0 */ \
118 __ASM_EMIT("vaddps " OFF " + 0x100 + %[knee], %%ymm1, %%ymm1") /* ymm1 = herm[0]*lx0+herm[1] */ \
119 __ASM_EMIT("vaddps " OFF " + 0x1c0 + %[knee], %%ymm2, %%ymm2") /* ymm2 = TV = tilt[0]*lx0+tilt[1] */ \
120 __ASM_EMIT("vcmpps $5, " OFF " + 0x040 + %[knee], %%ymm3, %%k4") /* k4 = [x0 >= end] */ \
121 __ASM_EMIT("vfmadd213ps " OFF " + 0x140 + %[knee], %%ymm1, %%ymm0") /* ymm0 = KV = (herm[0]*lx0+herm[1])*lx0+herm[2] */ \
122 __ASM_EMIT("vmovaps %%ymm2, %%ymm0 %{%%k4%}") /* ymm0 = [x0 >= end] ? TV : KV */ \
123 EXP_CORE_X8 /* ymm0 = EV = expf([x0 >= end] ? TV : KV) */ \
124 __ASM_EMIT("vmovaps 0x000 + %[mem], %%ymm1") /* ymm1 = x0 */ \
125 __ASM_EMIT("vcmpps $2, " OFF " + 0x000 + %[knee], %%ymm1, %%k4") /* k4 = [x0 <= start] */ \
126 __ASM_EMIT("vmovaps " OFF " + 0x80 + %[knee], %%ymm0 %{%%k4%}") /* ymm0 = [x0 <= start] ? GV : EV */ \
127 /* out: ymm0 = g0, ymm4 = g1 */
128
129 #define PROCESS_KNEE_SINGLE_X4(OFF) \
130 /* in: xmm0 = lx0 */ \
131 __ASM_EMIT("vmulps " OFF " + 0x0c0 + %[knee], %%xmm0, %%xmm1") /* xmm1 = herm[0]*lx0 */ \
132 __ASM_EMIT("vmulps " OFF " + 0x180 + %[knee], %%xmm0, %%xmm2") /* xmm2 = tilt[0]*lx0 */ \
133 __ASM_EMIT("vmovaps 0x000 + %[mem], %%xmm3") /* xmm3 = x0 */ \
134 __ASM_EMIT("vaddps " OFF " + 0x100 + %[knee], %%xmm1, %%xmm1") /* xmm1 = herm[0]*lx0+herm[1] */ \
135 __ASM_EMIT("vaddps " OFF " + 0x1c0 + %[knee], %%xmm2, %%xmm2") /* xmm2 = TV = tilt[0]*lx0+tilt[1] */ \
136 __ASM_EMIT("vcmpps $5, " OFF " + 0x040 + %[knee], %%xmm3, %%k4") /* k4 = [x0 >= end] */ \
137 __ASM_EMIT("vfmadd213ps " OFF " + 0x140 + %[knee], %%xmm1, %%xmm0") /* xmm0 = KV = (herm[0]*lx0+herm[1])*lx0+herm[2] */ \
138 __ASM_EMIT("vmovaps %%xmm2, %%xmm0 %{%%k4%}") /* xmm0 = [x0 >= end] ? TV : KV */ \
139 EXP_CORE_X4 /* xmm0 = EV = expf([x0 >= end] ? TV : KV) */ \
140 __ASM_EMIT("vmovaps 0x000 + %[mem], %%xmm1") /* xmm1 = x0 */ \
141 __ASM_EMIT("vcmpps $2, " OFF " + 0x000 + %[knee], %%xmm1, %%k4") /* k4 = [x0 <= start] */ \
142 __ASM_EMIT("vmovaps " OFF " + 0x80 + %[knee], %%xmm0 %{%%k4%}") /* xmm0 = [x0 <= start] ? GV : EV */ \
143 /* out: xmm0 = g0, xmm4 = g1 */
144
145 #define PROCESS_COMP_FULL_X32 \
146 /* in: zmm0 = x0, zmm4 = x1 */ \
147 __ASM_EMIT("vandps 0x00 + %[C2C], %%zmm0, %%zmm0") /* zmm0 = fabsf(x0) */ \
148 __ASM_EMIT("vandps 0x00 + %[C2C], %%zmm4, %%zmm4") \
149 __ASM_EMIT("vcmpps $6, 0x000 + %[knee], %%zmm0, %%k4") /* k4 = [fabsf(x0) > start[0] ] */ \
150 __ASM_EMIT("vcmpps $6, 0x000 + %[knee], %%zmm4, %%k5") \
151 __ASM_EMIT("vcmpps $6, 0x200 + %[knee], %%zmm0, %%k6") /* k6 = [fabsf(x0) > start[1] ] */ \
152 __ASM_EMIT("vcmpps $6, 0x200 + %[knee], %%zmm4, %%k7") \
153 __ASM_EMIT("korw %%k6, %%k4, %%k4") \
154 __ASM_EMIT("korw %%k7, %%k5, %%k5") \
155 __ASM_EMIT("korw %%k5, %%k4, %%k4") \
156 __ASM_EMIT("kmovw %%k4, %k[mask]") \
157 __ASM_EMIT("test %k[mask], %k[mask]") \
158 __ASM_EMIT("jnz 100f") \
159 __ASM_EMIT("vmovaps 0x080 + %[knee], %%zmm0") /* zmm0 = g0 */ \
160 __ASM_EMIT("vmulps 0x280 + %[knee], %%zmm0, %%zmm0") /* zmm0 = G0 = g0*g1 */ \
161 __ASM_EMIT("vmovaps %%zmm0, %%zmm4") /* zmm4 = G1 = g0*g1 */ \
162 __ASM_EMIT("jmp 200f") \
163 __ASM_EMIT("100:") \
164 __ASM_EMIT("vmovaps %%zmm0, 0x000 + %[mem]") /* *mem = fabsf(x0) */ \
165 __ASM_EMIT("vmovaps %%zmm4, 0x040 + %[mem]") \
166 LOGE_CORE_X32 /* zmm0 = lx0 = logf(fabsf(x0)) */ \
167 __ASM_EMIT("vmovaps %%zmm0, 0x080 + %[mem]") /* *mem = lx0 */ \
168 __ASM_EMIT("vmovaps %%zmm4, 0x0c0 + %[mem]") \
169 PROCESS_KNEE_SINGLE_X32("0x000") /* apply knee 0 */ \
170 __ASM_EMIT("vmovaps %%zmm0, 0x100 + %[mem]") /* *mem = g1 */ \
171 __ASM_EMIT("vmovaps %%zmm4, 0x140 + %[mem]") \
172 __ASM_EMIT("vmovaps 0x080 + %[mem], %%zmm0") /* zmm0 = lx0 */ \
173 __ASM_EMIT("vmovaps 0x0c0 + %[mem], %%zmm4") \
174 PROCESS_KNEE_SINGLE_X32("0x200") /* apply knee 1 */ \
175 __ASM_EMIT("vmulps 0x100 + %[mem], %%zmm0, %%zmm0") /* zmm0 = G = g0*g1 */ \
176 __ASM_EMIT("vmulps 0x140 + %[mem], %%zmm4, %%zmm4") \
177 __ASM_EMIT("200:") \
178 /* out: zmm0 = G0, zmm4 = G1 */
179
180 #define PROCESS_COMP_FULL_X16 \
181 /* in: zmm0 = x0 */ \
182 __ASM_EMIT("vandps 0x00 + %[C2C], %%zmm0, %%zmm0") /* zmm0 = fabsf(x0) */ \
183 __ASM_EMIT("vcmpps $6, 0x000 + %[knee], %%zmm0, %%k4") /* k4 = [fabsf(x0) > start[0] ] */ \
184 __ASM_EMIT("vcmpps $6, 0x200 + %[knee], %%zmm0, %%k6") /* k6 = [fabsf(x0) > start[1] ] */ \
185 __ASM_EMIT("korw %%k6, %%k4, %%k4") \
186 __ASM_EMIT("kmovw %%k4, %k[mask]") \
187 __ASM_EMIT("test %k[mask], %k[mask]") \
188 __ASM_EMIT("jnz 100f") \
189 __ASM_EMIT("vmovaps 0x080 + %[knee], %%zmm0") /* zmm0 = g0 */ \
190 __ASM_EMIT("vmulps 0x280 + %[knee], %%zmm0, %%zmm0") /* zmm0 = g0*g1 */ \
191 __ASM_EMIT("jmp 200f") \
192 __ASM_EMIT("100:") \
193 __ASM_EMIT("vmovaps %%zmm0, 0x000 + %[mem]") /* *mem = fabsf(x0) */ \
194 LOGE_CORE_X16 /* zmm0 = lx0 = logf(fabsf(x0)) */ \
195 __ASM_EMIT("vmovaps %%zmm0, 0x080 + %[mem]") /* *mem = lx0 */ \
196 PROCESS_KNEE_SINGLE_X16("0x000") /* apply knee 0 */ \
197 __ASM_EMIT("vmovaps %%zmm0, 0x100 + %[mem]") /* *mem = g1 */ \
198 __ASM_EMIT("vmovaps 0x080 + %[mem], %%zmm0") /* zmm0 = lx0 */ \
199 PROCESS_KNEE_SINGLE_X16("0x200") /* apply knee 1 */ \
200 __ASM_EMIT("vmulps 0x100 + %[mem], %%zmm0, %%zmm0") /* zmm0 = G = g0*g1 */ \
201 __ASM_EMIT("200:") \
202 /* out: zmm0 = G0 */
203
204 #define PROCESS_COMP_FULL_X8 \
205 /* in: ymm0 = x0 */ \
206 __ASM_EMIT("vandps 0x00 + %[C2C], %%ymm0, %%ymm0") /* ymm0 = fabsf(x0) */ \
207 __ASM_EMIT("vcmpps $6, 0x000 + %[knee], %%ymm0, %%k4") /* k4 = [fabsf(x0) > start[0] ] */ \
208 __ASM_EMIT("vcmpps $6, 0x200 + %[knee], %%ymm0, %%k6") /* k6 = [fabsf(x0) > start[1] ] */ \
209 __ASM_EMIT("korw %%k6, %%k4, %%k4") \
210 __ASM_EMIT("kmovw %%k4, %k[mask]") \
211 __ASM_EMIT("test %k[mask], %k[mask]") \
212 __ASM_EMIT("jnz 100f") \
213 __ASM_EMIT("vmovaps 0x080 + %[knee], %%ymm0") /* ymm0 = g0 */ \
214 __ASM_EMIT("vmulps 0x280 + %[knee], %%ymm0, %%ymm0") /* ymm0 = g0*g1 */ \
215 __ASM_EMIT("jmp 200f") \
216 __ASM_EMIT("100:") \
217 __ASM_EMIT("vmovaps %%ymm0, 0x000 + %[mem]") /* *mem = fabsf(x0) */ \
218 LOGE_CORE_X8 /* ymm0 = lx0 = logf(fabsf(x0)) */ \
219 __ASM_EMIT("vmovaps %%ymm0, 0x080 + %[mem]") /* *mem = lx0 */ \
220 PROCESS_KNEE_SINGLE_X8("0x000") /* apply knee 0 */ \
221 __ASM_EMIT("vmovaps %%ymm0, 0x100 + %[mem]") /* *mem = g1 */ \
222 __ASM_EMIT("vmovaps 0x080 + %[mem], %%ymm0") /* ymm0 = lx0 */ \
223 PROCESS_KNEE_SINGLE_X8("0x200") /* apply knee 1 */ \
224 __ASM_EMIT("vmulps 0x100 + %[mem], %%ymm0, %%ymm0") /* ymm0 = G = g0*g1 */ \
225 __ASM_EMIT("200:") \
226 /* out: ymm0 = G0 */
227
228 #define PROCESS_COMP_FULL_X4 \
229 /* in: xmm0 = x0 */ \
230 __ASM_EMIT("vandps 0x00 + %[C2C], %%xmm0, %%xmm0") /* xmm0 = fabsf(x0) */ \
231 __ASM_EMIT("vmovaps %%xmm0, 0x000 + %[mem]") /* *mem = fabsf(x0) */ \
232 LOGE_CORE_X4 /* xmm0 = lx0 = logf(fabsf(x0)) */ \
233 __ASM_EMIT("vmovaps %%xmm0, 0x080 + %[mem]") /* *mem = lx0 */ \
234 PROCESS_KNEE_SINGLE_X4("0x000") /* apply knee 0 */ \
235 __ASM_EMIT("vmovaps %%xmm0, 0x100 + %[mem]") /* *mem = g1 */ \
236 __ASM_EMIT("vmovaps 0x080 + %[mem], %%xmm0") /* xmm0 = lx0 */ \
237 PROCESS_KNEE_SINGLE_X4("0x200") /* apply knee 1 */ \
238 __ASM_EMIT("vmulps 0x100 + %[mem], %%xmm0, %%xmm0") /* xmm0 = G = g0*g1 */ \
239 /* out: xmm0 = G0 */
240
241 void compressor_x2_gain(float *dst, const float *src, const dsp::compressor_x2_t *c, size_t count)
242 {
243 IF_ARCH_X86(
244 comp_knee_t knee[2] __lsp_aligned64;
245 float mem[96] __lsp_aligned64;
246 float stub[16] __lsp_aligned64;
247 size_t mask;
248 );
249
250 ARCH_X86_ASM
251 (
252 // Prepare stuff
253 UNPACK_COMP_KNEE("knee", "0x000", "comp", "0x00")
254 UNPACK_COMP_KNEE("knee", "0x200", "comp", "0x20")
255
256 // 32x blocks
257 __ASM_EMIT("sub $32, %[count]")
258 __ASM_EMIT("jb 2f")
259 __ASM_EMIT("1:")
260 __ASM_EMIT("vmovups 0x00(%[src]), %%zmm0")
261 __ASM_EMIT("vmovups 0x40(%[src]), %%zmm4")
262 PROCESS_COMP_FULL_X32
263 __ASM_EMIT("vmovups %%zmm0, 0x00(%[dst])")
264 __ASM_EMIT("vmovups %%zmm4, 0x40(%[dst])")
265 __ASM_EMIT("add $0x80, %[src]")
266 __ASM_EMIT("add $0x80, %[dst]")
267 __ASM_EMIT("sub $32, %[count]")
268 __ASM_EMIT("jae 1b")
269 __ASM_EMIT("2:")
270 // 16x blocks
271 __ASM_EMIT("add $16, %[count]")
272 __ASM_EMIT("jl 4f")
273 __ASM_EMIT("vmovups 0x00(%[src]), %%zmm0")
274 PROCESS_COMP_FULL_X16
275 __ASM_EMIT("vmovups %%zmm0, 0x00(%[dst])")
276 __ASM_EMIT("add $0x40, %[src]")
277 __ASM_EMIT("add $0x40, %[dst]")
278 __ASM_EMIT("sub $16, %[count]")
279 __ASM_EMIT("4:")
280 // 8x blocks
281 __ASM_EMIT("add $8, %[count]")
282 __ASM_EMIT("jl 6f")
283 __ASM_EMIT("vmovups 0x00(%[src]), %%ymm0")
284 PROCESS_COMP_FULL_X8
285 __ASM_EMIT("vmovups %%ymm0, 0x00(%[dst])")
286 __ASM_EMIT("add $0x20, %[src]")
287 __ASM_EMIT("add $0x20, %[dst]")
288 __ASM_EMIT("sub $8, %[count]")
289 __ASM_EMIT("6:")
290 // 4x blocks
291 __ASM_EMIT("add $4, %[count]")
292 __ASM_EMIT("jl 8f")
293 __ASM_EMIT("vmovups 0x00(%[src]), %%xmm0")
294 PROCESS_COMP_FULL_X4
295 __ASM_EMIT("vmovups %%xmm0, 0x00(%[dst])")
296 __ASM_EMIT("sub $4, %[count]")
297 __ASM_EMIT("add $0x10, %[src]")
298 __ASM_EMIT("add $0x10, %[dst]")
299 __ASM_EMIT("8:")
300 // Tail: 1x-3x block
301 __ASM_EMIT("add $4, %[count]")
302 __ASM_EMIT("jle 16f")
303 __ASM_EMIT("test $1, %[count]")
304 __ASM_EMIT("jz 10f")
305 __ASM_EMIT("vmovss 0x00(%[src]), %%xmm0")
306 __ASM_EMIT("add $4, %[src]")
307 __ASM_EMIT("10:")
308 __ASM_EMIT("test $2, %[count]")
309 __ASM_EMIT("jz 12f")
310 __ASM_EMIT("vmovhps 0x00(%[src]), %%xmm0, %%xmm0")
311 __ASM_EMIT("12:")
312 PROCESS_COMP_FULL_X4
313 __ASM_EMIT("test $1, %[count]")
314 __ASM_EMIT("jz 14f")
315 __ASM_EMIT("vmovss %%xmm0, 0x00(%[dst])")
316 __ASM_EMIT("add $4, %[dst]")
317 __ASM_EMIT("14:")
318 __ASM_EMIT("test $2, %[count]")
319 __ASM_EMIT("jz 16f")
320 __ASM_EMIT("vmovhps %%xmm0, 0x00(%[dst])")
321 __ASM_EMIT("16:")
322
323 : [dst] "+r" (dst), [src] "+r" (src),
324 [count] "+r" (count),
325 [mask] "=&r" (mask)
326 : [comp] "r" (c),
327 [knee] "o" (knee),
328 [mem] "o" (mem),
329 [stub] "o" (stub),
330 [C2C] "o" (compressor_const),
331 [L2C] "o" (LOG2_CONST),
332 [LOGC] "o" (LOGE_C),
333 [E2C] "o" (EXP2_CONST),
334 [LOG2E] "m" (EXP_LOG2E)
335 : "cc", "memory",
336 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
337 "%xmm4", "%xmm5", "%xmm6", "%xmm7",
338 "%k4", "%k6"
339 );
340 }
341
342 void compressor_x2_curve(float *dst, const float *src, const dsp::compressor_x2_t *c, size_t count)
343 {
344 IF_ARCH_X86(
345 comp_knee_t knee[2] __lsp_aligned64;
346 float mem[96] __lsp_aligned64;
347 float stub[16] __lsp_aligned64;
348 size_t mask;
349 );
350
351 ARCH_X86_ASM
352 (
353 // Prepare stuff
354 UNPACK_COMP_KNEE("knee", "0x000", "comp", "0x00")
355 UNPACK_COMP_KNEE("knee", "0x200", "comp", "0x20")
356
357 // 32x blocks
358 __ASM_EMIT("sub $32, %[count]")
359 __ASM_EMIT("jb 2f")
360 __ASM_EMIT("1:")
361 __ASM_EMIT("vmovups 0x00(%[src]), %%zmm0")
362 __ASM_EMIT("vmovups 0x40(%[src]), %%zmm4")
363 PROCESS_COMP_FULL_X32
364 __ASM_EMIT("vmulps 0x00(%[src]), %%zmm0, %%zmm0")
365 __ASM_EMIT("vmulps 0x40(%[src]), %%zmm4, %%zmm4")
366 __ASM_EMIT("vmovups %%zmm0, 0x00(%[dst])")
367 __ASM_EMIT("vmovups %%zmm4, 0x40(%[dst])")
368 __ASM_EMIT("add $0x80, %[src]")
369 __ASM_EMIT("add $0x80, %[dst]")
370 __ASM_EMIT("sub $32, %[count]")
371 __ASM_EMIT("jae 1b")
372 __ASM_EMIT("2:")
373 // 16x blocks
374 __ASM_EMIT("add $16, %[count]")
375 __ASM_EMIT("jl 4f")
376 __ASM_EMIT("vmovups 0x00(%[src]), %%zmm0")
377 PROCESS_COMP_FULL_X16
378 __ASM_EMIT("vmulps 0x00(%[src]), %%zmm0, %%zmm0")
379 __ASM_EMIT("vmovups %%zmm0, 0x00(%[dst])")
380 __ASM_EMIT("add $0x40, %[src]")
381 __ASM_EMIT("add $0x40, %[dst]")
382 __ASM_EMIT("sub $16, %[count]")
383 __ASM_EMIT("4:")
384 // 8x blocks
385 __ASM_EMIT("add $8, %[count]")
386 __ASM_EMIT("jl 6f")
387 __ASM_EMIT("vmovups 0x00(%[src]), %%ymm0")
388 PROCESS_COMP_FULL_X8
389 __ASM_EMIT("vmulps 0x00(%[src]), %%ymm0, %%ymm0")
390 __ASM_EMIT("vmovups %%ymm0, 0x00(%[dst])")
391 __ASM_EMIT("add $0x20, %[src]")
392 __ASM_EMIT("add $0x20, %[dst]")
393 __ASM_EMIT("sub $8, %[count]")
394 __ASM_EMIT("6:")
395 // 4x blocks
396 __ASM_EMIT("add $4, %[count]")
397 __ASM_EMIT("jl 8f")
398 __ASM_EMIT("vmovups 0x00(%[src]), %%xmm0")
399 PROCESS_COMP_FULL_X4
400 __ASM_EMIT("vmulps 0x00(%[src]), %%xmm0, %%xmm0")
401 __ASM_EMIT("vmovups %%xmm0, 0x00(%[dst])")
402 __ASM_EMIT("sub $4, %[count]")
403 __ASM_EMIT("add $0x10, %[src]")
404 __ASM_EMIT("add $0x10, %[dst]")
405 __ASM_EMIT("8:")
406 // Tail: 1x-3x block
407 __ASM_EMIT("add $4, %[count]")
408 __ASM_EMIT("jle 16f")
409 __ASM_EMIT("test $1, %[count]")
410 __ASM_EMIT("jz 10f")
411 __ASM_EMIT("vmovss 0x00(%[src]), %%xmm0")
412 __ASM_EMIT("add $4, %[src]")
413 __ASM_EMIT("10:")
414 __ASM_EMIT("test $2, %[count]")
415 __ASM_EMIT("jz 12f")
416 __ASM_EMIT("vmovhps 0x00(%[src]), %%xmm0, %%xmm0")
417 __ASM_EMIT("12:")
418 __ASM_EMIT("vmovaps %%xmm0, %%xmm4")
419 PROCESS_COMP_FULL_X4
420 __ASM_EMIT("vmulps %%xmm4, %%xmm0, %%xmm0")
421 __ASM_EMIT("test $1, %[count]")
422 __ASM_EMIT("jz 14f")
423 __ASM_EMIT("vmovss %%xmm0, 0x00(%[dst])")
424 __ASM_EMIT("add $4, %[dst]")
425 __ASM_EMIT("14:")
426 __ASM_EMIT("test $2, %[count]")
427 __ASM_EMIT("jz 16f")
428 __ASM_EMIT("vmovhps %%xmm0, 0x00(%[dst])")
429 __ASM_EMIT("16:")
430
431 : [dst] "+r" (dst), [src] "+r" (src),
432 [count] "+r" (count),
433 [mask] "=&r" (mask)
434 : [comp] "r" (c),
435 [knee] "o" (knee),
436 [mem] "o" (mem),
437 [stub] "o" (stub),
438 [C2C] "o" (compressor_const),
439 [L2C] "o" (LOG2_CONST),
440 [LOGC] "o" (LOGE_C),
441 [E2C] "o" (EXP2_CONST),
442 [LOG2E] "m" (EXP_LOG2E)
443 : "cc", "memory",
444 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
445 "%xmm4", "%xmm5", "%xmm6", "%xmm7",
446 "%k4", "%k6"
447 );
448 }
449
450 #undef PROCESS_COMP_FULL_X4
451 #undef PROCESS_COMP_FULL_X8
452 #undef PROCESS_COMP_FULL_X16
453 #undef PROCESS_COMP_FULL_X32
454 #undef PROCESS_KNEE_SINGLE_X4
455 #undef PROCESS_KNEE_SINGLE_X8
456 #undef PROCESS_KNEE_SINGLE_X16
457 #undef PROCESS_KNEE_SINGLE_X32
458
459 #undef UNPACK_COMP_KNEE
460
461 } /* namespace avx512 */
462 } /* namespace lsp */
463
464
465
466
467
468 #endif /* PRIVATE_DSP_ARCH_X86_AVX512_DYNAMICS_COMPRESSOR_H_ */