Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/x86/avx512/dynamics/gate.h

raw
1 /*
2 * Copyright (C) 2023 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2023 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 31 окт. 2023 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_X86_AVX512_DYNAMICS_GATE_H_
23 #define PRIVATE_DSP_ARCH_X86_AVX512_DYNAMICS_GATE_H_
24
25 #ifndef PRIVATE_DSP_ARCH_X86_AVX512_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_X86_AVX512_IMPL */
28
29 #include <private/dsp/arch/x86/avx512/pmath/exp.h>
30 #include <private/dsp/arch/x86/avx512/pmath/log.h>
31
32 namespace lsp
33 {
34 namespace avx512
35 {
36 #pragma pack(push, 1)
37 typedef struct gate_knee_t
38 {
39 float start[16]; // +0x000
40 float end[16]; // +0x040
41 float gain_start[16]; // +0x080
42 float gain_end[16]; // +0x0c0
43 float herm[64]; // +0x100
44 } gate_knee_t;
45 #pragma pack(pop)
46
47 static const uint32_t gate_const[] __lsp_aligned64 =
48 {
49 LSP_DSP_VEC16(0x7fffffff)
50 };
51
52
53 #define UNPACK_GATE_KNEE(DST, SRC) \
54 __ASM_EMIT("vbroadcastss 0x00(%[" SRC "]), %%zmm0") \
55 __ASM_EMIT("vbroadcastss 0x04(%[" SRC "]), %%zmm1") \
56 __ASM_EMIT("vbroadcastss 0x08(%[" SRC "]), %%zmm2") \
57 __ASM_EMIT("vbroadcastss 0x0c(%[" SRC "]), %%zmm3") \
58 __ASM_EMIT("vbroadcastss 0x10(%[" SRC "]), %%zmm4") \
59 __ASM_EMIT("vbroadcastss 0x14(%[" SRC "]), %%zmm5") \
60 __ASM_EMIT("vbroadcastss 0x18(%[" SRC "]), %%zmm6") \
61 __ASM_EMIT("vbroadcastss 0x1c(%[" SRC "]), %%zmm7") \
62 __ASM_EMIT("vmovaps %%zmm0, 0x000 + %[" DST "]") \
63 __ASM_EMIT("vmovaps %%zmm1, 0x040 + %[" DST "]") \
64 __ASM_EMIT("vmovaps %%zmm2, 0x080 + %[" DST "]") \
65 __ASM_EMIT("vmovaps %%zmm3, 0x0c0 + %[" DST "]") \
66 __ASM_EMIT("vmovaps %%zmm4, 0x100 + %[" DST "]") \
67 __ASM_EMIT("vmovaps %%zmm5, 0x140 + %[" DST "]") \
68 __ASM_EMIT("vmovaps %%zmm6, 0x180 + %[" DST "]") \
69 __ASM_EMIT("vmovaps %%zmm7, 0x1c0 + %[" DST "]")
70
71 #define PROCESS_KNEE_SINGLE_X32 \
72 /* in: zmm0 = lx0, zmm4 = lx1 */ \
73 __ASM_EMIT("vmovaps 0x100 + %[knee], %%zmm1") /* zmm1 = herm[0] */ \
74 __ASM_EMIT("vmovaps 0x100 + %[knee], %%zmm5") \
75 __ASM_EMIT("vfmadd213ps 0x140 + %[knee], %%zmm0, %%zmm1") /* zmm1 = herm[0]*lx0+herm[1] */ \
76 __ASM_EMIT("vfmadd213ps 0x140 + %[knee], %%zmm4, %%zmm5") \
77 __ASM_EMIT("vfmadd213ps 0x180 + %[knee], %%zmm0, %%zmm1") /* zmm1 = (herm[0]*lx0+herm[1])*lx0+herm[2] */ \
78 __ASM_EMIT("vfmadd213ps 0x180 + %[knee], %%zmm4, %%zmm5") \
79 __ASM_EMIT("vfmadd213ps 0x1c0 + %[knee], %%zmm1, %%zmm0") /* zmm0 = KV = ((herm[0]*lx0+herm[1])*lx0+herm[2])*lx0+herm[3] */ \
80 __ASM_EMIT("vfmadd213ps 0x1c0 + %[knee], %%zmm5, %%zmm4") \
81 EXP_CORE_X32 /* zmm0 = EV = expf(KV) */ \
82 __ASM_EMIT("vmovaps 0x000 + %[mem], %%zmm1") /* zmm1 = x0 */ \
83 __ASM_EMIT("vmovaps 0x040 + %[mem], %%zmm5") \
84 __ASM_EMIT("vcmpps $2, 0x00 + %[knee], %%zmm1, %%k4") /* k4 = [x0 <= start] */ \
85 __ASM_EMIT("vcmpps $2, 0x00 + %[knee], %%zmm5, %%k6") \
86 __ASM_EMIT("vcmpps $5, 0x40 + %[knee], %%zmm1, %%k5") /* k5 = [x0 >= end] */ \
87 __ASM_EMIT("vcmpps $5, 0x40 + %[knee], %%zmm5, %%k7") \
88 __ASM_EMIT("vmovaps 0x080 + %[knee], %%zmm0 %{%%k4%}") /* zmm0 = [x0 <= start] ? gain_start : expf(KV) */ \
89 __ASM_EMIT("vmovaps 0x080 + %[knee], %%zmm4 %{%%k6%}") \
90 __ASM_EMIT("vmovaps 0x0c0 + %[knee], %%zmm0 %{%%k5%}") /* zmm0 = [x0 >= end] ? gain_end : [x0 <= start] ? gain_start : expf(KV) */ \
91 __ASM_EMIT("vmovaps 0x0c0 + %[knee], %%zmm4 %{%%k7%}") \
92 /* out: zmm0 = g0, zmm4 = g1 */
93
94 #define PROCESS_KNEE_SINGLE_X16 \
95 /* in: ymm0 = lx0 */ \
96 __ASM_EMIT("vmovaps 0x100 + %[knee], %%zmm1") /* zmm1 = herm[0] */ \
97 __ASM_EMIT("vfmadd213ps 0x140 + %[knee], %%zmm0, %%zmm1") /* zmm1 = herm[0]*lx0+herm[1] */ \
98 __ASM_EMIT("vfmadd213ps 0x180 + %[knee], %%zmm0, %%zmm1") /* zmm1 = (herm[0]*lx0+herm[1])*lx0+herm[2] */ \
99 __ASM_EMIT("vfmadd213ps 0x1c0 + %[knee], %%zmm1, %%zmm0") /* zmm0 = KV = ((herm[0]*lx0+herm[1])*lx0+herm[2])*lx0+herm[3] */ \
100 EXP_CORE_X16 /* zmm0 = EV = expf(KV) */ \
101 __ASM_EMIT("vmovaps 0x000 + %[mem], %%zmm1") /* zmm1 = x0 */ \
102 __ASM_EMIT("vcmpps $2, 0x00 + %[knee], %%zmm1, %%k4") /* k4 = [x0 <= start] */ \
103 __ASM_EMIT("vcmpps $5, 0x40 + %[knee], %%zmm1, %%k5") /* k5 = [x0 >= end] */ \
104 __ASM_EMIT("vmovaps 0x080 + %[knee], %%zmm0 %{%%k4%}") /* zmm0 = [x0 <= start] ? gain_start : expf(KV) */ \
105 __ASM_EMIT("vmovaps 0x0c0 + %[knee], %%zmm0 %{%%k5%}") /* zmm0 = [x0 >= end] ? gain_end : [x0 <= start] ? gain_start : expf(KV) */ \
106 /* out: ymm0 = g0 */
107
108 #define PROCESS_KNEE_SINGLE_X8 \
109 /* in: ymm0 = lx0 */ \
110 __ASM_EMIT("vmovaps 0x100 + %[knee], %%ymm1") /* ymm1 = herm[0] */ \
111 __ASM_EMIT("vfmadd213ps 0x140 + %[knee], %%ymm0, %%ymm1") /* ymm1 = herm[0]*lx0+herm[1] */ \
112 __ASM_EMIT("vfmadd213ps 0x180 + %[knee], %%ymm0, %%ymm1") /* ymm1 = (herm[0]*lx0+herm[1])*lx0+herm[2] */ \
113 __ASM_EMIT("vfmadd213ps 0x1c0 + %[knee], %%ymm1, %%ymm0") /* ymm0 = KV = ((herm[0]*lx0+herm[1])*lx0+herm[2])*lx0+herm[3] */ \
114 EXP_CORE_X8 /* ymm0 = EV = expf(KV) */ \
115 __ASM_EMIT("vmovaps 0x000 + %[mem], %%ymm1") /* ymm1 = x0 */ \
116 __ASM_EMIT("vcmpps $2, 0x00 + %[knee], %%ymm1, %%k4") /* k4 = [x0 <= start] */ \
117 __ASM_EMIT("vcmpps $5, 0x40 + %[knee], %%ymm1, %%k5") /* k5 = [x0 >= end] */ \
118 __ASM_EMIT("vmovaps 0x080 + %[knee], %%ymm0 %{%%k4%}") /* ymm0 = [x0 <= start] ? gain_start : expf(KV) */ \
119 __ASM_EMIT("vmovaps 0x0c0 + %[knee], %%ymm0 %{%%k5%}") /* ymm0 = [x0 >= end] ? gain_end : [x0 <= start] ? gain_start : expf(KV) */ \
120 /* out: ymm0 = g0 */
121
122 #define PROCESS_KNEE_SINGLE_X4 \
123 /* in: xmm0 = lx0 */ \
124 __ASM_EMIT("vmovaps 0x100 + %[knee], %%xmm1") /* xmm1 = herm[0] */ \
125 __ASM_EMIT("vfmadd213ps 0x140 + %[knee], %%xmm0, %%xmm1") /* xmm1 = herm[0]*lx0+herm[1] */ \
126 __ASM_EMIT("vfmadd213ps 0x180 + %[knee], %%xmm0, %%xmm1") /* xmm1 = (herm[0]*lx0+herm[1])*lx0+herm[2] */ \
127 __ASM_EMIT("vfmadd213ps 0x1c0 + %[knee], %%xmm1, %%xmm0") /* xmm0 = KV = ((herm[0]*lx0+herm[1])*lx0+herm[2])*lx0+herm[3] */ \
128 EXP_CORE_X4 /* xmm0 = EV = expf(KV) */ \
129 __ASM_EMIT("vmovaps 0x000 + %[mem], %%xmm1") /* xmm1 = x0 */ \
130 __ASM_EMIT("vcmpps $2, 0x00 + %[knee], %%xmm1, %%k4") /* k4 = [x0 <= start] */ \
131 __ASM_EMIT("vcmpps $5, 0x40 + %[knee], %%xmm1, %%k5") /* k5 = [x0 >= end] */ \
132 __ASM_EMIT("vmovaps 0x080 + %[knee], %%xmm0 %{%%k4%}") /* xmm0 = [x0 <= start] ? gain_start : expf(KV) */ \
133 __ASM_EMIT("vmovaps 0x0c0 + %[knee], %%xmm0 %{%%k5%}") /* xmm0 = [x0 >= end] ? gain_end : [x0 <= start] ? gain_start : expf(KV) */ \
134 /* out: xmm0 = g0 */
135
136 #define PROCESS_GATE_FULL_X32 \
137 /* in: zmm0 = x0, zmm4 = x1 */ \
138 __ASM_EMIT("vandps 0x00 + %[G2C], %%zmm0, %%zmm0") /* zmm0 = fabsf(x0) */ \
139 __ASM_EMIT("vandps 0x00 + %[G2C], %%zmm4, %%zmm4") \
140 __ASM_EMIT("vcmpps $6, 0x000 + %[knee], %%zmm0, %%k4") /* k4 = [x0 > start] */ \
141 __ASM_EMIT("vcmpps $6, 0x000 + %[knee], %%zmm4, %%k6") \
142 __ASM_EMIT("vcmpps $1, 0x040 + %[knee], %%zmm0, %%k5 %{%%k4%}") /* k5 = [x0 > start] && [x0 < end] */ \
143 __ASM_EMIT("vcmpps $1, 0x040 + %[knee], %%zmm4, %%k7 %{%%k6%}") \
144 __ASM_EMIT("korw %%k7, %%k5, %%k5") \
145 __ASM_EMIT("kmovw %%k5, %k[mask]") \
146 __ASM_EMIT("test %k[mask], %k[mask]") \
147 __ASM_EMIT("jnz 100f") \
148 __ASM_EMIT("vmovaps 0x080 + %[knee], %%zmm0") /* zmm0 = gain_start */ \
149 __ASM_EMIT("vmovaps 0x080 + %[knee], %%zmm4") \
150 __ASM_EMIT("vmovaps 0x0c0 + %[knee], %%zmm0 %{%%k4%}") /* zmm0 = [x0 > start] ? gain_end : gain_start */ \
151 __ASM_EMIT("vmovaps 0x0c0 + %[knee], %%zmm4 %{%%k6%}") \
152 __ASM_EMIT("jmp 200f") \
153 __ASM_EMIT("100:") \
154 __ASM_EMIT("vmovaps %%zmm0, 0x00 + %[mem]") /* store fabsf(x0) */ \
155 __ASM_EMIT("vmovaps %%zmm4, 0x40 + %[mem]") \
156 LOGE_CORE_X32 /* ymm0 = lx0 = logf(fabsf(x0)) */ \
157 PROCESS_KNEE_SINGLE_X32 /* apply knee */ \
158 __ASM_EMIT("200:") \
159 /* out: zmm0 = g0, zmm4 = g1 */
160
161 #define PROCESS_GATE_FULL_X16 \
162 /* in: zmm0 = x */ \
163 __ASM_EMIT("vandps 0x00 + %[G2C], %%zmm0, %%zmm0") /* zmm0 = fabsf(x0) */ \
164 __ASM_EMIT("vcmpps $6, 0x000 + %[knee], %%zmm0, %%k4") /* k4 = [x0 > start] */ \
165 __ASM_EMIT("vcmpps $1, 0x040 + %[knee], %%zmm0, %%k5 %{%%k4%}")/* k5 = [x0 > start] && [x0 < end] */ \
166 __ASM_EMIT("kmovw %%k5, %k[mask]") \
167 __ASM_EMIT("test %k[mask], %k[mask]") \
168 __ASM_EMIT("jnz 100f") \
169 __ASM_EMIT("vmovaps 0x080 + %[knee], %%zmm0") /* zmm0 = gain_start */ \
170 __ASM_EMIT("vmovaps 0x0c0 + %[knee], %%zmm0 %{%%k4%}") /* zmm0 = [x0 > start] ? gain_end : gain_start */ \
171 __ASM_EMIT("jmp 200f") \
172 __ASM_EMIT("100:") \
173 __ASM_EMIT("vmovaps %%zmm0, 0x00 + %[mem]") /* store fabsf(x0) */ \
174 LOGE_CORE_X16 /* ymm0 = lx0 = logf(fabsf(x0)) */ \
175 PROCESS_KNEE_SINGLE_X16 /* apply knee */ \
176 __ASM_EMIT("200:") \
177 /* out: zmm0 = g */
178
179 #define PROCESS_GATE_FULL_X8 \
180 /* in: ymm0 = x0 */ \
181 __ASM_EMIT("vandps 0x00 + %[G2C], %%ymm0, %%ymm0") /* ymm0 = fabsf(x0) */ \
182 __ASM_EMIT("vcmpps $6, 0x000 + %[knee], %%ymm0, %%k4") /* k4 = [x0 > start] */ \
183 __ASM_EMIT("vcmpps $1, 0x040 + %[knee], %%ymm0, %%k5 %{%%k4%}")/* k5 = [x0 > start] && [x0 < end] */ \
184 __ASM_EMIT("kmovw %%k5, %k[mask]") \
185 __ASM_EMIT("test %k[mask], %k[mask]") \
186 __ASM_EMIT("jnz 100f") \
187 __ASM_EMIT("vmovaps 0x080 + %[knee], %%ymm0") /* ymm0 = gain_start */ \
188 __ASM_EMIT("vmovaps 0x0c0 + %[knee], %%ymm0 %{%%k4%}") /* ymm0 = [x0 > start] ? gain_end : gain_start */ \
189 __ASM_EMIT("jmp 200f") \
190 __ASM_EMIT("100:") \
191 __ASM_EMIT("vandps 0x00 + %[G2C], %%ymm0, %%ymm0") /* ymm0 = fabsf(x0) */ \
192 __ASM_EMIT("vmovaps %%ymm0, 0x00 + %[mem]") /* store fabsf(x0) */ \
193 LOGE_CORE_X8 /* ymm0 = lx0 = logf(fabsf(x0)) */ \
194 PROCESS_KNEE_SINGLE_X8 /* apply knee */ \
195 __ASM_EMIT("200:") \
196 /* out: ymm0 = g */
197
198 #define PROCESS_GATE_FULL_X4 \
199 /* in: xmm0 = x0 */ \
200 __ASM_EMIT("vandps 0x00 + %[G2C], %%xmm0, %%xmm0") /* xmm0 = fabsf(x0) */ \
201 __ASM_EMIT("vmovaps %%xmm0, 0x00 + %[mem]") /* store fabsf(x0) */ \
202 LOGE_CORE_X4 /* xmm0 = lx0 = logf(fabsf(x0)) */ \
203 PROCESS_KNEE_SINGLE_X4 /* apply knee */ \
204 /* out: xmm0 = g */
205
206
207 void gate_x1_gain(float *dst, const float *src, const dsp::gate_knee_t *c, size_t count)
208 {
209 IF_ARCH_X86(
210 gate_knee_t knee __lsp_aligned64;
211 float mem[32] __lsp_aligned64;
212 float stub[16] __lsp_aligned64;
213 size_t mask;
214 );
215
216 ARCH_X86_ASM
217 (
218 // Prepare stuff
219 UNPACK_GATE_KNEE("knee", "gate")
220
221 // 32x blocks
222 __ASM_EMIT("sub $32, %[count]")
223 __ASM_EMIT("jb 2f")
224 __ASM_EMIT("1:")
225 __ASM_EMIT("vmovups 0x00(%[src]), %%zmm0")
226 __ASM_EMIT("vmovups 0x40(%[src]), %%zmm4")
227 PROCESS_GATE_FULL_X32
228 __ASM_EMIT("vmovups %%zmm0, 0x00(%[dst])")
229 __ASM_EMIT("vmovups %%zmm4, 0x40(%[dst])")
230 __ASM_EMIT("add $0x80, %[src]")
231 __ASM_EMIT("add $0x80, %[dst]")
232 __ASM_EMIT("sub $32, %[count]")
233 __ASM_EMIT("jae 1b")
234 __ASM_EMIT("2:")
235 // 16x blocks
236 __ASM_EMIT("add $16, %[count]")
237 __ASM_EMIT("jl 4f")
238 __ASM_EMIT("vmovups 0x00(%[src]), %%zmm0")
239 PROCESS_GATE_FULL_X16
240 __ASM_EMIT("vmovups %%zmm0, 0x00(%[dst])")
241 __ASM_EMIT("add $0x40, %[src]")
242 __ASM_EMIT("add $0x40, %[dst]")
243 __ASM_EMIT("sub $16, %[count]")
244 __ASM_EMIT("4:")
245 // 8x blocks
246 __ASM_EMIT("add $8, %[count]")
247 __ASM_EMIT("jl 6f")
248 __ASM_EMIT("vmovups 0x00(%[src]), %%ymm0")
249 PROCESS_GATE_FULL_X8
250 __ASM_EMIT("vmovups %%ymm0, 0x00(%[dst])")
251 __ASM_EMIT("add $0x20, %[src]")
252 __ASM_EMIT("add $0x20, %[dst]")
253 __ASM_EMIT("sub $8, %[count]")
254 __ASM_EMIT("6:")
255 // 4x blocks
256 __ASM_EMIT("add $4, %[count]")
257 __ASM_EMIT("jl 8f")
258 __ASM_EMIT("vmovups 0x00(%[src]), %%xmm0")
259 PROCESS_GATE_FULL_X4
260 __ASM_EMIT("vmovups %%xmm0, 0x00(%[dst])")
261 __ASM_EMIT("sub $4, %[count]")
262 __ASM_EMIT("add $0x10, %[src]")
263 __ASM_EMIT("add $0x10, %[dst]")
264 __ASM_EMIT("8:")
265 // Tail: 1x-3x block
266 __ASM_EMIT("add $4, %[count]")
267 __ASM_EMIT("jle 16f")
268 __ASM_EMIT("test $1, %[count]")
269 __ASM_EMIT("jz 10f")
270 __ASM_EMIT("vmovss 0x00(%[src]), %%xmm0")
271 __ASM_EMIT("add $4, %[src]")
272 __ASM_EMIT("10:")
273 __ASM_EMIT("test $2, %[count]")
274 __ASM_EMIT("jz 12f")
275 __ASM_EMIT("vmovhps 0x00(%[src]), %%xmm0, %%xmm0")
276 __ASM_EMIT("12:")
277 PROCESS_GATE_FULL_X4
278 __ASM_EMIT("test $1, %[count]")
279 __ASM_EMIT("jz 14f")
280 __ASM_EMIT("vmovss %%xmm0, 0x00(%[dst])")
281 __ASM_EMIT("add $4, %[dst]")
282 __ASM_EMIT("14:")
283 __ASM_EMIT("test $2, %[count]")
284 __ASM_EMIT("jz 16f")
285 __ASM_EMIT("vmovhps %%xmm0, 0x00(%[dst])")
286 __ASM_EMIT("16:")
287
288 : [dst] "+r" (dst), [src] "+r" (src),
289 [count] "+r" (count),
290 [mask] "=&r" (mask)
291 : [gate] "r" (c),
292 [knee] "o" (knee),
293 [mem] "o" (mem),
294 [stub] "o" (stub),
295 [G2C] "o" (gate_const),
296 [L2C] "o" (LOG2_CONST),
297 [LOGC] "o" (LOGE_C),
298 [E2C] "o" (EXP2_CONST),
299 [LOG2E] "m" (EXP_LOG2E)
300 : "cc", "memory",
301 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
302 "%xmm4", "%xmm5", "%xmm6", "%xmm7",
303 "%k4", "%k5", "%k6", "%k7"
304 );
305 }
306
307 void gate_x1_curve(float *dst, const float *src, const dsp::gate_knee_t *c, size_t count)
308 {
309 IF_ARCH_X86(
310 gate_knee_t knee __lsp_aligned64;
311 float mem[32] __lsp_aligned64;
312 float stub[16] __lsp_aligned64;
313 size_t mask;
314 );
315
316 ARCH_X86_ASM
317 (
318 // Prepare stuff
319 UNPACK_GATE_KNEE("knee", "gate")
320
321 // 32x blocks
322 __ASM_EMIT("sub $32, %[count]")
323 __ASM_EMIT("jb 2f")
324 __ASM_EMIT("1:")
325 __ASM_EMIT("vmovups 0x00(%[src]), %%zmm0")
326 __ASM_EMIT("vmovups 0x40(%[src]), %%zmm4")
327 PROCESS_GATE_FULL_X32
328 __ASM_EMIT("vmulps 0x00(%[src]), %%zmm0, %%zmm0")
329 __ASM_EMIT("vmulps 0x40(%[src]), %%zmm4, %%zmm4")
330 __ASM_EMIT("vmovups %%zmm0, 0x00(%[dst])")
331 __ASM_EMIT("vmovups %%zmm4, 0x40(%[dst])")
332 __ASM_EMIT("add $0x80, %[src]")
333 __ASM_EMIT("add $0x80, %[dst]")
334 __ASM_EMIT("sub $32, %[count]")
335 __ASM_EMIT("jae 1b")
336 __ASM_EMIT("2:")
337 // 16x blocks
338 __ASM_EMIT("add $16, %[count]")
339 __ASM_EMIT("jl 4f")
340 __ASM_EMIT("vmovups 0x00(%[src]), %%zmm0")
341 PROCESS_GATE_FULL_X16
342 __ASM_EMIT("vmulps 0x00(%[src]), %%zmm0, %%zmm0")
343 __ASM_EMIT("vmovups %%zmm0, 0x00(%[dst])")
344 __ASM_EMIT("add $0x40, %[src]")
345 __ASM_EMIT("add $0x40, %[dst]")
346 __ASM_EMIT("sub $16, %[count]")
347 __ASM_EMIT("4:")
348 // 8x blocks
349 __ASM_EMIT("add $8, %[count]")
350 __ASM_EMIT("jl 6f")
351 __ASM_EMIT("vmovups 0x00(%[src]), %%ymm0")
352 PROCESS_GATE_FULL_X8
353 __ASM_EMIT("vmulps 0x00(%[src]), %%ymm0, %%ymm0")
354 __ASM_EMIT("vmovups %%ymm0, 0x00(%[dst])")
355 __ASM_EMIT("add $0x20, %[src]")
356 __ASM_EMIT("add $0x20, %[dst]")
357 __ASM_EMIT("sub $8, %[count]")
358 __ASM_EMIT("6:")
359 // 4x blocks
360 __ASM_EMIT("add $4, %[count]")
361 __ASM_EMIT("jl 8f")
362 __ASM_EMIT("vmovups 0x00(%[src]), %%xmm0")
363 PROCESS_GATE_FULL_X4
364 __ASM_EMIT("vmulps 0x00(%[src]), %%xmm0, %%xmm0")
365 __ASM_EMIT("vmovups %%xmm0, 0x00(%[dst])")
366 __ASM_EMIT("sub $4, %[count]")
367 __ASM_EMIT("add $0x10, %[src]")
368 __ASM_EMIT("add $0x10, %[dst]")
369 __ASM_EMIT("8:")
370 // Tail: 1x-3x block
371 __ASM_EMIT("add $4, %[count]")
372 __ASM_EMIT("jle 16f")
373 __ASM_EMIT("test $1, %[count]")
374 __ASM_EMIT("jz 10f")
375 __ASM_EMIT("vmovss 0x00(%[src]), %%xmm0")
376 __ASM_EMIT("add $4, %[src]")
377 __ASM_EMIT("10:")
378 __ASM_EMIT("test $2, %[count]")
379 __ASM_EMIT("jz 12f")
380 __ASM_EMIT("vmovhps 0x00(%[src]), %%xmm0, %%xmm0")
381 __ASM_EMIT("12:")
382 __ASM_EMIT("vmovaps %%xmm0, %%xmm4")
383 PROCESS_GATE_FULL_X4
384 __ASM_EMIT("vmulps %%xmm4, %%xmm0, %%xmm0")
385 __ASM_EMIT("test $1, %[count]")
386 __ASM_EMIT("jz 14f")
387 __ASM_EMIT("vmovss %%xmm0, 0x00(%[dst])")
388 __ASM_EMIT("add $4, %[dst]")
389 __ASM_EMIT("14:")
390 __ASM_EMIT("test $2, %[count]")
391 __ASM_EMIT("jz 16f")
392 __ASM_EMIT("vmovhps %%xmm0, 0x00(%[dst])")
393 __ASM_EMIT("16:")
394
395 : [dst] "+r" (dst), [src] "+r" (src),
396 [count] "+r" (count),
397 [mask] "=&r" (mask)
398 : [gate] "r" (c),
399 [knee] "o" (knee),
400 [mem] "o" (mem),
401 [stub] "o" (stub),
402 [G2C] "o" (gate_const),
403 [L2C] "o" (LOG2_CONST),
404 [LOGC] "o" (LOGE_C),
405 [E2C] "o" (EXP2_CONST),
406 [LOG2E] "m" (EXP_LOG2E)
407 : "cc", "memory",
408 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
409 "%xmm4", "%xmm5", "%xmm6", "%xmm7",
410 "%k4", "%k5", "%k6", "%k7"
411 );
412 }
413
414 #undef PROCESS_GATE_FULL_X4
415 #undef PROCESS_GATE_FULL_X8
416 #undef PROCESS_GATE_FULL_X16
417 #undef PROCESS_GATE_FULL_X32
418 #undef PROCESS_KNEE_SINGLE_X4
419 #undef PROCESS_KNEE_SINGLE_X8
420 #undef PROCESS_KNEE_SINGLE_X16
421 #undef PROCESS_KNEE_SINGLE_X32
422
423 #undef UNPACK_GATE_KNEE
424
425 } /* namespace avx512 */
426 } /* namespace lsp */
427
428
429
430 #endif /* PRIVATE_DSP_ARCH_X86_AVX512_DYNAMICS_GATE_H_ */