Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/x86/avx2/dynamics/compressor.h

raw
1 /*
2 * Copyright (C) 2023 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2023 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 6 окт. 2023 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_X86_AVX2_DYNAMICS_COMPRESSOR_H_
23 #define PRIVATE_DSP_ARCH_X86_AVX2_DYNAMICS_COMPRESSOR_H_
24
25 #ifndef PRIVATE_DSP_ARCH_X86_AVX2_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_X86_AVX2_IMPL */
28
29 #include <private/dsp/arch/x86/avx2/pmath/exp.h>
30 #include <private/dsp/arch/x86/avx2/pmath/log.h>
31
32 namespace lsp
33 {
34 namespace avx2
35 {
36 #pragma pack(push, 1)
37 typedef struct comp_knee_t
38 {
39 float start[8]; // +0x00
40 float end[8]; // +0x20
41 float gain[8]; // +0x40
42 float herm[24]; // +0x60
43 float tilt[16]; // +0xc0
44 } comp_knee_t;
45 #pragma pack(pop)
46
47 static const uint32_t compressor_const[] __lsp_aligned32 =
48 {
49 LSP_DSP_VEC8(0x7fffffff)
50 };
51
52
53 #define UNPACK_COMP_KNEE(DST, DOFF, SRC, SOFF) \
54 __ASM_EMIT("vbroadcastss " SOFF " + 0x00(%[" SRC "]), %%ymm0") \
55 __ASM_EMIT("vbroadcastss " SOFF " + 0x04(%[" SRC "]), %%ymm1") \
56 __ASM_EMIT("vbroadcastss " SOFF " + 0x08(%[" SRC "]), %%ymm2") \
57 __ASM_EMIT("vbroadcastss " SOFF " + 0x0c(%[" SRC "]), %%ymm3") \
58 __ASM_EMIT("vbroadcastss " SOFF " + 0x10(%[" SRC "]), %%ymm4") \
59 __ASM_EMIT("vbroadcastss " SOFF " + 0x14(%[" SRC "]), %%ymm5") \
60 __ASM_EMIT("vbroadcastss " SOFF " + 0x18(%[" SRC "]), %%ymm6") \
61 __ASM_EMIT("vbroadcastss " SOFF " + 0x1c(%[" SRC "]), %%ymm7") \
62 __ASM_EMIT("vmovaps %%ymm0, " DOFF " + 0x000 + %[" DST "]") \
63 __ASM_EMIT("vmovaps %%ymm1, " DOFF " + 0x020 + %[" DST "]") \
64 __ASM_EMIT("vmovaps %%ymm2, " DOFF " + 0x040 + %[" DST "]") \
65 __ASM_EMIT("vmovaps %%ymm3, " DOFF " + 0x060 + %[" DST "]") \
66 __ASM_EMIT("vmovaps %%ymm4, " DOFF " + 0x080 + %[" DST "]") \
67 __ASM_EMIT("vmovaps %%ymm5, " DOFF " + 0x0a0 + %[" DST "]") \
68 __ASM_EMIT("vmovaps %%ymm6, " DOFF " + 0x0c0 + %[" DST "]") \
69 __ASM_EMIT("vmovaps %%ymm7, " DOFF " + 0x0e0 + %[" DST "]")
70
71 #define PROCESS_KNEE_SINGLE_X32(OFF) \
72 /* in: ymm0 = lx0, ymm4 = lx1 */ \
73 __ASM_EMIT("vmulps " OFF " + 0x60 + %[knee], %%ymm0, %%ymm1") /* ymm1 = herm[0]*lx0 */ \
74 __ASM_EMIT("vmulps " OFF " + 0x60 + %[knee], %%ymm4, %%ymm5") \
75 __ASM_EMIT("vmulps " OFF " + 0x60 + %[knee], %%ymm8, %%ymm9") \
76 __ASM_EMIT("vmulps " OFF " + 0x60 + %[knee], %%ymm12, %%ymm13") \
77 __ASM_EMIT("vmulps " OFF " + 0xc0 + %[knee], %%ymm0, %%ymm2") /* ymm2 = tilt[0]*lx0 */ \
78 __ASM_EMIT("vmulps " OFF " + 0xc0 + %[knee], %%ymm4, %%ymm6") \
79 __ASM_EMIT("vmulps " OFF " + 0xc0 + %[knee], %%ymm8, %%ymm10") \
80 __ASM_EMIT("vmulps " OFF " + 0xc0 + %[knee], %%ymm12, %%ymm14") \
81 __ASM_EMIT("vaddps " OFF " + 0x80 + %[knee], %%ymm1, %%ymm1") /* ymm1 = herm[0]*lx0+herm[1] */ \
82 __ASM_EMIT("vaddps " OFF " + 0x80 + %[knee], %%ymm5, %%ymm5") \
83 __ASM_EMIT("vaddps " OFF " + 0x80 + %[knee], %%ymm9, %%ymm9") \
84 __ASM_EMIT("vaddps " OFF " + 0x80 + %[knee], %%ymm13, %%ymm13") \
85 __ASM_EMIT("vaddps " OFF " + 0xe0 + %[knee], %%ymm2, %%ymm2") /* ymm2 = TV = tilt[0]*lx0+tilt[1] */ \
86 __ASM_EMIT("vaddps " OFF " + 0xe0 + %[knee], %%ymm6, %%ymm6") \
87 __ASM_EMIT("vaddps " OFF " + 0xe0 + %[knee], %%ymm10, %%ymm10") \
88 __ASM_EMIT("vaddps " OFF " + 0xe0 + %[knee], %%ymm14, %%ymm14") \
89 __ASM_EMIT("vmulps %%ymm1, %%ymm0, %%ymm1") /* ymm1 = (herm[0]*lx0+herm[1])*lx0 */ \
90 __ASM_EMIT("vmulps %%ymm5, %%ymm4, %%ymm5") \
91 __ASM_EMIT("vmulps %%ymm9, %%ymm8, %%ymm9") \
92 __ASM_EMIT("vmulps %%ymm13, %%ymm12, %%ymm13") \
93 __ASM_EMIT("vaddps " OFF " + 0xa0 + %[knee], %%ymm1, %%ymm1") /* ymm5 = KV = (herm[0]*lx0+herm[1])*lx0+herm[2] */ \
94 __ASM_EMIT("vaddps " OFF " + 0xa0 + %[knee], %%ymm5, %%ymm5") \
95 __ASM_EMIT("vaddps " OFF " + 0xa0 + %[knee], %%ymm9, %%ymm9") \
96 __ASM_EMIT("vaddps " OFF " + 0xa0 + %[knee], %%ymm13, %%ymm13") \
97 __ASM_EMIT("vmovaps 0x00 + %[mem], %%ymm0") /* ymm1 = x0 */ \
98 __ASM_EMIT("vmovaps 0x20 + %[mem], %%ymm4") \
99 __ASM_EMIT("vmovaps 0x40 + %[mem], %%ymm8") \
100 __ASM_EMIT("vmovaps 0x60 + %[mem], %%ymm12") \
101 __ASM_EMIT("vcmpps $5, " OFF " + 0x20 + %[knee], %%ymm0, %%ymm3") /* ymm3 = [x0 >= end] */ \
102 __ASM_EMIT("vcmpps $5, " OFF " + 0x20 + %[knee], %%ymm4, %%ymm7") \
103 __ASM_EMIT("vcmpps $5, " OFF " + 0x20 + %[knee], %%ymm8, %%ymm11") \
104 __ASM_EMIT("vcmpps $5, " OFF " + 0x20 + %[knee], %%ymm12, %%ymm15") \
105 __ASM_EMIT("vblendvps %%ymm3, %%ymm2, %%ymm1, %%ymm0") /* ymm0 = [x0 >= end] ? TV : KV */ \
106 __ASM_EMIT("vblendvps %%ymm7, %%ymm6, %%ymm5, %%ymm4") \
107 __ASM_EMIT("vblendvps %%ymm11, %%ymm10, %%ymm9, %%ymm8") \
108 __ASM_EMIT("vblendvps %%ymm15, %%ymm14, %%ymm13, %%ymm12") \
109 EXP_CORE_X32 /* ymm0 = EV = expf([x0 >= end] ? TV : KV) */ \
110 __ASM_EMIT("vmovaps 0x00 + %[mem], %%ymm1") /* ymm1 = x0 */ \
111 __ASM_EMIT("vmovaps 0x20 + %[mem], %%ymm5") \
112 __ASM_EMIT("vmovaps 0x40 + %[mem], %%ymm9") \
113 __ASM_EMIT("vmovaps 0x60 + %[mem], %%ymm13") \
114 __ASM_EMIT("vcmpps $2, " OFF " + 0x00 + %[knee], %%ymm1, %%ymm3") /* ymm3 = [x0 <= start] */ \
115 __ASM_EMIT("vcmpps $2, " OFF " + 0x00 + %[knee], %%ymm5, %%ymm7") \
116 __ASM_EMIT("vcmpps $2, " OFF " + 0x00 + %[knee], %%ymm9, %%ymm11") \
117 __ASM_EMIT("vcmpps $2, " OFF " + 0x00 + %[knee], %%ymm13, %%ymm15") \
118 __ASM_EMIT("vblendvps %%ymm3, " OFF " + 0x40 + %[knee], %%ymm0, %%ymm0") /* ymm0 = [x0 <= start] ? GV : EV */ \
119 __ASM_EMIT("vblendvps %%ymm7, " OFF " + 0x40 + %[knee], %%ymm4, %%ymm4") \
120 __ASM_EMIT("vblendvps %%ymm11, " OFF " + 0x40 + %[knee], %%ymm8, %%ymm8") \
121 __ASM_EMIT("vblendvps %%ymm15, " OFF " + 0x40 + %[knee], %%ymm12, %%ymm12") \
122 /* out: ymm0 = g0, ymm4 = g1 */
123
124 #define PROCESS_KNEE_SINGLE_X16(OFF) \
125 /* in: ymm0 = lx0, ymm4 = lx1 */ \
126 __ASM_EMIT("vmulps " OFF " + 0x60 + %[knee], %%ymm0, %%ymm1") /* ymm1 = herm[0]*lx0 */ \
127 __ASM_EMIT("vmulps " OFF " + 0x60 + %[knee], %%ymm4, %%ymm5") \
128 __ASM_EMIT("vmulps " OFF " + 0xc0 + %[knee], %%ymm0, %%ymm2") /* ymm2 = tilt[0]*lx0 */ \
129 __ASM_EMIT("vmulps " OFF " + 0xc0 + %[knee], %%ymm4, %%ymm6") \
130 __ASM_EMIT("vaddps " OFF " + 0x80 + %[knee], %%ymm1, %%ymm1") /* ymm1 = herm[0]*lx0+herm[1] */ \
131 __ASM_EMIT("vaddps " OFF " + 0x80 + %[knee], %%ymm5, %%ymm5") \
132 __ASM_EMIT("vaddps " OFF " + 0xe0 + %[knee], %%ymm2, %%ymm2") /* ymm2 = TV = tilt[0]*lx0+tilt[1] */ \
133 __ASM_EMIT("vaddps " OFF " + 0xe0 + %[knee], %%ymm6, %%ymm6") \
134 __ASM_EMIT("vmulps %%ymm1, %%ymm0, %%ymm1") /* ymm1 = (herm[0]*lx0+herm[1])*lx0 */ \
135 __ASM_EMIT("vmulps %%ymm5, %%ymm4, %%ymm5") \
136 __ASM_EMIT("vaddps " OFF " + 0xa0 + %[knee], %%ymm1, %%ymm1") /* ymm5 = KV = (herm[0]*lx0+herm[1])*lx0+herm[2] */ \
137 __ASM_EMIT("vaddps " OFF " + 0xa0 + %[knee], %%ymm5, %%ymm5") \
138 __ASM_EMIT("vmovaps 0x00 + %[mem], %%ymm0") /* ymm1 = x0 */ \
139 __ASM_EMIT("vmovaps 0x20 + %[mem], %%ymm4") \
140 __ASM_EMIT("vcmpps $5, " OFF " + 0x20 + %[knee], %%ymm0, %%ymm3") /* ymm3 = [x0 >= end] */ \
141 __ASM_EMIT("vcmpps $5, " OFF " + 0x20 + %[knee], %%ymm4, %%ymm7") \
142 __ASM_EMIT("vblendvps %%ymm3, %%ymm2, %%ymm1, %%ymm0") /* ymm0 = [x0 >= end] ? TV : KV */ \
143 __ASM_EMIT("vblendvps %%ymm7, %%ymm6, %%ymm5, %%ymm4") \
144 EXP_CORE_X16 /* ymm0 = EV = expf([x0 >= end] ? TV : KV) */ \
145 __ASM_EMIT("vmovaps 0x00 + %[mem], %%ymm1") /* ymm1 = x0 */ \
146 __ASM_EMIT("vmovaps 0x20 + %[mem], %%ymm5") \
147 __ASM_EMIT("vcmpps $2, " OFF " + 0x00 + %[knee], %%ymm1, %%ymm3") /* ymm3 = [x0 <= start] */ \
148 __ASM_EMIT("vcmpps $2, " OFF " + 0x00 + %[knee], %%ymm5, %%ymm7") \
149 __ASM_EMIT("vblendvps %%ymm3, " OFF " + 0x40 + %[knee], %%ymm0, %%ymm0") /* ymm0 = [x0 <= start] ? GV : EV */ \
150 __ASM_EMIT("vblendvps %%ymm7, " OFF " + 0x40 + %[knee], %%ymm4, %%ymm4") \
151 /* out: ymm0 = g0, ymm4 = g1 */
152
153 #define PROCESS_KNEE_SINGLE_X8(OFF) \
154 /* in: ymm0 = lx0 */ \
155 __ASM_EMIT("vmulps " OFF " + 0x60 + %[knee], %%ymm0, %%ymm1") /* ymm1 = herm[0]*lx0 */ \
156 __ASM_EMIT("vmulps " OFF " + 0xc0 + %[knee], %%ymm0, %%ymm2") /* ymm2 = tilt[0]*lx0 */ \
157 __ASM_EMIT("vaddps " OFF " + 0x80 + %[knee], %%ymm1, %%ymm1") /* ymm1 = herm[0]*lx0+herm[1] */ \
158 __ASM_EMIT("vaddps " OFF " + 0xe0 + %[knee], %%ymm2, %%ymm2") /* ymm2 = TV = tilt[0]*lx0+tilt[1] */ \
159 __ASM_EMIT("vmulps %%ymm1, %%ymm0, %%ymm1") /* ymm1 = (herm[0]*lx0+herm[1])*lx0 */ \
160 __ASM_EMIT("vaddps " OFF " + 0xa0 + %[knee], %%ymm1, %%ymm1") /* ymm5 = KV = (herm[0]*lx0+herm[1])*lx0+herm[2] */ \
161 __ASM_EMIT("vmovaps 0x00 + %[mem], %%ymm0") /* ymm1 = x0 */ \
162 __ASM_EMIT("vcmpps $5, " OFF " + 0x20 + %[knee], %%ymm0, %%ymm3") /* ymm3 = [x0 >= end] */ \
163 __ASM_EMIT("vblendvps %%ymm3, %%ymm2, %%ymm1, %%ymm0") /* ymm0 = [x0 >= end] ? TV : KV */ \
164 EXP_CORE_X8 /* ymm0 = EV = expf([x0 >= end] ? TV : KV) */ \
165 __ASM_EMIT("vmovaps 0x00 + %[mem], %%ymm1") /* ymm1 = x0 */ \
166 __ASM_EMIT("vcmpps $2, " OFF " + 0x00 + %[knee], %%ymm1, %%ymm3") /* ymm3 = [x0 <= start] */ \
167 __ASM_EMIT("vblendvps %%ymm3, " OFF " + 0x40 + %[knee], %%ymm0, %%ymm0") /* ymm0 = [x0 <= start] ? GV : EV */ \
168 /* out: ymm0 = g0, ymm4 = g1 */
169
170 #define PROCESS_KNEE_SINGLE_X4(OFF) \
171 /* in: xmm0 = lx0 */ \
172 __ASM_EMIT("vmulps " OFF " + 0x60 + %[knee], %%xmm0, %%xmm1") /* xmm1 = herm[0]*lx0 */ \
173 __ASM_EMIT("vmulps " OFF " + 0xc0 + %[knee], %%xmm0, %%xmm2") /* xmm2 = tilt[0]*lx0 */ \
174 __ASM_EMIT("vaddps " OFF " + 0x80 + %[knee], %%xmm1, %%xmm1") /* xmm1 = herm[0]*lx0+herm[1] */ \
175 __ASM_EMIT("vaddps " OFF " + 0xe0 + %[knee], %%xmm2, %%xmm2") /* xmm2 = TV = tilt[0]*lx0+tilt[1] */ \
176 __ASM_EMIT("vmulps %%xmm1, %%xmm0, %%xmm1") /* xmm1 = (herm[0]*lx0+herm[1])*lx0 */ \
177 __ASM_EMIT("vaddps " OFF " + 0xa0 + %[knee], %%xmm1, %%xmm1") /* xmm5 = KV = (herm[0]*lx0+herm[1])*lx0+herm[2] */ \
178 __ASM_EMIT("vmovaps 0x00 + %[mem], %%xmm0") /* xmm1 = x0 */ \
179 __ASM_EMIT("vcmpps $5, " OFF " + 0x20 + %[knee], %%xmm0, %%xmm3") /* xmm3 = [x0 >= end] */ \
180 __ASM_EMIT("vblendvps %%xmm3, %%xmm2, %%xmm1, %%xmm0") /* xmm0 = [x0 >= end] ? TV : KV */ \
181 EXP_CORE_X4 /* xmm0 = EV = expf([x0 >= end] ? TV : KV) */ \
182 __ASM_EMIT("vmovaps 0x00 + %[mem], %%xmm1") /* xmm1 = x0 */ \
183 __ASM_EMIT("vcmpps $2, " OFF " + 0x00 + %[knee], %%xmm1, %%xmm3") /* xmm3 = [x0 <= start] */ \
184 __ASM_EMIT("vblendvps %%xmm3, " OFF " + 0x40 + %[knee], %%xmm0, %%xmm0") /* xmm0 = [x0 <= start] ? GV : EV */ \
185 /* out: xmm0 = g0, xmm4 = g1 */
186
187 #define PROCESS_COMP_FULL_X32 \
188 /* in: ymm0 = x0, ymm4 = x1, ymm8 = x2, ymm12 = x4 */ \
189 __ASM_EMIT("vandps 0x00 + %[C2C], %%ymm0, %%ymm0") /* ymm0 = fabsf(x0) */ \
190 __ASM_EMIT("vandps 0x00 + %[C2C], %%ymm4, %%ymm4") \
191 __ASM_EMIT("vandps 0x00 + %[C2C], %%ymm8, %%ymm8") \
192 __ASM_EMIT("vandps 0x00 + %[C2C], %%ymm12, %%ymm12") \
193 __ASM_EMIT("vcmpps $6, 0x000 + 0x00 + %[knee], %%ymm0, %%ymm1") /* ymm1 = [x0 > k0.start] */ \
194 __ASM_EMIT("vcmpps $6, 0x000 + 0x00 + %[knee], %%ymm4, %%ymm5") \
195 __ASM_EMIT("vcmpps $6, 0x000 + 0x00 + %[knee], %%ymm8, %%ymm9") \
196 __ASM_EMIT("vcmpps $6, 0x000 + 0x00 + %[knee], %%ymm12, %%ymm13") \
197 __ASM_EMIT("vcmpps $6, 0x100 + 0x00 + %[knee], %%ymm0, %%ymm2") /* ymm2 = [x0 > k0.start] */ \
198 __ASM_EMIT("vcmpps $6, 0x100 + 0x00 + %[knee], %%ymm4, %%ymm6") \
199 __ASM_EMIT("vcmpps $6, 0x100 + 0x00 + %[knee], %%ymm8, %%ymm10") \
200 __ASM_EMIT("vcmpps $6, 0x100 + 0x00 + %[knee], %%ymm12, %%ymm14") \
201 __ASM_EMIT("vorps %%ymm2, %%ymm1, %%ymm1") \
202 __ASM_EMIT("vorps %%ymm6, %%ymm5, %%ymm5") \
203 __ASM_EMIT("vorps %%ymm10, %%ymm9, %%ymm9") \
204 __ASM_EMIT("vorps %%ymm14, %%ymm13, %%ymm13") \
205 __ASM_EMIT("vorps %%ymm5, %%ymm1, %%ymm1") \
206 __ASM_EMIT("vorps %%ymm13, %%ymm9, %%ymm9") \
207 __ASM_EMIT("vorps %%ymm9, %%ymm1, %%ymm1") \
208 __ASM_EMIT("vmovmskps %%ymm1, %[mask]") \
209 __ASM_EMIT("test %[mask], %[mask]") \
210 __ASM_EMIT("jnz 100f") \
211 __ASM_EMIT("vmovaps 0x000 + 0x40 + %[knee], %%ymm0") /* ymm0 = g1 */ \
212 __ASM_EMIT("vmovaps 0x000 + 0x40 + %[knee], %%ymm4") \
213 __ASM_EMIT("vmulps 0x100 + 0x40 + %[knee], %%ymm0, %%ymm0") /* ymm0 = G = g0*g1 */ \
214 __ASM_EMIT("vmulps 0x100 + 0x40 + %[knee], %%ymm4, %%ymm4") \
215 __ASM_EMIT("vmovaps %%ymm0, %%ymm8") \
216 __ASM_EMIT("vmovaps %%ymm4, %%ymm12") \
217 __ASM_EMIT("jmp 200f") \
218 __ASM_EMIT("100:") \
219 __ASM_EMIT("vmovaps %%ymm0, 0x00 + %[mem]") /* store fabsf(x0) */ \
220 __ASM_EMIT("vmovaps %%ymm4, 0x20 + %[mem]") \
221 __ASM_EMIT("vmovaps %%ymm8, 0x40 + %[mem]") \
222 __ASM_EMIT("vmovaps %%ymm12, 0x60 + %[mem]") \
223 LOGE_CORE_X32 /* ymm0 = lx0 = logf(fabsf(x0)) */ \
224 __ASM_EMIT("vmovaps %%ymm0, 0x80 + %[mem]") /* *mem = lx0 */ \
225 __ASM_EMIT("vmovaps %%ymm4, 0xa0 + %[mem]") \
226 __ASM_EMIT("vmovaps %%ymm8, 0xc0 + %[mem]") \
227 __ASM_EMIT("vmovaps %%ymm12, 0xe0 + %[mem]") \
228 PROCESS_KNEE_SINGLE_X32("0x000") /* apply knee 0 */ \
229 __ASM_EMIT("vmovaps %%ymm0, 0x100 + %[mem]") /* *mem = g1 */ \
230 __ASM_EMIT("vmovaps %%ymm4, 0x120 + %[mem]") \
231 __ASM_EMIT("vmovaps %%ymm8, 0x140 + %[mem]") \
232 __ASM_EMIT("vmovaps %%ymm12, 0x160 + %[mem]") \
233 __ASM_EMIT("vmovaps 0x80 + %[mem], %%ymm0") /* ymm0 = lx0 */ \
234 __ASM_EMIT("vmovaps 0xa0 + %[mem], %%ymm4") \
235 __ASM_EMIT("vmovaps 0xc0 + %[mem], %%ymm8") \
236 __ASM_EMIT("vmovaps 0xe0 + %[mem], %%ymm12") \
237 PROCESS_KNEE_SINGLE_X32("0x100") /* apply knee 1 */ \
238 __ASM_EMIT("vmulps 0x100 + %[mem], %%ymm0, %%ymm0") /* ymm0 = G = g0*g1 */ \
239 __ASM_EMIT("vmulps 0x120 + %[mem], %%ymm4, %%ymm4") \
240 __ASM_EMIT("vmulps 0x140 + %[mem], %%ymm8, %%ymm8") \
241 __ASM_EMIT("vmulps 0x160 + %[mem], %%ymm12, %%ymm12") \
242 __ASM_EMIT("200:") \
243 /* out: ymm0 = G0, ymm4 = G1, ymm8 = G2, ymm12 = G3 */
244
245 #define PROCESS_COMP_FULL_X16 \
246 /* in: ymm0 = x0, ymm4 = x1 */ \
247 __ASM_EMIT("vandps 0x00 + %[C2C], %%ymm0, %%ymm0") /* ymm0 = fabsf(x0) */ \
248 __ASM_EMIT("vandps 0x00 + %[C2C], %%ymm4, %%ymm4") \
249 __ASM_EMIT("vcmpps $6, 0x000 + 0x00 + %[knee], %%ymm0, %%ymm1") /* ymm1 = [x0 > k0.start] */ \
250 __ASM_EMIT("vcmpps $6, 0x000 + 0x00 + %[knee], %%ymm4, %%ymm5") \
251 __ASM_EMIT("vcmpps $6, 0x100 + 0x00 + %[knee], %%ymm0, %%ymm2") /* ymm2 = [x0 > k0.start] */ \
252 __ASM_EMIT("vcmpps $6, 0x100 + 0x00 + %[knee], %%ymm4, %%ymm6") \
253 __ASM_EMIT("vorps %%ymm2, %%ymm1, %%ymm1") \
254 __ASM_EMIT("vorps %%ymm6, %%ymm5, %%ymm5") \
255 __ASM_EMIT("vorps %%ymm5, %%ymm1, %%ymm1") \
256 __ASM_EMIT("vmovmskps %%ymm1, %[mask]") \
257 __ASM_EMIT("test %[mask], %[mask]") \
258 __ASM_EMIT("jnz 100f") \
259 __ASM_EMIT("vmovaps 0x000 + 0x40 + %[knee], %%ymm0") /* ymm0 = g1 */ \
260 __ASM_EMIT("vmovaps 0x000 + 0x40 + %[knee], %%ymm4") \
261 __ASM_EMIT("vmulps 0x100 + 0x40 + %[knee], %%ymm0, %%ymm0") /* ymm0 = G = g0*g1 */ \
262 __ASM_EMIT("vmulps 0x100 + 0x40 + %[knee], %%ymm4, %%ymm4") \
263 __ASM_EMIT("jmp 200f") \
264 __ASM_EMIT("100:") \
265 __ASM_EMIT("vmovaps %%ymm0, 0x00 + %[mem]") /* store fabsf(x0) */ \
266 __ASM_EMIT("vmovaps %%ymm4, 0x20 + %[mem]") \
267 LOGE_CORE_X16 /* ymm0 = lx0 = logf(fabsf(x0)) */ \
268 __ASM_EMIT("vmovaps %%ymm0, 0x40 + %[mem]") /* *mem = lx0 */ \
269 __ASM_EMIT("vmovaps %%ymm4, 0x60 + %[mem]") \
270 PROCESS_KNEE_SINGLE_X16("0x000") /* apply knee 0 */ \
271 __ASM_EMIT("vmovaps %%ymm0, 0x80 + %[mem]") /* *mem = g1 */ \
272 __ASM_EMIT("vmovaps %%ymm4, 0xa0 + %[mem]") \
273 __ASM_EMIT("vmovaps 0x40 + %[mem], %%ymm0") /* ymm0 = lx0 */ \
274 __ASM_EMIT("vmovaps 0x60 + %[mem], %%ymm4") \
275 PROCESS_KNEE_SINGLE_X16("0x100") /* apply knee 1 */ \
276 __ASM_EMIT("vmulps 0x80 + %[mem], %%ymm0, %%ymm0") /* ymm0 = G = g0*g1 */ \
277 __ASM_EMIT("vmulps 0xa0 + %[mem], %%ymm4, %%ymm4") \
278 __ASM_EMIT("200:") \
279 /* out: ymm0 = G0, ymm4 = G1 */
280
281 #define PROCESS_COMP_FULL_X8 \
282 /* in: ymm0 = x0, ymm4 = x1 */ \
283 __ASM_EMIT("vandps 0x00 + %[C2C], %%ymm0, %%ymm0") /* ymm0 = fabsf(x0) */ \
284 __ASM_EMIT("vmovaps %%ymm0, 0x00 + %[mem]") /* store fabsf(x0) */ \
285 LOGE_CORE_X8 /* ymm0 = lx0 = logf(fabsf(x0)) */ \
286 __ASM_EMIT("vmovaps %%ymm0, 0x40 + %[mem]") /* *mem = lx0 */ \
287 PROCESS_KNEE_SINGLE_X8("0x000") /* apply knee 0 */ \
288 __ASM_EMIT("vmovaps %%ymm0, 0x80 + %[mem]") /* *mem = g1 */ \
289 __ASM_EMIT("vmovaps 0x40 + %[mem], %%ymm0") /* ymm0 = lx0 */ \
290 PROCESS_KNEE_SINGLE_X8("0x100") /* apply knee 1 */ \
291 __ASM_EMIT("vmulps 0x80 + %[mem], %%ymm0, %%ymm0") /* ymm0 = G = g0*g1 */ \
292 /* out: ymm0 = G0 */
293
294 #define PROCESS_COMP_FULL_X4 \
295 /* in: xmm0 = x0, xmm4 = x1 */ \
296 __ASM_EMIT("vandps 0x00 + %[C2C], %%xmm0, %%xmm0") /* xmm0 = fabsf(x0) */ \
297 __ASM_EMIT("vmovaps %%xmm0, 0x00 + %[mem]") /* store fabsf(x0) */ \
298 LOGE_CORE_X4 /* xmm0 = lx0 = logf(fabsf(x0)) */ \
299 __ASM_EMIT("vmovaps %%xmm0, 0x40 + %[mem]") /* *mem = lx0 */ \
300 PROCESS_KNEE_SINGLE_X4("0x000") /* apply knee 0 */ \
301 __ASM_EMIT("vmovaps %%xmm0, 0x80 + %[mem]") /* *mem = g1 */ \
302 __ASM_EMIT("vmovaps 0x40 + %[mem], %%xmm0") /* xmm0 = lx0 */ \
303 PROCESS_KNEE_SINGLE_X4("0x100") /* apply knee 1 */ \
304 __ASM_EMIT("vmulps 0x80 + %[mem], %%xmm0, %%xmm0") /* xmm0 = G = g0*g1 */ \
305 /* out: xmm0 = G0 */
306
307 void compressor_x2_gain(float *dst, const float *src, const dsp::compressor_x2_t *c, size_t count)
308 {
309 IF_ARCH_X86(
310 comp_knee_t knee[2] __lsp_aligned32;
311 float mem[48] __lsp_aligned32;
312 float stub[16] __lsp_aligned32;
313 size_t mask;
314 );
315
316 ARCH_X86_ASM
317 (
318 // Prepare stuff
319 UNPACK_COMP_KNEE("knee", "0x000", "comp", "0x00")
320 UNPACK_COMP_KNEE("knee", "0x100", "comp", "0x20")
321
322 // 16x blocks
323 __ASM_EMIT("sub $16, %[count]")
324 __ASM_EMIT("jb 2f")
325 __ASM_EMIT("1:")
326 __ASM_EMIT("vmovups 0x00(%[src]), %%ymm0")
327 __ASM_EMIT("vmovups 0x20(%[src]), %%ymm4")
328 PROCESS_COMP_FULL_X16
329 __ASM_EMIT("vmovups %%ymm0, 0x00(%[dst])")
330 __ASM_EMIT("vmovups %%ymm4, 0x20(%[dst])")
331 __ASM_EMIT("add $0x40, %[src]")
332 __ASM_EMIT("add $0x40, %[dst]")
333 __ASM_EMIT("sub $16, %[count]")
334 __ASM_EMIT("jae 1b")
335 __ASM_EMIT("2:")
336 // 8x blocks
337 __ASM_EMIT("add $8, %[count]")
338 __ASM_EMIT("jl 4f")
339 __ASM_EMIT("vmovups 0x00(%[src]), %%ymm0")
340 PROCESS_COMP_FULL_X8
341 __ASM_EMIT("vmovups %%ymm0, 0x00(%[dst])")
342 __ASM_EMIT("add $0x20, %[src]")
343 __ASM_EMIT("add $0x20, %[dst]")
344 __ASM_EMIT("sub $8, %[count]")
345 __ASM_EMIT("4:")
346 // 4x blocks
347 __ASM_EMIT("add $4, %[count]")
348 __ASM_EMIT("jl 6f")
349 __ASM_EMIT("vmovups 0x00(%[src]), %%xmm0")
350 PROCESS_COMP_FULL_X4
351 __ASM_EMIT("vmovups %%xmm0, 0x00(%[dst])")
352 __ASM_EMIT("sub $4, %[count]")
353 __ASM_EMIT("add $0x10, %[src]")
354 __ASM_EMIT("add $0x10, %[dst]")
355 __ASM_EMIT("6:")
356 // Tail: 1x-3x block
357 __ASM_EMIT("add $4, %[count]")
358 __ASM_EMIT("jle 14f")
359 __ASM_EMIT("test $1, %[count]")
360 __ASM_EMIT("jz 8f")
361 __ASM_EMIT("vmovss 0x00(%[src]), %%xmm0")
362 __ASM_EMIT("add $4, %[src]")
363 __ASM_EMIT("8:")
364 __ASM_EMIT("test $2, %[count]")
365 __ASM_EMIT("jz 10f")
366 __ASM_EMIT("vmovhps 0x00(%[src]), %%xmm0, %%xmm0")
367 __ASM_EMIT("10:")
368 PROCESS_COMP_FULL_X4
369 __ASM_EMIT("test $1, %[count]")
370 __ASM_EMIT("jz 12f")
371 __ASM_EMIT("vmovss %%xmm0, 0x00(%[dst])")
372 __ASM_EMIT("add $4, %[dst]")
373 __ASM_EMIT("12:")
374 __ASM_EMIT("test $2, %[count]")
375 __ASM_EMIT("jz 14f")
376 __ASM_EMIT("vmovhps %%xmm0, 0x00(%[dst])")
377 __ASM_EMIT("14:")
378
379 : [dst] "+r" (dst), [src] "+r" (src),
380 [count] "+r" (count),
381 [mask] "=&r" (mask)
382 : [comp] "r" (c),
383 [knee] "o" (knee),
384 [mem] "o" (mem),
385 [stub] "o" (stub),
386 [C2C] "o" (compressor_const),
387 [L2C] "o" (LOG2_CONST),
388 [LOGC] "o" (LOGE_C),
389 [E2C] "o" (EXP2_CONST),
390 [LOG2E] "m" (EXP_LOG2E)
391 : "cc", "memory",
392 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
393 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
394 );
395 }
396
397 IF_ARCH_X86_64(
398 void x64_compressor_x2_gain(float *dst, const float *src, const dsp::compressor_x2_t *c, size_t count)
399 {
400 IF_ARCH_X86(
401 comp_knee_t knee[2] __lsp_aligned32;
402 float mem[96] __lsp_aligned32;
403 float stub[16] __lsp_aligned32;
404 size_t mask;
405 );
406
407 ARCH_X86_ASM
408 (
409 // Prepare stuff
410 UNPACK_COMP_KNEE("knee", "0x000", "comp", "0x00")
411 UNPACK_COMP_KNEE("knee", "0x100", "comp", "0x20")
412
413 // 32x blocks
414 __ASM_EMIT("sub $32, %[count]")
415 __ASM_EMIT("jb 2f")
416 __ASM_EMIT("1:")
417 __ASM_EMIT("vmovups 0x00(%[src]), %%ymm0")
418 __ASM_EMIT("vmovups 0x20(%[src]), %%ymm4")
419 __ASM_EMIT("vmovups 0x40(%[src]), %%ymm8")
420 __ASM_EMIT("vmovups 0x60(%[src]), %%ymm12")
421 PROCESS_COMP_FULL_X32
422 __ASM_EMIT("vmovups %%ymm0, 0x00(%[dst])")
423 __ASM_EMIT("vmovups %%ymm4, 0x20(%[dst])")
424 __ASM_EMIT("vmovups %%ymm8, 0x40(%[dst])")
425 __ASM_EMIT("vmovups %%ymm12, 0x60(%[dst])")
426 __ASM_EMIT("add $0x80, %[src]")
427 __ASM_EMIT("add $0x80, %[dst]")
428 __ASM_EMIT("sub $32, %[count]")
429 __ASM_EMIT("jae 1b")
430 __ASM_EMIT("2:")
431 // 16x blocks
432 __ASM_EMIT("add $16, %[count]")
433 __ASM_EMIT("jl 4f")
434 __ASM_EMIT("vmovups 0x00(%[src]), %%ymm0")
435 __ASM_EMIT("vmovups 0x20(%[src]), %%ymm4")
436 PROCESS_COMP_FULL_X16
437 __ASM_EMIT("vmovups %%ymm0, 0x00(%[dst])")
438 __ASM_EMIT("vmovups %%ymm4, 0x20(%[dst])")
439 __ASM_EMIT("add $0x40, %[src]")
440 __ASM_EMIT("add $0x40, %[dst]")
441 __ASM_EMIT("sub $16, %[count]")
442 __ASM_EMIT("4:")
443 // 8x blocks
444 __ASM_EMIT("add $8, %[count]")
445 __ASM_EMIT("jl 6f")
446 __ASM_EMIT("vmovups 0x00(%[src]), %%ymm0")
447 PROCESS_COMP_FULL_X8
448 __ASM_EMIT("vmovups %%ymm0, 0x00(%[dst])")
449 __ASM_EMIT("add $0x20, %[src]")
450 __ASM_EMIT("add $0x20, %[dst]")
451 __ASM_EMIT("sub $8, %[count]")
452 __ASM_EMIT("6:")
453 // 4x blocks
454 __ASM_EMIT("add $4, %[count]")
455 __ASM_EMIT("jl 8f")
456 __ASM_EMIT("vmovups 0x00(%[src]), %%xmm0")
457 PROCESS_COMP_FULL_X4
458 __ASM_EMIT("vmovups %%xmm0, 0x00(%[dst])")
459 __ASM_EMIT("sub $4, %[count]")
460 __ASM_EMIT("add $0x10, %[src]")
461 __ASM_EMIT("add $0x10, %[dst]")
462 __ASM_EMIT("8:")
463 // Tail: 1x-3x block
464 __ASM_EMIT("add $4, %[count]")
465 __ASM_EMIT("jle 16f")
466 __ASM_EMIT("test $1, %[count]")
467 __ASM_EMIT("jz 10f")
468 __ASM_EMIT("vmovss 0x00(%[src]), %%xmm0")
469 __ASM_EMIT("add $4, %[src]")
470 __ASM_EMIT("10:")
471 __ASM_EMIT("test $2, %[count]")
472 __ASM_EMIT("jz 12f")
473 __ASM_EMIT("vmovhps 0x00(%[src]), %%xmm0, %%xmm0")
474 __ASM_EMIT("12:")
475 PROCESS_COMP_FULL_X4
476 __ASM_EMIT("test $1, %[count]")
477 __ASM_EMIT("jz 14f")
478 __ASM_EMIT("vmovss %%xmm0, 0x00(%[dst])")
479 __ASM_EMIT("add $4, %[dst]")
480 __ASM_EMIT("14:")
481 __ASM_EMIT("test $2, %[count]")
482 __ASM_EMIT("jz 16f")
483 __ASM_EMIT("vmovhps %%xmm0, 0x00(%[dst])")
484 __ASM_EMIT("16:")
485
486 : [dst] "+r" (dst), [src] "+r" (src),
487 [count] "+r" (count),
488 [mask] "=&r" (mask)
489 : [comp] "r" (c),
490 [knee] "o" (knee),
491 [mem] "o" (mem),
492 [stub] "o" (stub),
493 [C2C] "o" (compressor_const),
494 [L2C] "o" (LOG2_CONST),
495 [LOGC] "o" (LOGE_C),
496 [E2C] "o" (EXP2_CONST),
497 [LOG2E] "m" (EXP_LOG2E)
498 : "cc", "memory",
499 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
500 "%xmm4", "%xmm5", "%xmm6", "%xmm7",
501 "%xmm8", "%xmm9", "%xmm10", "%xmm11",
502 "%xmm12", "%xmm13", "%xmm14", "%xmm15"
503 );
504 }
505 )
506
507 void compressor_x2_curve(float *dst, const float *src, const dsp::compressor_x2_t *c, size_t count)
508 {
509 IF_ARCH_X86(
510 comp_knee_t knee[2] __lsp_aligned32;
511 float mem[48] __lsp_aligned32;
512 float stub[16] __lsp_aligned32;
513 size_t mask;
514 );
515
516 ARCH_X86_ASM
517 (
518 // Prepare stuff
519 UNPACK_COMP_KNEE("knee", "0x000", "comp", "0x00")
520 UNPACK_COMP_KNEE("knee", "0x100", "comp", "0x20")
521
522 // 16x blocks
523 __ASM_EMIT("sub $16, %[count]")
524 __ASM_EMIT("jb 2f")
525 __ASM_EMIT("1:")
526 __ASM_EMIT("vmovups 0x00(%[src]), %%ymm0")
527 __ASM_EMIT("vmovups 0x20(%[src]), %%ymm4")
528 PROCESS_COMP_FULL_X16
529 __ASM_EMIT("vmulps 0x00(%[src]), %%ymm0, %%ymm0")
530 __ASM_EMIT("vmulps 0x20(%[src]), %%ymm4, %%ymm4")
531 __ASM_EMIT("vmovups %%ymm0, 0x00(%[dst])")
532 __ASM_EMIT("vmovups %%ymm4, 0x20(%[dst])")
533 __ASM_EMIT("add $0x40, %[src]")
534 __ASM_EMIT("add $0x40, %[dst]")
535 __ASM_EMIT("sub $16, %[count]")
536 __ASM_EMIT("jae 1b")
537 __ASM_EMIT("2:")
538 // 8x blocks
539 __ASM_EMIT("add $8, %[count]")
540 __ASM_EMIT("jl 4f")
541 __ASM_EMIT("vmovups 0x00(%[src]), %%ymm0")
542 PROCESS_COMP_FULL_X8
543 __ASM_EMIT("vmulps 0x00(%[src]), %%ymm0, %%ymm0")
544 __ASM_EMIT("vmovups %%ymm0, 0x00(%[dst])")
545 __ASM_EMIT("add $0x20, %[src]")
546 __ASM_EMIT("add $0x20, %[dst]")
547 __ASM_EMIT("sub $8, %[count]")
548 __ASM_EMIT("4:")
549 // 4x blocks
550 __ASM_EMIT("add $4, %[count]")
551 __ASM_EMIT("jl 6f")
552 __ASM_EMIT("vmovups 0x00(%[src]), %%xmm0")
553 PROCESS_COMP_FULL_X4
554 __ASM_EMIT("vmulps 0x00(%[src]), %%xmm0, %%xmm0")
555 __ASM_EMIT("vmovups %%xmm0, 0x00(%[dst])")
556 __ASM_EMIT("sub $4, %[count]")
557 __ASM_EMIT("add $0x10, %[src]")
558 __ASM_EMIT("add $0x10, %[dst]")
559 __ASM_EMIT("6:")
560 // Tail: 1x-3x block
561 __ASM_EMIT("add $4, %[count]")
562 __ASM_EMIT("jle 14f")
563 __ASM_EMIT("test $1, %[count]")
564 __ASM_EMIT("jz 8f")
565 __ASM_EMIT("vmovss 0x00(%[src]), %%xmm0")
566 __ASM_EMIT("add $4, %[src]")
567 __ASM_EMIT("8:")
568 __ASM_EMIT("test $2, %[count]")
569 __ASM_EMIT("jz 10f")
570 __ASM_EMIT("vmovhps 0x00(%[src]), %%xmm0, %%xmm0")
571 __ASM_EMIT("10:")
572 __ASM_EMIT("vmovaps %%xmm0, %%xmm4")
573 PROCESS_COMP_FULL_X4
574 __ASM_EMIT("vmulps %%xmm4, %%xmm0, %%xmm0")
575 __ASM_EMIT("test $1, %[count]")
576 __ASM_EMIT("jz 12f")
577 __ASM_EMIT("vmovss %%xmm0, 0x00(%[dst])")
578 __ASM_EMIT("add $4, %[dst]")
579 __ASM_EMIT("12:")
580 __ASM_EMIT("test $2, %[count]")
581 __ASM_EMIT("jz 14f")
582 __ASM_EMIT("vmovhps %%xmm0, 0x00(%[dst])")
583 __ASM_EMIT("14:")
584
585 : [dst] "+r" (dst), [src] "+r" (src),
586 [count] "+r" (count),
587 [mask] "=&r" (mask)
588 : [comp] "r" (c),
589 [knee] "o" (knee),
590 [mem] "o" (mem),
591 [stub] "o" (stub),
592 [C2C] "o" (compressor_const),
593 [L2C] "o" (LOG2_CONST),
594 [LOGC] "o" (LOGE_C),
595 [E2C] "o" (EXP2_CONST),
596 [LOG2E] "m" (EXP_LOG2E)
597 : "cc", "memory",
598 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
599 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
600 );
601 }
602
603 IF_ARCH_X86_64(
604 void x64_compressor_x2_curve(float *dst, const float *src, const dsp::compressor_x2_t *c, size_t count)
605 {
606 IF_ARCH_X86(
607 comp_knee_t knee[2] __lsp_aligned32;
608 float mem[96] __lsp_aligned32;
609 float stub[16] __lsp_aligned32;
610 size_t mask;
611 );
612
613 ARCH_X86_ASM
614 (
615 // Prepare stuff
616 UNPACK_COMP_KNEE("knee", "0x000", "comp", "0x00")
617 UNPACK_COMP_KNEE("knee", "0x100", "comp", "0x20")
618
619 // 32x blocks
620 __ASM_EMIT("sub $32, %[count]")
621 __ASM_EMIT("jb 2f")
622 __ASM_EMIT("1:")
623 __ASM_EMIT("vmovups 0x00(%[src]), %%ymm0")
624 __ASM_EMIT("vmovups 0x20(%[src]), %%ymm4")
625 __ASM_EMIT("vmovups 0x40(%[src]), %%ymm8")
626 __ASM_EMIT("vmovups 0x60(%[src]), %%ymm12")
627 PROCESS_COMP_FULL_X32
628 __ASM_EMIT("vmulps 0x00(%[src]), %%ymm0, %%ymm0")
629 __ASM_EMIT("vmulps 0x20(%[src]), %%ymm4, %%ymm4")
630 __ASM_EMIT("vmulps 0x40(%[src]), %%ymm8, %%ymm8")
631 __ASM_EMIT("vmulps 0x60(%[src]), %%ymm12, %%ymm12")
632 __ASM_EMIT("vmovups %%ymm0, 0x00(%[dst])")
633 __ASM_EMIT("vmovups %%ymm4, 0x20(%[dst])")
634 __ASM_EMIT("vmovups %%ymm8, 0x40(%[dst])")
635 __ASM_EMIT("vmovups %%ymm12, 0x60(%[dst])")
636 __ASM_EMIT("add $0x80, %[src]")
637 __ASM_EMIT("add $0x80, %[dst]")
638 __ASM_EMIT("sub $32, %[count]")
639 __ASM_EMIT("jae 1b")
640 __ASM_EMIT("2:")
641 // 16x blocks
642 __ASM_EMIT("add $16, %[count]")
643 __ASM_EMIT("jl 4f")
644 __ASM_EMIT("vmovups 0x00(%[src]), %%ymm0")
645 __ASM_EMIT("vmovups 0x20(%[src]), %%ymm4")
646 PROCESS_COMP_FULL_X16
647 __ASM_EMIT("vmulps 0x00(%[src]), %%ymm0, %%ymm0")
648 __ASM_EMIT("vmulps 0x20(%[src]), %%ymm4, %%ymm4")
649 __ASM_EMIT("vmovups %%ymm0, 0x00(%[dst])")
650 __ASM_EMIT("vmovups %%ymm4, 0x20(%[dst])")
651 __ASM_EMIT("add $0x40, %[src]")
652 __ASM_EMIT("add $0x40, %[dst]")
653 __ASM_EMIT("sub $16, %[count]")
654 __ASM_EMIT("4:")
655 // 8x blocks
656 __ASM_EMIT("add $8, %[count]")
657 __ASM_EMIT("jl 6f")
658 __ASM_EMIT("vmovups 0x00(%[src]), %%ymm0")
659 PROCESS_COMP_FULL_X8
660 __ASM_EMIT("vmulps 0x00(%[src]), %%ymm0, %%ymm0")
661 __ASM_EMIT("vmovups %%ymm0, 0x00(%[dst])")
662 __ASM_EMIT("add $0x20, %[src]")
663 __ASM_EMIT("add $0x20, %[dst]")
664 __ASM_EMIT("sub $8, %[count]")
665 __ASM_EMIT("6:")
666 // 4x blocks
667 __ASM_EMIT("add $4, %[count]")
668 __ASM_EMIT("jl 8f")
669 __ASM_EMIT("vmovups 0x00(%[src]), %%xmm0")
670 PROCESS_COMP_FULL_X4
671 __ASM_EMIT("vmulps 0x00(%[src]), %%xmm0, %%xmm0")
672 __ASM_EMIT("vmovups %%xmm0, 0x00(%[dst])")
673 __ASM_EMIT("sub $4, %[count]")
674 __ASM_EMIT("add $0x10, %[src]")
675 __ASM_EMIT("add $0x10, %[dst]")
676 __ASM_EMIT("8:")
677 // Tail: 1x-3x block
678 __ASM_EMIT("add $4, %[count]")
679 __ASM_EMIT("jle 16f")
680 __ASM_EMIT("test $1, %[count]")
681 __ASM_EMIT("jz 10f")
682 __ASM_EMIT("vmovss 0x00(%[src]), %%xmm0")
683 __ASM_EMIT("add $4, %[src]")
684 __ASM_EMIT("10:")
685 __ASM_EMIT("test $2, %[count]")
686 __ASM_EMIT("jz 12f")
687 __ASM_EMIT("vmovhps 0x00(%[src]), %%xmm0, %%xmm0")
688 __ASM_EMIT("12:")
689 __ASM_EMIT("vmovaps %%xmm0, %%xmm4")
690 PROCESS_COMP_FULL_X4
691 __ASM_EMIT("vmulps %%xmm4, %%xmm0, %%xmm0")
692 __ASM_EMIT("test $1, %[count]")
693 __ASM_EMIT("jz 14f")
694 __ASM_EMIT("vmovss %%xmm0, 0x00(%[dst])")
695 __ASM_EMIT("add $4, %[dst]")
696 __ASM_EMIT("14:")
697 __ASM_EMIT("test $2, %[count]")
698 __ASM_EMIT("jz 16f")
699 __ASM_EMIT("vmovhps %%xmm0, 0x00(%[dst])")
700 __ASM_EMIT("16:")
701
702 : [dst] "+r" (dst), [src] "+r" (src),
703 [count] "+r" (count),
704 [mask] "=&r" (mask)
705 : [comp] "r" (c),
706 [knee] "o" (knee),
707 [mem] "o" (mem),
708 [stub] "o" (stub),
709 [C2C] "o" (compressor_const),
710 [L2C] "o" (LOG2_CONST),
711 [LOGC] "o" (LOGE_C),
712 [E2C] "o" (EXP2_CONST),
713 [LOG2E] "m" (EXP_LOG2E)
714 : "cc", "memory",
715 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
716 "%xmm4", "%xmm5", "%xmm6", "%xmm7",
717 "%xmm8", "%xmm9", "%xmm10", "%xmm11",
718 "%xmm12", "%xmm13", "%xmm14", "%xmm15"
719 );
720 }
721 )
722
723 #undef PROCESS_COMP_FULL_X4
724 #undef PROCESS_COMP_FULL_X8
725 #undef PROCESS_COMP_FULL_X16
726 #undef PROCESS_COMP_FULL_X32
727 #undef PROCESS_KNEE_SINGLE_X4
728 #undef PROCESS_KNEE_SINGLE_X8
729 #undef PROCESS_KNEE_SINGLE_X16
730 #undef PROCESS_KNEE_SINGLE_X32
731
732 #define PROCESS_KNEE_SINGLE_X32_FMA3(OFF) \
733 /* in: ymm0 = lx0, ymm4 = lx1 */ \
734 __ASM_EMIT("vmulps " OFF " + 0x60 + %[knee], %%ymm0, %%ymm1") /* ymm1 = herm[0]*lx0 */ \
735 __ASM_EMIT("vmulps " OFF " + 0x60 + %[knee], %%ymm4, %%ymm5") \
736 __ASM_EMIT("vmulps " OFF " + 0x60 + %[knee], %%ymm8, %%ymm9") \
737 __ASM_EMIT("vmulps " OFF " + 0x60 + %[knee], %%ymm12, %%ymm13") \
738 __ASM_EMIT("vmulps " OFF " + 0xc0 + %[knee], %%ymm0, %%ymm2") /* ymm2 = tilt[0]*lx0 */ \
739 __ASM_EMIT("vmulps " OFF " + 0xc0 + %[knee], %%ymm4, %%ymm6") \
740 __ASM_EMIT("vmulps " OFF " + 0xc0 + %[knee], %%ymm8, %%ymm10") \
741 __ASM_EMIT("vmulps " OFF " + 0xc0 + %[knee], %%ymm12, %%ymm14") \
742 __ASM_EMIT("vaddps " OFF " + 0x80 + %[knee], %%ymm1, %%ymm1") /* ymm1 = herm[0]*lx0+herm[1] */ \
743 __ASM_EMIT("vaddps " OFF " + 0x80 + %[knee], %%ymm5, %%ymm5") \
744 __ASM_EMIT("vaddps " OFF " + 0x80 + %[knee], %%ymm9, %%ymm9") \
745 __ASM_EMIT("vaddps " OFF " + 0x80 + %[knee], %%ymm13, %%ymm13") \
746 __ASM_EMIT("vaddps " OFF " + 0xe0 + %[knee], %%ymm2, %%ymm2") /* ymm2 = TV = tilt[0]*lx0+tilt[1] */ \
747 __ASM_EMIT("vaddps " OFF " + 0xe0 + %[knee], %%ymm6, %%ymm6") \
748 __ASM_EMIT("vaddps " OFF " + 0xe0 + %[knee], %%ymm10, %%ymm10") \
749 __ASM_EMIT("vaddps " OFF " + 0xe0 + %[knee], %%ymm14, %%ymm14") \
750 __ASM_EMIT("vfmadd213ps " OFF " + 0xa0 + %[knee], %%ymm0, %%ymm1") /* ymm1 = KV = (herm[0]*lx0+herm[1])*lx0+herm[2] */ \
751 __ASM_EMIT("vfmadd213ps " OFF " + 0xa0 + %[knee], %%ymm4, %%ymm5") \
752 __ASM_EMIT("vfmadd213ps " OFF " + 0xa0 + %[knee], %%ymm8, %%ymm9") \
753 __ASM_EMIT("vfmadd213ps " OFF " + 0xa0 + %[knee], %%ymm12, %%ymm13") \
754 __ASM_EMIT("vmovaps 0x00 + %[mem], %%ymm0") /* ymm1 = x0 */ \
755 __ASM_EMIT("vmovaps 0x20 + %[mem], %%ymm4") \
756 __ASM_EMIT("vmovaps 0x40 + %[mem], %%ymm8") \
757 __ASM_EMIT("vmovaps 0x60 + %[mem], %%ymm12") \
758 __ASM_EMIT("vcmpps $5, " OFF " + 0x20 + %[knee], %%ymm0, %%ymm3") /* ymm3 = [x0 >= end] */ \
759 __ASM_EMIT("vcmpps $5, " OFF " + 0x20 + %[knee], %%ymm4, %%ymm7") \
760 __ASM_EMIT("vcmpps $5, " OFF " + 0x20 + %[knee], %%ymm8, %%ymm11") \
761 __ASM_EMIT("vcmpps $5, " OFF " + 0x20 + %[knee], %%ymm12, %%ymm15") \
762 __ASM_EMIT("vblendvps %%ymm3, %%ymm2, %%ymm1, %%ymm0") /* ymm0 = [x0 >= end] ? TV : KV */ \
763 __ASM_EMIT("vblendvps %%ymm7, %%ymm6, %%ymm5, %%ymm4") \
764 __ASM_EMIT("vblendvps %%ymm11, %%ymm10, %%ymm9, %%ymm8") \
765 __ASM_EMIT("vblendvps %%ymm15, %%ymm14, %%ymm13, %%ymm12") \
766 EXP_CORE_X32_FMA3 /* ymm0 = EV = expf([x0 >= end] ? TV : KV) */ \
767 __ASM_EMIT("vmovaps 0x00 + %[mem], %%ymm1") /* ymm1 = x0 */ \
768 __ASM_EMIT("vmovaps 0x20 + %[mem], %%ymm5") \
769 __ASM_EMIT("vmovaps 0x40 + %[mem], %%ymm9") \
770 __ASM_EMIT("vmovaps 0x60 + %[mem], %%ymm13") \
771 __ASM_EMIT("vcmpps $2, " OFF " + 0x00 + %[knee], %%ymm1, %%ymm3") /* ymm3 = [x0 <= start] */ \
772 __ASM_EMIT("vcmpps $2, " OFF " + 0x00 + %[knee], %%ymm5, %%ymm7") \
773 __ASM_EMIT("vcmpps $2, " OFF " + 0x00 + %[knee], %%ymm9, %%ymm11") \
774 __ASM_EMIT("vcmpps $2, " OFF " + 0x00 + %[knee], %%ymm13, %%ymm15") \
775 __ASM_EMIT("vblendvps %%ymm3, " OFF " + 0x40 + %[knee], %%ymm0, %%ymm0") /* ymm0 = [x0 <= start] ? GV : EV */ \
776 __ASM_EMIT("vblendvps %%ymm7, " OFF " + 0x40 + %[knee], %%ymm4, %%ymm4") \
777 __ASM_EMIT("vblendvps %%ymm11, " OFF " + 0x40 + %[knee], %%ymm8, %%ymm8") \
778 __ASM_EMIT("vblendvps %%ymm15, " OFF " + 0x40 + %[knee], %%ymm12, %%ymm12") \
779
780 #define PROCESS_KNEE_SINGLE_X16_FMA3(OFF) \
781 /* in: ymm0 = lx0, ymm4 = lx1 */ \
782 __ASM_EMIT("vmulps " OFF " + 0x60 + %[knee], %%ymm0, %%ymm1") /* ymm1 = herm[0]*lx0 */ \
783 __ASM_EMIT("vmulps " OFF " + 0x60 + %[knee], %%ymm4, %%ymm5") \
784 __ASM_EMIT("vmulps " OFF " + 0xc0 + %[knee], %%ymm0, %%ymm2") /* ymm2 = tilt[0]*lx0 */ \
785 __ASM_EMIT("vmulps " OFF " + 0xc0 + %[knee], %%ymm4, %%ymm6") \
786 __ASM_EMIT("vaddps " OFF " + 0x80 + %[knee], %%ymm1, %%ymm1") /* ymm1 = herm[0]*lx0+herm[1] */ \
787 __ASM_EMIT("vaddps " OFF " + 0x80 + %[knee], %%ymm5, %%ymm5") \
788 __ASM_EMIT("vaddps " OFF " + 0xe0 + %[knee], %%ymm2, %%ymm2") /* ymm2 = TV = tilt[0]*lx0+tilt[1] */ \
789 __ASM_EMIT("vaddps " OFF " + 0xe0 + %[knee], %%ymm6, %%ymm6") \
790 __ASM_EMIT("vfmadd213ps " OFF " + 0xa0 + %[knee], %%ymm0, %%ymm1") /* ymm1 = KV = (herm[0]*lx0+herm[1])*lx0+herm[2] */ \
791 __ASM_EMIT("vfmadd213ps " OFF " + 0xa0 + %[knee], %%ymm4, %%ymm5") \
792 __ASM_EMIT("vmovaps 0x00 + %[mem], %%ymm0") /* ymm1 = x0 */ \
793 __ASM_EMIT("vmovaps 0x20 + %[mem], %%ymm4") \
794 __ASM_EMIT("vcmpps $5, " OFF " + 0x20 + %[knee], %%ymm0, %%ymm3") /* ymm3 = [x0 >= end] */ \
795 __ASM_EMIT("vcmpps $5, " OFF " + 0x20 + %[knee], %%ymm4, %%ymm7") \
796 __ASM_EMIT("vblendvps %%ymm3, %%ymm2, %%ymm1, %%ymm0") /* ymm0 = [x0 >= end] ? TV : KV */ \
797 __ASM_EMIT("vblendvps %%ymm7, %%ymm6, %%ymm5, %%ymm4") \
798 EXP_CORE_X16_FMA3 /* ymm0 = EV = expf([x0 >= end] ? TV : KV) */ \
799 __ASM_EMIT("vmovaps 0x00 + %[mem], %%ymm1") /* ymm1 = x0 */ \
800 __ASM_EMIT("vmovaps 0x20 + %[mem], %%ymm5") \
801 __ASM_EMIT("vcmpps $2, " OFF " + 0x00 + %[knee], %%ymm1, %%ymm3") /* ymm3 = [x0 <= start] */ \
802 __ASM_EMIT("vcmpps $2, " OFF " + 0x00 + %[knee], %%ymm5, %%ymm7") \
803 __ASM_EMIT("vblendvps %%ymm3, " OFF " + 0x40 + %[knee], %%ymm0, %%ymm0") /* ymm0 = [x0 <= start] ? GV : EV */ \
804 __ASM_EMIT("vblendvps %%ymm7, " OFF " + 0x40 + %[knee], %%ymm4, %%ymm4") \
805 /* out: ymm0 = g0, ymm4 = g1 */
806
807 #define PROCESS_KNEE_SINGLE_X8_FMA3(OFF) \
808 /* in: ymm0 = lx0 */ \
809 __ASM_EMIT("vmulps " OFF " + 0x60 + %[knee], %%ymm0, %%ymm1") /* ymm1 = herm[0]*lx0 */ \
810 __ASM_EMIT("vmulps " OFF " + 0xc0 + %[knee], %%ymm0, %%ymm2") /* ymm2 = tilt[0]*lx0 */ \
811 __ASM_EMIT("vaddps " OFF " + 0x80 + %[knee], %%ymm1, %%ymm1") /* ymm1 = herm[0]*lx0+herm[1] */ \
812 __ASM_EMIT("vaddps " OFF " + 0xe0 + %[knee], %%ymm2, %%ymm2") /* ymm2 = TV = tilt[0]*lx0+tilt[1] */ \
813 __ASM_EMIT("vfmadd213ps " OFF " + 0xa0 + %[knee], %%ymm0, %%ymm1") /* ymm1 = KV = (herm[0]*lx0+herm[1])*lx0+herm[2] */ \
814 __ASM_EMIT("vmovaps 0x00 + %[mem], %%ymm0") /* ymm1 = x0 */ \
815 __ASM_EMIT("vcmpps $5, " OFF " + 0x20 + %[knee], %%ymm0, %%ymm3") /* ymm3 = [x0 >= end] */ \
816 __ASM_EMIT("vblendvps %%ymm3, %%ymm2, %%ymm1, %%ymm0") /* ymm0 = [x0 >= end] ? TV : KV */ \
817 EXP_CORE_X8_FMA3 /* ymm0 = EV = expf([x0 >= end] ? TV : KV) */ \
818 __ASM_EMIT("vmovaps 0x00 + %[mem], %%ymm1") /* ymm1 = x0 */ \
819 __ASM_EMIT("vcmpps $2, " OFF " + 0x00 + %[knee], %%ymm1, %%ymm3") /* ymm3 = [x0 <= start] */ \
820 __ASM_EMIT("vblendvps %%ymm3, " OFF " + 0x40 + %[knee], %%ymm0, %%ymm0") /* ymm0 = [x0 <= start] ? GV : EV */ \
821 /* out: ymm0 = g0, ymm4 = g1 */
822
823 #define PROCESS_KNEE_SINGLE_X4_FMA3(OFF) \
824 /* in: xmm0 = lx0 */ \
825 __ASM_EMIT("vmulps " OFF " + 0x60 + %[knee], %%xmm0, %%xmm1") /* xmm1 = herm[0]*lx0 */ \
826 __ASM_EMIT("vmulps " OFF " + 0xc0 + %[knee], %%xmm0, %%xmm2") /* xmm2 = tilt[0]*lx0 */ \
827 __ASM_EMIT("vaddps " OFF " + 0x80 + %[knee], %%xmm1, %%xmm1") /* xmm1 = herm[0]*lx0+herm[1] */ \
828 __ASM_EMIT("vaddps " OFF " + 0xe0 + %[knee], %%xmm2, %%xmm2") /* xmm2 = TV = tilt[0]*lx0+tilt[1] */ \
829 __ASM_EMIT("vfmadd213ps " OFF " + 0xa0 + %[knee], %%xmm0, %%xmm1") /* xmm1 = KV = (herm[0]*lx0+herm[1])*lx0+herm[2] */ \
830 __ASM_EMIT("vmovaps 0x00 + %[mem], %%xmm0") /* xmm1 = x0 */ \
831 __ASM_EMIT("vcmpps $5, " OFF " + 0x20 + %[knee], %%xmm0, %%xmm3") /* xmm3 = [x0 >= end] */ \
832 __ASM_EMIT("vblendvps %%xmm3, %%xmm2, %%xmm1, %%xmm0") /* xmm0 = [x0 >= end] ? TV : KV */ \
833 EXP_CORE_X4_FMA3 /* xmm0 = EV = expf([x0 >= end] ? TV : KV) */ \
834 __ASM_EMIT("vmovaps 0x00 + %[mem], %%xmm1") /* xmm1 = x0 */ \
835 __ASM_EMIT("vcmpps $2, " OFF " + 0x00 + %[knee], %%xmm1, %%xmm3") /* xmm3 = [x0 <= start] */ \
836 __ASM_EMIT("vblendvps %%xmm3, " OFF " + 0x40 + %[knee], %%xmm0, %%xmm0") /* xmm0 = [x0 <= start] ? GV : EV */ \
837 /* out: xmm0 = g0, xmm4 = g1 */
838
839 #define PROCESS_COMP_FULL_X32_FMA3 \
840 /* in: ymm0 = x0, ymm4 = x1, ymm8 = x2, ymm12 = x4 */ \
841 __ASM_EMIT("vandps 0x00 + %[C2C], %%ymm0, %%ymm0") /* ymm0 = fabsf(x0) */ \
842 __ASM_EMIT("vandps 0x00 + %[C2C], %%ymm4, %%ymm4") \
843 __ASM_EMIT("vandps 0x00 + %[C2C], %%ymm8, %%ymm8") \
844 __ASM_EMIT("vandps 0x00 + %[C2C], %%ymm12, %%ymm12") \
845 __ASM_EMIT("vcmpps $6, 0x000 + 0x00 + %[knee], %%ymm0, %%ymm1") /* ymm1 = [x0 > k0.start] */ \
846 __ASM_EMIT("vcmpps $6, 0x000 + 0x00 + %[knee], %%ymm4, %%ymm5") \
847 __ASM_EMIT("vcmpps $6, 0x000 + 0x00 + %[knee], %%ymm8, %%ymm9") \
848 __ASM_EMIT("vcmpps $6, 0x000 + 0x00 + %[knee], %%ymm12, %%ymm13") \
849 __ASM_EMIT("vcmpps $6, 0x100 + 0x00 + %[knee], %%ymm0, %%ymm2") /* ymm2 = [x0 > k0.start] */ \
850 __ASM_EMIT("vcmpps $6, 0x100 + 0x00 + %[knee], %%ymm4, %%ymm6") \
851 __ASM_EMIT("vcmpps $6, 0x100 + 0x00 + %[knee], %%ymm8, %%ymm10") \
852 __ASM_EMIT("vcmpps $6, 0x100 + 0x00 + %[knee], %%ymm12, %%ymm14") \
853 __ASM_EMIT("vorps %%ymm2, %%ymm1, %%ymm1") \
854 __ASM_EMIT("vorps %%ymm6, %%ymm5, %%ymm5") \
855 __ASM_EMIT("vorps %%ymm10, %%ymm9, %%ymm9") \
856 __ASM_EMIT("vorps %%ymm14, %%ymm13, %%ymm13") \
857 __ASM_EMIT("vorps %%ymm5, %%ymm1, %%ymm1") \
858 __ASM_EMIT("vorps %%ymm13, %%ymm9, %%ymm9") \
859 __ASM_EMIT("vorps %%ymm9, %%ymm1, %%ymm1") \
860 __ASM_EMIT("vmovmskps %%ymm1, %[mask]") \
861 __ASM_EMIT("test %[mask], %[mask]") \
862 __ASM_EMIT("jnz 100f") \
863 __ASM_EMIT("vmovaps 0x000 + 0x40 + %[knee], %%ymm0") /* ymm0 = g1 */ \
864 __ASM_EMIT("vmovaps 0x000 + 0x40 + %[knee], %%ymm4") \
865 __ASM_EMIT("vmulps 0x100 + 0x40 + %[knee], %%ymm0, %%ymm0") /* ymm0 = G = g0*g1 */ \
866 __ASM_EMIT("vmulps 0x100 + 0x40 + %[knee], %%ymm4, %%ymm4") \
867 __ASM_EMIT("vmovaps %%ymm0, %%ymm8") \
868 __ASM_EMIT("vmovaps %%ymm4, %%ymm12") \
869 __ASM_EMIT("jmp 200f") \
870 __ASM_EMIT("100:") \
871 __ASM_EMIT("vmovaps %%ymm0, 0x00 + %[mem]") /* store fabsf(x0) */ \
872 __ASM_EMIT("vmovaps %%ymm4, 0x20 + %[mem]") \
873 __ASM_EMIT("vmovaps %%ymm8, 0x40 + %[mem]") \
874 __ASM_EMIT("vmovaps %%ymm12, 0x60 + %[mem]") \
875 LOGE_CORE_X32_FMA3 /* ymm0 = lx0 = logf(fabsf(x0)) */ \
876 __ASM_EMIT("vmovaps %%ymm0, 0x80 + %[mem]") /* *mem = lx0 */ \
877 __ASM_EMIT("vmovaps %%ymm4, 0xa0 + %[mem]") \
878 __ASM_EMIT("vmovaps %%ymm8, 0xc0 + %[mem]") \
879 __ASM_EMIT("vmovaps %%ymm12, 0xe0 + %[mem]") \
880 PROCESS_KNEE_SINGLE_X32_FMA3("0x000") /* apply knee 0 */ \
881 __ASM_EMIT("vmovaps %%ymm0, 0x100 + %[mem]") /* *mem = g1 */ \
882 __ASM_EMIT("vmovaps %%ymm4, 0x120 + %[mem]") \
883 __ASM_EMIT("vmovaps %%ymm8, 0x140 + %[mem]") \
884 __ASM_EMIT("vmovaps %%ymm12, 0x160 + %[mem]") \
885 __ASM_EMIT("vmovaps 0x80 + %[mem], %%ymm0") /* ymm0 = lx0 */ \
886 __ASM_EMIT("vmovaps 0xa0 + %[mem], %%ymm4") \
887 __ASM_EMIT("vmovaps 0xc0 + %[mem], %%ymm8") \
888 __ASM_EMIT("vmovaps 0xe0 + %[mem], %%ymm12") \
889 PROCESS_KNEE_SINGLE_X32_FMA3("0x100") /* apply knee 1 */ \
890 __ASM_EMIT("vmulps 0x100 + %[mem], %%ymm0, %%ymm0") /* ymm0 = G = g0*g1 */ \
891 __ASM_EMIT("vmulps 0x120 + %[mem], %%ymm4, %%ymm4") \
892 __ASM_EMIT("vmulps 0x140 + %[mem], %%ymm8, %%ymm8") \
893 __ASM_EMIT("vmulps 0x160 + %[mem], %%ymm12, %%ymm12") \
894 __ASM_EMIT("200:") \
895 /* out: ymm0 = G0, ymm4 = G1, ymm8 = G2, ymm12 = G3 */
896
897 #define PROCESS_COMP_FULL_X16_FMA3 \
898 /* in: ymm0 = x0, ymm4 = x1 */ \
899 __ASM_EMIT("vandps 0x00 + %[C2C], %%ymm0, %%ymm0") /* ymm0 = fabsf(x0) */ \
900 __ASM_EMIT("vandps 0x00 + %[C2C], %%ymm4, %%ymm4") \
901 __ASM_EMIT("vcmpps $6, 0x000 + 0x00 + %[knee], %%ymm0, %%ymm1") /* ymm1 = [x0 > k0.start] */ \
902 __ASM_EMIT("vcmpps $6, 0x000 + 0x00 + %[knee], %%ymm4, %%ymm5") \
903 __ASM_EMIT("vcmpps $6, 0x100 + 0x00 + %[knee], %%ymm0, %%ymm2") /* ymm2 = [x0 > k0.start] */ \
904 __ASM_EMIT("vcmpps $6, 0x100 + 0x00 + %[knee], %%ymm4, %%ymm6") \
905 __ASM_EMIT("vorps %%ymm2, %%ymm1, %%ymm1") \
906 __ASM_EMIT("vorps %%ymm6, %%ymm5, %%ymm5") \
907 __ASM_EMIT("vorps %%ymm5, %%ymm1, %%ymm1") \
908 __ASM_EMIT("vmovmskps %%ymm1, %[mask]") \
909 __ASM_EMIT("test %[mask], %[mask]") \
910 __ASM_EMIT("jnz 100f") \
911 __ASM_EMIT("vmovaps 0x000 + 0x40 + %[knee], %%ymm0") /* ymm0 = g1 */ \
912 __ASM_EMIT("vmovaps 0x000 + 0x40 + %[knee], %%ymm4") \
913 __ASM_EMIT("vmulps 0x100 + 0x40 + %[knee], %%ymm0, %%ymm0") /* ymm0 = G = g0*g1 */ \
914 __ASM_EMIT("vmulps 0x100 + 0x40 + %[knee], %%ymm4, %%ymm4") \
915 __ASM_EMIT("jmp 200f") \
916 __ASM_EMIT("100:") \
917 __ASM_EMIT("vmovaps %%ymm0, 0x00 + %[mem]") /* store fabsf(x0) */ \
918 __ASM_EMIT("vmovaps %%ymm4, 0x20 + %[mem]") \
919 LOGE_CORE_X16_FMA3 /* ymm0 = lx0 = logf(fabsf(x0)) */ \
920 __ASM_EMIT("vmovaps %%ymm0, 0x40 + %[mem]") /* *mem = lx0 */ \
921 __ASM_EMIT("vmovaps %%ymm4, 0x60 + %[mem]") \
922 PROCESS_KNEE_SINGLE_X16_FMA3("0x000") /* apply knee 0 */ \
923 __ASM_EMIT("vmovaps %%ymm0, 0x80 + %[mem]") /* *mem = g1 */ \
924 __ASM_EMIT("vmovaps %%ymm4, 0xa0 + %[mem]") \
925 __ASM_EMIT("vmovaps 0x40 + %[mem], %%ymm0") /* ymm0 = lx0 */ \
926 __ASM_EMIT("vmovaps 0x60 + %[mem], %%ymm4") \
927 PROCESS_KNEE_SINGLE_X16_FMA3("0x100") /* apply knee 1 */ \
928 __ASM_EMIT("vmulps 0x80 + %[mem], %%ymm0, %%ymm0") /* ymm0 = G = g0*g1 */ \
929 __ASM_EMIT("vmulps 0xa0 + %[mem], %%ymm4, %%ymm4") \
930 __ASM_EMIT("200:") \
931 /* out: ymm0 = G0, ymm4 = G1 */
932
933 #define PROCESS_COMP_FULL_X8_FMA3 \
934 /* in: ymm0 = x0, ymm4 = x1 */ \
935 __ASM_EMIT("vandps 0x00 + %[C2C], %%ymm0, %%ymm0") /* ymm0 = fabsf(x0) */ \
936 __ASM_EMIT("vmovaps %%ymm0, 0x00 + %[mem]") /* store fabsf(x0) */ \
937 LOGE_CORE_X8_FMA3 /* ymm0 = lx0 = logf(fabsf(x0)) */ \
938 __ASM_EMIT("vmovaps %%ymm0, 0x40 + %[mem]") /* *mem = lx0 */ \
939 PROCESS_KNEE_SINGLE_X8_FMA3("0x000") /* apply knee 0 */ \
940 __ASM_EMIT("vmovaps %%ymm0, 0x80 + %[mem]") /* *mem = g1 */ \
941 __ASM_EMIT("vmovaps 0x40 + %[mem], %%ymm0") /* ymm0 = lx0 */ \
942 PROCESS_KNEE_SINGLE_X8_FMA3("0x100") /* apply knee 1 */ \
943 __ASM_EMIT("vmulps 0x80 + %[mem], %%ymm0, %%ymm0") /* ymm0 = G = g0*g1 */ \
944 /* out: ymm0 = G0 */
945
946 #define PROCESS_COMP_FULL_X4_FMA3 \
947 /* in: xmm0 = x0, xmm4 = x1 */ \
948 __ASM_EMIT("vandps 0x00 + %[C2C], %%xmm0, %%xmm0") /* xmm0 = fabsf(x0) */ \
949 __ASM_EMIT("vmovaps %%xmm0, 0x00 + %[mem]") /* store fabsf(x0) */ \
950 LOGE_CORE_X4_FMA3 /* xmm0 = lx0 = logf(fabsf(x0)) */ \
951 __ASM_EMIT("vmovaps %%xmm0, 0x40 + %[mem]") /* *mem = lx0 */ \
952 PROCESS_KNEE_SINGLE_X4_FMA3("0x000") /* apply knee 0 */ \
953 __ASM_EMIT("vmovaps %%xmm0, 0x80 + %[mem]") /* *mem = g1 */ \
954 __ASM_EMIT("vmovaps 0x40 + %[mem], %%xmm0") /* xmm0 = lx0 */ \
955 PROCESS_KNEE_SINGLE_X4_FMA3("0x100") /* apply knee 1 */ \
956 __ASM_EMIT("vmulps 0x80 + %[mem], %%xmm0, %%xmm0") /* xmm0 = G = g0*g1 */ \
957 /* out: xmm0 = G0 */
958
959 void compressor_x2_gain_fma3(float *dst, const float *src, const dsp::compressor_x2_t *c, size_t count)
960 {
961 IF_ARCH_X86(
962 comp_knee_t knee[2] __lsp_aligned32;
963 float mem[48] __lsp_aligned32;
964 float stub[16] __lsp_aligned32;
965 size_t mask;
966 );
967
968 ARCH_X86_ASM
969 (
970 // Prepare stuff
971 UNPACK_COMP_KNEE("knee", "0x000", "comp", "0x00")
972 UNPACK_COMP_KNEE("knee", "0x100", "comp", "0x20")
973
974 // 16x blocks
975 __ASM_EMIT("sub $16, %[count]")
976 __ASM_EMIT("jb 2f")
977 __ASM_EMIT("1:")
978 __ASM_EMIT("vmovups 0x00(%[src]), %%ymm0")
979 __ASM_EMIT("vmovups 0x20(%[src]), %%ymm4")
980 PROCESS_COMP_FULL_X16_FMA3
981 __ASM_EMIT("vmovups %%ymm0, 0x00(%[dst])")
982 __ASM_EMIT("vmovups %%ymm4, 0x20(%[dst])")
983 __ASM_EMIT("add $0x40, %[src]")
984 __ASM_EMIT("add $0x40, %[dst]")
985 __ASM_EMIT("sub $16, %[count]")
986 __ASM_EMIT("jae 1b")
987 __ASM_EMIT("2:")
988 // 8x blocks
989 __ASM_EMIT("add $8, %[count]")
990 __ASM_EMIT("jl 4f")
991 __ASM_EMIT("vmovups 0x00(%[src]), %%ymm0")
992 PROCESS_COMP_FULL_X8_FMA3
993 __ASM_EMIT("vmovups %%ymm0, 0x00(%[dst])")
994 __ASM_EMIT("add $0x20, %[src]")
995 __ASM_EMIT("add $0x20, %[dst]")
996 __ASM_EMIT("sub $8, %[count]")
997 __ASM_EMIT("4:")
998 // 4x blocks
999 __ASM_EMIT("add $4, %[count]")
1000 __ASM_EMIT("jl 6f")
1001 __ASM_EMIT("vmovups 0x00(%[src]), %%xmm0")
1002 PROCESS_COMP_FULL_X4_FMA3
1003 __ASM_EMIT("vmovups %%xmm0, 0x00(%[dst])")
1004 __ASM_EMIT("sub $4, %[count]")
1005 __ASM_EMIT("add $0x10, %[src]")
1006 __ASM_EMIT("add $0x10, %[dst]")
1007 __ASM_EMIT("6:")
1008 // Tail: 1x-3x block
1009 __ASM_EMIT("add $4, %[count]")
1010 __ASM_EMIT("jle 14f")
1011 __ASM_EMIT("test $1, %[count]")
1012 __ASM_EMIT("jz 8f")
1013 __ASM_EMIT("vmovss 0x00(%[src]), %%xmm0")
1014 __ASM_EMIT("add $4, %[src]")
1015 __ASM_EMIT("8:")
1016 __ASM_EMIT("test $2, %[count]")
1017 __ASM_EMIT("jz 10f")
1018 __ASM_EMIT("vmovhps 0x00(%[src]), %%xmm0, %%xmm0")
1019 __ASM_EMIT("10:")
1020 PROCESS_COMP_FULL_X4_FMA3
1021 __ASM_EMIT("test $1, %[count]")
1022 __ASM_EMIT("jz 12f")
1023 __ASM_EMIT("vmovss %%xmm0, 0x00(%[dst])")
1024 __ASM_EMIT("add $4, %[dst]")
1025 __ASM_EMIT("12:")
1026 __ASM_EMIT("test $2, %[count]")
1027 __ASM_EMIT("jz 14f")
1028 __ASM_EMIT("vmovhps %%xmm0, 0x00(%[dst])")
1029 __ASM_EMIT("14:")
1030
1031 : [dst] "+r" (dst), [src] "+r" (src),
1032 [count] "+r" (count),
1033 [mask] "=&r" (mask)
1034 : [comp] "r" (c),
1035 [knee] "o" (knee),
1036 [mem] "o" (mem),
1037 [stub] "o" (stub),
1038 [C2C] "o" (compressor_const),
1039 [L2C] "o" (LOG2_CONST),
1040 [LOGC] "o" (LOGE_C),
1041 [E2C] "o" (EXP2_CONST),
1042 [LOG2E] "m" (EXP_LOG2E)
1043 : "cc", "memory",
1044 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
1045 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
1046 );
1047 }
1048
1049 IF_ARCH_X86_64(
1050 void x64_compressor_x2_gain_fma3(float *dst, const float *src, const dsp::compressor_x2_t *c, size_t count)
1051 {
1052 IF_ARCH_X86(
1053 comp_knee_t knee[2] __lsp_aligned32;
1054 float mem[96] __lsp_aligned32;
1055 float stub[16] __lsp_aligned32;
1056 size_t mask;
1057 );
1058
1059 ARCH_X86_ASM
1060 (
1061 // Prepare stuff
1062 UNPACK_COMP_KNEE("knee", "0x000", "comp", "0x00")
1063 UNPACK_COMP_KNEE("knee", "0x100", "comp", "0x20")
1064
1065 // 32x blocks
1066 __ASM_EMIT("sub $32, %[count]")
1067 __ASM_EMIT("jb 2f")
1068 __ASM_EMIT("1:")
1069 __ASM_EMIT("vmovups 0x00(%[src]), %%ymm0")
1070 __ASM_EMIT("vmovups 0x20(%[src]), %%ymm4")
1071 __ASM_EMIT("vmovups 0x40(%[src]), %%ymm8")
1072 __ASM_EMIT("vmovups 0x60(%[src]), %%ymm12")
1073 PROCESS_COMP_FULL_X32_FMA3
1074 __ASM_EMIT("vmovups %%ymm0, 0x00(%[dst])")
1075 __ASM_EMIT("vmovups %%ymm4, 0x20(%[dst])")
1076 __ASM_EMIT("vmovups %%ymm8, 0x40(%[dst])")
1077 __ASM_EMIT("vmovups %%ymm12, 0x60(%[dst])")
1078 __ASM_EMIT("add $0x80, %[src]")
1079 __ASM_EMIT("add $0x80, %[dst]")
1080 __ASM_EMIT("sub $32, %[count]")
1081 __ASM_EMIT("jae 1b")
1082 __ASM_EMIT("2:")
1083 // 16x blocks
1084 __ASM_EMIT("add $16, %[count]")
1085 __ASM_EMIT("jl 4f")
1086 __ASM_EMIT("vmovups 0x00(%[src]), %%ymm0")
1087 __ASM_EMIT("vmovups 0x20(%[src]), %%ymm4")
1088 PROCESS_COMP_FULL_X16_FMA3
1089 __ASM_EMIT("vmovups %%ymm0, 0x00(%[dst])")
1090 __ASM_EMIT("vmovups %%ymm4, 0x20(%[dst])")
1091 __ASM_EMIT("add $0x40, %[src]")
1092 __ASM_EMIT("add $0x40, %[dst]")
1093 __ASM_EMIT("sub $16, %[count]")
1094 __ASM_EMIT("4:")
1095 // 8x blocks
1096 __ASM_EMIT("add $8, %[count]")
1097 __ASM_EMIT("jl 6f")
1098 __ASM_EMIT("vmovups 0x00(%[src]), %%ymm0")
1099 PROCESS_COMP_FULL_X8_FMA3
1100 __ASM_EMIT("vmovups %%ymm0, 0x00(%[dst])")
1101 __ASM_EMIT("add $0x20, %[src]")
1102 __ASM_EMIT("add $0x20, %[dst]")
1103 __ASM_EMIT("sub $8, %[count]")
1104 __ASM_EMIT("6:")
1105 // 4x blocks
1106 __ASM_EMIT("add $4, %[count]")
1107 __ASM_EMIT("jl 8f")
1108 __ASM_EMIT("vmovups 0x00(%[src]), %%xmm0")
1109 PROCESS_COMP_FULL_X4_FMA3
1110 __ASM_EMIT("vmovups %%xmm0, 0x00(%[dst])")
1111 __ASM_EMIT("sub $4, %[count]")
1112 __ASM_EMIT("add $0x10, %[src]")
1113 __ASM_EMIT("add $0x10, %[dst]")
1114 __ASM_EMIT("8:")
1115 // Tail: 1x-3x block
1116 __ASM_EMIT("add $4, %[count]")
1117 __ASM_EMIT("jle 16f")
1118 __ASM_EMIT("test $1, %[count]")
1119 __ASM_EMIT("jz 10f")
1120 __ASM_EMIT("vmovss 0x00(%[src]), %%xmm0")
1121 __ASM_EMIT("add $4, %[src]")
1122 __ASM_EMIT("10:")
1123 __ASM_EMIT("test $2, %[count]")
1124 __ASM_EMIT("jz 12f")
1125 __ASM_EMIT("vmovhps 0x00(%[src]), %%xmm0, %%xmm0")
1126 __ASM_EMIT("12:")
1127 PROCESS_COMP_FULL_X4_FMA3
1128 __ASM_EMIT("test $1, %[count]")
1129 __ASM_EMIT("jz 14f")
1130 __ASM_EMIT("vmovss %%xmm0, 0x00(%[dst])")
1131 __ASM_EMIT("add $4, %[dst]")
1132 __ASM_EMIT("14:")
1133 __ASM_EMIT("test $2, %[count]")
1134 __ASM_EMIT("jz 16f")
1135 __ASM_EMIT("vmovhps %%xmm0, 0x00(%[dst])")
1136 __ASM_EMIT("16:")
1137
1138 : [dst] "+r" (dst), [src] "+r" (src),
1139 [count] "+r" (count),
1140 [mask] "=&r" (mask)
1141 : [comp] "r" (c),
1142 [knee] "o" (knee),
1143 [mem] "o" (mem),
1144 [stub] "o" (stub),
1145 [C2C] "o" (compressor_const),
1146 [L2C] "o" (LOG2_CONST),
1147 [LOGC] "o" (LOGE_C),
1148 [E2C] "o" (EXP2_CONST),
1149 [LOG2E] "m" (EXP_LOG2E)
1150 : "cc", "memory",
1151 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
1152 "%xmm4", "%xmm5", "%xmm6", "%xmm7",
1153 "%xmm8", "%xmm9", "%xmm10", "%xmm11",
1154 "%xmm12", "%xmm13", "%xmm14", "%xmm15"
1155 );
1156 }
1157 )
1158
1159 void compressor_x2_curve_fma3(float *dst, const float *src, const dsp::compressor_x2_t *c, size_t count)
1160 {
1161 IF_ARCH_X86(
1162 comp_knee_t knee[2] __lsp_aligned32;
1163 float mem[48] __lsp_aligned32;
1164 float stub[16] __lsp_aligned32;
1165 size_t mask;
1166 );
1167
1168 ARCH_X86_ASM
1169 (
1170 // Prepare stuff
1171 UNPACK_COMP_KNEE("knee", "0x000", "comp", "0x00")
1172 UNPACK_COMP_KNEE("knee", "0x100", "comp", "0x20")
1173
1174 // 16x blocks
1175 __ASM_EMIT("sub $16, %[count]")
1176 __ASM_EMIT("jb 2f")
1177 __ASM_EMIT("1:")
1178 __ASM_EMIT("vmovups 0x00(%[src]), %%ymm0")
1179 __ASM_EMIT("vmovups 0x20(%[src]), %%ymm4")
1180 PROCESS_COMP_FULL_X16_FMA3
1181 __ASM_EMIT("vmulps 0x00(%[src]), %%ymm0, %%ymm0")
1182 __ASM_EMIT("vmulps 0x20(%[src]), %%ymm4, %%ymm4")
1183 __ASM_EMIT("vmovups %%ymm0, 0x00(%[dst])")
1184 __ASM_EMIT("vmovups %%ymm4, 0x20(%[dst])")
1185 __ASM_EMIT("add $0x40, %[src]")
1186 __ASM_EMIT("add $0x40, %[dst]")
1187 __ASM_EMIT("sub $16, %[count]")
1188 __ASM_EMIT("jae 1b")
1189 __ASM_EMIT("2:")
1190 // 8x blocks
1191 __ASM_EMIT("add $8, %[count]")
1192 __ASM_EMIT("jl 4f")
1193 __ASM_EMIT("vmovups 0x00(%[src]), %%ymm0")
1194 PROCESS_COMP_FULL_X8_FMA3
1195 __ASM_EMIT("vmulps 0x00(%[src]), %%ymm0, %%ymm0")
1196 __ASM_EMIT("vmovups %%ymm0, 0x00(%[dst])")
1197 __ASM_EMIT("add $0x20, %[src]")
1198 __ASM_EMIT("add $0x20, %[dst]")
1199 __ASM_EMIT("sub $8, %[count]")
1200 __ASM_EMIT("4:")
1201 // 4x blocks
1202 __ASM_EMIT("add $4, %[count]")
1203 __ASM_EMIT("jl 6f")
1204 __ASM_EMIT("vmovups 0x00(%[src]), %%xmm0")
1205 PROCESS_COMP_FULL_X4_FMA3
1206 __ASM_EMIT("vmulps 0x00(%[src]), %%xmm0, %%xmm0")
1207 __ASM_EMIT("vmovups %%xmm0, 0x00(%[dst])")
1208 __ASM_EMIT("sub $4, %[count]")
1209 __ASM_EMIT("add $0x10, %[src]")
1210 __ASM_EMIT("add $0x10, %[dst]")
1211 __ASM_EMIT("6:")
1212 // Tail: 1x-3x block
1213 __ASM_EMIT("add $4, %[count]")
1214 __ASM_EMIT("jle 14f")
1215 __ASM_EMIT("test $1, %[count]")
1216 __ASM_EMIT("jz 8f")
1217 __ASM_EMIT("vmovss 0x00(%[src]), %%xmm0")
1218 __ASM_EMIT("add $4, %[src]")
1219 __ASM_EMIT("8:")
1220 __ASM_EMIT("test $2, %[count]")
1221 __ASM_EMIT("jz 10f")
1222 __ASM_EMIT("vmovhps 0x00(%[src]), %%xmm0, %%xmm0")
1223 __ASM_EMIT("10:")
1224 __ASM_EMIT("vmovaps %%xmm0, %%xmm4")
1225 PROCESS_COMP_FULL_X4_FMA3
1226 __ASM_EMIT("vmulps %%xmm4, %%xmm0, %%xmm0")
1227 __ASM_EMIT("test $1, %[count]")
1228 __ASM_EMIT("jz 12f")
1229 __ASM_EMIT("vmovss %%xmm0, 0x00(%[dst])")
1230 __ASM_EMIT("add $4, %[dst]")
1231 __ASM_EMIT("12:")
1232 __ASM_EMIT("test $2, %[count]")
1233 __ASM_EMIT("jz 14f")
1234 __ASM_EMIT("vmovhps %%xmm0, 0x00(%[dst])")
1235 __ASM_EMIT("14:")
1236
1237 : [dst] "+r" (dst), [src] "+r" (src),
1238 [count] "+r" (count),
1239 [mask] "=&r" (mask)
1240 : [comp] "r" (c),
1241 [knee] "o" (knee),
1242 [mem] "o" (mem),
1243 [stub] "o" (stub),
1244 [C2C] "o" (compressor_const),
1245 [L2C] "o" (LOG2_CONST),
1246 [LOGC] "o" (LOGE_C),
1247 [E2C] "o" (EXP2_CONST),
1248 [LOG2E] "m" (EXP_LOG2E)
1249 : "cc", "memory",
1250 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
1251 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
1252 );
1253 }
1254
1255 IF_ARCH_X86_64(
1256 void x64_compressor_x2_curve_fma3(float *dst, const float *src, const dsp::compressor_x2_t *c, size_t count)
1257 {
1258 IF_ARCH_X86(
1259 comp_knee_t knee[2] __lsp_aligned32;
1260 float mem[96] __lsp_aligned32;
1261 float stub[16] __lsp_aligned32;
1262 size_t mask;
1263 );
1264
1265 ARCH_X86_ASM
1266 (
1267 // Prepare stuff
1268 UNPACK_COMP_KNEE("knee", "0x000", "comp", "0x00")
1269 UNPACK_COMP_KNEE("knee", "0x100", "comp", "0x20")
1270
1271 // 32x blocks
1272 __ASM_EMIT("sub $32, %[count]")
1273 __ASM_EMIT("jb 2f")
1274 __ASM_EMIT("1:")
1275 __ASM_EMIT("vmovups 0x00(%[src]), %%ymm0")
1276 __ASM_EMIT("vmovups 0x20(%[src]), %%ymm4")
1277 __ASM_EMIT("vmovups 0x40(%[src]), %%ymm8")
1278 __ASM_EMIT("vmovups 0x60(%[src]), %%ymm12")
1279 PROCESS_COMP_FULL_X32_FMA3
1280 __ASM_EMIT("vmulps 0x00(%[src]), %%ymm0, %%ymm0")
1281 __ASM_EMIT("vmulps 0x20(%[src]), %%ymm4, %%ymm4")
1282 __ASM_EMIT("vmulps 0x40(%[src]), %%ymm8, %%ymm8")
1283 __ASM_EMIT("vmulps 0x60(%[src]), %%ymm12, %%ymm12")
1284 __ASM_EMIT("vmovups %%ymm0, 0x00(%[dst])")
1285 __ASM_EMIT("vmovups %%ymm4, 0x20(%[dst])")
1286 __ASM_EMIT("vmovups %%ymm8, 0x40(%[dst])")
1287 __ASM_EMIT("vmovups %%ymm12, 0x60(%[dst])")
1288 __ASM_EMIT("add $0x80, %[src]")
1289 __ASM_EMIT("add $0x80, %[dst]")
1290 __ASM_EMIT("sub $32, %[count]")
1291 __ASM_EMIT("jae 1b")
1292 __ASM_EMIT("2:")
1293 // 16x blocks
1294 __ASM_EMIT("add $16, %[count]")
1295 __ASM_EMIT("jl 4f")
1296 __ASM_EMIT("vmovups 0x00(%[src]), %%ymm0")
1297 __ASM_EMIT("vmovups 0x20(%[src]), %%ymm4")
1298 PROCESS_COMP_FULL_X16_FMA3
1299 __ASM_EMIT("vmulps 0x00(%[src]), %%ymm0, %%ymm0")
1300 __ASM_EMIT("vmulps 0x20(%[src]), %%ymm4, %%ymm4")
1301 __ASM_EMIT("vmovups %%ymm0, 0x00(%[dst])")
1302 __ASM_EMIT("vmovups %%ymm4, 0x20(%[dst])")
1303 __ASM_EMIT("add $0x40, %[src]")
1304 __ASM_EMIT("add $0x40, %[dst]")
1305 __ASM_EMIT("sub $16, %[count]")
1306 __ASM_EMIT("4:")
1307 // 8x blocks
1308 __ASM_EMIT("add $8, %[count]")
1309 __ASM_EMIT("jl 6f")
1310 __ASM_EMIT("vmovups 0x00(%[src]), %%ymm0")
1311 PROCESS_COMP_FULL_X8_FMA3
1312 __ASM_EMIT("vmulps 0x00(%[src]), %%ymm0, %%ymm0")
1313 __ASM_EMIT("vmovups %%ymm0, 0x00(%[dst])")
1314 __ASM_EMIT("add $0x20, %[src]")
1315 __ASM_EMIT("add $0x20, %[dst]")
1316 __ASM_EMIT("sub $8, %[count]")
1317 __ASM_EMIT("6:")
1318 // 4x blocks
1319 __ASM_EMIT("add $4, %[count]")
1320 __ASM_EMIT("jl 8f")
1321 __ASM_EMIT("vmovups 0x00(%[src]), %%xmm0")
1322 PROCESS_COMP_FULL_X4_FMA3
1323 __ASM_EMIT("vmulps 0x00(%[src]), %%xmm0, %%xmm0")
1324 __ASM_EMIT("vmovups %%xmm0, 0x00(%[dst])")
1325 __ASM_EMIT("sub $4, %[count]")
1326 __ASM_EMIT("add $0x10, %[src]")
1327 __ASM_EMIT("add $0x10, %[dst]")
1328 __ASM_EMIT("8:")
1329 // Tail: 1x-3x block
1330 __ASM_EMIT("add $4, %[count]")
1331 __ASM_EMIT("jle 16f")
1332 __ASM_EMIT("test $1, %[count]")
1333 __ASM_EMIT("jz 10f")
1334 __ASM_EMIT("vmovss 0x00(%[src]), %%xmm0")
1335 __ASM_EMIT("add $4, %[src]")
1336 __ASM_EMIT("10:")
1337 __ASM_EMIT("test $2, %[count]")
1338 __ASM_EMIT("jz 12f")
1339 __ASM_EMIT("vmovhps 0x00(%[src]), %%xmm0, %%xmm0")
1340 __ASM_EMIT("12:")
1341 __ASM_EMIT("vmovaps %%xmm0, %%xmm4")
1342 PROCESS_COMP_FULL_X4_FMA3
1343 __ASM_EMIT("vmulps %%xmm4, %%xmm0, %%xmm0")
1344 __ASM_EMIT("test $1, %[count]")
1345 __ASM_EMIT("jz 14f")
1346 __ASM_EMIT("vmovss %%xmm0, 0x00(%[dst])")
1347 __ASM_EMIT("add $4, %[dst]")
1348 __ASM_EMIT("14:")
1349 __ASM_EMIT("test $2, %[count]")
1350 __ASM_EMIT("jz 16f")
1351 __ASM_EMIT("vmovhps %%xmm0, 0x00(%[dst])")
1352 __ASM_EMIT("16:")
1353
1354 : [dst] "+r" (dst), [src] "+r" (src),
1355 [count] "+r" (count),
1356 [mask] "=&r" (mask)
1357 : [comp] "r" (c),
1358 [knee] "o" (knee),
1359 [mem] "o" (mem),
1360 [stub] "o" (stub),
1361 [C2C] "o" (compressor_const),
1362 [L2C] "o" (LOG2_CONST),
1363 [LOGC] "o" (LOGE_C),
1364 [E2C] "o" (EXP2_CONST),
1365 [LOG2E] "m" (EXP_LOG2E)
1366 : "cc", "memory",
1367 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
1368 "%xmm4", "%xmm5", "%xmm6", "%xmm7",
1369 "%xmm8", "%xmm9", "%xmm10", "%xmm11",
1370 "%xmm12", "%xmm13", "%xmm14", "%xmm15"
1371 );
1372 }
1373 )
1374
1375 #undef PROCESS_COMP_FULL_X4_FMA3
1376 #undef PROCESS_COMP_FULL_X8_FMA3
1377 #undef PROCESS_COMP_FULL_X16_FMA3
1378 #undef PROCESS_COMP_FULL_X32_FMA3
1379 #undef PROCESS_KNEE_SINGLE_X4_FMA3
1380 #undef PROCESS_KNEE_SINGLE_X8_FMA3
1381 #undef PROCESS_KNEE_SINGLE_X16_FMA3
1382 #undef PROCESS_KNEE_SINGLE_X32_FMA3
1383
1384 #undef UNPACK_COMP_KNEE
1385
1386 } /* namespace avx2 */
1387 } /* namespace lsp */
1388
1389
1390
1391 #endif /* PRIVATE_DSP_ARCH_X86_AVX2_DYNAMICS_COMPRESSOR_H_ */