Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/x86/avx512/pan.h

raw
1 /*
2 * Copyright (C) 2024 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2024 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 28 нояб. 2024 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_X86_AVX512_PAN_H_
23 #define PRIVATE_DSP_ARCH_X86_AVX512_PAN_H_
24
25 #ifndef PRIVATE_DSP_ARCH_X86_AVX512_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_X86_AVX512_IMPL */
28
29 namespace lsp
30 {
31 namespace avx512
32 {
33 IF_ARCH_X86(
34 static const uint32_t depan_lin_const_i[] __lsp_aligned64 =
35 {
36 LSP_DSP_VEC16(0x7fffffff)
37 };
38 static const float depan_lin_const_f[] __lsp_aligned64 =
39 {
40 LSP_DSP_VEC16(1e-18f)
41 };
42 );
43
44 void depan_lin(float *dst, const float *l, const float *r, float dfl, size_t count)
45 {
46 /*
47 const float sl = fabsf(l[i]);
48 const float sr = fabsf(r[i]);
49 const float den = sl + sr;
50 dst[i] = (den >= 1e-18f) ? sr / den : dfl;
51 */
52 IF_ARCH_X86(size_t off);
53 ARCH_X86_ASM
54 (
55 __ASM_EMIT("xor %[off], %[off]")
56 __ASM_EMIT("vbroadcastss %%xmm0, %%zmm6") /* zmm6 = dfl */
57 __ASM_EMIT("vmovaps %[CI], %%zmm7") /* zmm7 = mask */
58 __ASM_EMIT("sub $32, %[count]")
59 __ASM_EMIT("vmovaps %[CF], %%zmm5") /* zmm5 = thresh */
60 __ASM_EMIT("jb 2f")
61 /* 32x blocks */
62 __ASM_EMIT("1:")
63 __ASM_EMIT("vandps 0x00(%[l],%[off]), %%zmm7, %%zmm2") /* zmm2 = fabsf(l) */
64 __ASM_EMIT("vandps 0x40(%[l],%[off]), %%zmm7, %%zmm3")
65 __ASM_EMIT("vandps 0x00(%[r],%[off]), %%zmm7, %%zmm0") /* xmm0 = fabsf(r) */
66 __ASM_EMIT("vandps 0x40(%[r],%[off]), %%zmm7, %%zmm1")
67 __ASM_EMIT("vaddps %%zmm0, %%zmm2, %%zmm2") /* zmm2 = den = fabsf(l) + fabsf(r) */
68 __ASM_EMIT("vaddps %%zmm1, %%zmm3, %%zmm3")
69 __ASM_EMIT("vdivps %%zmm2, %%zmm0, %%zmm0") /* zmm0 = pan = fabsf(r) / den */
70 __ASM_EMIT("vdivps %%zmm3, %%zmm1, %%zmm1")
71 __ASM_EMIT("vcmpps $1, %%zmm5, %%zmm2, %%k2") /* k2 = [den < thresh] */
72 __ASM_EMIT("vcmpps $1, %%zmm5, %%zmm3, %%k3")
73 __ASM_EMIT("vmovaps %%zmm6, %%zmm0 %{%%k2%}") /* zmm0 = (den >= thresh) ? pan : dfl */
74 __ASM_EMIT("vmovaps %%zmm6, %%zmm1 %{%%k3%}")
75 __ASM_EMIT("vmovups %%zmm0, 0x00(%[dst],%[off])")
76 __ASM_EMIT("vmovups %%zmm1, 0x40(%[dst],%[off])")
77 __ASM_EMIT("add $0x80, %[off]")
78 __ASM_EMIT("sub $32, %[count]")
79 __ASM_EMIT("jae 1b")
80 /* 16x blocks */
81 __ASM_EMIT("2:")
82 __ASM_EMIT("add $16, %[count]")
83 __ASM_EMIT("jl 4f")
84 __ASM_EMIT("vandps 0x00(%[l],%[off]), %%ymm7, %%ymm2") /* ymm2 = fabsf(l) */
85 __ASM_EMIT("vandps 0x20(%[l],%[off]), %%ymm7, %%ymm3")
86 __ASM_EMIT("vandps 0x00(%[r],%[off]), %%ymm7, %%ymm0") /* xmm0 = fabsf(r) */
87 __ASM_EMIT("vandps 0x20(%[r],%[off]), %%ymm7, %%ymm1")
88 __ASM_EMIT("vaddps %%ymm0, %%ymm2, %%ymm2") /* ymm2 = den = fabsf(l) + fabsf(r) */
89 __ASM_EMIT("vaddps %%ymm1, %%ymm3, %%ymm3")
90 __ASM_EMIT("vdivps %%ymm2, %%ymm0, %%ymm0") /* ymm0 = pan = fabsf(r) / den */
91 __ASM_EMIT("vdivps %%ymm3, %%ymm1, %%ymm1")
92 __ASM_EMIT("vcmpps $5, %%ymm5, %%ymm2, %%ymm2") /* ymm2 = [den >= thresh] */
93 __ASM_EMIT("vcmpps $5, %%ymm5, %%ymm3, %%ymm3")
94 __ASM_EMIT("vblendvps %%ymm2, %%ymm0, %%ymm6, %%ymm0") /* ymm0 = den >= thresh) ? pan : dfl */
95 __ASM_EMIT("vblendvps %%ymm3, %%ymm1, %%ymm6, %%ymm1")
96 __ASM_EMIT("vmovups %%ymm0, 0x00(%[dst],%[off])")
97 __ASM_EMIT("vmovups %%ymm1, 0x20(%[dst],%[off])")
98 __ASM_EMIT("sub $16, %[count]")
99 __ASM_EMIT("add $0x40, %[off]")
100 /* 8x block */
101 __ASM_EMIT("4:")
102 __ASM_EMIT("add $8, %[count]")
103 __ASM_EMIT("jl 6f")
104 __ASM_EMIT("vandps 0x00(%[l],%[off]), %%xmm7, %%xmm2") /* xmm2 = fabsf(l) */
105 __ASM_EMIT("vandps 0x10(%[l],%[off]), %%xmm7, %%xmm3")
106 __ASM_EMIT("vandps 0x00(%[r],%[off]), %%xmm7, %%xmm0") /* xmm0 = fabsf(r) */
107 __ASM_EMIT("vandps 0x10(%[r],%[off]), %%xmm7, %%xmm1")
108 __ASM_EMIT("vaddps %%xmm0, %%xmm2, %%xmm2") /* xmm2 = den = fabsf(l) + fabsf(r) */
109 __ASM_EMIT("vaddps %%xmm1, %%xmm3, %%xmm3")
110 __ASM_EMIT("vdivps %%xmm2, %%xmm0, %%xmm0") /* xmm0 = pan = fabsf(r) / den */
111 __ASM_EMIT("vdivps %%xmm3, %%xmm1, %%xmm1")
112 __ASM_EMIT("vcmpps $5, %%xmm5, %%xmm2, %%xmm2") /* xmm2 = [den >= thresh] */
113 __ASM_EMIT("vcmpps $5, %%xmm5, %%xmm3, %%xmm3")
114 __ASM_EMIT("vblendvps %%xmm2, %%xmm0, %%xmm6, %%xmm0") /* xmm0 = den >= thresh) ? pan : dfl */
115 __ASM_EMIT("vblendvps %%xmm3, %%xmm1, %%xmm6, %%xmm1")
116 __ASM_EMIT("vmovups %%xmm0, 0x00(%[dst],%[off])")
117 __ASM_EMIT("vmovups %%xmm1, 0x10(%[dst],%[off])")
118 __ASM_EMIT("sub $8, %[count]")
119 __ASM_EMIT("add $0x20, %[off]")
120 /* 4x block */
121 __ASM_EMIT("6:")
122 __ASM_EMIT("add $4, %[count]")
123 __ASM_EMIT("jl 8f")
124 __ASM_EMIT("vandps 0x00(%[l],%[off]), %%xmm7, %%xmm2") /* xmm2 = fabsf(l) */
125 __ASM_EMIT("vandps 0x00(%[r],%[off]), %%xmm7, %%xmm0") /* xmm0 = fabsf(r) */
126 __ASM_EMIT("vaddps %%xmm0, %%xmm2, %%xmm2") /* xmm2 = den = fabsf(l) + fabsf(r) */
127 __ASM_EMIT("vdivps %%xmm2, %%xmm0, %%xmm0") /* xmm0 = pan = fabsf(r) / den */
128 __ASM_EMIT("vcmpps $5, %%xmm5, %%xmm2, %%xmm2") /* xmm2 = [den >= thresh] */
129 __ASM_EMIT("vblendvps %%xmm2, %%xmm0, %%xmm6, %%xmm0") /* xmm0 = den >= thresh) ? pan : dfl */
130 __ASM_EMIT("vmovups %%xmm0, 0x00(%[dst],%[off])")
131 __ASM_EMIT("add $0x10, %[off]")
132 __ASM_EMIT("sub $4, %[count]")
133 /* 1x blocks */
134 __ASM_EMIT("8:")
135 __ASM_EMIT("add $3, %[count]")
136 __ASM_EMIT("jl 10f")
137 __ASM_EMIT("9:")
138 __ASM_EMIT("vmovss 0x00(%[l],%[off]), %%xmm2")
139 __ASM_EMIT("vmovss 0x00(%[r],%[off]), %%xmm0")
140 __ASM_EMIT("vandps %%xmm2, %%xmm7, %%xmm2") /* xmm2 = fabsf(l) */
141 __ASM_EMIT("vandps %%xmm0, %%xmm7, %%xmm0") /* xmm2 = fabsf(r) */
142 __ASM_EMIT("vaddps %%xmm0, %%xmm2, %%xmm2") /* xmm2 = den = fabsf(l) + fabsf(r) */
143 __ASM_EMIT("vdivps %%xmm2, %%xmm0, %%xmm0") /* xmm0 = pan = fabsf(r) / den */
144 __ASM_EMIT("vcmpps $5, %%xmm5, %%xmm2, %%xmm2") /* xmm2 = [den >= thresh] */
145 __ASM_EMIT("vblendvps %%xmm2, %%xmm0, %%xmm6, %%xmm0") /* xmm0 = den >= thresh) ? pan : dfl */
146 __ASM_EMIT("vmovss %%xmm0, 0x00(%[dst],%[off])")
147 __ASM_EMIT("add $0x04, %[off]")
148 __ASM_EMIT("dec %[count]")
149 __ASM_EMIT("jge 9b")
150 /* End */
151 __ASM_EMIT("10:")
152 : [count] "+r" (count), [off] "=&r" (off),
153 [dfl] "+Yz" (dfl)
154 : [dst] "r" (dst), [l] "r" (l), [r] "r" (r),
155 [CI] "m" (depan_lin_const_i),
156 [CF] "m" (depan_lin_const_f)
157 : "%xmm1", "%xmm2", "%xmm3",
158 "%xmm5", "%xmm6", "%xmm7",
159 "%k2", "%k3"
160 );
161 }
162
163 IF_ARCH_X86(
164 static const float depan_eqpow_const_f[] __lsp_aligned64 =
165 {
166 LSP_DSP_VEC16(1e-36f)
167 };
168 );
169
170 void depan_eqpow(float *dst, const float *l, const float *r, float dfl, size_t count)
171 {
172 /*
173 const float sl = l[i] * l[i];
174 const float sr = r[i] * r[i];
175 const float den = sl + sr;
176 dst[i] = (den >= 1e-36f) ? sr / den : dfl;
177 */
178 IF_ARCH_X86(size_t off);
179 ARCH_X86_ASM
180 (
181 __ASM_EMIT("xor %[off], %[off]")
182 __ASM_EMIT("vbroadcastss %%xmm0, %%zmm7") /* zmm7 = dfl */
183 __ASM_EMIT("sub $32, %[count]")
184 __ASM_EMIT("vmovaps %[CF], %%zmm6") /* zmm6 = thresh */
185 __ASM_EMIT("jb 2f")
186 /* 32x blocks */
187 __ASM_EMIT("1:")
188 __ASM_EMIT("vmovups 0x00(%[r],%[off]), %%zmm0") /* zmm0 = r */
189 __ASM_EMIT("vmovups 0x40(%[r],%[off]), %%zmm1")
190 __ASM_EMIT("vmovups 0x00(%[l],%[off]), %%zmm2") /* zmm2 = l */
191 __ASM_EMIT("vmovups 0x40(%[l],%[off]), %%zmm3")
192 __ASM_EMIT("vmulps %%zmm0, %%zmm0, %%zmm0") /* zmm0 = r*r */
193 __ASM_EMIT("vmulps %%zmm1, %%zmm1, %%zmm1")
194 __ASM_EMIT("vfmadd132ps %%zmm2, %%zmm0, %%zmm2") /* zmm2 = den = l*l + r*r */
195 __ASM_EMIT("vfmadd132ps %%zmm3, %%zmm1, %%zmm3")
196 __ASM_EMIT("vdivps %%zmm2, %%zmm0, %%zmm0") /* zmm0 = pan = r*r / den */
197 __ASM_EMIT("vdivps %%zmm3, %%zmm1, %%zmm1")
198 __ASM_EMIT("vcmpps $1, %%zmm6, %%zmm2, %%k2") /* zmm2 = [den < thresh] */
199 __ASM_EMIT("vcmpps $1, %%zmm6, %%zmm3, %%k3")
200 __ASM_EMIT("vmovaps %%zmm7, %%zmm0 %{%%k2%}") /* zmm0 = (den >= thresh) ? pan : dfl */
201 __ASM_EMIT("vmovaps %%zmm7, %%zmm1 %{%%k3%}")
202 __ASM_EMIT("vmovups %%zmm0, 0x00(%[dst],%[off])")
203 __ASM_EMIT("vmovups %%zmm1, 0x40(%[dst],%[off])")
204 __ASM_EMIT("add $0x80, %[off]")
205 __ASM_EMIT("sub $32, %[count]")
206 __ASM_EMIT("jae 1b")
207 /* 16x block */
208 __ASM_EMIT("2:")
209 __ASM_EMIT("add $16, %[count]")
210 __ASM_EMIT("jl 4f")
211 __ASM_EMIT("vmovups 0x00(%[r],%[off]), %%ymm0") /* ymm0 = r */
212 __ASM_EMIT("vmovups 0x20(%[r],%[off]), %%ymm1")
213 __ASM_EMIT("vmovups 0x00(%[l],%[off]), %%ymm2") /* ymm2 = l */
214 __ASM_EMIT("vmovups 0x20(%[l],%[off]), %%ymm3")
215 __ASM_EMIT("vmulps %%ymm0, %%ymm0, %%ymm0") /* ymm0 = r*r */
216 __ASM_EMIT("vmulps %%ymm1, %%ymm1, %%ymm1")
217 __ASM_EMIT("vfmadd132ps %%ymm2, %%ymm0, %%ymm2") /* ymm2 = den = l*l + r*r */
218 __ASM_EMIT("vfmadd132ps %%ymm3, %%ymm1, %%ymm3")
219 __ASM_EMIT("vdivps %%ymm2, %%ymm0, %%ymm0") /* ymm0 = pan = r*r / den */
220 __ASM_EMIT("vdivps %%ymm3, %%ymm1, %%ymm1")
221 __ASM_EMIT("vcmpps $5, %%ymm6, %%ymm2, %%ymm2") /* ymm2 = [den >= thresh] */
222 __ASM_EMIT("vcmpps $5, %%ymm6, %%ymm3, %%ymm3")
223 __ASM_EMIT("vblendvps %%ymm2, %%ymm0, %%ymm7, %%ymm0") /* ymm0 = den >= thresh) ? pan : dfl */
224 __ASM_EMIT("vblendvps %%ymm3, %%ymm1, %%ymm7, %%ymm1")
225 __ASM_EMIT("vmovups %%ymm0, 0x00(%[dst],%[off])")
226 __ASM_EMIT("vmovups %%ymm1, 0x20(%[dst],%[off])")
227 __ASM_EMIT("sub $16, %[count]")
228 __ASM_EMIT("add $0x40, %[off]")
229 /* 8x block */
230 __ASM_EMIT("4:")
231 __ASM_EMIT("add $8, %[count]")
232 __ASM_EMIT("jl 6f")
233 __ASM_EMIT("vmovups 0x00(%[r],%[off]), %%xmm0") /* xmm0 = r */
234 __ASM_EMIT("vmovups 0x10(%[r],%[off]), %%xmm1")
235 __ASM_EMIT("vmovups 0x00(%[l],%[off]), %%xmm2") /* xmm2 = l */
236 __ASM_EMIT("vmovups 0x10(%[l],%[off]), %%xmm3")
237 __ASM_EMIT("vmulps %%xmm0, %%xmm0, %%xmm0") /* xmm0 = r*r */
238 __ASM_EMIT("vmulps %%xmm1, %%xmm1, %%xmm1")
239 __ASM_EMIT("vfmadd132ps %%xmm2, %%xmm0, %%xmm2") /* xmm2 = den = l*l + r*r */
240 __ASM_EMIT("vfmadd132ps %%xmm3, %%xmm1, %%xmm3")
241 __ASM_EMIT("vdivps %%xmm2, %%xmm0, %%xmm0") /* xmm0 = pan = r*r / den */
242 __ASM_EMIT("vdivps %%xmm3, %%xmm1, %%xmm1")
243 __ASM_EMIT("vcmpps $5, %%xmm6, %%xmm2, %%xmm2") /* xmm2 = [den >= thresh] */
244 __ASM_EMIT("vcmpps $5, %%xmm6, %%xmm3, %%xmm3")
245 __ASM_EMIT("vblendvps %%xmm2, %%xmm0, %%xmm7, %%xmm0") /* xmm0 = den >= thresh) ? pan : dfl */
246 __ASM_EMIT("vblendvps %%xmm3, %%xmm1, %%xmm7, %%xmm1")
247 __ASM_EMIT("vmovups %%xmm0, 0x00(%[dst],%[off])")
248 __ASM_EMIT("vmovups %%xmm1, 0x10(%[dst],%[off])")
249 __ASM_EMIT("sub $8, %[count]")
250 __ASM_EMIT("add $0x20, %[off]")
251 /* 4x block */
252 __ASM_EMIT("6:")
253 __ASM_EMIT("add $4, %[count]")
254 __ASM_EMIT("jl 8f")
255 __ASM_EMIT("vmovups 0x00(%[r],%[off]), %%xmm0") /* xmm0 = r */
256 __ASM_EMIT("vmovups 0x00(%[l],%[off]), %%xmm2") /* xmm2 = l */
257 __ASM_EMIT("vmulps %%xmm0, %%xmm0, %%xmm0") /* xmm0 = r*r */
258 __ASM_EMIT("vfmadd132ps %%xmm2, %%xmm0, %%xmm2") /* xmm2 = den = l*l + r*r */
259 __ASM_EMIT("vdivps %%xmm2, %%xmm0, %%xmm0") /* xmm0 = pan = r*r / den */
260 __ASM_EMIT("vcmpps $5, %%xmm6, %%xmm2, %%xmm2") /* xmm2 = [den >= thresh] */
261 __ASM_EMIT("vblendvps %%xmm2, %%xmm0, %%xmm7, %%xmm0") /* xmm0 = den >= thresh) ? pan : dfl */
262 __ASM_EMIT("vmovups %%xmm0, 0x00(%[dst],%[off])")
263 __ASM_EMIT("sub $4, %[count]")
264 __ASM_EMIT("add $0x10, %[off]")
265 /* 1x blocks */
266 __ASM_EMIT("8:")
267 __ASM_EMIT("add $3, %[count]")
268 __ASM_EMIT("jl 10f")
269 __ASM_EMIT("9:")
270 __ASM_EMIT("vmovss 0x00(%[l],%[off]), %%xmm2")
271 __ASM_EMIT("vmovss 0x00(%[r],%[off]), %%xmm0")
272 __ASM_EMIT("vmulps %%xmm0, %%xmm0, %%xmm0") /* xmm0 = r*r */
273 __ASM_EMIT("vfmadd132ps %%xmm2, %%xmm0, %%xmm2") /* xmm2 = den = l*l + r*r */
274 __ASM_EMIT("vdivps %%xmm2, %%xmm0, %%xmm0") /* xmm0 = pan = r*r / den */
275 __ASM_EMIT("vcmpps $5, %%xmm6, %%xmm2, %%xmm2") /* xmm2 = [den >= thresh] */
276 __ASM_EMIT("vblendvps %%xmm2, %%xmm0, %%xmm7, %%xmm0") /* xmm0 = den >= thresh) ? pan : dfl */
277 __ASM_EMIT("vmovss %%xmm0, 0x00(%[dst],%[off])")
278 __ASM_EMIT("add $0x04, %[off]")
279 __ASM_EMIT("dec %[count]")
280 __ASM_EMIT("jge 9b")
281 /* End */
282 __ASM_EMIT("10:")
283 : [count] "+r" (count), [off] "=&r" (off),
284 [dfl] "+Yz" (dfl)
285 : [dst] "r" (dst), [l] "r" (l), [r] "r" (r),
286 [CF] "m" (depan_eqpow_const_f)
287 : "%xmm1", "%xmm2", "%xmm3",
288 "%xmm6", "%xmm7",
289 "%k2", "%k3"
290 );
291 }
292
293 } /* namespaec avx512 */
294 } /* namespace lsp */
295
296 #endif /* PRIVATE_DSP_ARCH_X86_AVX512_PAN_H_ */