Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/x86/sse/pmath/abs_vv.h

raw
1 /*
2 * Copyright (C) 2024 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2024 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 31 мар. 2020 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_X86_SSE_PMATH_ABS_VV_H_
23 #define PRIVATE_DSP_ARCH_X86_SSE_PMATH_ABS_VV_H_
24
25 #ifndef PRIVATE_DSP_ARCH_X86_SSE_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_X86_SSE_IMPL */
28
29 namespace lsp
30 {
31 namespace sse
32 {
33 IF_ARCH_X86(
34 static const uint32_t abs_vv_const[] __lsp_aligned16 =
35 {
36 LSP_DSP_VEC4(0x7fffffff)
37 };
38 )
39
40 #define OP_DSEL(a, b) a
41 #define OP_RSEL(a, b) b
42
43 #define ABS_CORE(DST, SRC) \
44 __ASM_EMIT("xor %[off], %[off]") \
45 __ASM_EMIT("movaps %[X_SIGN], %%xmm0") \
46 __ASM_EMIT("sub $24, %[count]") \
47 __ASM_EMIT("movaps %%xmm0, %%xmm1") \
48 __ASM_EMIT("jb 2f") \
49 /* 24x blocks */ \
50 __ASM_EMIT("1:") \
51 __ASM_EMIT("movups 0x00(%[" SRC "], %[off]), %%xmm2") \
52 __ASM_EMIT("movups 0x10(%[" SRC "], %[off]), %%xmm3") \
53 __ASM_EMIT("movups 0x20(%[" SRC "], %[off]), %%xmm4") \
54 __ASM_EMIT("movups 0x30(%[" SRC "], %[off]), %%xmm5") \
55 __ASM_EMIT("movups 0x40(%[" SRC "], %[off]), %%xmm6") \
56 __ASM_EMIT("movups 0x50(%[" SRC "], %[off]), %%xmm7") \
57 __ASM_EMIT("andps %%xmm0, %%xmm2") \
58 __ASM_EMIT("andps %%xmm1, %%xmm3") \
59 __ASM_EMIT("andps %%xmm0, %%xmm4") \
60 __ASM_EMIT("andps %%xmm1, %%xmm5") \
61 __ASM_EMIT("andps %%xmm0, %%xmm6") \
62 __ASM_EMIT("andps %%xmm1, %%xmm7") \
63 __ASM_EMIT("movups %%xmm2, 0x00(%[" DST "], %[off])") \
64 __ASM_EMIT("movups %%xmm3, 0x10(%[" DST "], %[off])") \
65 __ASM_EMIT("movups %%xmm4, 0x20(%[" DST "], %[off])") \
66 __ASM_EMIT("movups %%xmm5, 0x30(%[" DST "], %[off])") \
67 __ASM_EMIT("movups %%xmm6, 0x40(%[" DST "], %[off])") \
68 __ASM_EMIT("movups %%xmm7, 0x50(%[" DST "], %[off])") \
69 __ASM_EMIT("add $0x60, %[off]") \
70 __ASM_EMIT("sub $24, %[count]") \
71 __ASM_EMIT("jae 1b") \
72 /* 16x block */ \
73 __ASM_EMIT("2:") \
74 __ASM_EMIT("add $8, %[count]") \
75 __ASM_EMIT("jl 4f") \
76 __ASM_EMIT("movups 0x00(%[" SRC "], %[off]), %%xmm2") \
77 __ASM_EMIT("movups 0x10(%[" SRC "], %[off]), %%xmm3") \
78 __ASM_EMIT("movups 0x20(%[" SRC "], %[off]), %%xmm4") \
79 __ASM_EMIT("movups 0x30(%[" SRC "], %[off]), %%xmm5") \
80 __ASM_EMIT("andps %%xmm0, %%xmm2") \
81 __ASM_EMIT("andps %%xmm1, %%xmm3") \
82 __ASM_EMIT("andps %%xmm0, %%xmm4") \
83 __ASM_EMIT("andps %%xmm1, %%xmm5") \
84 __ASM_EMIT("movups %%xmm2, 0x00(%[" DST "], %[off])") \
85 __ASM_EMIT("movups %%xmm3, 0x10(%[" DST "], %[off])") \
86 __ASM_EMIT("movups %%xmm4, 0x20(%[" DST "], %[off])") \
87 __ASM_EMIT("movups %%xmm5, 0x30(%[" DST "], %[off])") \
88 __ASM_EMIT("sub $16, %[count]") \
89 __ASM_EMIT("add $0x40, %[off]") \
90 /* 8x block */ \
91 __ASM_EMIT("4:") \
92 __ASM_EMIT("add $8, %[count]") \
93 __ASM_EMIT("jl 6f") \
94 __ASM_EMIT("movups 0x00(%[" SRC "], %[off]), %%xmm2") \
95 __ASM_EMIT("movups 0x10(%[" SRC "], %[off]), %%xmm3") \
96 __ASM_EMIT("andps %%xmm0, %%xmm2") \
97 __ASM_EMIT("andps %%xmm1, %%xmm3") \
98 __ASM_EMIT("movups %%xmm2, 0x00(%[" DST "], %[off])") \
99 __ASM_EMIT("movups %%xmm3, 0x10(%[" DST "], %[off])") \
100 __ASM_EMIT("sub $8, %[count]") \
101 __ASM_EMIT("add $0x20, %[off]") \
102 /* 4x block */ \
103 __ASM_EMIT("6:") \
104 __ASM_EMIT("add $4, %[count]") \
105 __ASM_EMIT("jl 8f") \
106 __ASM_EMIT("movups 0x00(%[" SRC "], %[off]), %%xmm2") \
107 __ASM_EMIT("andps %%xmm0, %%xmm2") \
108 __ASM_EMIT("movups %%xmm2, 0x00(%[" DST "], %[off])") \
109 __ASM_EMIT("sub $4, %[count]") \
110 __ASM_EMIT("add $0x10, %[off]") \
111 /* 1x block */ \
112 __ASM_EMIT("8:") \
113 __ASM_EMIT("add $3, %[count]") \
114 __ASM_EMIT("jl 10f") \
115 __ASM_EMIT("9:") \
116 __ASM_EMIT("movss 0x00(%[" SRC "], %[off]), %%xmm2") \
117 __ASM_EMIT("andps %%xmm0, %%xmm2") \
118 __ASM_EMIT("movss %%xmm2, 0x00(%[" DST "], %[off])") \
119 __ASM_EMIT("add $0x04, %[off]") \
120 __ASM_EMIT("dec %[count]") \
121 __ASM_EMIT("jge 9b") \
122 __ASM_EMIT("10:")
123
124 void abs1(float *dst, size_t count)
125 {
126 IF_ARCH_X86(size_t off);
127 ARCH_X86_ASM
128 (
129 ABS_CORE("dst", "dst")
130 : [off] "=&r" (off), [count] "+r" (count)
131 : [dst] "r" (dst),
132 [X_SIGN] "m" (abs_vv_const)
133 : "%xmm0", "%xmm1", "%xmm2", "%xmm3",
134 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
135 );
136 }
137
138 void abs2(float *dst, const float *src, size_t count)
139 {
140 IF_ARCH_X86(size_t off);
141 ARCH_X86_ASM
142 (
143 ABS_CORE("dst", "src")
144 : [off] "=&r" (off), [count] "+r" (count)
145 : [dst] "r" (dst), [src] "r" (src),
146 [X_SIGN] "m" (abs_vv_const)
147 : "%xmm0", "%xmm1", "%xmm2", "%xmm3",
148 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
149 );
150 }
151
152 #undef ABS_CORE
153
154 #define ABS_OP2_CORE(OP, DST, SRC1, SRC2, SEL) \
155 __ASM_EMIT("xor %[off], %[off]") \
156 __ASM_EMIT("movaps %[X_SIGN], %%xmm0") \
157 __ASM_EMIT("sub $12, %[count]") \
158 __ASM_EMIT("movaps %%xmm0, %%xmm1") \
159 __ASM_EMIT("jb 2f") \
160 /* 12x blocks */ \
161 __ASM_EMIT("1:") \
162 __ASM_EMIT("movups 0x00(%[" SEL(SRC2, SRC1) "], %[off]), %%xmm2") \
163 __ASM_EMIT("movups 0x10(%[" SEL(SRC2, SRC1) "], %[off]), %%xmm3") \
164 __ASM_EMIT("movups 0x20(%[" SEL(SRC2, SRC1) "], %[off]), %%xmm4") \
165 __ASM_EMIT("movups 0x00(%[" SEL(SRC1, SRC2) "], %[off]), %%xmm5") \
166 __ASM_EMIT("movups 0x10(%[" SEL(SRC1, SRC2) "], %[off]), %%xmm6") \
167 __ASM_EMIT("movups 0x20(%[" SEL(SRC1, SRC2) "], %[off]), %%xmm7") \
168 __ASM_EMIT("andps %%xmm0, " SEL("%%xmm2", "%%xmm5")) \
169 __ASM_EMIT("andps %%xmm1, " SEL("%%xmm3", "%%xmm6")) \
170 __ASM_EMIT("andps %%xmm0, " SEL("%%xmm4", "%%xmm7")) \
171 __ASM_EMIT(OP "ps %%xmm2, %%xmm5") \
172 __ASM_EMIT(OP "ps %%xmm3, %%xmm6") \
173 __ASM_EMIT(OP "ps %%xmm4, %%xmm7") \
174 __ASM_EMIT("movups %%xmm5, 0x00(%[" DST "], %[off])") \
175 __ASM_EMIT("movups %%xmm6, 0x10(%[" DST "], %[off])") \
176 __ASM_EMIT("movups %%xmm7, 0x20(%[" DST "], %[off])") \
177 __ASM_EMIT("add $0x30, %[off]") \
178 __ASM_EMIT("sub $12, %[count]") \
179 __ASM_EMIT("jae 1b") \
180 /* 8x block */ \
181 __ASM_EMIT("2:") \
182 __ASM_EMIT("add $4, %[count]") \
183 __ASM_EMIT("jl 4f") \
184 __ASM_EMIT("movups 0x00(%[" SEL(SRC2, SRC1) "], %[off]), %%xmm2") \
185 __ASM_EMIT("movups 0x10(%[" SEL(SRC2, SRC1) "], %[off]), %%xmm3") \
186 __ASM_EMIT("movups 0x00(%[" SEL(SRC1, SRC2) "], %[off]), %%xmm4") \
187 __ASM_EMIT("movups 0x10(%[" SEL(SRC1, SRC2) "], %[off]), %%xmm5") \
188 __ASM_EMIT("andps %%xmm0, " SEL("%%xmm2", "%%xmm4")) \
189 __ASM_EMIT("andps %%xmm1, " SEL("%%xmm3", "%%xmm5")) \
190 __ASM_EMIT(OP "ps %%xmm2, %%xmm4") \
191 __ASM_EMIT(OP "ps %%xmm3, %%xmm5") \
192 __ASM_EMIT("movups %%xmm4, 0x00(%[" DST "], %[off])") \
193 __ASM_EMIT("movups %%xmm5, 0x10(%[" DST "], %[off])") \
194 __ASM_EMIT("sub $8, %[count]") \
195 __ASM_EMIT("add $0x20, %[off]") \
196 /* 4x block */ \
197 __ASM_EMIT("4:") \
198 __ASM_EMIT("add $4, %[count]") \
199 __ASM_EMIT("jl 6f") \
200 __ASM_EMIT("movups 0x00(%[" SEL(SRC2, SRC1) "], %[off]), %%xmm2") \
201 __ASM_EMIT("movups 0x00(%[" SEL(SRC1, SRC2) "], %[off]), %%xmm4") \
202 __ASM_EMIT("andps %%xmm0, " SEL("%%xmm2", "%%xmm4")) \
203 __ASM_EMIT(OP "ps %%xmm2, %%xmm4") \
204 __ASM_EMIT("movups %%xmm4, 0x00(%[" DST "], %[off])") \
205 __ASM_EMIT("sub $4, %[count]") \
206 __ASM_EMIT("add $0x10, %[off]") \
207 /* 1x blocks */ \
208 __ASM_EMIT("6:") \
209 __ASM_EMIT("add $3, %[count]") \
210 __ASM_EMIT("jl 8f") \
211 __ASM_EMIT("7:") \
212 __ASM_EMIT("movss 0x00(%[" SEL(SRC2, SRC1) "], %[off]), %%xmm2") \
213 __ASM_EMIT("movss 0x00(%[" SEL(SRC1, SRC2) "], %[off]), %%xmm4") \
214 __ASM_EMIT("andps %%xmm0, " SEL("%%xmm2", "%%xmm4")) \
215 __ASM_EMIT(OP "ss %%xmm2, %%xmm4") \
216 __ASM_EMIT("movss %%xmm4, 0x00(%[" DST "], %[off])") \
217 __ASM_EMIT("add $0x04, %[off]") \
218 __ASM_EMIT("dec %[count]") \
219 __ASM_EMIT("jge 7b") \
220 __ASM_EMIT("8:") \
221
222 void abs_add2(float *dst, const float *src, size_t count)
223 {
224 IF_ARCH_X86(size_t off);
225 ARCH_X86_ASM
226 (
227 ABS_OP2_CORE("add", "dst", "dst", "src", OP_DSEL)
228 : [off] "=&r" (off), [count] "+r" (count)
229 : [dst] "r" (dst), [src] "r" (src),
230 [X_SIGN] "m" (abs_vv_const)
231 : "%xmm0", "%xmm1", "%xmm2", "%xmm3",
232 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
233 );
234 }
235
236 void abs_sub2(float *dst, const float *src, size_t count)
237 {
238 IF_ARCH_X86(size_t off);
239 ARCH_X86_ASM
240 (
241 ABS_OP2_CORE("sub", "dst", "dst", "src", OP_DSEL)
242 : [off] "=&r" (off), [count] "+r" (count)
243 : [dst] "r" (dst), [src] "r" (src),
244 [X_SIGN] "m" (abs_vv_const)
245 : "%xmm0", "%xmm1", "%xmm2", "%xmm3",
246 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
247 );
248 }
249
250 void abs_rsub2(float *dst, const float *src, size_t count)
251 {
252 IF_ARCH_X86(size_t off);
253 ARCH_X86_ASM
254 (
255 ABS_OP2_CORE("sub", "dst", "dst", "src", OP_RSEL)
256 : [off] "=&r" (off), [count] "+r" (count)
257 : [dst] "r" (dst), [src] "r" (src),
258 [X_SIGN] "m" (abs_vv_const)
259 : "%xmm0", "%xmm1", "%xmm2", "%xmm3",
260 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
261 );
262 }
263
264 void abs_mul2(float *dst, const float *src, size_t count)
265 {
266 IF_ARCH_X86(size_t off);
267 ARCH_X86_ASM
268 (
269 ABS_OP2_CORE("mul", "dst", "dst", "src", OP_DSEL)
270 : [off] "=&r" (off), [count] "+r" (count)
271 : [dst] "r" (dst), [src] "r" (src),
272 [X_SIGN] "m" (abs_vv_const)
273 : "%xmm0", "%xmm1", "%xmm2", "%xmm3",
274 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
275 );
276 }
277
278 void abs_div2(float *dst, const float *src, size_t count)
279 {
280 IF_ARCH_X86(size_t off);
281 ARCH_X86_ASM
282 (
283 ABS_OP2_CORE("div", "dst", "dst", "src", OP_DSEL)
284 : [off] "=&r" (off), [count] "+r" (count)
285 : [dst] "r" (dst), [src] "r" (src),
286 [X_SIGN] "m" (abs_vv_const)
287 : "%xmm0", "%xmm1", "%xmm2", "%xmm3",
288 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
289 );
290 }
291
292 void abs_rdiv2(float *dst, const float *src, size_t count)
293 {
294 IF_ARCH_X86(size_t off);
295 ARCH_X86_ASM
296 (
297 ABS_OP2_CORE("div", "dst", "dst", "src", OP_RSEL)
298 : [off] "=&r" (off), [count] "+r" (count)
299 : [dst] "r" (dst), [src] "r" (src),
300 [X_SIGN] "m" (abs_vv_const)
301 : "%xmm0", "%xmm1", "%xmm2", "%xmm3",
302 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
303 );
304 }
305
306 void abs_max2(float *dst, const float *src, size_t count)
307 {
308 IF_ARCH_X86(size_t off);
309 ARCH_X86_ASM
310 (
311 ABS_OP2_CORE("max", "dst", "dst", "src", OP_DSEL)
312 : [off] "=&r" (off), [count] "+r" (count)
313 : [dst] "r" (dst), [src] "r" (src),
314 [X_SIGN] "m" (abs_vv_const)
315 : "%xmm0", "%xmm1", "%xmm2", "%xmm3",
316 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
317 );
318 }
319
320 void abs_min2(float *dst, const float *src, size_t count)
321 {
322 IF_ARCH_X86(size_t off);
323 ARCH_X86_ASM
324 (
325 ABS_OP2_CORE("min", "dst", "dst", "src", OP_DSEL)
326 : [off] "=&r" (off), [count] "+r" (count)
327 : [dst] "r" (dst), [src] "r" (src),
328 [X_SIGN] "m" (abs_vv_const)
329 : "%xmm0", "%xmm1", "%xmm2", "%xmm3",
330 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
331 );
332 }
333
334 void abs_add3(float *dst, const float *src1, const float *src2, size_t count)
335 {
336 IF_ARCH_X86(size_t off);
337 ARCH_X86_ASM
338 (
339 ABS_OP2_CORE("add", "dst", "src1", "src2", OP_DSEL)
340 : [off] "=&r" (off), [count] "+r" (count)
341 : [dst] "r" (dst), [src1] "r" (src1), [src2] "r" (src2),
342 [X_SIGN] "m" (abs_vv_const)
343 : "%xmm0", "%xmm1", "%xmm2", "%xmm3",
344 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
345 );
346 }
347
348 void abs_sub3(float *dst, const float *src1, const float *src2, size_t count)
349 {
350 IF_ARCH_X86(size_t off);
351 ARCH_X86_ASM
352 (
353 ABS_OP2_CORE("sub", "dst", "src1", "src2", OP_DSEL)
354 : [off] "=&r" (off), [count] "+r" (count)
355 : [dst] "r" (dst), [src1] "r" (src1), [src2] "r" (src2),
356 [X_SIGN] "m" (abs_vv_const)
357 : "%xmm0", "%xmm1", "%xmm2", "%xmm3",
358 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
359 );
360 }
361
362 void abs_rsub3(float *dst, const float *src1, const float *src2, size_t count)
363 {
364 IF_ARCH_X86(size_t off);
365 ARCH_X86_ASM
366 (
367 ABS_OP2_CORE("sub", "dst", "src1", "src2", OP_RSEL)
368 : [off] "=&r" (off), [count] "+r" (count)
369 : [dst] "r" (dst), [src1] "r" (src1), [src2] "r" (src2),
370 [X_SIGN] "m" (abs_vv_const)
371 : "%xmm0", "%xmm1", "%xmm2", "%xmm3",
372 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
373 );
374 }
375
376 void abs_mul3(float *dst, const float *src1, const float *src2, size_t count)
377 {
378 IF_ARCH_X86(size_t off);
379 ARCH_X86_ASM
380 (
381 ABS_OP2_CORE("mul", "dst", "src1", "src2", OP_DSEL)
382 : [off] "=&r" (off), [count] "+r" (count)
383 : [dst] "r" (dst), [src1] "r" (src1), [src2] "r" (src2),
384 [X_SIGN] "m" (abs_vv_const)
385 : "%xmm0", "%xmm1", "%xmm2", "%xmm3",
386 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
387 );
388 }
389
390 void abs_div3(float *dst, const float *src1, const float *src2, size_t count)
391 {
392 IF_ARCH_X86(size_t off);
393 ARCH_X86_ASM
394 (
395 ABS_OP2_CORE("div", "dst", "src1", "src2", OP_DSEL)
396 : [off] "=&r" (off), [count] "+r" (count)
397 : [dst] "r" (dst), [src1] "r" (src1), [src2] "r" (src2),
398 [X_SIGN] "m" (abs_vv_const)
399 : "%xmm0", "%xmm1", "%xmm2", "%xmm3",
400 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
401 );
402 }
403
404 void abs_rdiv3(float *dst, const float *src1, const float *src2, size_t count)
405 {
406 IF_ARCH_X86(size_t off);
407 ARCH_X86_ASM
408 (
409 ABS_OP2_CORE("div", "dst", "src1", "src2", OP_RSEL)
410 : [off] "=&r" (off), [count] "+r" (count)
411 : [dst] "r" (dst), [src1] "r" (src1), [src2] "r" (src2),
412 [X_SIGN] "m" (abs_vv_const)
413 : "%xmm0", "%xmm1", "%xmm2", "%xmm3",
414 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
415 );
416 }
417
418 void abs_max3(float *dst, const float *src1, const float *src2, size_t count)
419 {
420 IF_ARCH_X86(size_t off);
421 ARCH_X86_ASM
422 (
423 ABS_OP2_CORE("max", "dst", "src1", "src2", OP_DSEL)
424 : [off] "=&r" (off), [count] "+r" (count)
425 : [dst] "r" (dst), [src1] "r" (src1), [src2] "r" (src2),
426 [X_SIGN] "m" (abs_vv_const)
427 : "%xmm0", "%xmm1", "%xmm2", "%xmm3",
428 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
429 );
430 }
431
432 void abs_min3(float *dst, const float *src1, const float *src2, size_t count)
433 {
434 IF_ARCH_X86(size_t off);
435 ARCH_X86_ASM
436 (
437 ABS_OP2_CORE("min", "dst", "src1", "src2", OP_DSEL)
438 : [off] "=&r" (off), [count] "+r" (count)
439 : [dst] "r" (dst), [src1] "r" (src1), [src2] "r" (src2),
440 [X_SIGN] "m" (abs_vv_const)
441 : "%xmm0", "%xmm1", "%xmm2", "%xmm3",
442 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
443 );
444 }
445
446 #undef ABS_OP2_CORE
447
448 #undef OP_DSEL
449 #undef OP_RSEL
450 } /* namespace sse */
451 } /* namespace lsp */
452
453 #endif /* PRIVATE_DSP_ARCH_X86_SSE_PMATH_ABS_VV_H_ */