Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/x86/avx512/pmath/abs_vv.h

raw
1 /*
2 * Copyright (C) 2024 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2024 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 24 окт. 2023 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_X86_AVX512_PMATH_ABS_VV_H_
23 #define PRIVATE_DSP_ARCH_X86_AVX512_PMATH_ABS_VV_H_
24
25 #ifndef PRIVATE_DSP_ARCH_X86_AVX512_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_X86_AVX512_IMPL */
28
29 namespace lsp
30 {
31 namespace avx512
32 {
33 IF_ARCH_X86(
34 static const uint32_t abs_vv_const[] __lsp_aligned64 =
35 {
36 LSP_DSP_VEC16(0x7fffffff)
37 };
38 )
39
40 #define OP_DSEL(a, b) a
41 #define OP_RSEL(a, b) b
42
43 #define ABS_OP_CORE(DST, SRC1, SRC2, OP, SEL) \
44 __ASM_EMIT("vmovaps %[SIGN], %%zmm6") \
45 __ASM_EMIT("xor %[off], %[off]") \
46 __ASM_EMIT("vmovaps %%zmm6, %%zmm7") \
47 /* 48x blocks */ \
48 __ASM_EMIT("sub $48, %[count]") \
49 __ASM_EMIT("jb 2f") \
50 __ASM_EMIT("1:") \
51 __ASM_EMIT(SEL("vmovups 0x000(%[" SRC1 "], %[off]), %%zmm0", "")) \
52 __ASM_EMIT(SEL("vmovups 0x040(%[" SRC1 "], %[off]), %%zmm1", "")) \
53 __ASM_EMIT(SEL("vmovups 0x080(%[" SRC1 "], %[off]), %%zmm2", "")) \
54 __ASM_EMIT("vandps 0x000(%[" SRC2 "], %[off]), %%zmm6, %%zmm3") \
55 __ASM_EMIT("vandps 0x040(%[" SRC2 "], %[off]), %%zmm7, %%zmm4") \
56 __ASM_EMIT("vandps 0x080(%[" SRC2 "], %[off]), %%zmm6, %%zmm5") \
57 __ASM_EMIT(OP "ps " SEL("%%zmm3", "0x000(%[" SRC1 "], %[off])") ", " SEL("%%zmm0", "%%zmm3") ", %%zmm0") \
58 __ASM_EMIT(OP "ps " SEL("%%zmm4", "0x040(%[" SRC1 "], %[off])") ", " SEL("%%zmm1", "%%zmm4") ", %%zmm1") \
59 __ASM_EMIT(OP "ps " SEL("%%zmm5", "0x080(%[" SRC1 "], %[off])") ", " SEL("%%zmm2", "%%zmm5") ", %%zmm2") \
60 __ASM_EMIT("vmovups %%zmm0, 0x000(%[" DST "], %[off])") \
61 __ASM_EMIT("vmovups %%zmm1, 0x040(%[" DST "], %[off])") \
62 __ASM_EMIT("vmovups %%zmm2, 0x080(%[" DST "], %[off])") \
63 __ASM_EMIT("add $0xc0, %[off]") \
64 __ASM_EMIT("sub $48, %[count]") \
65 __ASM_EMIT("jae 1b") \
66 __ASM_EMIT("2:") \
67 /* 16x blocks */ \
68 __ASM_EMIT("add $32, %[count]") \
69 __ASM_EMIT("jl 4f") \
70 __ASM_EMIT("3:") \
71 __ASM_EMIT(SEL("vmovups 0x000(%[" SRC1 "], %[off]), %%ymm0", "")) \
72 __ASM_EMIT(SEL("vmovups 0x020(%[" SRC1 "], %[off]), %%ymm1", "")) \
73 __ASM_EMIT("vandps 0x000(%[" SRC2 "], %[off]), %%ymm6, %%ymm2") \
74 __ASM_EMIT("vandps 0x020(%[" SRC2 "], %[off]), %%ymm7, %%ymm3") \
75 __ASM_EMIT(OP "ps " SEL("%%ymm2", "0x000(%[" SRC1 "], %[off])") ", " SEL("%%ymm0", "%%ymm2") ", %%ymm0") \
76 __ASM_EMIT(OP "ps " SEL("%%ymm3", "0x020(%[" SRC1 "], %[off])") ", " SEL("%%ymm1", "%%ymm3") ", %%ymm1") \
77 __ASM_EMIT("vmovups %%ymm0, 0x000(%[" DST "], %[off])") \
78 __ASM_EMIT("vmovups %%ymm1, 0x020(%[" DST "], %[off])") \
79 __ASM_EMIT("add $0x40, %[off]") \
80 __ASM_EMIT("sub $16, %[count]") \
81 __ASM_EMIT("jge 3b") \
82 __ASM_EMIT("4:") \
83 /* 8x block */ \
84 __ASM_EMIT("add $8, %[count]") \
85 __ASM_EMIT("jl 6f") \
86 __ASM_EMIT(SEL("vmovups 0x000(%[" SRC1 "], %[off]), %%xmm0", "")) \
87 __ASM_EMIT(SEL("vmovups 0x010(%[" SRC1 "], %[off]), %%xmm1", "")) \
88 __ASM_EMIT("vandps 0x000(%[" SRC2 "], %[off]), %%xmm6, %%xmm2") \
89 __ASM_EMIT("vandps 0x010(%[" SRC2 "], %[off]), %%xmm7, %%xmm3") \
90 __ASM_EMIT(OP "ps " SEL("%%xmm2", "0x000(%[" SRC1 "], %[off])") ", " SEL("%%xmm0", "%%xmm2") ", %%xmm0") \
91 __ASM_EMIT(OP "ps " SEL("%%xmm3", "0x010(%[" SRC1 "], %[off])") ", " SEL("%%xmm1", "%%xmm3") ", %%xmm1") \
92 __ASM_EMIT("vmovups %%xmm0, 0x000(%[" DST "], %[off])") \
93 __ASM_EMIT("vmovups %%xmm1, 0x010(%[" DST "], %[off])") \
94 __ASM_EMIT("add $0x20, %[off]") \
95 __ASM_EMIT("sub $8, %[count]") \
96 __ASM_EMIT("6:") \
97 /* 4x block */ \
98 __ASM_EMIT("add $4, %[count]") \
99 __ASM_EMIT("jl 8f") \
100 __ASM_EMIT(SEL("vmovups 0x000(%[" SRC1 "], %[off]), %%xmm0", "")) \
101 __ASM_EMIT("vandps 0x000(%[" SRC2 "], %[off]), %%xmm6, %%xmm2") \
102 __ASM_EMIT(OP "ps " SEL("%%xmm2", "0x000(%[" SRC1 "], %[off])") ", " SEL("%%xmm0", "%%xmm2") ", %%xmm0") \
103 __ASM_EMIT("vmovups %%xmm0, 0x000(%[" DST "], %[off])") \
104 __ASM_EMIT("add $0x10, %[off]") \
105 __ASM_EMIT("sub $4, %[count]") \
106 __ASM_EMIT("8:") \
107 /* 1x blocks */ \
108 __ASM_EMIT("add $3, %[count]") \
109 __ASM_EMIT("jl 10f") \
110 __ASM_EMIT("9:") \
111 __ASM_EMIT(SEL("vmovss 0x000(%[" SRC1 "], %[off]), %%xmm0", "")) \
112 __ASM_EMIT("vandps 0x000(%[" SRC2 "], %[off]), %%xmm6, %%xmm2") \
113 __ASM_EMIT(OP "ss " SEL("%%xmm2", "0x000(%[" SRC1 "], %[off])") ", " SEL("%%xmm0", "%%xmm2") ", %%xmm0") \
114 __ASM_EMIT("vmovss %%xmm0, 0x000(%[" DST "], %[off])") \
115 __ASM_EMIT("add $0x04, %[off]") \
116 __ASM_EMIT("dec %[count]") \
117 __ASM_EMIT("jge 9b") \
118 __ASM_EMIT("10:")
119
120 void abs_add2(float *dst, const float *src, size_t count)
121 {
122 IF_ARCH_X86_64(size_t off);
123 ARCH_X86_64_ASM
124 (
125 ABS_OP_CORE("dst", "dst", "src", "vadd", OP_DSEL)
126 : [off] "=&r" (off), [count] "+r" (count)
127 : [dst] "r"(dst), [src] "r"(src),
128 [SIGN] "m" (abs_vv_const)
129 : "cc", "memory",
130 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
131 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
132 );
133 }
134
135 void abs_sub2(float *dst, const float *src, size_t count)
136 {
137 IF_ARCH_X86_64(size_t off);
138 ARCH_X86_64_ASM
139 (
140 ABS_OP_CORE("dst", "dst", "src", "vsub", OP_DSEL)
141 : [off] "=&r" (off), [count] "+r" (count)
142 : [dst] "r"(dst), [src] "r"(src),
143 [SIGN] "m" (abs_vv_const)
144 : "cc", "memory",
145 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
146 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
147 );
148 }
149
150 void abs_rsub2(float *dst, const float *src, size_t count)
151 {
152 IF_ARCH_X86_64(size_t off);
153 ARCH_X86_64_ASM
154 (
155 ABS_OP_CORE("dst", "dst", "src", "vsub", OP_RSEL)
156 : [off] "=&r" (off), [count] "+r" (count)
157 : [dst] "r"(dst), [src] "r"(src),
158 [SIGN] "m" (abs_vv_const)
159 : "cc", "memory",
160 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
161 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
162 );
163 }
164
165 void abs_mul2(float *dst, const float *src, size_t count)
166 {
167 IF_ARCH_X86_64(size_t off);
168 ARCH_X86_64_ASM
169 (
170 ABS_OP_CORE("dst", "dst", "src", "vmul", OP_DSEL)
171 : [off] "=&r" (off), [count] "+r" (count)
172 : [dst] "r"(dst), [src] "r"(src),
173 [SIGN] "m" (abs_vv_const)
174 : "cc", "memory",
175 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
176 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
177 );
178 }
179
180 void abs_div2(float *dst, const float *src, size_t count)
181 {
182 IF_ARCH_X86_64(size_t off);
183 ARCH_X86_64_ASM
184 (
185 ABS_OP_CORE("dst", "dst", "src", "vdiv", OP_DSEL)
186 : [off] "=&r" (off), [count] "+r" (count)
187 : [dst] "r"(dst), [src] "r"(src),
188 [SIGN] "m" (abs_vv_const)
189 : "cc", "memory",
190 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
191 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
192 );
193 }
194
195 void abs_rdiv2(float *dst, const float *src, size_t count)
196 {
197 IF_ARCH_X86_64(size_t off);
198 ARCH_X86_64_ASM
199 (
200 ABS_OP_CORE("dst", "dst", "src", "vdiv", OP_RSEL)
201 : [off] "=&r" (off), [count] "+r" (count)
202 : [dst] "r"(dst), [src] "r"(src),
203 [SIGN] "m" (abs_vv_const)
204 : "cc", "memory",
205 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
206 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
207 );
208 }
209
210 void abs_max2(float *dst, const float *src, size_t count)
211 {
212 IF_ARCH_X86_64(size_t off);
213 ARCH_X86_64_ASM
214 (
215 ABS_OP_CORE("dst", "dst", "src", "vmax", OP_DSEL)
216 : [off] "=&r" (off), [count] "+r" (count)
217 : [dst] "r"(dst), [src] "r"(src),
218 [SIGN] "m" (abs_vv_const)
219 : "cc", "memory",
220 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
221 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
222 );
223 }
224
225 void abs_min2(float *dst, const float *src, size_t count)
226 {
227 IF_ARCH_X86_64(size_t off);
228 ARCH_X86_64_ASM
229 (
230 ABS_OP_CORE("dst", "dst", "src", "vmin", OP_DSEL)
231 : [off] "=&r" (off), [count] "+r" (count)
232 : [dst] "r"(dst), [src] "r"(src),
233 [SIGN] "m" (abs_vv_const)
234 : "cc", "memory",
235 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
236 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
237 );
238 }
239
240 void abs_add3(float *dst, const float *src1, const float *src2, size_t count)
241 {
242 IF_ARCH_X86_64(size_t off);
243 ARCH_X86_64_ASM
244 (
245 ABS_OP_CORE("dst", "src1", "src2", "vadd", OP_DSEL)
246 : [off] "=&r" (off), [count] "+r" (count)
247 : [dst] "r"(dst), [src1] "r" (src1), [src2] "r" (src2),
248 [SIGN] "m" (abs_vv_const)
249 : "cc", "memory",
250 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
251 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
252 );
253 }
254
255 void abs_sub3(float *dst, const float *src1, const float *src2, size_t count)
256 {
257 IF_ARCH_X86_64(size_t off);
258 ARCH_X86_64_ASM
259 (
260 ABS_OP_CORE("dst", "src1", "src2", "vsub", OP_DSEL)
261 : [off] "=&r" (off), [count] "+r" (count)
262 : [dst] "r"(dst), [src1] "r" (src1), [src2] "r" (src2),
263 [SIGN] "m" (abs_vv_const)
264 : "cc", "memory",
265 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
266 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
267 );
268 }
269
270 void abs_rsub3(float *dst, const float *src1, const float *src2, size_t count)
271 {
272 IF_ARCH_X86_64(size_t off);
273 ARCH_X86_64_ASM
274 (
275 ABS_OP_CORE("dst", "src1", "src2", "vsub", OP_RSEL)
276 : [off] "=&r" (off), [count] "+r" (count)
277 : [dst] "r"(dst), [src1] "r" (src1), [src2] "r" (src2),
278 [SIGN] "m" (abs_vv_const)
279 : "cc", "memory",
280 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
281 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
282 );
283 }
284
285 void abs_mul3(float *dst, const float *src1, const float *src2, size_t count)
286 {
287 IF_ARCH_X86_64(size_t off);
288 ARCH_X86_64_ASM
289 (
290 ABS_OP_CORE("dst", "src1", "src2", "vmul", OP_DSEL)
291 : [off] "=&r" (off), [count] "+r" (count)
292 : [dst] "r"(dst), [src1] "r" (src1), [src2] "r" (src2),
293 [SIGN] "m" (abs_vv_const)
294 : "cc", "memory",
295 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
296 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
297 );
298 }
299
300 void abs_div3(float *dst, const float *src1, const float *src2, size_t count)
301 {
302 IF_ARCH_X86_64(size_t off);
303 ARCH_X86_64_ASM
304 (
305 ABS_OP_CORE("dst", "src1", "src2", "vdiv", OP_DSEL)
306 : [off] "=&r" (off), [count] "+r" (count)
307 : [dst] "r"(dst), [src1] "r" (src1), [src2] "r" (src2),
308 [SIGN] "m" (abs_vv_const)
309 : "cc", "memory",
310 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
311 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
312 );
313 }
314
315 void abs_rdiv3(float *dst, const float *src1, const float *src2, size_t count)
316 {
317 IF_ARCH_X86_64(size_t off);
318 ARCH_X86_64_ASM
319 (
320 ABS_OP_CORE("dst", "src1", "src2", "vdiv", OP_RSEL)
321 : [off] "=&r" (off), [count] "+r" (count)
322 : [dst] "r"(dst), [src1] "r" (src1), [src2] "r" (src2),
323 [SIGN] "m" (abs_vv_const)
324 : "cc", "memory",
325 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
326 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
327 );
328 }
329
330 void abs_max3(float *dst, const float *src1, const float *src2, size_t count)
331 {
332 IF_ARCH_X86_64(size_t off);
333 ARCH_X86_64_ASM
334 (
335 ABS_OP_CORE("dst", "src1", "src2", "vmax", OP_DSEL)
336 : [off] "=&r" (off), [count] "+r" (count)
337 : [dst] "r"(dst), [src1] "r" (src1), [src2] "r" (src2),
338 [SIGN] "m" (abs_vv_const)
339 : "cc", "memory",
340 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
341 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
342 );
343 }
344
345 void abs_min3(float *dst, const float *src1, const float *src2, size_t count)
346 {
347 IF_ARCH_X86_64(size_t off);
348 ARCH_X86_64_ASM
349 (
350 ABS_OP_CORE("dst", "src1", "src2", "vmin", OP_DSEL)
351 : [off] "=&r" (off), [count] "+r" (count)
352 : [dst] "r"(dst), [src1] "r" (src1), [src2] "r" (src2),
353 [SIGN] "m" (abs_vv_const)
354 : "cc", "memory",
355 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
356 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
357 );
358 }
359
360 #undef ABS_OP_CORE
361
362 #define ABS_CORE(DST, SRC) \
363 __ASM_EMIT("vmovaps %[SIGN], %%zmm6") \
364 __ASM_EMIT("xor %[off], %[off]") \
365 __ASM_EMIT("vmovaps %%zmm6, %%zmm7") \
366 /* 96x blocks */ \
367 __ASM_EMIT("sub $96, %[count]") \
368 __ASM_EMIT("jb 2f") \
369 __ASM_EMIT("1:") \
370 __ASM_EMIT("vandps 0x000(%[" SRC "], %[off]), %%zmm6, %%zmm0") \
371 __ASM_EMIT("vandps 0x040(%[" SRC "], %[off]), %%zmm7, %%zmm1") \
372 __ASM_EMIT("vandps 0x080(%[" SRC "], %[off]), %%zmm6, %%zmm2") \
373 __ASM_EMIT("vandps 0x0c0(%[" SRC "], %[off]), %%zmm7, %%zmm3") \
374 __ASM_EMIT("vandps 0x100(%[" SRC "], %[off]), %%zmm6, %%zmm4") \
375 __ASM_EMIT("vandps 0x140(%[" SRC "], %[off]), %%zmm7, %%zmm5") \
376 __ASM_EMIT("vmovups %%zmm0, 0x000(%[" DST "], %[off])") \
377 __ASM_EMIT("vmovups %%zmm1, 0x040(%[" DST "], %[off])") \
378 __ASM_EMIT("vmovups %%zmm2, 0x080(%[" DST "], %[off])") \
379 __ASM_EMIT("vmovups %%zmm3, 0x0c0(%[" DST "], %[off])") \
380 __ASM_EMIT("vmovups %%zmm4, 0x100(%[" DST "], %[off])") \
381 __ASM_EMIT("vmovups %%zmm5, 0x140(%[" DST "], %[off])") \
382 __ASM_EMIT("add $0x180, %[off]") \
383 __ASM_EMIT("sub $96, %[count]") \
384 __ASM_EMIT("jae 1b") \
385 __ASM_EMIT("2:") \
386 /* 32x blocks */ \
387 __ASM_EMIT("add $64, %[count]") \
388 __ASM_EMIT("jl 4f") \
389 __ASM_EMIT("3:") \
390 __ASM_EMIT("vandps 0x000(%[" SRC "], %[off]), %%zmm6, %%zmm0") \
391 __ASM_EMIT("vandps 0x040(%[" SRC "], %[off]), %%zmm7, %%zmm1") \
392 __ASM_EMIT("vmovups %%zmm0, 0x000(%[" DST "], %[off])") \
393 __ASM_EMIT("vmovups %%zmm1, 0x040(%[" DST "], %[off])") \
394 __ASM_EMIT("add $0x80, %[off]") \
395 __ASM_EMIT("sub $32, %[count]") \
396 __ASM_EMIT("jge 3b") \
397 __ASM_EMIT("4:") \
398 /* 16x block */ \
399 __ASM_EMIT("add $16, %[count]") \
400 __ASM_EMIT("jl 6f") \
401 __ASM_EMIT("vandps 0x000(%[" SRC "], %[off]), %%ymm6, %%ymm0") \
402 __ASM_EMIT("vandps 0x020(%[" SRC "], %[off]), %%ymm7, %%ymm1") \
403 __ASM_EMIT("vmovups %%ymm0, 0x000(%[" DST "], %[off])") \
404 __ASM_EMIT("vmovups %%ymm1, 0x020(%[" DST "], %[off])") \
405 __ASM_EMIT("sub $16, %[count]") \
406 __ASM_EMIT("add $0x40, %[off]") \
407 __ASM_EMIT("6:") \
408 /* 8x block */ \
409 __ASM_EMIT("add $8, %[count]") \
410 __ASM_EMIT("jl 8f") \
411 __ASM_EMIT("vandps 0x000(%[" SRC "], %[off]), %%xmm6, %%xmm0") \
412 __ASM_EMIT("vandps 0x010(%[" SRC "], %[off]), %%xmm7, %%xmm1") \
413 __ASM_EMIT("vmovups %%xmm0, 0x000(%[" DST "], %[off])") \
414 __ASM_EMIT("vmovups %%xmm1, 0x010(%[" DST "], %[off])") \
415 __ASM_EMIT("sub $8, %[count]") \
416 __ASM_EMIT("add $0x20, %[off]") \
417 __ASM_EMIT("8:") \
418 /* 4x block */ \
419 __ASM_EMIT("add $4, %[count]") \
420 __ASM_EMIT("jl 10f") \
421 __ASM_EMIT("vandps 0x000(%[" SRC "], %[off]), %%xmm6, %%xmm0") \
422 __ASM_EMIT("vmovups %%xmm0, 0x000(%[" DST "], %[off])") \
423 __ASM_EMIT("sub $4, %[count]") \
424 __ASM_EMIT("add $0x10, %[off]") \
425 __ASM_EMIT("10:") \
426 /* 1x block */ \
427 __ASM_EMIT("add $3, %[count]") \
428 __ASM_EMIT("jl 12f") \
429 __ASM_EMIT("11:") \
430 __ASM_EMIT("vmovss 0x00(%[" SRC "], %[off]), %%xmm0") \
431 __ASM_EMIT("vandps %%xmm0, %%xmm6, %%xmm0") \
432 __ASM_EMIT("vmovss %%xmm0, 0x00(%[" DST "], %[off])") \
433 __ASM_EMIT("add $0x04, %[off]") \
434 __ASM_EMIT("dec %[count]") \
435 __ASM_EMIT("jge 11b") \
436 __ASM_EMIT("12:")
437
438 void abs1(float *dst, size_t count)
439 {
440 IF_ARCH_X86_64(size_t off);
441 ARCH_X86_64_ASM
442 (
443 ABS_CORE("dst", "dst")
444 : [off] "=&r" (off), [count] "+r" (count)
445 : [dst] "r" (dst),
446 [SIGN] "m" (abs_vv_const)
447 : "%xmm0", "%xmm1", "%xmm2", "%xmm3",
448 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
449 );
450 }
451
452 void abs2(float *dst, const float *src, size_t count)
453 {
454 IF_ARCH_X86_64(size_t off);
455 ARCH_X86_64_ASM
456 (
457 ABS_CORE("dst", "src")
458 : [off] "=&r" (off), [count] "+r" (count)
459 : [dst] "r" (dst), [src] "r" (src),
460 [SIGN] "m" (abs_vv_const)
461 : "%xmm0", "%xmm1", "%xmm2", "%xmm3",
462 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
463 );
464 }
465
466 #undef ABS_CORE
467
468 #undef OP_DSEL
469 #undef OP_RSEL
470
471 } /* namespace avx512 */
472 } /* namespace lsp */
473
474
475 #endif /* PRIVATE_DSP_ARCH_X86_AVX512_PMATH_ABS_VV_H_ */