Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/x86/avx512/pmath/fmop_kx.h

raw
1 /*
2 * Copyright (C) 2023 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2023 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 26 окт. 2023 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_X86_AVX512_PMATH_FMOP_KX_H_
23 #define PRIVATE_DSP_ARCH_X86_AVX512_PMATH_FMOP_KX_H_
24
25 #ifndef PRIVATE_DSP_ARCH_X86_AVX512_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_X86_AVX512_IMPL */
28
29 namespace lsp
30 {
31 namespace avx512
32 {
33 #define OP_DSEL(a, b) a
34 #define OP_RSEL(a, b) b
35
36 #define FMADDSUB_K3_CORE(DST, SRC1, SRC2, OP) \
37 __ASM_EMIT("vbroadcastss %[k], %%zmm0") \
38 __ASM_EMIT("xor %[off], %[off]") \
39 __ASM_EMIT("vmovaps %%zmm0, %%zmm1") \
40 /* 96x blocks */ \
41 __ASM_EMIT("sub $96, %[count]") \
42 __ASM_EMIT("jb 2f") \
43 __ASM_EMIT("1:") \
44 __ASM_EMIT("vmovups 0x000(%[" SRC1 "], %[off]), %%zmm2") \
45 __ASM_EMIT("vmovups 0x040(%[" SRC1 "], %[off]), %%zmm3") \
46 __ASM_EMIT("vmovups 0x080(%[" SRC1 "], %[off]), %%zmm4") \
47 __ASM_EMIT("vmovups 0x0c0(%[" SRC1 "], %[off]), %%zmm5") \
48 __ASM_EMIT("vmovups 0x100(%[" SRC1 "], %[off]), %%zmm6") \
49 __ASM_EMIT("vmovups 0x140(%[" SRC1 "], %[off]), %%zmm7") \
50 __ASM_EMIT(OP "ps 0x000(%[" SRC2 "], %[off]), %%zmm0, %%zmm2") \
51 __ASM_EMIT(OP "ps 0x040(%[" SRC2 "], %[off]), %%zmm1, %%zmm3") \
52 __ASM_EMIT(OP "ps 0x080(%[" SRC2 "], %[off]), %%zmm0, %%zmm4") \
53 __ASM_EMIT(OP "ps 0x0c0(%[" SRC2 "], %[off]), %%zmm1, %%zmm5") \
54 __ASM_EMIT(OP "ps 0x100(%[" SRC2 "], %[off]), %%zmm0, %%zmm6") \
55 __ASM_EMIT(OP "ps 0x140(%[" SRC2 "], %[off]), %%zmm1, %%zmm7") \
56 __ASM_EMIT("vmovups %%zmm2, 0x000(%[" DST "], %[off])") \
57 __ASM_EMIT("vmovups %%zmm3, 0x040(%[" DST "], %[off])") \
58 __ASM_EMIT("vmovups %%zmm4, 0x080(%[" DST "], %[off])") \
59 __ASM_EMIT("vmovups %%zmm5, 0x0c0(%[" DST "], %[off])") \
60 __ASM_EMIT("vmovups %%zmm6, 0x100(%[" DST "], %[off])") \
61 __ASM_EMIT("vmovups %%zmm7, 0x140(%[" DST "], %[off])") \
62 __ASM_EMIT("add $0x180, %[off]") \
63 __ASM_EMIT("sub $96, %[count]") \
64 __ASM_EMIT("jae 1b") \
65 __ASM_EMIT("2:") \
66 /* 32x blocks */ \
67 __ASM_EMIT("add $64, %[count]") \
68 __ASM_EMIT("jl 4f") \
69 __ASM_EMIT("3:") \
70 __ASM_EMIT("vmovups 0x000(%[" SRC1 "], %[off]), %%ymm4") \
71 __ASM_EMIT("vmovups 0x020(%[" SRC1 "], %[off]), %%ymm5") \
72 __ASM_EMIT("vmovups 0x040(%[" SRC1 "], %[off]), %%ymm6") \
73 __ASM_EMIT("vmovups 0x060(%[" SRC1 "], %[off]), %%ymm7") \
74 __ASM_EMIT(OP "ps 0x000(%[" SRC2 "], %[off]), %%ymm0, %%ymm4") \
75 __ASM_EMIT(OP "ps 0x020(%[" SRC2 "], %[off]), %%ymm1, %%ymm5") \
76 __ASM_EMIT(OP "ps 0x040(%[" SRC2 "], %[off]), %%ymm0, %%ymm6") \
77 __ASM_EMIT(OP "ps 0x060(%[" SRC2 "], %[off]), %%ymm1, %%ymm7") \
78 __ASM_EMIT("vmovups %%ymm4, 0x000(%[" DST "], %[off])") \
79 __ASM_EMIT("vmovups %%ymm5, 0x020(%[" DST "], %[off])") \
80 __ASM_EMIT("vmovups %%ymm6, 0x040(%[" DST "], %[off])") \
81 __ASM_EMIT("vmovups %%ymm7, 0x060(%[" DST "], %[off])") \
82 __ASM_EMIT("add $0x80, %[off]") \
83 __ASM_EMIT("sub $32, %[count]") \
84 __ASM_EMIT("jge 3b") \
85 __ASM_EMIT("4:") \
86 /* 16x block */ \
87 __ASM_EMIT("add $16, %[count]") \
88 __ASM_EMIT("jl 6f") \
89 __ASM_EMIT("vmovups 0x000(%[" SRC1 "], %[off]), %%ymm4") \
90 __ASM_EMIT("vmovups 0x020(%[" SRC1 "], %[off]), %%ymm5") \
91 __ASM_EMIT(OP "ps 0x000(%[" SRC2 "], %[off]), %%ymm0, %%ymm4") \
92 __ASM_EMIT(OP "ps 0x020(%[" SRC2 "], %[off]), %%ymm1, %%ymm5") \
93 __ASM_EMIT("vmovups %%ymm4, 0x000(%[" DST "], %[off])") \
94 __ASM_EMIT("vmovups %%ymm5, 0x020(%[" DST "], %[off])") \
95 __ASM_EMIT("add $0x40, %[off]") \
96 __ASM_EMIT("sub $16, %[count]") \
97 __ASM_EMIT("6:") \
98 /* 8x block */ \
99 __ASM_EMIT("add $8, %[count]") \
100 __ASM_EMIT("jl 8f") \
101 __ASM_EMIT("vmovups 0x000(%[" SRC1 "], %[off]), %%ymm4") \
102 __ASM_EMIT(OP "ps 0x000(%[" SRC2 "], %[off]), %%ymm0, %%ymm4") \
103 __ASM_EMIT("vmovups %%ymm4, 0x000(%[" DST "], %[off])") \
104 __ASM_EMIT("add $0x20, %[off]") \
105 __ASM_EMIT("sub $8, %[count]") \
106 __ASM_EMIT("8:") \
107 /* 4x block */ \
108 __ASM_EMIT("add $4, %[count]") \
109 __ASM_EMIT("jl 10f") \
110 __ASM_EMIT("vmovups 0x000(%[" SRC1 "], %[off]), %%xmm4") \
111 __ASM_EMIT(OP "ps 0x000(%[" SRC2 "], %[off]), %%xmm0, %%xmm4") \
112 __ASM_EMIT("vmovups %%xmm4, 0x000(%[" DST "], %[off])") \
113 __ASM_EMIT("add $0x10, %[off]") \
114 __ASM_EMIT("sub $4, %[count]") \
115 __ASM_EMIT("10:") \
116 /* 1x blocks */ \
117 __ASM_EMIT("add $3, %[count]") \
118 __ASM_EMIT("jl 12f") \
119 __ASM_EMIT("11:") \
120 __ASM_EMIT("vmovss 0x000(%[" SRC1 "], %[off]), %%xmm4") \
121 __ASM_EMIT(OP "ss 0x000(%[" SRC2 "], %[off]), %%xmm0, %%xmm4") \
122 __ASM_EMIT("vmovss %%xmm4, 0x000(%[" DST "], %[off])") \
123 __ASM_EMIT("add $0x04, %[off]") \
124 __ASM_EMIT("dec %[count]") \
125 __ASM_EMIT("jge 11b") \
126 __ASM_EMIT("12:")
127
128 void fmadd_k3(float *dst, const float *src, float k, size_t count)
129 {
130 IF_ARCH_X86(size_t off);
131 ARCH_X86_ASM
132 (
133 FMADDSUB_K3_CORE("dst", "dst", "src", "vfmadd231")
134 : [off] "=&r" (off), [count] "+r" (count),
135 [k] "+Yz" (k)
136 : [dst] "r"(dst), [src] "r"(src)
137 : "cc", "memory",
138 "%xmm1", "%xmm2", "%xmm3",
139 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
140 );
141 }
142
143 void fmsub_k3(float *dst, const float *src, float k, size_t count)
144 {
145 IF_ARCH_X86(size_t off);
146 ARCH_X86_ASM
147 (
148 FMADDSUB_K3_CORE("dst", "dst", "src", "vfnmadd231")
149 : [off] "=&r" (off), [count] "+r" (count),
150 [k] "+Yz" (k)
151 : [dst] "r"(dst), [src] "r"(src)
152 : "cc", "memory",
153 "%xmm1", "%xmm2", "%xmm3",
154 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
155 );
156 }
157
158 void fmrsub_k3(float *dst, const float *src, float k, size_t count)
159 {
160 IF_ARCH_X86(size_t off);
161 ARCH_X86_ASM
162 (
163 FMADDSUB_K3_CORE("dst", "dst", "src", "vfmsub231")
164 : [off] "=&r" (off), [count] "+r" (count),
165 [k] "+Yz" (k)
166 : [dst] "r"(dst), [src] "r"(src)
167 : "cc", "memory",
168 "%xmm1", "%xmm2", "%xmm3",
169 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
170 );
171 }
172
173 void fmadd_k4(float *dst, const float *src1, const float *src2, float k, size_t count)
174 {
175 IF_ARCH_X86(size_t off);
176 ARCH_X86_ASM
177 (
178 FMADDSUB_K3_CORE("dst", "src1", "src2", "vfmadd231")
179 : [off] "=&r" (off), [count] "+r" (count)
180 : [dst] "r"(dst), [src1] "r" (src1), [src2] "r" (src2),
181 [k] "m" (k)
182 : "cc", "memory",
183 "%xmm1", "%xmm2", "%xmm3",
184 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
185 );
186 }
187
188 void fmsub_k4(float *dst, const float *src1, const float *src2, float k, size_t count)
189 {
190 IF_ARCH_X86(size_t off);
191 ARCH_X86_ASM
192 (
193 FMADDSUB_K3_CORE("dst", "src1", "src2", "vfnmadd231")
194 : [off] "=&r" (off), [count] "+r" (count),
195 [k] "+Yz" (k)
196 : [dst] "r"(dst), [src1] "r" (src1), [src2] "r" (src2)
197 : "cc", "memory",
198 "%xmm1", "%xmm2", "%xmm3",
199 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
200 );
201 }
202
203 void fmrsub_k4(float *dst, const float *src1, const float *src2, float k, size_t count)
204 {
205 IF_ARCH_X86(size_t off);
206 ARCH_X86_ASM
207 (
208 FMADDSUB_K3_CORE("dst", "src1", "src2", "vfmsub231")
209 : [off] "=&r" (off), [count] "+r" (count),
210 [k] "+Yz" (k)
211 : [dst] "r"(dst), [src1] "r" (src1), [src2] "r" (src2)
212 : "cc", "memory",
213 "%xmm1", "%xmm2", "%xmm3",
214 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
215 );
216 }
217
218 #undef FMADDSUB_K3_CORE
219
220 #define FMOP_K4_CORE(DST, SRC1, SRC2, OP, SEL) \
221 __ASM_EMIT("vbroadcastss %[k], %%zmm0") \
222 __ASM_EMIT("xor %[off], %[off]") \
223 __ASM_EMIT("vmovaps %%zmm0, %%zmm1") \
224 /* 48x blocks */ \
225 __ASM_EMIT("sub $48, %[count]") \
226 __ASM_EMIT("jbe 2f") \
227 __ASM_EMIT("1:") \
228 __ASM_EMIT("vmovups 0x000(%[" SRC1 "], %[off]), %%zmm5") \
229 __ASM_EMIT("vmovups 0x040(%[" SRC1 "], %[off]), %%zmm6") \
230 __ASM_EMIT("vmovups 0x080(%[" SRC1 "], %[off]), %%zmm7") \
231 __ASM_EMIT("vmulps 0x000(%[" SRC2 "], %[off]), %%zmm0, %%zmm2") \
232 __ASM_EMIT("vmulps 0x040(%[" SRC2 "], %[off]), %%zmm1, %%zmm3") \
233 __ASM_EMIT("vmulps 0x080(%[" SRC2 "], %[off]), %%zmm0, %%zmm4") \
234 __ASM_EMIT(OP "ps " SEL("%%zmm2", "%%zmm5") ", " SEL("%%zmm5", "%%zmm2") ", %%zmm5") \
235 __ASM_EMIT(OP "ps " SEL("%%zmm3", "%%zmm6") ", " SEL("%%zmm6", "%%zmm3") ", %%zmm6") \
236 __ASM_EMIT(OP "ps " SEL("%%zmm4", "%%zmm7") ", " SEL("%%zmm7", "%%zmm4") ", %%zmm7") \
237 __ASM_EMIT("vmovups %%zmm5, 0x000(%[" DST "], %[off])") \
238 __ASM_EMIT("vmovups %%zmm6, 0x040(%[" DST "], %[off])") \
239 __ASM_EMIT("vmovups %%zmm7, 0x080(%[" DST "], %[off])") \
240 __ASM_EMIT("add $0xc0, %[off]") \
241 __ASM_EMIT("sub $48, %[count]") \
242 __ASM_EMIT("jae 1b") \
243 __ASM_EMIT("2:") \
244 /* 16x blocks */ \
245 __ASM_EMIT("add $32, %[count]") \
246 __ASM_EMIT("jl 4f") \
247 __ASM_EMIT("3:") \
248 __ASM_EMIT("vmovups 0x000(%[" SRC1 "], %[off]), %%ymm6") \
249 __ASM_EMIT("vmovups 0x020(%[" SRC1 "], %[off]), %%ymm7") \
250 __ASM_EMIT("vmulps 0x000(%[" SRC2 "], %[off]), %%ymm0, %%ymm2") \
251 __ASM_EMIT("vmulps 0x020(%[" SRC2 "], %[off]), %%ymm1, %%ymm3") \
252 __ASM_EMIT(OP "ps " SEL("%%ymm2", "%%ymm6") ", " SEL("%%ymm6", "%%ymm2") ", %%ymm6") \
253 __ASM_EMIT(OP "ps " SEL("%%ymm3", "%%ymm7") ", " SEL("%%ymm7", "%%ymm3") ", %%ymm7") \
254 __ASM_EMIT("vmovups %%ymm6, 0x000(%[" DST "], %[off])") \
255 __ASM_EMIT("vmovups %%ymm7, 0x020(%[" DST "], %[off])") \
256 __ASM_EMIT("add $0x40, %[off]") \
257 __ASM_EMIT("sub $16, %[count]") \
258 __ASM_EMIT("jge 3b") \
259 __ASM_EMIT("4:") \
260 /* 8x block */ \
261 __ASM_EMIT("add $8, %[count]") \
262 __ASM_EMIT("jl 6f") \
263 __ASM_EMIT("vmovups 0x000(%[" SRC1 "], %[off]), %%ymm6") \
264 __ASM_EMIT("vmulps 0x000(%[" SRC2 "], %[off]), %%ymm0, %%ymm2") \
265 __ASM_EMIT(OP "ps " SEL("%%ymm2", "%%ymm6") ", " SEL("%%ymm6", "%%ymm2") ", %%ymm6") \
266 __ASM_EMIT("vmovups %%ymm6, 0x000(%[" DST "], %[off])") \
267 __ASM_EMIT("add $0x20, %[off]") \
268 __ASM_EMIT("sub $8, %[count]") \
269 __ASM_EMIT("6:") \
270 /* 4x block */ \
271 __ASM_EMIT("add $4, %[count]") /* 8 - 4 */ \
272 __ASM_EMIT("jl 8f") \
273 __ASM_EMIT("vmovups 0x000(%[" SRC1 "], %[off]), %%xmm6") \
274 __ASM_EMIT("vmulps 0x000(%[" SRC2 "], %[off]), %%xmm0, %%xmm2") \
275 __ASM_EMIT(OP "ps " SEL("%%xmm2", "%%xmm6") ", " SEL("%%xmm6", "%%xmm2") ", %%xmm6") \
276 __ASM_EMIT("vmovups %%xmm6, 0x000(%[" DST "], %[off])") \
277 __ASM_EMIT("add $0x10, %[off]") \
278 __ASM_EMIT("sub $4, %[count]") \
279 __ASM_EMIT("8:") \
280 /* 1x blocks */ \
281 __ASM_EMIT("add $3, %[count]") /* 8 - 4 */ \
282 __ASM_EMIT("jl 10f") \
283 __ASM_EMIT("9:") \
284 __ASM_EMIT("vmovss 0x000(%[" SRC1 "], %[off]), %%xmm6") \
285 __ASM_EMIT("vmulss 0x000(%[" SRC2 "], %[off]), %%xmm0, %%xmm2") \
286 __ASM_EMIT(OP "ss " SEL("%%xmm2", "%%xmm6") ", " SEL("%%xmm6", "%%xmm2") ", %%xmm6") \
287 __ASM_EMIT("vmovss %%xmm6, 0x000(%[" DST "], %[off])") \
288 __ASM_EMIT("add $0x04, %[off]") \
289 __ASM_EMIT("dec %[count]") \
290 __ASM_EMIT("jge 9b") \
291 __ASM_EMIT("10:")
292
293 void fmmul_k3(float *dst, const float *src, float k, size_t count)
294 {
295 IF_ARCH_X86(size_t off);
296 ARCH_X86_ASM
297 (
298 FMOP_K4_CORE("dst", "dst", "src", "vmul", OP_DSEL)
299 : [off] "=&r" (off), [count] "+r" (count),
300 [k] "+Yz" (k)
301 : [dst] "r"(dst), [src] "r"(src)
302 : "cc", "memory",
303 "%xmm1", "%xmm2", "%xmm3",
304 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
305 );
306 }
307
308 void fmdiv_k3(float *dst, const float *src, float k, size_t count)
309 {
310 IF_ARCH_X86(size_t off);
311 ARCH_X86_ASM
312 (
313 FMOP_K4_CORE("dst", "dst", "src", "vdiv", OP_DSEL)
314 : [off] "=&r" (off), [count] "+r" (count),
315 [k] "+Yz" (k)
316 : [dst] "r"(dst), [src] "r"(src)
317 : "cc", "memory",
318 "%xmm1", "%xmm2", "%xmm3",
319 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
320 );
321 }
322
323 void fmrdiv_k3(float *dst, const float *src, float k, size_t count)
324 {
325 IF_ARCH_X86(size_t off);
326 ARCH_X86_ASM
327 (
328 FMOP_K4_CORE("dst", "dst", "src", "vdiv", OP_RSEL)
329 : [off] "=&r" (off), [count] "+r" (count),
330 [k] "+Yz" (k)
331 : [dst] "r"(dst), [src] "r"(src)
332 : "cc", "memory",
333 "%xmm1", "%xmm2", "%xmm3",
334 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
335 );
336 }
337
338 void fmmul_k4(float *dst, const float *src1, const float *src2, float k, size_t count)
339 {
340 IF_ARCH_X86(size_t off);
341 ARCH_X86_ASM
342 (
343 FMOP_K4_CORE("dst", "src1", "src2", "vmul", OP_DSEL)
344 : [off] "=&r" (off), [count] "+r" (count),
345 [k] "+Yz" (k)
346 : [dst] "r"(dst), [src1] "r" (src1), [src2] "r" (src2)
347 : "cc", "memory",
348 "%xmm1", "%xmm2", "%xmm3",
349 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
350 );
351 }
352
353 void fmdiv_k4(float *dst, const float *src1, const float *src2, float k, size_t count)
354 {
355 IF_ARCH_X86(size_t off);
356 ARCH_X86_ASM
357 (
358 FMOP_K4_CORE("dst", "src1", "src2", "vdiv", OP_DSEL)
359 : [off] "=&r" (off), [count] "+r" (count),
360 [k] "+Yz" (k)
361 : [dst] "r"(dst), [src1] "r" (src1), [src2] "r" (src2)
362 : "cc", "memory",
363 "%xmm1", "%xmm2", "%xmm3",
364 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
365 );
366 }
367
368 void fmrdiv_k4(float *dst, const float *src1, const float *src2, float k, size_t count)
369 {
370 IF_ARCH_X86(size_t off);
371 ARCH_X86_ASM
372 (
373 FMOP_K4_CORE("dst", "src1", "src2", "vdiv", OP_RSEL)
374 : [off] "=&r" (off), [count] "+r" (count),
375 [k] "+Yz" (k)
376 : [dst] "r"(dst), [src1] "r" (src1), [src2] "r" (src2)
377 : "cc", "memory",
378 "%xmm1", "%xmm2", "%xmm3",
379 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
380 );
381 }
382
383 #undef FMOP_K4_CORE
384
385 #define FMOD_KX_CORE(DST, SRC1, SRC2, OP_SEL) \
386 __ASM_EMIT("vbroadcastss %[k], %%zmm0") \
387 __ASM_EMIT("xor %[off], %[off]") \
388 __ASM_EMIT("vmovaps %%zmm0, %%zmm1") \
389 /* x32 blocks */ \
390 __ASM_EMIT("sub $32, %[count]") \
391 __ASM_EMIT("jb 2f") \
392 __ASM_EMIT("1:") \
393 __ASM_EMIT("vmovups 0x00(%[" SRC1 "], %[off]), " OP_SEL("%%zmm2", "%%zmm6")) \
394 __ASM_EMIT("vmovups 0x40(%[" SRC1 "], %[off]), " OP_SEL("%%zmm3", "%%zmm7")) \
395 __ASM_EMIT("vmulps 0x00(%[" SRC2 "], %[off]), %%zmm0, " OP_SEL("%%zmm6", "%%zmm2")) \
396 __ASM_EMIT("vmulps 0x40(%[" SRC2 "], %[off]), %%zmm1, " OP_SEL("%%zmm7", "%%zmm3")) \
397 __ASM_EMIT("vdivps %%zmm6, %%zmm2, %%zmm4") \
398 __ASM_EMIT("vdivps %%zmm7, %%zmm3, %%zmm5") \
399 __ASM_EMIT("vcvttps2dq %%zmm4, %%zmm4") \
400 __ASM_EMIT("vcvttps2dq %%zmm5, %%zmm5") \
401 __ASM_EMIT("vcvtdq2ps %%zmm4, %%zmm4") \
402 __ASM_EMIT("vcvtdq2ps %%zmm5, %%zmm5") \
403 __ASM_EMIT("vfnmadd231ps %%zmm6, %%zmm4, %%zmm2") \
404 __ASM_EMIT("vfnmadd231ps %%zmm7, %%zmm5, %%zmm3") \
405 __ASM_EMIT("vmovups %%zmm2, 0x00(%[" DST "], %[off])") \
406 __ASM_EMIT("vmovups %%zmm3, 0x40(%[" DST "], %[off])") \
407 __ASM_EMIT("add $0x80, %[off]") \
408 __ASM_EMIT("sub $32, %[count]") \
409 __ASM_EMIT("jae 1b") \
410 __ASM_EMIT("2:") \
411 /* x16 blocks */ \
412 __ASM_EMIT("add $16, %[count]") \
413 __ASM_EMIT("jl 4f") \
414 __ASM_EMIT("vmovups 0x00(%[" SRC1 "], %[off]), " OP_SEL("%%ymm2", "%%ymm6")) \
415 __ASM_EMIT("vmovups 0x20(%[" SRC1 "], %[off]), " OP_SEL("%%ymm3", "%%ymm7")) \
416 __ASM_EMIT("vmulps 0x00(%[" SRC2 "], %[off]), %%ymm0, " OP_SEL("%%ymm6", "%%ymm2")) \
417 __ASM_EMIT("vmulps 0x20(%[" SRC2 "], %[off]), %%ymm1, " OP_SEL("%%ymm7", "%%ymm3")) \
418 __ASM_EMIT("vdivps %%ymm6, %%ymm2, %%ymm4") \
419 __ASM_EMIT("vdivps %%ymm7, %%ymm3, %%ymm5") \
420 __ASM_EMIT("vcvttps2dq %%ymm4, %%ymm4") \
421 __ASM_EMIT("vcvttps2dq %%ymm5, %%ymm5") \
422 __ASM_EMIT("vcvtdq2ps %%ymm4, %%ymm4") \
423 __ASM_EMIT("vcvtdq2ps %%ymm5, %%ymm5") \
424 __ASM_EMIT("vfnmadd231ps %%ymm6, %%ymm4, %%ymm2") \
425 __ASM_EMIT("vfnmadd231ps %%ymm7, %%ymm5, %%ymm3") \
426 __ASM_EMIT("vmovups %%ymm2, 0x00(%[" DST "], %[off])") \
427 __ASM_EMIT("vmovups %%ymm3, 0x20(%[" DST "], %[off])") \
428 __ASM_EMIT("add $0x40, %[off]") \
429 __ASM_EMIT("sub $16, %[count]") \
430 __ASM_EMIT("jae 1b") \
431 __ASM_EMIT("4:") \
432 /* x8 block */ \
433 __ASM_EMIT("add $8, %[count]") \
434 __ASM_EMIT("jl 6f") \
435 __ASM_EMIT("vmovups 0x00(%[" SRC1 "], %[off]), " OP_SEL("%%xmm2", "%%xmm6")) \
436 __ASM_EMIT("vmovups 0x10(%[" SRC1 "], %[off]), " OP_SEL("%%xmm3", "%%xmm7")) \
437 __ASM_EMIT("vmulps 0x00(%[" SRC2 "], %[off]), %%xmm0, " OP_SEL("%%xmm6", "%%xmm2")) \
438 __ASM_EMIT("vmulps 0x10(%[" SRC2 "], %[off]), %%xmm1, " OP_SEL("%%xmm7", "%%xmm3")) \
439 __ASM_EMIT("vdivps %%xmm6, %%xmm2, %%xmm4") \
440 __ASM_EMIT("vdivps %%xmm7, %%xmm3, %%xmm5") \
441 __ASM_EMIT("vcvttps2dq %%xmm4, %%xmm4") \
442 __ASM_EMIT("vcvttps2dq %%xmm5, %%xmm5") \
443 __ASM_EMIT("vcvtdq2ps %%xmm4, %%xmm4") \
444 __ASM_EMIT("vcvtdq2ps %%xmm5, %%xmm5") \
445 __ASM_EMIT("vfnmadd231ps %%xmm6, %%xmm4, %%xmm2") \
446 __ASM_EMIT("vfnmadd231ps %%xmm7, %%xmm5, %%xmm3") \
447 __ASM_EMIT("vmovups %%xmm2, 0x00(%[" DST "], %[off])") \
448 __ASM_EMIT("vmovups %%xmm3, 0x10(%[" DST "], %[off])") \
449 __ASM_EMIT("sub $8, %[count]") \
450 __ASM_EMIT("add $0x20, %[off]") \
451 __ASM_EMIT("6:") \
452 /* x4 block */ \
453 __ASM_EMIT("add $4, %[count]") \
454 __ASM_EMIT("jl 8f") \
455 __ASM_EMIT("vmovups 0x00(%[" SRC1 "], %[off]), " OP_SEL("%%xmm2", "%%xmm6")) \
456 __ASM_EMIT("vmulps 0x00(%[" SRC2 "], %[off]), %%xmm0, " OP_SEL("%%xmm6", "%%xmm2")) \
457 __ASM_EMIT("vdivps %%xmm6, %%xmm2, %%xmm4") \
458 __ASM_EMIT("vcvttps2dq %%xmm4, %%xmm4") \
459 __ASM_EMIT("vcvtdq2ps %%xmm4, %%xmm4") \
460 __ASM_EMIT("vfnmadd231ps %%xmm6, %%xmm4, %%xmm2") \
461 __ASM_EMIT("vmovups %%xmm2, 0x00(%[" DST "], %[off])") \
462 __ASM_EMIT("sub $4, %[count]") \
463 __ASM_EMIT("add $0x10, %[off]") \
464 __ASM_EMIT("8:") \
465 /* x1 blocks */ \
466 __ASM_EMIT("add $3, %[count]") \
467 __ASM_EMIT("jl 10f") \
468 __ASM_EMIT("9:") \
469 __ASM_EMIT("vmovss 0x00(%[" SRC1 "], %[off]), " OP_SEL("%%xmm2", "%%xmm6")) \
470 __ASM_EMIT("vmulss 0x00(%[" SRC2 "], %[off]), %%xmm0, " OP_SEL("%%xmm6", "%%xmm2")) \
471 __ASM_EMIT("vdivss %%xmm6, %%xmm2, %%xmm4") \
472 __ASM_EMIT("vcvttps2dq %%xmm4, %%xmm4") \
473 __ASM_EMIT("vcvtdq2ps %%xmm4, %%xmm4") \
474 __ASM_EMIT("vfnmadd231ss %%xmm6, %%xmm4, %%xmm2") \
475 __ASM_EMIT("vmovss %%xmm2, 0x00(%[" DST "], %[off])") \
476 __ASM_EMIT("add $0x04, %[off]") \
477 __ASM_EMIT("dec %[count]") \
478 __ASM_EMIT("jge 9b") \
479 __ASM_EMIT("10:")
480
481 void fmmod_k3(float *dst, const float *src, float k, size_t count)
482 {
483 IF_ARCH_X86(size_t off);
484 ARCH_X86_ASM
485 (
486 FMOD_KX_CORE("dst", "dst", "src", OP_DSEL)
487 : [off] "=&r" (off), [count] "+r" (count),
488 [k] "+Yz" (k)
489 : [dst] "r"(dst), [src] "r" (src)
490 : "cc", "memory",
491 "%xmm1", "%xmm2", "%xmm3",
492 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
493 );
494 }
495
496 void fmrmod_k3(float *dst, const float *src, float k, size_t count)
497 {
498 IF_ARCH_X86(size_t off);
499 ARCH_X86_ASM
500 (
501 FMOD_KX_CORE("dst", "dst", "src", OP_RSEL)
502 : [off] "=&r" (off), [count] "+r" (count),
503 [k] "+Yz" (k)
504 : [dst] "r"(dst), [src] "r" (src)
505 : "cc", "memory",
506 "%xmm1", "%xmm2", "%xmm3",
507 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
508 );
509 }
510
511 void fmmod_k4(float *dst, const float *src1, const float *src2, float k, size_t count)
512 {
513 IF_ARCH_X86(size_t off);
514 ARCH_X86_ASM
515 (
516 FMOD_KX_CORE("dst", "src1", "src2", OP_DSEL)
517 : [off] "=&r" (off), [count] "+r" (count),
518 [k] "+Yz" (k)
519 : [dst] "r"(dst), [src1] "r" (src1), [src2] "r" (src2)
520 : "cc", "memory",
521 "%xmm1", "%xmm2", "%xmm3",
522 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
523 );
524 }
525
526 void fmrmod_k4(float *dst, const float *src1, const float *src2, float k, size_t count)
527 {
528 IF_ARCH_X86(size_t off);
529 ARCH_X86_ASM
530 (
531 FMOD_KX_CORE("dst", "src1", "src2", OP_RSEL)
532 : [off] "=&r" (off), [count] "+r" (count),
533 [k] "+Yz" (k)
534 : [dst] "r"(dst), [src1] "r" (src1), [src2] "r" (src2)
535 : "cc", "memory",
536 "%xmm1", "%xmm2", "%xmm3",
537 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
538 );
539 }
540
541 #undef FMOD_KX_CORE
542
543 #undef OP_DSEL
544 #undef OP_RSEL
545
546 } /* namespace avx512 */
547 } /* namespace lsp */
548
549
550
551
552 #endif /* PRIVATE_DSP_ARCH_X86_AVX512_PMATH_FMOP_KX_H_ */