Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/x86/avx/pmath/fmop_kx.h

raw
1 /*
2 * Copyright (C) 2023 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2023 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 31 мар. 2020 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_X86_AVX_PMATH_FMOP_KX_H_
23 #define PRIVATE_DSP_ARCH_X86_AVX_PMATH_FMOP_KX_H_
24
25 #ifndef PRIVATE_DSP_ARCH_X86_AVX_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_X86_AVX_IMPL */
28
29 namespace lsp
30 {
31 namespace avx
32 {
33 #define OP_DSEL(a, b) a
34 #define OP_RSEL(a, b) b
35
36 #define FMADDSUB_K3_CORE(DST, SRC1, SRC2, OP) \
37 __ASM_EMIT("xor %[off], %[off]") \
38 __ASM_EMIT("vbroadcastss %[k], %%ymm0") \
39 __ASM_EMIT("sub $32, %[count]") \
40 __ASM_EMIT("vmovaps %%ymm0, %%ymm1") \
41 __ASM_EMIT("jb 2f") \
42 /* 32x blocks */ \
43 __ASM_EMIT("1:") \
44 __ASM_EMIT("vmovups 0x000(%[" SRC1 "], %[off]), %%ymm4") \
45 __ASM_EMIT("vmovups 0x020(%[" SRC1 "], %[off]), %%ymm5") \
46 __ASM_EMIT("vmovups 0x040(%[" SRC1 "], %[off]), %%ymm6") \
47 __ASM_EMIT("vmovups 0x060(%[" SRC1 "], %[off]), %%ymm7") \
48 __ASM_EMIT(OP "ps 0x000(%[" SRC2 "], %[off]), %%ymm0, %%ymm4") \
49 __ASM_EMIT(OP "ps 0x020(%[" SRC2 "], %[off]), %%ymm1, %%ymm5") \
50 __ASM_EMIT(OP "ps 0x040(%[" SRC2 "], %[off]), %%ymm0, %%ymm6") \
51 __ASM_EMIT(OP "ps 0x060(%[" SRC2 "], %[off]), %%ymm1, %%ymm7") \
52 __ASM_EMIT("vmovups %%ymm4, 0x000(%[" DST "], %[off])") \
53 __ASM_EMIT("vmovups %%ymm5, 0x020(%[" DST "], %[off])") \
54 __ASM_EMIT("vmovups %%ymm6, 0x040(%[" DST "], %[off])") \
55 __ASM_EMIT("vmovups %%ymm7, 0x060(%[" DST "], %[off])") \
56 __ASM_EMIT("add $0x80, %[off]") \
57 __ASM_EMIT("sub $32, %[count]") \
58 __ASM_EMIT("jae 1b") \
59 /* 16x block */ \
60 __ASM_EMIT("2:") \
61 __ASM_EMIT("add $16, %[count]") /* 32 - 16 */ \
62 __ASM_EMIT("jl 4f") \
63 __ASM_EMIT("vmovups 0x000(%[" SRC1 "], %[off]), %%ymm4") \
64 __ASM_EMIT("vmovups 0x020(%[" SRC1 "], %[off]), %%ymm5") \
65 __ASM_EMIT(OP "ps 0x000(%[" SRC2 "], %[off]), %%ymm0, %%ymm4") \
66 __ASM_EMIT(OP "ps 0x020(%[" SRC2 "], %[off]), %%ymm1, %%ymm5") \
67 __ASM_EMIT("vmovups %%ymm4, 0x000(%[" DST "], %[off])") \
68 __ASM_EMIT("vmovups %%ymm5, 0x020(%[" DST "], %[off])") \
69 __ASM_EMIT("add $0x40, %[off]") \
70 __ASM_EMIT("sub $16, %[count]") \
71 /* 8x block */ \
72 __ASM_EMIT("4:") \
73 __ASM_EMIT("add $8, %[count]") /* 16 - 8 */ \
74 __ASM_EMIT("jl 6f") \
75 __ASM_EMIT("vmovups 0x000(%[" SRC1 "], %[off]), %%ymm4") \
76 __ASM_EMIT(OP "ps 0x000(%[" SRC2 "], %[off]), %%ymm0, %%ymm4") \
77 __ASM_EMIT("vmovups %%ymm4, 0x000(%[" DST "], %[off])") \
78 __ASM_EMIT("add $0x20, %[off]") \
79 __ASM_EMIT("sub $8, %[count]") \
80 \
81 /* 4x block */ \
82 __ASM_EMIT("6:") \
83 __ASM_EMIT("add $4, %[count]") /* 8 - 4 */ \
84 __ASM_EMIT("jl 8f") \
85 __ASM_EMIT("vmovups 0x000(%[" SRC1 "], %[off]), %%xmm4") \
86 __ASM_EMIT(OP "ps 0x000(%[" SRC2 "], %[off]), %%xmm0, %%xmm4") \
87 __ASM_EMIT("vmovups %%xmm4, 0x000(%[" DST "], %[off])") \
88 __ASM_EMIT("add $0x10, %[off]") \
89 __ASM_EMIT("sub $4, %[count]") \
90 \
91 /* 1x blocks */ \
92 __ASM_EMIT("8:") \
93 __ASM_EMIT("add $3, %[count]") /* 8 - 4 */ \
94 __ASM_EMIT("jl 10f") \
95 __ASM_EMIT("9:") \
96 __ASM_EMIT("vmovss 0x000(%[" SRC1 "], %[off]), %%xmm4") \
97 __ASM_EMIT(OP "ss 0x000(%[" SRC2 "], %[off]), %%xmm0, %%xmm4") \
98 __ASM_EMIT("vmovss %%xmm4, 0x000(%[" DST "], %[off])") \
99 __ASM_EMIT("add $0x04, %[off]") \
100 __ASM_EMIT("dec %[count]") \
101 __ASM_EMIT("jge 9b") \
102 __ASM_EMIT("10:")
103
104 void fmadd_k3_fma3(float *dst, const float *src, float k, size_t count)
105 {
106 IF_ARCH_X86(size_t off);
107 ARCH_X86_ASM
108 (
109 FMADDSUB_K3_CORE("dst", "dst", "src", "vfmadd231")
110 : [off] "=&r" (off), [count] "+r" (count)
111 : [dst] "r"(dst), [src] "r"(src),
112 [k] "m" (k)
113 : "cc", "memory",
114 "%xmm0", "%xmm1",
115 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
116 );
117 }
118
119 void fmsub_k3_fma3(float *dst, const float *src, float k, size_t count)
120 {
121 IF_ARCH_X86(size_t off);
122 ARCH_X86_ASM
123 (
124 FMADDSUB_K3_CORE("dst", "dst", "src", "vfnmadd231")
125 : [off] "=&r" (off), [count] "+r" (count)
126 : [dst] "r"(dst), [src] "r"(src),
127 [k] "m" (k)
128 : "cc", "memory",
129 "%xmm0", "%xmm1",
130 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
131 );
132 }
133
134 void fmrsub_k3_fma3(float *dst, const float *src, float k, size_t count)
135 {
136 IF_ARCH_X86(size_t off);
137 ARCH_X86_ASM
138 (
139 FMADDSUB_K3_CORE("dst", "dst", "src", "vfmsub231")
140 : [off] "=&r" (off), [count] "+r" (count)
141 : [dst] "r"(dst), [src] "r"(src),
142 [k] "m" (k)
143 : "cc", "memory",
144 "%xmm0", "%xmm1",
145 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
146 );
147 }
148
149 void fmadd_k4_fma3(float *dst, const float *src1, const float *src2, float k, size_t count)
150 {
151 IF_ARCH_X86(size_t off);
152 ARCH_X86_ASM
153 (
154 FMADDSUB_K3_CORE("dst", "src1", "src2", "vfmadd231")
155 : [off] "=&r" (off), [count] "+r" (count)
156 : [dst] "r"(dst), [src1] "r" (src1), [src2] "r" (src2),
157 [k] "m" (k)
158 : "cc", "memory",
159 "%xmm0", "%xmm1",
160 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
161 );
162 }
163
164 void fmsub_k4_fma3(float *dst, const float *src1, const float *src2, float k, size_t count)
165 {
166 IF_ARCH_X86(size_t off);
167 ARCH_X86_ASM
168 (
169 FMADDSUB_K3_CORE("dst", "src1", "src2", "vfnmadd231")
170 : [off] "=&r" (off), [count] "+r" (count)
171 : [dst] "r"(dst), [src1] "r" (src1), [src2] "r" (src2),
172 [k] "m" (k)
173 : "cc", "memory",
174 "%xmm0", "%xmm1",
175 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
176 );
177 }
178
179 void fmrsub_k4_fma3(float *dst, const float *src1, const float *src2, float k, size_t count)
180 {
181 IF_ARCH_X86(size_t off);
182 ARCH_X86_ASM
183 (
184 FMADDSUB_K3_CORE("dst", "src1", "src2", "vfmsub231")
185 : [off] "=&r" (off), [count] "+r" (count)
186 : [dst] "r"(dst), [src1] "r" (src1), [src2] "r" (src2),
187 [k] "m" (k)
188 : "cc", "memory",
189 "%xmm0", "%xmm1",
190 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
191 );
192 }
193
194 #undef FMADDSUB_K3_CORE
195
196 #define FMOP_K4_CORE(DST, SRC1, SRC2, OP, SEL) \
197 __ASM_EMIT("xor %[off], %[off]") \
198 __ASM_EMIT("vbroadcastss %[k], %%ymm0") \
199 __ASM_EMIT("sub $24, %[count]") \
200 __ASM_EMIT("vmovaps %%ymm0, %%ymm1") \
201 __ASM_EMIT("jb 2f") \
202 /* 24x blocks */ \
203 __ASM_EMIT("1:") \
204 __ASM_EMIT("vmovups 0x000(%[" SRC1 "], %[off]), %%ymm5") \
205 __ASM_EMIT("vmovups 0x020(%[" SRC1 "], %[off]), %%ymm6") \
206 __ASM_EMIT("vmovups 0x040(%[" SRC1 "], %[off]), %%ymm7") \
207 __ASM_EMIT("vmulps 0x000(%[" SRC2 "], %[off]), %%ymm0, %%ymm2") \
208 __ASM_EMIT("vmulps 0x020(%[" SRC2 "], %[off]), %%ymm1, %%ymm3") \
209 __ASM_EMIT("vmulps 0x040(%[" SRC2 "], %[off]), %%ymm0, %%ymm4") \
210 __ASM_EMIT(OP "ps " SEL("%%ymm2", "%%ymm5") ", " SEL("%%ymm5", "%%ymm2") ", %%ymm5") \
211 __ASM_EMIT(OP "ps " SEL("%%ymm3", "%%ymm6") ", " SEL("%%ymm6", "%%ymm3") ", %%ymm6") \
212 __ASM_EMIT(OP "ps " SEL("%%ymm4", "%%ymm7") ", " SEL("%%ymm7", "%%ymm4") ", %%ymm7") \
213 __ASM_EMIT("vmovups %%ymm5, 0x000(%[" DST "], %[off])") \
214 __ASM_EMIT("vmovups %%ymm6, 0x020(%[" DST "], %[off])") \
215 __ASM_EMIT("vmovups %%ymm7, 0x040(%[" DST "], %[off])") \
216 __ASM_EMIT("add $0x60, %[off]") \
217 __ASM_EMIT("sub $24, %[count]") \
218 __ASM_EMIT("jae 1b") \
219 /* 16x block */ \
220 __ASM_EMIT("2:") \
221 __ASM_EMIT("add $8, %[count]") /* 24 - 16 */ \
222 __ASM_EMIT("jl 4f") \
223 __ASM_EMIT("vmovups 0x000(%[" SRC1 "], %[off]), %%ymm6") \
224 __ASM_EMIT("vmovups 0x020(%[" SRC1 "], %[off]), %%ymm7") \
225 __ASM_EMIT("vmulps 0x000(%[" SRC2 "], %[off]), %%ymm0, %%ymm2") \
226 __ASM_EMIT("vmulps 0x020(%[" SRC2 "], %[off]), %%ymm1, %%ymm3") \
227 __ASM_EMIT(OP "ps " SEL("%%ymm2", "%%ymm6") ", " SEL("%%ymm6", "%%ymm2") ", %%ymm6") \
228 __ASM_EMIT(OP "ps " SEL("%%ymm3", "%%ymm7") ", " SEL("%%ymm7", "%%ymm3") ", %%ymm7") \
229 __ASM_EMIT("vmovups %%ymm6, 0x000(%[" DST "], %[off])") \
230 __ASM_EMIT("vmovups %%ymm7, 0x020(%[" DST "], %[off])") \
231 __ASM_EMIT("add $0x40, %[off]") \
232 __ASM_EMIT("sub $16, %[count]") \
233 /* 8x block */ \
234 __ASM_EMIT("4:") \
235 __ASM_EMIT("add $8, %[count]") /* 16 - 8 */ \
236 __ASM_EMIT("jl 6f") \
237 __ASM_EMIT("vmovups 0x000(%[" SRC1 "], %[off]), %%ymm6") \
238 __ASM_EMIT("vmulps 0x000(%[" SRC2 "], %[off]), %%ymm0, %%ymm2") \
239 __ASM_EMIT(OP "ps " SEL("%%ymm2", "%%ymm6") ", " SEL("%%ymm6", "%%ymm2") ", %%ymm6") \
240 __ASM_EMIT("vmovups %%ymm6, 0x000(%[" DST "], %[off])") \
241 __ASM_EMIT("add $0x20, %[off]") \
242 __ASM_EMIT("sub $8, %[count]") \
243 \
244 /* 4x block */ \
245 __ASM_EMIT("6:") \
246 __ASM_EMIT("add $4, %[count]") /* 8 - 4 */ \
247 __ASM_EMIT("jl 8f") \
248 __ASM_EMIT("vmovups 0x000(%[" SRC1 "], %[off]), %%xmm6") \
249 __ASM_EMIT("vmulps 0x000(%[" SRC2 "], %[off]), %%xmm0, %%xmm2") \
250 __ASM_EMIT(OP "ps " SEL("%%xmm2", "%%xmm6") ", " SEL("%%xmm6", "%%xmm2") ", %%xmm6") \
251 __ASM_EMIT("vmovups %%xmm6, 0x000(%[" DST "], %[off])") \
252 __ASM_EMIT("add $0x10, %[off]") \
253 __ASM_EMIT("sub $4, %[count]") \
254 \
255 /* 1x blocks */ \
256 __ASM_EMIT("8:") \
257 __ASM_EMIT("add $3, %[count]") /* 8 - 4 */ \
258 __ASM_EMIT("jl 10f") \
259 __ASM_EMIT("9:") \
260 __ASM_EMIT("vmovss 0x000(%[" SRC1 "], %[off]), %%xmm6") \
261 __ASM_EMIT("vmulss 0x000(%[" SRC2 "], %[off]), %%xmm0, %%xmm2") \
262 __ASM_EMIT(OP "ss " SEL("%%xmm2", "%%xmm6") ", " SEL("%%xmm6", "%%xmm2") ", %%xmm6") \
263 __ASM_EMIT("vmovss %%xmm6, 0x000(%[" DST "], %[off])") \
264 __ASM_EMIT("add $0x04, %[off]") \
265 __ASM_EMIT("dec %[count]") \
266 __ASM_EMIT("jge 9b") \
267 __ASM_EMIT("10:")
268
269 void fmadd_k3(float *dst, const float *src, float k, size_t count)
270 {
271 IF_ARCH_X86(size_t off);
272 ARCH_X86_ASM
273 (
274 FMOP_K4_CORE("dst", "dst", "src", "vadd", OP_DSEL)
275 : [off] "=&r" (off), [count] "+r" (count)
276 : [dst] "r"(dst), [src] "r"(src),
277 [k] "m" (k)
278 : "cc", "memory",
279 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
280 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
281 );
282 }
283
284 void fmsub_k3(float *dst, const float *src, float k, size_t count)
285 {
286 IF_ARCH_X86(size_t off);
287 ARCH_X86_ASM
288 (
289 FMOP_K4_CORE("dst", "dst", "src", "vsub", OP_DSEL)
290 : [off] "=&r" (off), [count] "+r" (count)
291 : [dst] "r"(dst), [src] "r"(src),
292 [k] "m" (k)
293 : "cc", "memory",
294 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
295 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
296 );
297 }
298
299 void fmrsub_k3(float *dst, const float *src, float k, size_t count)
300 {
301 IF_ARCH_X86(size_t off);
302 ARCH_X86_ASM
303 (
304 FMOP_K4_CORE("dst", "dst", "src", "vsub", OP_RSEL)
305 : [off] "=&r" (off), [count] "+r" (count)
306 : [dst] "r"(dst), [src] "r"(src),
307 [k] "m" (k)
308 : "cc", "memory",
309 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
310 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
311 );
312 }
313
314 void fmmul_k3(float *dst, const float *src, float k, size_t count)
315 {
316 IF_ARCH_X86(size_t off);
317 ARCH_X86_ASM
318 (
319 FMOP_K4_CORE("dst", "dst", "src", "vmul", OP_DSEL)
320 : [off] "=&r" (off), [count] "+r" (count)
321 : [dst] "r"(dst), [src] "r"(src),
322 [k] "m" (k)
323 : "cc", "memory",
324 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
325 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
326 );
327 }
328
329 void fmdiv_k3(float *dst, const float *src, float k, size_t count)
330 {
331 IF_ARCH_X86(size_t off);
332 ARCH_X86_ASM
333 (
334 FMOP_K4_CORE("dst", "dst", "src", "vdiv", OP_DSEL)
335 : [off] "=&r" (off), [count] "+r" (count)
336 : [dst] "r"(dst), [src] "r"(src),
337 [k] "m" (k)
338 : "cc", "memory",
339 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
340 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
341 );
342 }
343
344 void fmrdiv_k3(float *dst, const float *src, float k, size_t count)
345 {
346 IF_ARCH_X86(size_t off);
347 ARCH_X86_ASM
348 (
349 FMOP_K4_CORE("dst", "dst", "src", "vdiv", OP_RSEL)
350 : [off] "=&r" (off), [count] "+r" (count)
351 : [dst] "r"(dst), [src] "r"(src),
352 [k] "m" (k)
353 : "cc", "memory",
354 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
355 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
356 );
357 }
358
359 void fmadd_k4(float *dst, const float *src1, const float *src2, float k, size_t count)
360 {
361 IF_ARCH_X86(size_t off);
362 ARCH_X86_ASM
363 (
364 FMOP_K4_CORE("dst", "src1", "src2", "vadd", OP_DSEL)
365 : [off] "=&r" (off), [count] "+r" (count)
366 : [dst] "r"(dst), [src1] "r" (src1), [src2] "r" (src2),
367 [k] "m" (k)
368 : "cc", "memory",
369 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
370 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
371 );
372 }
373
374 void fmsub_k4(float *dst, const float *src1, const float *src2, float k, size_t count)
375 {
376 IF_ARCH_X86(size_t off);
377 ARCH_X86_ASM
378 (
379 FMOP_K4_CORE("dst", "src1", "src2", "vsub", OP_DSEL)
380 : [off] "=&r" (off), [count] "+r" (count)
381 : [dst] "r"(dst), [src1] "r" (src1), [src2] "r" (src2),
382 [k] "m" (k)
383 : "cc", "memory",
384 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
385 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
386 );
387 }
388
389 void fmrsub_k4(float *dst, const float *src1, const float *src2, float k, size_t count)
390 {
391 IF_ARCH_X86(size_t off);
392 ARCH_X86_ASM
393 (
394 FMOP_K4_CORE("dst", "src1", "src2", "vsub", OP_RSEL)
395 : [off] "=&r" (off), [count] "+r" (count)
396 : [dst] "r"(dst), [src1] "r" (src1), [src2] "r" (src2),
397 [k] "m" (k)
398 : "cc", "memory",
399 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
400 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
401 );
402 }
403
404 void fmmul_k4(float *dst, const float *src1, const float *src2, float k, size_t count)
405 {
406 IF_ARCH_X86(size_t off);
407 ARCH_X86_ASM
408 (
409 FMOP_K4_CORE("dst", "src1", "src2", "vmul", OP_DSEL)
410 : [off] "=&r" (off), [count] "+r" (count)
411 : [dst] "r"(dst), [src1] "r" (src1), [src2] "r" (src2),
412 [k] "m" (k)
413 : "cc", "memory",
414 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
415 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
416 );
417 }
418
419 void fmdiv_k4(float *dst, const float *src1, const float *src2, float k, size_t count)
420 {
421 IF_ARCH_X86(size_t off);
422 ARCH_X86_ASM
423 (
424 FMOP_K4_CORE("dst", "src1", "src2", "vdiv", OP_DSEL)
425 : [off] "=&r" (off), [count] "+r" (count)
426 : [dst] "r"(dst), [src1] "r" (src1), [src2] "r" (src2),
427 [k] "m" (k)
428 : "cc", "memory",
429 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
430 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
431 );
432 }
433
434 void fmrdiv_k4(float *dst, const float *src1, const float *src2, float k, size_t count)
435 {
436 IF_ARCH_X86(size_t off);
437 ARCH_X86_ASM
438 (
439 FMOP_K4_CORE("dst", "src1", "src2", "vdiv", OP_RSEL)
440 : [off] "=&r" (off), [count] "+r" (count)
441 : [dst] "r"(dst), [src1] "r" (src1), [src2] "r" (src2),
442 [k] "m" (k)
443 : "cc", "memory",
444 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
445 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
446 );
447 }
448
449 #undef FMOP_K4_CORE
450
451 #define FMA_OFF(a, b) a
452 #define FMA_ON(a, b) b
453
454 #define FMOD_KX_CORE(DST, SRC1, SRC2, OP_SEL, FMA_SEL) \
455 __ASM_EMIT("vbroadcastss %[k], %%ymm0") \
456 __ASM_EMIT("xor %[off], %[off]") \
457 __ASM_EMIT("vmovaps %%ymm0, %%ymm1") \
458 /* x16 blocks */ \
459 __ASM_EMIT("sub $16, %[count]") \
460 __ASM_EMIT("jb 2f") \
461 __ASM_EMIT("1:") \
462 __ASM_EMIT("vmovups 0x00(%[" SRC1 "], %[off]), " OP_SEL("%%ymm2", "%%ymm6")) \
463 __ASM_EMIT("vmovups 0x20(%[" SRC1 "], %[off]), " OP_SEL("%%ymm3", "%%ymm7")) \
464 __ASM_EMIT("vmulps 0x00(%[" SRC2 "], %[off]), %%ymm0, " OP_SEL("%%ymm6", "%%ymm2")) \
465 __ASM_EMIT("vmulps 0x20(%[" SRC2 "], %[off]), %%ymm1, " OP_SEL("%%ymm7", "%%ymm3")) \
466 __ASM_EMIT("vdivps %%ymm6, %%ymm2, %%ymm4") \
467 __ASM_EMIT("vdivps %%ymm7, %%ymm3, %%ymm5") \
468 __ASM_EMIT("vcvttps2dq %%ymm4, %%ymm4") \
469 __ASM_EMIT("vcvttps2dq %%ymm5, %%ymm5") \
470 __ASM_EMIT("vcvtdq2ps %%ymm4, %%ymm4") \
471 __ASM_EMIT("vcvtdq2ps %%ymm5, %%ymm5") \
472 __ASM_EMIT(FMA_SEL("vmulps %%ymm6, %%ymm4, %%ymm4", "")) \
473 __ASM_EMIT(FMA_SEL("vmulps %%ymm7, %%ymm5, %%ymm5", "")) \
474 __ASM_EMIT(FMA_SEL("vsubps %%ymm4, %%ymm2, %%ymm2", "vfnmadd231ps %%ymm6, %%ymm4, %%ymm2")) \
475 __ASM_EMIT(FMA_SEL("vsubps %%ymm5, %%ymm3, %%ymm3", "vfnmadd231ps %%ymm7, %%ymm5, %%ymm3")) \
476 __ASM_EMIT("vmovups %%ymm2, 0x00(%[" DST "], %[off])") \
477 __ASM_EMIT("vmovups %%ymm3, 0x20(%[" DST "], %[off])") \
478 __ASM_EMIT("add $0x40, %[off]") \
479 __ASM_EMIT("sub $16, %[count]") \
480 __ASM_EMIT("jae 1b") \
481 __ASM_EMIT("2:") \
482 /* x8 block */ \
483 __ASM_EMIT("add $8, %[count]") \
484 __ASM_EMIT("jl 4f") \
485 __ASM_EMIT("vmovups 0x00(%[" SRC1 "], %[off]), " OP_SEL("%%xmm2", "%%xmm6")) \
486 __ASM_EMIT("vmovups 0x10(%[" SRC1 "], %[off]), " OP_SEL("%%xmm3", "%%xmm7")) \
487 __ASM_EMIT("vmulps 0x00(%[" SRC2 "], %[off]), %%xmm0, " OP_SEL("%%xmm6", "%%xmm2")) \
488 __ASM_EMIT("vmulps 0x10(%[" SRC2 "], %[off]), %%xmm1, " OP_SEL("%%xmm7", "%%xmm3")) \
489 __ASM_EMIT("vdivps %%xmm6, %%xmm2, %%xmm4") \
490 __ASM_EMIT("vdivps %%xmm7, %%xmm3, %%xmm5") \
491 __ASM_EMIT("vcvttps2dq %%xmm4, %%xmm4") \
492 __ASM_EMIT("vcvttps2dq %%xmm5, %%xmm5") \
493 __ASM_EMIT("vcvtdq2ps %%xmm4, %%xmm4") \
494 __ASM_EMIT("vcvtdq2ps %%xmm5, %%xmm5") \
495 __ASM_EMIT(FMA_SEL("vmulps %%xmm6, %%xmm4, %%xmm4", "")) \
496 __ASM_EMIT(FMA_SEL("vmulps %%xmm7, %%xmm5, %%xmm5", "")) \
497 __ASM_EMIT(FMA_SEL("vsubps %%xmm4, %%xmm2, %%xmm2", "vfnmadd231ps %%xmm6, %%xmm4, %%xmm2")) \
498 __ASM_EMIT(FMA_SEL("vsubps %%xmm5, %%xmm3, %%xmm3", "vfnmadd231ps %%xmm7, %%xmm5, %%xmm3")) \
499 __ASM_EMIT("vmovups %%xmm2, 0x00(%[" DST "], %[off])") \
500 __ASM_EMIT("vmovups %%xmm3, 0x10(%[" DST "], %[off])") \
501 __ASM_EMIT("sub $8, %[count]") \
502 __ASM_EMIT("add $0x20, %[off]") \
503 __ASM_EMIT("4:") \
504 /* x4 block */ \
505 __ASM_EMIT("add $4, %[count]") \
506 __ASM_EMIT("jl 6f") \
507 __ASM_EMIT("vmovups 0x00(%[" SRC1 "], %[off]), " OP_SEL("%%xmm2", "%%xmm6")) \
508 __ASM_EMIT("vmulps 0x00(%[" SRC2 "], %[off]), %%xmm0, " OP_SEL("%%xmm6", "%%xmm2")) \
509 __ASM_EMIT("vdivps %%xmm6, %%xmm2, %%xmm4") \
510 __ASM_EMIT("vcvttps2dq %%xmm4, %%xmm4") \
511 __ASM_EMIT("vcvtdq2ps %%xmm4, %%xmm4") \
512 __ASM_EMIT(FMA_SEL("vmulps %%xmm6, %%xmm4, %%xmm4", "")) \
513 __ASM_EMIT(FMA_SEL("vsubps %%xmm4, %%xmm2, %%xmm2", "vfnmadd231ps %%xmm6, %%xmm4, %%xmm2")) \
514 __ASM_EMIT("vmovups %%xmm2, 0x00(%[" DST "], %[off])") \
515 __ASM_EMIT("sub $4, %[count]") \
516 __ASM_EMIT("add $0x10, %[off]") \
517 __ASM_EMIT("6:") \
518 /* x1 blocks */ \
519 __ASM_EMIT("add $3, %[count]") \
520 __ASM_EMIT("jl 8f") \
521 __ASM_EMIT("7:") \
522 __ASM_EMIT("vmovss 0x00(%[" SRC1 "], %[off]), " OP_SEL("%%xmm2", "%%xmm6")) \
523 __ASM_EMIT("vmulss 0x00(%[" SRC2 "], %[off]), %%xmm0, " OP_SEL("%%xmm6", "%%xmm2")) \
524 __ASM_EMIT("vdivss %%xmm6, %%xmm2, %%xmm4") \
525 __ASM_EMIT("vcvttps2dq %%xmm4, %%xmm4") \
526 __ASM_EMIT("vcvtdq2ps %%xmm4, %%xmm4") \
527 __ASM_EMIT(FMA_SEL("vmulss %%xmm6, %%xmm4, %%xmm4", "")) \
528 __ASM_EMIT(FMA_SEL("vsubss %%xmm4, %%xmm2, %%xmm2", "vfnmadd231ss %%xmm6, %%xmm4, %%xmm2")) \
529 __ASM_EMIT("vmovss %%xmm2, 0x00(%[" DST "], %[off])") \
530 __ASM_EMIT("add $0x04, %[off]") \
531 __ASM_EMIT("dec %[count]") \
532 __ASM_EMIT("jge 7b") \
533 __ASM_EMIT("8:")
534
535 void fmmod_k3(float *dst, const float *src, float k, size_t count)
536 {
537 IF_ARCH_X86(size_t off);
538 ARCH_X86_ASM
539 (
540 FMOD_KX_CORE("dst", "dst", "src", OP_DSEL, FMA_OFF)
541 : [off] "=&r" (off), [count] "+r" (count)
542 : [dst] "r"(dst), [src] "r" (src),
543 [k] "m" (k)
544 : "cc", "memory",
545 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
546 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
547 );
548 }
549
550 void fmmod_k3_fma3(float *dst, const float *src, float k, size_t count)
551 {
552 IF_ARCH_X86(size_t off);
553 ARCH_X86_ASM
554 (
555 FMOD_KX_CORE("dst", "dst", "src", OP_DSEL, FMA_ON)
556 : [off] "=&r" (off), [count] "+r" (count)
557 : [dst] "r"(dst), [src] "r" (src),
558 [k] "m" (k)
559 : "cc", "memory",
560 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
561 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
562 );
563 }
564
565 void fmrmod_k3(float *dst, const float *src, float k, size_t count)
566 {
567 IF_ARCH_X86(size_t off);
568 ARCH_X86_ASM
569 (
570 FMOD_KX_CORE("dst", "dst", "src", OP_RSEL, FMA_OFF)
571 : [off] "=&r" (off), [count] "+r" (count)
572 : [dst] "r"(dst), [src] "r" (src),
573 [k] "m" (k)
574 : "cc", "memory",
575 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
576 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
577 );
578 }
579
580 void fmrmod_k3_fma3(float *dst, const float *src, float k, size_t count)
581 {
582 IF_ARCH_X86(size_t off);
583 ARCH_X86_ASM
584 (
585 FMOD_KX_CORE("dst", "dst", "src", OP_RSEL, FMA_ON)
586 : [off] "=&r" (off), [count] "+r" (count)
587 : [dst] "r"(dst), [src] "r" (src),
588 [k] "m" (k)
589 : "cc", "memory",
590 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
591 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
592 );
593 }
594
595 void fmmod_k4(float *dst, const float *src1, const float *src2, float k, size_t count)
596 {
597 IF_ARCH_X86(size_t off);
598 ARCH_X86_ASM
599 (
600 FMOD_KX_CORE("dst", "src1", "src2", OP_DSEL, FMA_OFF)
601 : [off] "=&r" (off), [count] "+r" (count)
602 : [dst] "r"(dst), [src1] "r" (src1), [src2] "r" (src2),
603 [k] "m" (k)
604 : "cc", "memory",
605 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
606 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
607 );
608 }
609
610 void fmmod_k4_fma3(float *dst, const float *src1, const float *src2, float k, size_t count)
611 {
612 IF_ARCH_X86(size_t off);
613 ARCH_X86_ASM
614 (
615 FMOD_KX_CORE("dst", "src1", "src2", OP_DSEL, FMA_ON)
616 : [off] "=&r" (off), [count] "+r" (count)
617 : [dst] "r"(dst), [src1] "r" (src1), [src2] "r" (src2),
618 [k] "m" (k)
619 : "cc", "memory",
620 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
621 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
622 );
623 }
624
625 void fmrmod_k4(float *dst, const float *src1, const float *src2, float k, size_t count)
626 {
627 IF_ARCH_X86(size_t off);
628 ARCH_X86_ASM
629 (
630 FMOD_KX_CORE("dst", "src1", "src2", OP_RSEL, FMA_OFF)
631 : [off] "=&r" (off), [count] "+r" (count)
632 : [dst] "r"(dst), [src1] "r" (src1), [src2] "r" (src2),
633 [k] "m" (k)
634 : "cc", "memory",
635 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
636 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
637 );
638 }
639
640 void fmrmod_k4_fma3(float *dst, const float *src1, const float *src2, float k, size_t count)
641 {
642 IF_ARCH_X86(size_t off);
643 ARCH_X86_ASM
644 (
645 FMOD_KX_CORE("dst", "src1", "src2", OP_RSEL, FMA_ON)
646 : [off] "=&r" (off), [count] "+r" (count)
647 : [dst] "r"(dst), [src1] "r" (src1), [src2] "r" (src2),
648 [k] "m" (k)
649 : "cc", "memory",
650 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
651 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
652 );
653 }
654
655 #undef FMOD_KX_CORE
656
657 #undef FMA_OFF
658 #undef FMA_ON
659
660 #undef OP_DSEL
661 #undef OP_RSEL
662 } /* namespace avx */
663 } /* namespace lsp */
664
665 #endif /* PRIVATE_DSP_ARCH_X86_AVX_PMATH_FMOP_KX_H_ */