Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/x86/avx2/pmath/fmop_kx.h

raw
1 /*
2 * Copyright (C) 2020 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2020 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 31 мар. 2020 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_X86_AVX2_PMATH_FMOP_KX_H_
23 #define PRIVATE_DSP_ARCH_X86_AVX2_PMATH_FMOP_KX_H_
24
25 #ifndef PRIVATE_DSP_ARCH_X86_AVX2_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_X86_AVX2_IMPL */
28
29 namespace lsp
30 {
31 namespace avx2
32 {
33 #define OP_DSEL(a, b) a
34 #define OP_RSEL(a, b) b
35
36 #define FMADDSUB_K3_CORE(DST, SRC1, SRC2, OP) \
37 __ASM_EMIT("xor %[off], %[off]") \
38 __ASM_EMIT("vbroadcastss %%xmm0, %%ymm0") \
39 __ASM_EMIT("sub $32, %[count]") \
40 __ASM_EMIT("vmovaps %%ymm0, %%ymm1") \
41 __ASM_EMIT("jb 2f") \
42 /* 32x blocks */ \
43 __ASM_EMIT("1:") \
44 __ASM_EMIT("vmovups 0x000(%[" SRC1 "], %[off]), %%ymm4") \
45 __ASM_EMIT("vmovups 0x020(%[" SRC1 "], %[off]), %%ymm5") \
46 __ASM_EMIT("vmovups 0x040(%[" SRC1 "], %[off]), %%ymm6") \
47 __ASM_EMIT("vmovups 0x060(%[" SRC1 "], %[off]), %%ymm7") \
48 __ASM_EMIT(OP "ps 0x000(%[" SRC2 "], %[off]), %%ymm0, %%ymm4") \
49 __ASM_EMIT(OP "ps 0x020(%[" SRC2 "], %[off]), %%ymm1, %%ymm5") \
50 __ASM_EMIT(OP "ps 0x040(%[" SRC2 "], %[off]), %%ymm0, %%ymm6") \
51 __ASM_EMIT(OP "ps 0x060(%[" SRC2 "], %[off]), %%ymm1, %%ymm7") \
52 __ASM_EMIT("vmovups %%ymm4, 0x000(%[" DST "], %[off])") \
53 __ASM_EMIT("vmovups %%ymm5, 0x020(%[" DST "], %[off])") \
54 __ASM_EMIT("vmovups %%ymm6, 0x040(%[" DST "], %[off])") \
55 __ASM_EMIT("vmovups %%ymm7, 0x060(%[" DST "], %[off])") \
56 __ASM_EMIT("add $0x80, %[off]") \
57 __ASM_EMIT("sub $32, %[count]") \
58 __ASM_EMIT("jae 1b") \
59 /* 16x block */ \
60 __ASM_EMIT("2:") \
61 __ASM_EMIT("add $16, %[count]") /* 32 - 16 */ \
62 __ASM_EMIT("jl 4f") \
63 __ASM_EMIT("vmovups 0x000(%[" SRC1 "], %[off]), %%ymm4") \
64 __ASM_EMIT("vmovups 0x020(%[" SRC1 "], %[off]), %%ymm5") \
65 __ASM_EMIT(OP "ps 0x000(%[" SRC2 "], %[off]), %%ymm0, %%ymm4") \
66 __ASM_EMIT(OP "ps 0x020(%[" SRC2 "], %[off]), %%ymm1, %%ymm5") \
67 __ASM_EMIT("vmovups %%ymm4, 0x000(%[" DST "], %[off])") \
68 __ASM_EMIT("vmovups %%ymm5, 0x020(%[" DST "], %[off])") \
69 __ASM_EMIT("add $0x40, %[off]") \
70 __ASM_EMIT("sub $16, %[count]") \
71 /* 8x block */ \
72 __ASM_EMIT("4:") \
73 __ASM_EMIT("add $8, %[count]") /* 16 - 8 */ \
74 __ASM_EMIT("jl 6f") \
75 __ASM_EMIT("vmovups 0x000(%[" SRC1 "], %[off]), %%ymm4") \
76 __ASM_EMIT(OP "ps 0x000(%[" SRC2 "], %[off]), %%ymm0, %%ymm4") \
77 __ASM_EMIT("vmovups %%ymm4, 0x000(%[" DST "], %[off])") \
78 __ASM_EMIT("add $0x20, %[off]") \
79 __ASM_EMIT("sub $8, %[count]") \
80 \
81 /* 4x block */ \
82 __ASM_EMIT("6:") \
83 __ASM_EMIT("add $4, %[count]") /* 8 - 4 */ \
84 __ASM_EMIT("jl 8f") \
85 __ASM_EMIT("vmovups 0x000(%[" SRC1 "], %[off]), %%xmm4") \
86 __ASM_EMIT(OP "ps 0x000(%[" SRC2 "], %[off]), %%xmm0, %%xmm4") \
87 __ASM_EMIT("vmovups %%xmm4, 0x000(%[" DST "], %[off])") \
88 __ASM_EMIT("add $0x10, %[off]") \
89 __ASM_EMIT("sub $4, %[count]") \
90 \
91 /* 1x blocks */ \
92 __ASM_EMIT("8:") \
93 __ASM_EMIT("add $3, %[count]") /* 8 - 4 */ \
94 __ASM_EMIT("jl 10f") \
95 __ASM_EMIT("9:") \
96 __ASM_EMIT("vmovss 0x000(%[" SRC1 "], %[off]), %%xmm4") \
97 __ASM_EMIT(OP "ss 0x000(%[" SRC2 "], %[off]), %%xmm0, %%xmm4") \
98 __ASM_EMIT("vmovss %%xmm4, 0x000(%[" DST "], %[off])") \
99 __ASM_EMIT("add $0x04, %[off]") \
100 __ASM_EMIT("dec %[count]") \
101 __ASM_EMIT("jge 9b") \
102 __ASM_EMIT("10:")
103
104 void fmadd_k3_fma3(float *dst, const float *src, float k, size_t count)
105 {
106 IF_ARCH_X86(size_t off);
107 ARCH_X86_ASM
108 (
109 FMADDSUB_K3_CORE("dst", "dst", "src", "vfmadd231")
110 : [off] "=&r" (off), [count] "+r" (count),
111 [k] "+Yz"(k)
112 : [dst] "r"(dst), [src] "r"(src)
113 : "cc", "memory",
114 "%xmm1",
115 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
116 );
117 }
118
119 void fmsub_k3_fma3(float *dst, const float *src, float k, size_t count)
120 {
121 IF_ARCH_X86(size_t off);
122 ARCH_X86_ASM
123 (
124 FMADDSUB_K3_CORE("dst", "dst", "src", "vfnmadd231")
125 : [off] "=&r" (off), [count] "+r" (count),
126 [k] "+Yz"(k)
127 : [dst] "r"(dst), [src] "r"(src)
128 : "cc", "memory",
129 "%xmm1",
130 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
131 );
132 }
133
134 void fmrsub_k3_fma3(float *dst, const float *src, float k, size_t count)
135 {
136 IF_ARCH_X86(size_t off);
137 ARCH_X86_ASM
138 (
139 FMADDSUB_K3_CORE("dst", "dst", "src", "vfmsub231")
140 : [off] "=&r" (off), [count] "+r" (count),
141 [k] "+Yz"(k)
142 : [dst] "r"(dst), [src] "r"(src)
143 : "cc", "memory",
144 "%xmm1",
145 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
146 );
147 }
148
149 void fmadd_k4_fma3(float *dst, const float *src1, const float *src2, float k, size_t count)
150 {
151 IF_ARCH_X86(size_t off);
152 ARCH_X86_ASM
153 (
154 FMADDSUB_K3_CORE("dst", "src1", "src2", "vfmadd231")
155 : [off] "=&r" (off), [count] "+r" (count),
156 [k] "+Yz" (k)
157 : [dst] "r"(dst), [src1] "r" (src1), [src2] "r" (src2)
158 : "cc", "memory",
159 "%xmm1",
160 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
161 );
162 }
163
164 void fmsub_k4_fma3(float *dst, const float *src1, const float *src2, float k, size_t count)
165 {
166 IF_ARCH_X86(size_t off);
167 ARCH_X86_ASM
168 (
169 FMADDSUB_K3_CORE("dst", "src1", "src2", "vfnmadd231")
170 : [off] "=&r" (off), [count] "+r" (count),
171 [k] "+Yz" (k)
172 : [dst] "r"(dst), [src1] "r" (src1), [src2] "r" (src2)
173 : "cc", "memory",
174 "%xmm1",
175 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
176 );
177 }
178
179 void fmrsub_k4_fma3(float *dst, const float *src1, const float *src2, float k, size_t count)
180 {
181 IF_ARCH_X86(size_t off);
182 ARCH_X86_ASM
183 (
184 FMADDSUB_K3_CORE("dst", "src1", "src2", "vfmsub231")
185 : [off] "=&r" (off), [count] "+r" (count),
186 [k] "+Yz" (k)
187 : [dst] "r"(dst), [src1] "r" (src1), [src2] "r" (src2)
188 : "cc", "memory",
189 "%xmm1",
190 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
191 );
192 }
193
194 #undef FMADDSUB_K3_CORE
195
196 #define FMOP_K4_CORE(DST, SRC1, SRC2, OP, SEL) \
197 __ASM_EMIT("xor %[off], %[off]") \
198 __ASM_EMIT("vbroadcastss %%xmm0, %%ymm0") \
199 __ASM_EMIT("sub $32, %[count]") \
200 __ASM_EMIT("vmovaps %%ymm0, %%ymm1") \
201 __ASM_EMIT("jb 2f") \
202 /* 24x blocks */ \
203 __ASM_EMIT("1:") \
204 __ASM_EMIT("vmovups 0x000(%[" SRC1 "], %[off]), %%ymm5") \
205 __ASM_EMIT("vmovups 0x020(%[" SRC1 "], %[off]), %%ymm6") \
206 __ASM_EMIT("vmovups 0x040(%[" SRC1 "], %[off]), %%ymm7") \
207 __ASM_EMIT("vmulps 0x000(%[" SRC2 "], %[off]), %%ymm0, %%ymm2") \
208 __ASM_EMIT("vmulps 0x020(%[" SRC2 "], %[off]), %%ymm1, %%ymm3") \
209 __ASM_EMIT("vmulps 0x040(%[" SRC2 "], %[off]), %%ymm0, %%ymm4") \
210 __ASM_EMIT(OP "ps " SEL("%%ymm2", "%%ymm5") ", " SEL("%%ymm5", "%%ymm2") ", %%ymm5") \
211 __ASM_EMIT(OP "ps " SEL("%%ymm3", "%%ymm6") ", " SEL("%%ymm6", "%%ymm3") ", %%ymm6") \
212 __ASM_EMIT(OP "ps " SEL("%%ymm4", "%%ymm7") ", " SEL("%%ymm7", "%%ymm4") ", %%ymm7") \
213 __ASM_EMIT("vmovups %%ymm5, 0x000(%[" DST "], %[off])") \
214 __ASM_EMIT("vmovups %%ymm6, 0x020(%[" DST "], %[off])") \
215 __ASM_EMIT("vmovups %%ymm7, 0x040(%[" DST "], %[off])") \
216 __ASM_EMIT("add $0x60, %[off]") \
217 __ASM_EMIT("sub $24, %[count]") \
218 __ASM_EMIT("jae 1b") \
219 /* 16x block */ \
220 __ASM_EMIT("2:") \
221 __ASM_EMIT("add $16, %[count]") /* 32 - 16 */ \
222 __ASM_EMIT("jl 4f") \
223 __ASM_EMIT("vmovups 0x000(%[" SRC1 "], %[off]), %%ymm6") \
224 __ASM_EMIT("vmovups 0x020(%[" SRC1 "], %[off]), %%ymm7") \
225 __ASM_EMIT("vmulps 0x000(%[" SRC2 "], %[off]), %%ymm0, %%ymm2") \
226 __ASM_EMIT("vmulps 0x020(%[" SRC2 "], %[off]), %%ymm1, %%ymm3") \
227 __ASM_EMIT(OP "ps " SEL("%%ymm2", "%%ymm6") ", " SEL("%%ymm6", "%%ymm2") ", %%ymm6") \
228 __ASM_EMIT(OP "ps " SEL("%%ymm3", "%%ymm7") ", " SEL("%%ymm7", "%%ymm3") ", %%ymm7") \
229 __ASM_EMIT("vmovups %%ymm6, 0x000(%[" DST "], %[off])") \
230 __ASM_EMIT("vmovups %%ymm7, 0x020(%[" DST "], %[off])") \
231 __ASM_EMIT("add $0x40, %[off]") \
232 __ASM_EMIT("sub $16, %[count]") \
233 /* 8x block */ \
234 __ASM_EMIT("4:") \
235 __ASM_EMIT("add $8, %[count]") /* 16 - 8 */ \
236 __ASM_EMIT("jl 6f") \
237 __ASM_EMIT("vmovups 0x000(%[" SRC1 "], %[off]), %%ymm6") \
238 __ASM_EMIT("vmulps 0x000(%[" SRC2 "], %[off]), %%ymm0, %%ymm2") \
239 __ASM_EMIT(OP "ps " SEL("%%ymm2", "%%ymm6") ", " SEL("%%ymm6", "%%ymm2") ", %%ymm6") \
240 __ASM_EMIT("vmovups %%ymm6, 0x000(%[" DST "], %[off])") \
241 __ASM_EMIT("add $0x20, %[off]") \
242 __ASM_EMIT("sub $8, %[count]") \
243 \
244 /* 4x block */ \
245 __ASM_EMIT("6:") \
246 __ASM_EMIT("add $4, %[count]") /* 8 - 4 */ \
247 __ASM_EMIT("jl 8f") \
248 __ASM_EMIT("vmovups 0x000(%[" SRC1 "], %[off]), %%xmm6") \
249 __ASM_EMIT("vmulps 0x000(%[" SRC2 "], %[off]), %%xmm0, %%xmm2") \
250 __ASM_EMIT(OP "ps " SEL("%%xmm2", "%%xmm6") ", " SEL("%%xmm6", "%%xmm2") ", %%xmm6") \
251 __ASM_EMIT("vmovups %%xmm6, 0x000(%[" DST "], %[off])") \
252 __ASM_EMIT("add $0x10, %[off]") \
253 __ASM_EMIT("sub $4, %[count]") \
254 \
255 /* 1x blocks */ \
256 __ASM_EMIT("8:") \
257 __ASM_EMIT("add $3, %[count]") /* 8 - 4 */ \
258 __ASM_EMIT("jl 10f") \
259 __ASM_EMIT("9:") \
260 __ASM_EMIT("vmovss 0x000(%[" SRC1 "], %[off]), %%xmm6") \
261 __ASM_EMIT("vmulss 0x000(%[" SRC2 "], %[off]), %%xmm0, %%xmm2") \
262 __ASM_EMIT(OP "ss " SEL("%%xmm2", "%%xmm6") ", " SEL("%%xmm6", "%%xmm2") ", %%xmm6") \
263 __ASM_EMIT("vmovss %%xmm6, 0x000(%[" DST "], %[off])") \
264 __ASM_EMIT("add $0x04, %[off]") \
265 __ASM_EMIT("dec %[count]") \
266 __ASM_EMIT("jge 9b") \
267 __ASM_EMIT("10:")
268
269 void fmadd_k3(float *dst, const float *src, float k, size_t count)
270 {
271 IF_ARCH_X86(size_t off);
272 ARCH_X86_ASM
273 (
274 FMOP_K4_CORE("dst", "dst", "src", "vadd", OP_DSEL)
275 : [off] "=&r" (off), [count] "+r" (count),
276 [k] "+Yz"(k)
277 : [dst] "r"(dst), [src] "r"(src)
278 : "cc", "memory",
279 "%xmm1", "%xmm2", "%xmm3",
280 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
281 );
282 }
283
284 void fmsub_k3(float *dst, const float *src, float k, size_t count)
285 {
286 IF_ARCH_X86(size_t off);
287 ARCH_X86_ASM
288 (
289 FMOP_K4_CORE("dst", "dst", "src", "vsub", OP_DSEL)
290 : [off] "=&r" (off), [count] "+r" (count),
291 [k] "+Yz"(k)
292 : [dst] "r"(dst), [src] "r"(src)
293 : "cc", "memory",
294 "%xmm1", "%xmm2", "%xmm3",
295 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
296 );
297 }
298
299 void fmrsub_k3(float *dst, const float *src, float k, size_t count)
300 {
301 IF_ARCH_X86(size_t off);
302 ARCH_X86_ASM
303 (
304 FMOP_K4_CORE("dst", "dst", "src", "vsub", OP_RSEL)
305 : [off] "=&r" (off), [count] "+r" (count),
306 [k] "+Yz"(k)
307 : [dst] "r"(dst), [src] "r"(src)
308 : "cc", "memory",
309 "%xmm1", "%xmm2", "%xmm3",
310 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
311 );
312 }
313
314 void fmmul_k3(float *dst, const float *src, float k, size_t count)
315 {
316 IF_ARCH_X86(size_t off);
317 ARCH_X86_ASM
318 (
319 FMOP_K4_CORE("dst", "dst", "src", "vmul", OP_DSEL)
320 : [off] "=&r" (off), [count] "+r" (count),
321 [k] "+Yz"(k)
322 : [dst] "r"(dst), [src] "r"(src)
323 : "cc", "memory",
324 "%xmm1", "%xmm2", "%xmm3",
325 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
326 );
327 }
328
329 void fmdiv_k3(float *dst, const float *src, float k, size_t count)
330 {
331 IF_ARCH_X86(size_t off);
332 ARCH_X86_ASM
333 (
334 FMOP_K4_CORE("dst", "dst", "src", "vdiv", OP_DSEL)
335 : [off] "=&r" (off), [count] "+r" (count),
336 [k] "+Yz"(k)
337 : [dst] "r"(dst), [src] "r"(src)
338 : "cc", "memory",
339 "%xmm1", "%xmm2", "%xmm3",
340 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
341 );
342 }
343
344 void fmrdiv_k3(float *dst, const float *src, float k, size_t count)
345 {
346 IF_ARCH_X86(size_t off);
347 ARCH_X86_ASM
348 (
349 FMOP_K4_CORE("dst", "dst", "src", "vdiv", OP_RSEL)
350 : [off] "=&r" (off), [count] "+r" (count),
351 [k] "+Yz"(k)
352 : [dst] "r"(dst), [src] "r"(src)
353 : "cc", "memory",
354 "%xmm1", "%xmm2", "%xmm3",
355 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
356 );
357 }
358
359 void fmadd_k4(float *dst, const float *src1, const float *src2, float k, size_t count)
360 {
361 IF_ARCH_X86(size_t off);
362 ARCH_X86_ASM
363 (
364 FMOP_K4_CORE("dst", "src1", "src2", "vadd", OP_DSEL)
365 : [off] "=&r" (off), [count] "+r" (count),
366 [k] "+Yz" (k)
367 : [dst] "r"(dst), [src1] "r" (src1), [src2] "r" (src2)
368 : "cc", "memory",
369 "%xmm1", "%xmm2", "%xmm3",
370 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
371 );
372 }
373
374 void fmsub_k4(float *dst, const float *src1, const float *src2, float k, size_t count)
375 {
376 IF_ARCH_X86(size_t off);
377 ARCH_X86_ASM
378 (
379 FMOP_K4_CORE("dst", "src1", "src2", "vsub", OP_DSEL)
380 : [off] "=&r" (off), [count] "+r" (count),
381 [k] "+Yz" (k)
382 : [dst] "r"(dst), [src1] "r" (src1), [src2] "r" (src2)
383 : "cc", "memory",
384 "%xmm1", "%xmm2", "%xmm3",
385 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
386 );
387 }
388
389 void fmrsub_k4(float *dst, const float *src1, const float *src2, float k, size_t count)
390 {
391 IF_ARCH_X86(size_t off);
392 ARCH_X86_ASM
393 (
394 FMOP_K4_CORE("dst", "src1", "src2", "vsub", OP_RSEL)
395 : [off] "=&r" (off), [count] "+r" (count),
396 [k] "+Yz" (k)
397 : [dst] "r"(dst), [src1] "r" (src1), [src2] "r" (src2)
398 : "cc", "memory",
399 "%xmm1", "%xmm2", "%xmm3",
400 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
401 );
402 }
403
404 void fmmul_k4(float *dst, const float *src1, const float *src2, float k, size_t count)
405 {
406 IF_ARCH_X86(size_t off);
407 ARCH_X86_ASM
408 (
409 FMOP_K4_CORE("dst", "src1", "src2", "vmul", OP_DSEL)
410 : [off] "=&r" (off), [count] "+r" (count),
411 [k] "+Yz" (k)
412 : [dst] "r"(dst), [src1] "r" (src1), [src2] "r" (src2)
413 : "cc", "memory",
414 "%xmm1", "%xmm2", "%xmm3",
415 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
416 );
417 }
418
419 void fmdiv_k4(float *dst, const float *src1, const float *src2, float k, size_t count)
420 {
421 IF_ARCH_X86(size_t off);
422 ARCH_X86_ASM
423 (
424 FMOP_K4_CORE("dst", "src1", "src2", "vdiv", OP_DSEL)
425 : [off] "=&r" (off), [count] "+r" (count),
426 [k] "+Yz" (k)
427 : [dst] "r"(dst), [src1] "r" (src1), [src2] "r" (src2)
428 : "cc", "memory",
429 "%xmm1", "%xmm2", "%xmm3",
430 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
431 );
432 }
433
434 void fmrdiv_k4(float *dst, const float *src1, const float *src2, float k, size_t count)
435 {
436 IF_ARCH_X86(size_t off);
437 ARCH_X86_ASM
438 (
439 FMOP_K4_CORE("dst", "src1", "src2", "vdiv", OP_RSEL)
440 : [off] "=&r" (off), [count] "+r" (count),
441 [k] "+Yz" (k)
442 : [dst] "r"(dst), [src1] "r" (src1), [src2] "r" (src2)
443 : "cc", "memory",
444 "%xmm1", "%xmm2", "%xmm3",
445 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
446 );
447 }
448
449 #undef FMOP_K4_CORE
450
451 #define FMA_OFF(a, b) a
452 #define FMA_ON(a, b) b
453
454 #define FMOD_KX_CORE(DST, SRC1, SRC2, OP_SEL, FMA_SEL) \
455 __ASM_EMIT("vbroadcastss %%xmm0, %%ymm0") \
456 __ASM_EMIT("xor %[off], %[off]") \
457 __ASM_EMIT("vmovaps %%ymm0, %%ymm1") \
458 /* x16 blocks */ \
459 __ASM_EMIT("sub $16, %[count]") \
460 __ASM_EMIT("jb 2f") \
461 __ASM_EMIT("1:") \
462 __ASM_EMIT("vmovups 0x00(%[" SRC1 "], %[off]), " OP_SEL("%%ymm2", "%%ymm6")) \
463 __ASM_EMIT("vmovups 0x20(%[" SRC1 "], %[off]), " OP_SEL("%%ymm3", "%%ymm7")) \
464 __ASM_EMIT("vmulps 0x00(%[" SRC2 "], %[off]), %%ymm0, " OP_SEL("%%ymm6", "%%ymm2")) \
465 __ASM_EMIT("vmulps 0x20(%[" SRC2 "], %[off]), %%ymm1, " OP_SEL("%%ymm7", "%%ymm3")) \
466 __ASM_EMIT("vdivps %%ymm6, %%ymm2, %%ymm4") \
467 __ASM_EMIT("vdivps %%ymm7, %%ymm3, %%ymm5") \
468 __ASM_EMIT("vcvttps2dq %%ymm4, %%ymm4") \
469 __ASM_EMIT("vcvttps2dq %%ymm5, %%ymm5") \
470 __ASM_EMIT("vcvtdq2ps %%ymm4, %%ymm4") \
471 __ASM_EMIT("vcvtdq2ps %%ymm5, %%ymm5") \
472 __ASM_EMIT(FMA_SEL("vmulps %%ymm6, %%ymm4, %%ymm4", "")) \
473 __ASM_EMIT(FMA_SEL("vmulps %%ymm7, %%ymm5, %%ymm5", "")) \
474 __ASM_EMIT(FMA_SEL("vsubps %%ymm4, %%ymm2, %%ymm2", "vfnmadd231ps %%ymm6, %%ymm4, %%ymm2")) \
475 __ASM_EMIT(FMA_SEL("vsubps %%ymm5, %%ymm3, %%ymm3", "vfnmadd231ps %%ymm7, %%ymm5, %%ymm3")) \
476 __ASM_EMIT("vmovups %%ymm2, 0x00(%[" DST "], %[off])") \
477 __ASM_EMIT("vmovups %%ymm3, 0x20(%[" DST "], %[off])") \
478 __ASM_EMIT("add $0x40, %[off]") \
479 __ASM_EMIT("sub $16, %[count]") \
480 __ASM_EMIT("jae 1b") \
481 __ASM_EMIT("2:") \
482 /* x8 block */ \
483 __ASM_EMIT("add $8, %[count]") \
484 __ASM_EMIT("jl 4f") \
485 __ASM_EMIT("vmovups 0x00(%[" SRC1 "], %[off]), " OP_SEL("%%xmm2", "%%xmm6")) \
486 __ASM_EMIT("vmovups 0x10(%[" SRC1 "], %[off]), " OP_SEL("%%xmm3", "%%xmm7")) \
487 __ASM_EMIT("vmulps 0x00(%[" SRC2 "], %[off]), %%xmm0, " OP_SEL("%%xmm6", "%%xmm2")) \
488 __ASM_EMIT("vmulps 0x10(%[" SRC2 "], %[off]), %%xmm1, " OP_SEL("%%xmm7", "%%xmm3")) \
489 __ASM_EMIT("vdivps %%xmm6, %%xmm2, %%xmm4") \
490 __ASM_EMIT("vdivps %%xmm7, %%xmm3, %%xmm5") \
491 __ASM_EMIT("vcvttps2dq %%xmm4, %%xmm4") \
492 __ASM_EMIT("vcvttps2dq %%xmm5, %%xmm5") \
493 __ASM_EMIT("vcvtdq2ps %%xmm4, %%xmm4") \
494 __ASM_EMIT("vcvtdq2ps %%xmm5, %%xmm5") \
495 __ASM_EMIT(FMA_SEL("vmulps %%xmm6, %%xmm4, %%xmm4", "")) \
496 __ASM_EMIT(FMA_SEL("vmulps %%xmm7, %%xmm5, %%xmm5", "")) \
497 __ASM_EMIT(FMA_SEL("vsubps %%xmm4, %%xmm2, %%xmm2", "vfnmadd231ps %%xmm6, %%xmm4, %%xmm2")) \
498 __ASM_EMIT(FMA_SEL("vsubps %%xmm5, %%xmm3, %%xmm3", "vfnmadd231ps %%xmm7, %%xmm5, %%xmm3")) \
499 __ASM_EMIT("vmovups %%xmm2, 0x00(%[" DST "], %[off])") \
500 __ASM_EMIT("vmovups %%xmm3, 0x10(%[" DST "], %[off])") \
501 __ASM_EMIT("sub $8, %[count]") \
502 __ASM_EMIT("add $0x20, %[off]") \
503 __ASM_EMIT("4:") \
504 /* x4 block */ \
505 __ASM_EMIT("add $4, %[count]") \
506 __ASM_EMIT("jl 6f") \
507 __ASM_EMIT("vmovups 0x00(%[" SRC1 "], %[off]), " OP_SEL("%%xmm2", "%%xmm6")) \
508 __ASM_EMIT("vmulps 0x00(%[" SRC2 "], %[off]), %%xmm0, " OP_SEL("%%xmm6", "%%xmm2")) \
509 __ASM_EMIT("vdivps %%xmm6, %%xmm2, %%xmm4") \
510 __ASM_EMIT("vcvttps2dq %%xmm4, %%xmm4") \
511 __ASM_EMIT("vcvtdq2ps %%xmm4, %%xmm4") \
512 __ASM_EMIT(FMA_SEL("vmulps %%xmm6, %%xmm4, %%xmm4", "")) \
513 __ASM_EMIT(FMA_SEL("vsubps %%xmm4, %%xmm2, %%xmm2", "vfnmadd231ps %%xmm6, %%xmm4, %%xmm2")) \
514 __ASM_EMIT("vmovups %%xmm2, 0x00(%[" DST "], %[off])") \
515 __ASM_EMIT("sub $4, %[count]") \
516 __ASM_EMIT("add $0x10, %[off]") \
517 __ASM_EMIT("6:") \
518 /* x1 blocks */ \
519 __ASM_EMIT("add $3, %[count]") \
520 __ASM_EMIT("jl 8f") \
521 __ASM_EMIT("7:") \
522 __ASM_EMIT("vmovss 0x00(%[" SRC1 "], %[off]), " OP_SEL("%%xmm2", "%%xmm6")) \
523 __ASM_EMIT("vmulss 0x00(%[" SRC2 "], %[off]), %%xmm0, " OP_SEL("%%xmm6", "%%xmm2")) \
524 __ASM_EMIT("vdivss %%xmm6, %%xmm2, %%xmm4") \
525 __ASM_EMIT("vcvttps2dq %%xmm4, %%xmm4") \
526 __ASM_EMIT("vcvtdq2ps %%xmm4, %%xmm4") \
527 __ASM_EMIT(FMA_SEL("vmulss %%xmm6, %%xmm4, %%xmm4", "")) \
528 __ASM_EMIT(FMA_SEL("vsubss %%xmm4, %%xmm2, %%xmm2", "vfnmadd231ss %%xmm6, %%xmm4, %%xmm2")) \
529 __ASM_EMIT("vmovss %%xmm2, 0x00(%[" DST "], %[off])") \
530 __ASM_EMIT("add $0x04, %[off]") \
531 __ASM_EMIT("dec %[count]") \
532 __ASM_EMIT("jge 7b") \
533 __ASM_EMIT("8:")
534
535 void fmmod_k3(float *dst, const float *src, float k, size_t count)
536 {
537 IF_ARCH_X86(size_t off);
538 ARCH_X86_ASM
539 (
540 FMOD_KX_CORE("dst", "dst", "src", OP_DSEL, FMA_OFF)
541 : [off] "=&r" (off), [count] "+r" (count), [k] "+Yz" (k)
542 : [dst] "r"(dst), [src] "r" (src)
543 : "cc", "memory",
544 "%xmm1", "%xmm2", "%xmm3",
545 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
546 );
547 }
548
549 void fmmod_k3_fma3(float *dst, const float *src, float k, size_t count)
550 {
551 IF_ARCH_X86(size_t off);
552 ARCH_X86_ASM
553 (
554 FMOD_KX_CORE("dst", "dst", "src", OP_DSEL, FMA_ON)
555 : [off] "=&r" (off), [count] "+r" (count), [k] "+Yz" (k)
556 : [dst] "r"(dst), [src] "r" (src)
557 : "cc", "memory",
558 "%xmm1", "%xmm2", "%xmm3",
559 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
560 );
561 }
562
563 void fmrmod_k3(float *dst, const float *src, float k, size_t count)
564 {
565 IF_ARCH_X86(size_t off);
566 ARCH_X86_ASM
567 (
568 FMOD_KX_CORE("dst", "dst", "src", OP_RSEL, FMA_OFF)
569 : [off] "=&r" (off), [count] "+r" (count), [k] "+Yz" (k)
570 : [dst] "r"(dst), [src] "r" (src)
571 : "cc", "memory",
572 "%xmm1", "%xmm2", "%xmm3",
573 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
574 );
575 }
576
577 void fmrmod_k3_fma3(float *dst, const float *src, float k, size_t count)
578 {
579 IF_ARCH_X86(size_t off);
580 ARCH_X86_ASM
581 (
582 FMOD_KX_CORE("dst", "dst", "src", OP_RSEL, FMA_ON)
583 : [off] "=&r" (off), [count] "+r" (count), [k] "+Yz" (k)
584 : [dst] "r"(dst), [src] "r" (src)
585 : "cc", "memory",
586 "%xmm1", "%xmm2", "%xmm3",
587 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
588 );
589 }
590
591 void fmmod_k4(float *dst, const float *src1, const float *src2, float k, size_t count)
592 {
593 IF_ARCH_X86(size_t off);
594 ARCH_X86_ASM
595 (
596 FMOD_KX_CORE("dst", "src1", "src2", OP_DSEL, FMA_OFF)
597 : [off] "=&r" (off), [count] "+r" (count), [k] "+Yz" (k)
598 : [dst] "r"(dst), [src1] "r" (src1), [src2] "r" (src2)
599 : "cc", "memory",
600 "%xmm1", "%xmm2", "%xmm3",
601 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
602 );
603 }
604
605 void fmmod_k4_fma3(float *dst, const float *src1, const float *src2, float k, size_t count)
606 {
607 IF_ARCH_X86(size_t off);
608 ARCH_X86_ASM
609 (
610 FMOD_KX_CORE("dst", "src1", "src2", OP_DSEL, FMA_ON)
611 : [off] "=&r" (off), [count] "+r" (count), [k] "+Yz" (k)
612 : [dst] "r"(dst), [src1] "r" (src1), [src2] "r" (src2)
613 : "cc", "memory",
614 "%xmm1", "%xmm2", "%xmm3",
615 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
616 );
617 }
618
619 void fmrmod_k4(float *dst, const float *src1, const float *src2, float k, size_t count)
620 {
621 IF_ARCH_X86(size_t off);
622 ARCH_X86_ASM
623 (
624 FMOD_KX_CORE("dst", "src1", "src2", OP_RSEL, FMA_OFF)
625 : [off] "=&r" (off), [count] "+r" (count), [k] "+Yz" (k)
626 : [dst] "r"(dst), [src1] "r" (src1), [src2] "r" (src2)
627 : "cc", "memory",
628 "%xmm1", "%xmm2", "%xmm3",
629 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
630 );
631 }
632
633 void fmrmod_k4_fma3(float *dst, const float *src1, const float *src2, float k, size_t count)
634 {
635 IF_ARCH_X86(size_t off);
636 ARCH_X86_ASM
637 (
638 FMOD_KX_CORE("dst", "src1", "src2", OP_RSEL, FMA_ON)
639 : [off] "=&r" (off), [count] "+r" (count), [k] "+Yz" (k)
640 : [dst] "r"(dst), [src1] "r" (src1), [src2] "r" (src2)
641 : "cc", "memory",
642 "%xmm1", "%xmm2", "%xmm3",
643 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
644 );
645 }
646
647 #undef FMOD_KX_CORE
648
649 #undef OP_DSEL
650 #undef OP_RSEL
651 }
652 }
653
654 #endif /* PRIVATE_DSP_ARCH_X86_AVX2_PMATH_FMOP_KX_H_ */