Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/x86/avx2/pmath/op_kx.h

raw
1 /*
2 * Copyright (C) 2020 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2020 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 31 мар. 2020 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_X86_AVX2_PMATH_OP_KX_H_
23 #define PRIVATE_DSP_ARCH_X86_AVX2_PMATH_OP_KX_H_
24
25 #ifndef PRIVATE_DSP_ARCH_X86_AVX2_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_X86_AVX2_IMPL */
28
29 namespace lsp
30 {
31 namespace avx2
32 {
33 #define OP_DSEL(a, b) a
34 #define OP_RSEL(a, b) b
35
36 #define OP_K4_CORE(DST, SRC, OP, SEL) \
37 __ASM_EMIT("xor %[off], %[off]") \
38 __ASM_EMIT("vbroadcastss %%xmm0, %%ymm0") \
39 __ASM_EMIT("sub $32, %[count]") \
40 __ASM_EMIT("vmovaps %%ymm0, %%ymm1") \
41 __ASM_EMIT("jb 2f") \
42 /* 32x blocks */ \
43 __ASM_EMIT("1:") \
44 __ASM_EMIT(SEL("", "vmovups 0x000(%[" SRC "], %[off]), %%ymm4")) \
45 __ASM_EMIT(SEL("", "vmovups 0x020(%[" SRC "], %[off]), %%ymm5")) \
46 __ASM_EMIT(SEL("", "vmovups 0x040(%[" SRC "], %[off]), %%ymm6")) \
47 __ASM_EMIT(SEL("", "vmovups 0x060(%[" SRC "], %[off]), %%ymm7")) \
48 __ASM_EMIT(OP "ps " SEL("0x000(%[" SRC "], %[off])", "%%ymm0") ", " SEL("%%ymm0", "%%ymm4") ", %%ymm4") \
49 __ASM_EMIT(OP "ps " SEL("0x020(%[" SRC "], %[off])", "%%ymm1") ", " SEL("%%ymm1", "%%ymm5") ", %%ymm5") \
50 __ASM_EMIT(OP "ps " SEL("0x040(%[" SRC "], %[off])", "%%ymm0") ", " SEL("%%ymm0", "%%ymm6") ", %%ymm6") \
51 __ASM_EMIT(OP "ps " SEL("0x060(%[" SRC "], %[off])", "%%ymm1") ", " SEL("%%ymm1", "%%ymm7") ", %%ymm7") \
52 __ASM_EMIT("vmovups %%ymm4, 0x000(%[" DST "], %[off])") \
53 __ASM_EMIT("vmovups %%ymm5, 0x020(%[" DST "], %[off])") \
54 __ASM_EMIT("vmovups %%ymm6, 0x040(%[" DST "], %[off])") \
55 __ASM_EMIT("vmovups %%ymm7, 0x060(%[" DST "], %[off])") \
56 __ASM_EMIT("add $0x80, %[off]") \
57 __ASM_EMIT("sub $32, %[count]") \
58 __ASM_EMIT("jae 1b") \
59 /* 16x block */ \
60 __ASM_EMIT("2:") \
61 __ASM_EMIT("add $16, %[count]") /* 32 - 16 */ \
62 __ASM_EMIT("jl 4f") \
63 __ASM_EMIT(SEL("", "vmovups 0x000(%[" SRC "], %[off]), %%ymm4")) \
64 __ASM_EMIT(SEL("", "vmovups 0x020(%[" SRC "], %[off]), %%ymm5")) \
65 __ASM_EMIT(OP "ps " SEL("0x000(%[" SRC "], %[off])", "%%ymm0") ", " SEL("%%ymm0", "%%ymm4") ", %%ymm4") \
66 __ASM_EMIT(OP "ps " SEL("0x020(%[" SRC "], %[off])", "%%ymm1") ", " SEL("%%ymm1", "%%ymm5") ", %%ymm5") \
67 __ASM_EMIT("vmovups %%ymm4, 0x000(%[" DST "], %[off])") \
68 __ASM_EMIT("vmovups %%ymm5, 0x020(%[" DST "], %[off])") \
69 __ASM_EMIT("add $0x40, %[off]") \
70 __ASM_EMIT("sub $16, %[count]") \
71 /* 8x block */ \
72 __ASM_EMIT("4:") \
73 __ASM_EMIT("add $8, %[count]") /* 16 - 8 */ \
74 __ASM_EMIT("jl 6f") \
75 __ASM_EMIT(SEL("", "vmovups 0x000(%[" SRC "], %[off]), %%ymm4")) \
76 __ASM_EMIT(OP "ps " SEL("0x000(%[" SRC "], %[off])", "%%ymm0") ", " SEL("%%ymm0", "%%ymm4") ", %%ymm4") \
77 __ASM_EMIT("vmovups %%ymm4, 0x000(%[" DST "], %[off])") \
78 __ASM_EMIT("add $0x20, %[off]") \
79 __ASM_EMIT("sub $8, %[count]") \
80 \
81 /* 4x block */ \
82 __ASM_EMIT("6:") \
83 __ASM_EMIT("add $4, %[count]") /* 8 - 4 */ \
84 __ASM_EMIT("jl 8f") \
85 __ASM_EMIT(SEL("", "vmovups 0x000(%[" SRC "], %[off]), %%xmm4")) \
86 __ASM_EMIT(OP "ps " SEL("0x000(%[" SRC "], %[off])", "%%xmm0") ", " SEL("%%xmm0", "%%xmm4") ", %%xmm4") \
87 __ASM_EMIT("vmovups %%xmm4, 0x000(%[" DST "], %[off])") \
88 __ASM_EMIT("add $0x10, %[off]") \
89 __ASM_EMIT("sub $4, %[count]") \
90 \
91 /* 1x blocks */ \
92 __ASM_EMIT("8:") \
93 __ASM_EMIT("add $3, %[count]") /* 8 - 4 */ \
94 __ASM_EMIT("jl 10f") \
95 __ASM_EMIT("9:") \
96 __ASM_EMIT(SEL("", "vmovss 0x000(%[" SRC "], %[off]), %%xmm4")) \
97 __ASM_EMIT(OP "ss " SEL("0x000(%[" SRC "], %[off])", "%%xmm0") ", " SEL("%%xmm0", "%%xmm4") ", %%xmm4") \
98 __ASM_EMIT("vmovss %%xmm4, 0x000(%[" DST "], %[off])") \
99 __ASM_EMIT("add $0x04, %[off]") \
100 __ASM_EMIT("dec %[count]") \
101 __ASM_EMIT("jge 9b") \
102 __ASM_EMIT("10:")
103
104 void add_k2(float *dst, float k, size_t count)
105 {
106 IF_ARCH_X86(size_t off);
107 ARCH_X86_ASM
108 (
109 OP_K4_CORE("dst", "dst", "vadd", OP_DSEL)
110 : [count] "+r" (count), [off] "=&r" (off),
111 [k] "+Yz" (k)
112 : [dst] "r" (dst)
113 : "cc", "memory",
114 "%xmm1",
115 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
116 );
117 }
118
119 void sub_k2(float *dst, float k, size_t count)
120 {
121 IF_ARCH_X86(size_t off);
122 ARCH_X86_ASM
123 (
124 OP_K4_CORE("dst", "dst", "vsub", OP_RSEL)
125 : [count] "+r" (count), [off] "=&r" (off),
126 [k] "+Yz" (k)
127 : [dst] "r" (dst)
128 : "cc", "memory",
129 "%xmm1",
130 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
131 );
132 }
133
134 void rsub_k2(float *dst, float k, size_t count)
135 {
136 IF_ARCH_X86(size_t off);
137 ARCH_X86_ASM
138 (
139 OP_K4_CORE("dst", "dst", "vsub", OP_DSEL)
140 : [count] "+r" (count), [off] "=&r" (off),
141 [k] "+Yz" (k)
142 : [dst] "r" (dst)
143 : "cc", "memory",
144 "%xmm1",
145 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
146 );
147 }
148
149 void mul_k2(float *dst, float k, size_t count)
150 {
151 IF_ARCH_X86(size_t off);
152 ARCH_X86_ASM
153 (
154 OP_K4_CORE("dst", "dst", "vmul", OP_DSEL)
155 : [count] "+r" (count), [off] "=&r" (off),
156 [k] "+Yz" (k)
157 : [dst] "r" (dst)
158 : "cc", "memory",
159 "%xmm1",
160 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
161 );
162 }
163
164 void div_k2(float *dst, float k, size_t count)
165 {
166 mul_k2(dst, 1.0f/k, count);
167 }
168
169 void rdiv_k2(float *dst, float k, size_t count)
170 {
171 IF_ARCH_X86(size_t off);
172 ARCH_X86_ASM
173 (
174 OP_K4_CORE("dst", "dst", "vdiv", OP_DSEL)
175 : [count] "+r" (count), [off] "=&r" (off),
176 [k] "+Yz" (k)
177 : [dst] "r" (dst)
178 : "cc", "memory",
179 "%xmm1",
180 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
181 );
182 }
183
184 void add_k3(float *dst, const float *src, float k, size_t count)
185 {
186 IF_ARCH_X86(size_t off);
187 ARCH_X86_ASM
188 (
189 OP_K4_CORE("dst", "src", "vadd", OP_DSEL)
190 : [count] "+r" (count), [off] "=&r" (off),
191 [k] "+Yz" (k)
192 : [dst] "r" (dst), [src] "r" (src)
193 : "cc", "memory",
194 "%xmm1",
195 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
196 );
197 }
198
199 void sub_k3(float *dst, const float *src, float k, size_t count)
200 {
201 IF_ARCH_X86(size_t off);
202 ARCH_X86_ASM
203 (
204 OP_K4_CORE("dst", "src", "vsub", OP_RSEL)
205 : [count] "+r" (count), [off] "=&r" (off),
206 [k] "+Yz" (k)
207 : [dst] "r" (dst), [src] "r" (src)
208 : "cc", "memory",
209 "%xmm1",
210 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
211 );
212 }
213
214 void rsub_k3(float *dst, const float *src, float k, size_t count)
215 {
216 IF_ARCH_X86(size_t off);
217 ARCH_X86_ASM
218 (
219 OP_K4_CORE("dst", "src", "vsub", OP_DSEL)
220 : [count] "+r" (count), [off] "=&r" (off),
221 [k] "+Yz" (k)
222 : [dst] "r" (dst), [src] "r" (src)
223 : "cc", "memory",
224 "%xmm1",
225 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
226 );
227 }
228
229 void mul_k3(float *dst, const float *src, float k, size_t count)
230 {
231 IF_ARCH_X86(size_t off);
232 ARCH_X86_ASM
233 (
234 OP_K4_CORE("dst", "src", "vmul", OP_DSEL)
235 : [count] "+r" (count), [off] "=&r" (off),
236 [k] "+Yz" (k)
237 : [dst] "r" (dst), [src] "r" (src)
238 : "cc", "memory",
239 "%xmm1",
240 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
241 );
242 }
243
244 void div_k3(float *dst, const float *src, float k, size_t count)
245 {
246 mul_k3(dst, src, 1.0f/k, count);
247 }
248
249 void rdiv_k3(float *dst, const float *src, float k, size_t count)
250 {
251 IF_ARCH_X86(size_t off);
252 ARCH_X86_ASM
253 (
254 OP_K4_CORE("dst", "src", "vdiv", OP_DSEL)
255 : [count] "+r" (count), [off] "=&r" (off),
256 [k] "+Yz" (k)
257 : [dst] "r" (dst), [src] "r" (src)
258 : "cc", "memory",
259 "%xmm1",
260 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
261 );
262 }
263
264 #undef OP_K4_CORE
265 #undef OP_DSEL
266 #undef OP_RSEL
267
268 #define FMA_OFF(a, b) a
269 #define FMA_ON(a, b) b
270
271 #define FMOD_KX_CORE(DST, SRC, FMA_SEL) \
272 __ASM_EMIT("vbroadcastss %%xmm0, %%ymm0") \
273 __ASM_EMIT("xor %[off], %[off]") \
274 __ASM_EMIT("vmovaps %%ymm0, %%ymm1") \
275 /* x16 blocks */ \
276 __ASM_EMIT("sub $16, %[count]") \
277 __ASM_EMIT("jb 2f") \
278 __ASM_EMIT("1:") \
279 __ASM_EMIT("vmovups 0x00(%[" SRC "], %[off]), %%ymm2") \
280 __ASM_EMIT("vmovups 0x20(%[" SRC "], %[off]), %%ymm3") \
281 __ASM_EMIT("vdivps %%ymm0, %%ymm2, %%ymm4") \
282 __ASM_EMIT("vdivps %%ymm1, %%ymm3, %%ymm5") \
283 __ASM_EMIT("vcvttps2dq %%ymm4, %%ymm4") \
284 __ASM_EMIT("vcvttps2dq %%ymm5, %%ymm5") \
285 __ASM_EMIT("vcvtdq2ps %%ymm4, %%ymm4") \
286 __ASM_EMIT("vcvtdq2ps %%ymm5, %%ymm5") \
287 __ASM_EMIT(FMA_SEL("vmulps %%ymm0, %%ymm4, %%ymm4", "")) \
288 __ASM_EMIT(FMA_SEL("vmulps %%ymm1, %%ymm5, %%ymm5", "")) \
289 __ASM_EMIT(FMA_SEL("vsubps %%ymm4, %%ymm2, %%ymm2", "vfnmadd231ps %%ymm0, %%ymm4, %%ymm2")) \
290 __ASM_EMIT(FMA_SEL("vsubps %%ymm5, %%ymm3, %%ymm3", "vfnmadd231ps %%ymm1, %%ymm5, %%ymm3")) \
291 __ASM_EMIT("vmovups %%ymm2, 0x00(%[" DST "], %[off])") \
292 __ASM_EMIT("vmovups %%ymm3, 0x20(%[" DST "], %[off])") \
293 __ASM_EMIT("add $0x40, %[off]") \
294 __ASM_EMIT("sub $16, %[count]") \
295 __ASM_EMIT("jae 1b") \
296 __ASM_EMIT("2:") \
297 /* x8 block */ \
298 __ASM_EMIT("add $8, %[count]") \
299 __ASM_EMIT("jl 4f") \
300 __ASM_EMIT("vmovups 0x00(%[" SRC "], %[off]), %%xmm2") \
301 __ASM_EMIT("vmovups 0x10(%[" SRC "], %[off]), %%xmm3") \
302 __ASM_EMIT("vdivps %%xmm0, %%xmm2, %%xmm4") \
303 __ASM_EMIT("vdivps %%xmm1, %%xmm3, %%xmm5") \
304 __ASM_EMIT("vcvttps2dq %%xmm4, %%xmm4") \
305 __ASM_EMIT("vcvttps2dq %%xmm5, %%xmm5") \
306 __ASM_EMIT("vcvtdq2ps %%xmm4, %%xmm4") \
307 __ASM_EMIT("vcvtdq2ps %%xmm5, %%xmm5") \
308 __ASM_EMIT(FMA_SEL("vmulps %%xmm0, %%xmm4, %%xmm4", "")) \
309 __ASM_EMIT(FMA_SEL("vmulps %%xmm1, %%xmm5, %%xmm5", "")) \
310 __ASM_EMIT(FMA_SEL("vsubps %%xmm4, %%xmm2, %%xmm2", "vfnmadd231ps %%xmm0, %%xmm4, %%xmm2")) \
311 __ASM_EMIT(FMA_SEL("vsubps %%xmm5, %%xmm3, %%xmm3", "vfnmadd231ps %%xmm1, %%xmm5, %%xmm3")) \
312 __ASM_EMIT("vmovups %%xmm2, 0x00(%[" DST "], %[off])") \
313 __ASM_EMIT("vmovups %%xmm3, 0x10(%[" DST "], %[off])") \
314 __ASM_EMIT("sub $8, %[count]") \
315 __ASM_EMIT("add $0x20, %[off]") \
316 __ASM_EMIT("4:") \
317 /* x4 block */ \
318 __ASM_EMIT("add $4, %[count]") \
319 __ASM_EMIT("jl 6f") \
320 __ASM_EMIT("vmovups 0x00(%[" SRC "], %[off]), %%xmm2") \
321 __ASM_EMIT("vdivps %%xmm0, %%xmm2, %%xmm4") \
322 __ASM_EMIT("vcvttps2dq %%xmm4, %%xmm4") \
323 __ASM_EMIT("vcvtdq2ps %%xmm4, %%xmm4") \
324 __ASM_EMIT(FMA_SEL("vmulps %%xmm0, %%xmm4, %%xmm4", "")) \
325 __ASM_EMIT(FMA_SEL("vsubps %%xmm4, %%xmm2, %%xmm2", "vfnmadd231ps %%xmm0, %%xmm4, %%xmm2")) \
326 __ASM_EMIT("vmovups %%xmm2, 0x00(%[" DST "], %[off])") \
327 __ASM_EMIT("sub $4, %[count]") \
328 __ASM_EMIT("add $0x10, %[off]") \
329 __ASM_EMIT("6:") \
330 /* x1 blocks */ \
331 __ASM_EMIT("add $3, %[count]") \
332 __ASM_EMIT("jl 8f") \
333 __ASM_EMIT("7:") \
334 __ASM_EMIT("vmovss 0x00(%[" SRC "], %[off]), %%xmm2") \
335 __ASM_EMIT("vdivps %%xmm0, %%xmm2, %%xmm4") \
336 __ASM_EMIT("vcvttps2dq %%xmm4, %%xmm4") \
337 __ASM_EMIT("vcvtdq2ps %%xmm4, %%xmm4") \
338 __ASM_EMIT(FMA_SEL("vmulps %%xmm0, %%xmm4, %%xmm4", "")) \
339 __ASM_EMIT(FMA_SEL("vsubps %%xmm4, %%xmm2, %%xmm2", "vfnmadd231ps %%xmm0, %%xmm4, %%xmm2")) \
340 __ASM_EMIT("vmovss %%xmm2, 0x00(%[" DST "], %[off])") \
341 __ASM_EMIT("add $0x04, %[off]") \
342 __ASM_EMIT("dec %[count]") \
343 __ASM_EMIT("jge 7b") \
344 __ASM_EMIT("8:")
345
346 void mod_k2(float *dst, float k, size_t count)
347 {
348 IF_ARCH_X86(size_t off);
349 ARCH_X86_ASM
350 (
351 FMOD_KX_CORE("dst", "dst", FMA_OFF)
352 : [count] "+r" (count), [off] "=&r" (off),
353 [k] "+Yz"(k)
354 : [dst] "r" (dst)
355 : "cc", "memory",
356 "%xmm1", "%xmm2", "%xmm3",
357 "%xmm4", "%xmm5"
358 );
359 }
360
361 void mod_k2_fma3(float *dst, float k, size_t count)
362 {
363 IF_ARCH_X86(size_t off);
364 ARCH_X86_ASM
365 (
366 FMOD_KX_CORE("dst", "dst", FMA_ON)
367 : [count] "+r" (count), [off] "=&r" (off),
368 [k] "+Yz"(k)
369 : [dst] "r" (dst)
370 : "cc", "memory",
371 "%xmm1", "%xmm2", "%xmm3",
372 "%xmm4", "%xmm5"
373 );
374 }
375
376 void mod_k3(float *dst, const float *src, float k, size_t count)
377 {
378 IF_ARCH_X86(size_t off);
379 ARCH_X86_ASM
380 (
381 FMOD_KX_CORE("dst", "src", FMA_OFF)
382 : [count] "+r" (count), [off] "=&r" (off),
383 [k] "+Yz"(k)
384 : [dst] "r" (dst), [src] "r" (src)
385 : "cc", "memory",
386 "%xmm1", "%xmm2", "%xmm3",
387 "%xmm4", "%xmm5"
388 );
389 }
390
391 void mod_k3_fma3(float *dst, const float *src, float k, size_t count)
392 {
393 IF_ARCH_X86(size_t off);
394 ARCH_X86_ASM
395 (
396 FMOD_KX_CORE("dst", "src", FMA_ON)
397 : [count] "+r" (count), [off] "=&r" (off),
398 [k] "+Yz"(k)
399 : [dst] "r" (dst), [src] "r" (src)
400 : "cc", "memory",
401 "%xmm1", "%xmm2", "%xmm3",
402 "%xmm4", "%xmm5"
403 );
404 }
405
406 #undef FMOD_KX_CORE
407
408 #define FRMOD_KX_CORE(DST, SRC, FMA_SEL) \
409 __ASM_EMIT("vbroadcastss %%xmm0, %%ymm0") \
410 __ASM_EMIT("xor %[off], %[off]") \
411 __ASM_EMIT("vmovaps %%ymm0, %%ymm1") \
412 /* x16 blocks */ \
413 __ASM_EMIT("sub $16, %[count]") \
414 __ASM_EMIT("jb 2f") \
415 __ASM_EMIT("1:") \
416 __ASM_EMIT("vmovups 0x00(%[" SRC "], %[off]), %%ymm2") \
417 __ASM_EMIT("vmovups 0x20(%[" SRC "], %[off]), %%ymm3") \
418 __ASM_EMIT("vdivps %%ymm2, %%ymm0, %%ymm4") \
419 __ASM_EMIT("vdivps %%ymm3, %%ymm1, %%ymm5") \
420 __ASM_EMIT("vcvttps2dq %%ymm4, %%ymm4") \
421 __ASM_EMIT("vcvttps2dq %%ymm5, %%ymm5") \
422 __ASM_EMIT("vcvtdq2ps %%ymm4, %%ymm4") \
423 __ASM_EMIT("vcvtdq2ps %%ymm5, %%ymm5") \
424 __ASM_EMIT(FMA_SEL("vmulps %%ymm2, %%ymm4, %%ymm4", "")) \
425 __ASM_EMIT(FMA_SEL("vmulps %%ymm3, %%ymm5, %%ymm5", "")) \
426 __ASM_EMIT(FMA_SEL("vsubps %%ymm4, %%ymm0, %%ymm2", "vfnmadd132ps %%ymm4, %%ymm0, %%ymm2")) \
427 __ASM_EMIT(FMA_SEL("vsubps %%ymm5, %%ymm1, %%ymm3", "vfnmadd132ps %%ymm5, %%ymm1, %%ymm3")) \
428 __ASM_EMIT("vmovups %%ymm2, 0x00(%[" DST "], %[off])") \
429 __ASM_EMIT("vmovups %%ymm3, 0x20(%[" DST "], %[off])") \
430 __ASM_EMIT("add $0x40, %[off]") \
431 __ASM_EMIT("sub $16, %[count]") \
432 __ASM_EMIT("jae 1b") \
433 __ASM_EMIT("2:") \
434 /* x8 block */ \
435 __ASM_EMIT("add $8, %[count]") \
436 __ASM_EMIT("jl 4f") \
437 __ASM_EMIT("vmovups 0x00(%[" SRC "], %[off]), %%xmm2") \
438 __ASM_EMIT("vmovups 0x10(%[" SRC "], %[off]), %%xmm3") \
439 __ASM_EMIT("vdivps %%xmm2, %%xmm0, %%xmm4") \
440 __ASM_EMIT("vdivps %%xmm3, %%xmm1, %%xmm5") \
441 __ASM_EMIT("vcvttps2dq %%xmm4, %%xmm4") \
442 __ASM_EMIT("vcvttps2dq %%xmm5, %%xmm5") \
443 __ASM_EMIT("vcvtdq2ps %%xmm4, %%xmm4") \
444 __ASM_EMIT("vcvtdq2ps %%xmm5, %%xmm5") \
445 __ASM_EMIT(FMA_SEL("vmulps %%xmm2, %%xmm4, %%xmm4", "")) \
446 __ASM_EMIT(FMA_SEL("vmulps %%xmm3, %%xmm5, %%xmm5", "")) \
447 __ASM_EMIT(FMA_SEL("vsubps %%xmm4, %%xmm0, %%xmm2", "vfnmadd132ps %%xmm4, %%xmm0, %%xmm2")) \
448 __ASM_EMIT(FMA_SEL("vsubps %%xmm5, %%xmm1, %%xmm3", "vfnmadd132ps %%xmm5, %%xmm1, %%xmm3")) \
449 __ASM_EMIT("vmovups %%xmm2, 0x00(%[" DST "], %[off])") \
450 __ASM_EMIT("vmovups %%xmm3, 0x10(%[" DST "], %[off])") \
451 __ASM_EMIT("sub $8, %[count]") \
452 __ASM_EMIT("add $0x20, %[off]") \
453 __ASM_EMIT("4:") \
454 /* x4 block */ \
455 __ASM_EMIT("add $4, %[count]") \
456 __ASM_EMIT("jl 6f") \
457 __ASM_EMIT("vmovups 0x00(%[" SRC "], %[off]), %%xmm2") \
458 __ASM_EMIT("vdivps %%xmm2, %%xmm0, %%xmm4") \
459 __ASM_EMIT("vcvttps2dq %%xmm4, %%xmm4") \
460 __ASM_EMIT("vcvtdq2ps %%xmm4, %%xmm4") \
461 __ASM_EMIT(FMA_SEL("vmulps %%xmm2, %%xmm4, %%xmm4", "")) \
462 __ASM_EMIT(FMA_SEL("vsubps %%xmm4, %%xmm0, %%xmm2", "vfnmadd132ps %%xmm4, %%xmm0, %%xmm2")) \
463 __ASM_EMIT("vmovups %%xmm2, 0x00(%[" DST "], %[off])") \
464 __ASM_EMIT("sub $4, %[count]") \
465 __ASM_EMIT("add $0x10, %[off]") \
466 __ASM_EMIT("6:") \
467 /* x1 blocks */ \
468 __ASM_EMIT("add $3, %[count]") \
469 __ASM_EMIT("jl 8f") \
470 __ASM_EMIT("7:") \
471 __ASM_EMIT("vmovss 0x00(%[" SRC "], %[off]), %%xmm2") \
472 __ASM_EMIT("vdivps %%xmm2, %%xmm0, %%xmm4") \
473 __ASM_EMIT("vcvttps2dq %%xmm4, %%xmm4") \
474 __ASM_EMIT("vcvtdq2ps %%xmm4, %%xmm4") \
475 __ASM_EMIT(FMA_SEL("vmulps %%xmm2, %%xmm4, %%xmm4", "")) \
476 __ASM_EMIT(FMA_SEL("vsubps %%xmm4, %%xmm0, %%xmm2", "vfnmadd132ps %%xmm4, %%xmm0, %%xmm2")) \
477 __ASM_EMIT("vmovss %%xmm2, 0x00(%[" DST "], %[off])") \
478 __ASM_EMIT("add $0x04, %[off]") \
479 __ASM_EMIT("dec %[count]") \
480 __ASM_EMIT("jge 7b") \
481 __ASM_EMIT("8:")
482
483 void rmod_k2(float *dst, float k, size_t count)
484 {
485 IF_ARCH_X86(size_t off);
486 ARCH_X86_ASM
487 (
488 FRMOD_KX_CORE("dst", "dst", FMA_OFF)
489 : [count] "+r" (count), [off] "=&r" (off),
490 [k] "+Yz"(k)
491 : [dst] "r" (dst)
492 : "cc", "memory",
493 "%xmm1", "%xmm2", "%xmm3",
494 "%xmm4", "%xmm5"
495 );
496 }
497
498 void rmod_k2_fma3(float *dst, float k, size_t count)
499 {
500 IF_ARCH_X86(size_t off);
501 ARCH_X86_ASM
502 (
503 FRMOD_KX_CORE("dst", "dst", FMA_ON)
504 : [count] "+r" (count), [off] "=&r" (off),
505 [k] "+Yz"(k)
506 : [dst] "r" (dst)
507 : "cc", "memory",
508 "%xmm1", "%xmm2", "%xmm3",
509 "%xmm4", "%xmm5"
510 );
511 }
512
513 void rmod_k3(float *dst, const float *src, float k, size_t count)
514 {
515 IF_ARCH_X86(size_t off);
516 ARCH_X86_ASM
517 (
518 FRMOD_KX_CORE("dst", "src", FMA_OFF)
519 : [count] "+r" (count), [off] "=&r" (off),
520 [k] "+Yz"(k)
521 : [dst] "r" (dst), [src] "r" (src)
522 : "cc", "memory",
523 "%xmm1", "%xmm2", "%xmm3",
524 "%xmm4", "%xmm5"
525 );
526 }
527
528 void rmod_k3_fma3(float *dst, const float *src, float k, size_t count)
529 {
530 IF_ARCH_X86(size_t off);
531 ARCH_X86_ASM
532 (
533 FRMOD_KX_CORE("dst", "src", FMA_ON)
534 : [count] "+r" (count), [off] "=&r" (off),
535 [k] "+Yz"(k)
536 : [dst] "r" (dst), [src] "r" (src)
537 : "cc", "memory",
538 "%xmm1", "%xmm2", "%xmm3",
539 "%xmm4", "%xmm5"
540 );
541 }
542
543 #undef FRMOD_KX_CORE
544 }
545 }
546
547 #endif /* PRIVATE_DSP_ARCH_X86_AVX2_PMATH_OP_KX_H_ */