Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/x86/avx/pmath/fmop_vv.h

raw
1 /*
2 * Copyright (C) 2020 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2020 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 31 мар. 2020 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_X86_AVX_PMATH_FMOP_VV_H_
23 #define PRIVATE_DSP_ARCH_X86_AVX_PMATH_FMOP_VV_H_
24
25 #ifndef PRIVATE_DSP_ARCH_X86_AVX_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_X86_AVX_IMPL */
28
29 namespace lsp
30 {
31 namespace avx
32 {
33 #define OP_DSEL(a, b) a
34 #define OP_RSEL(a, b) b
35
36 #define FMADDSUB_VV_CORE(DST, A, B, C, OP) \
37 __ASM_EMIT("xor %[off], %[off]") \
38 __ASM_EMIT("sub $32, %[count]") \
39 __ASM_EMIT("jb 2f") \
40 /* 32x blocks */ \
41 __ASM_EMIT("1:") \
42 __ASM_EMIT("vmovups 0x000(%[" A "], %[off]), %%ymm0") \
43 __ASM_EMIT("vmovups 0x020(%[" A "], %[off]), %%ymm1") \
44 __ASM_EMIT("vmovups 0x040(%[" A "], %[off]), %%ymm2") \
45 __ASM_EMIT("vmovups 0x060(%[" A "], %[off]), %%ymm3") \
46 __ASM_EMIT("vmovups 0x000(%[" B "], %[off]), %%ymm4") \
47 __ASM_EMIT("vmovups 0x020(%[" B "], %[off]), %%ymm5") \
48 __ASM_EMIT("vmovups 0x040(%[" B "], %[off]), %%ymm6") \
49 __ASM_EMIT("vmovups 0x060(%[" B "], %[off]), %%ymm7") \
50 __ASM_EMIT(OP "ps 0x000(%[" C "], %[off]), %%ymm4, %%ymm0") \
51 __ASM_EMIT(OP "ps 0x020(%[" C "], %[off]), %%ymm5, %%ymm1") \
52 __ASM_EMIT(OP "ps 0x040(%[" C "], %[off]), %%ymm6, %%ymm2") \
53 __ASM_EMIT(OP "ps 0x060(%[" C "], %[off]), %%ymm7, %%ymm3") \
54 __ASM_EMIT("vmovups %%ymm0, 0x000(%[" DST "], %[off])") \
55 __ASM_EMIT("vmovups %%ymm1, 0x020(%[" DST "], %[off])") \
56 __ASM_EMIT("vmovups %%ymm2, 0x040(%[" DST "], %[off])") \
57 __ASM_EMIT("vmovups %%ymm3, 0x060(%[" DST "], %[off])") \
58 __ASM_EMIT("add $0x80, %[off]") \
59 __ASM_EMIT("sub $32, %[count]") \
60 __ASM_EMIT("jae 1b") \
61 /* 16x block */ \
62 __ASM_EMIT("2:") \
63 __ASM_EMIT("add $16, %[count]") /* 32 - 16 */ \
64 __ASM_EMIT("jl 4f") \
65 __ASM_EMIT("vmovups 0x000(%[" A "], %[off]), %%ymm0") \
66 __ASM_EMIT("vmovups 0x020(%[" A "], %[off]), %%ymm1") \
67 __ASM_EMIT("vmovups 0x000(%[" B "], %[off]), %%ymm4") \
68 __ASM_EMIT("vmovups 0x020(%[" B "], %[off]), %%ymm5") \
69 __ASM_EMIT(OP "ps 0x000(%[" C "], %[off]), %%ymm4, %%ymm0") \
70 __ASM_EMIT(OP "ps 0x020(%[" C "], %[off]), %%ymm5, %%ymm1") \
71 __ASM_EMIT("vmovups %%ymm0, 0x000(%[" DST "], %[off])") \
72 __ASM_EMIT("vmovups %%ymm1, 0x020(%[" DST "], %[off])") \
73 __ASM_EMIT("add $0x40, %[off]") \
74 __ASM_EMIT("sub $16, %[count]") \
75 /* 8x block */ \
76 __ASM_EMIT("4:") \
77 __ASM_EMIT("add $8, %[count]") /* 16 - 8 */ \
78 __ASM_EMIT("jl 6f") \
79 __ASM_EMIT("vmovups 0x000(%[" A "], %[off]), %%ymm0") \
80 __ASM_EMIT("vmovups 0x000(%[" B "], %[off]), %%ymm4") \
81 __ASM_EMIT(OP "ps 0x000(%[" C "], %[off]), %%ymm4, %%ymm0") \
82 __ASM_EMIT("vmovups %%ymm0, 0x000(%[" DST "], %[off])") \
83 __ASM_EMIT("add $0x20, %[off]") \
84 __ASM_EMIT("sub $8, %[count]") \
85 \
86 /* 4x block */ \
87 __ASM_EMIT("6:") \
88 __ASM_EMIT("add $4, %[count]") /* 8 - 4 */ \
89 __ASM_EMIT("jl 8f") \
90 __ASM_EMIT("vmovups 0x000(%[" A "], %[off]), %%xmm0") \
91 __ASM_EMIT("vmovups 0x000(%[" B "], %[off]), %%xmm4") \
92 __ASM_EMIT(OP "ps 0x000(%[" C "], %[off]), %%xmm4, %%xmm0") \
93 __ASM_EMIT("vmovups %%xmm0, 0x000(%[" DST "], %[off])") \
94 __ASM_EMIT("add $0x10, %[off]") \
95 __ASM_EMIT("sub $4, %[count]") \
96 \
97 /* 1x blocks */ \
98 __ASM_EMIT("8:") \
99 __ASM_EMIT("add $3, %[count]") /* 8 - 4 */ \
100 __ASM_EMIT("jl 10f") \
101 __ASM_EMIT("9:") \
102 __ASM_EMIT("vmovss 0x000(%[" A "], %[off]), %%xmm0") \
103 __ASM_EMIT("vmovss 0x000(%[" B "], %[off]), %%xmm4") \
104 __ASM_EMIT(OP "ss 0x000(%[" C "], %[off]), %%xmm4, %%xmm0") \
105 __ASM_EMIT("vmovss %%xmm0, 0x000(%[" DST "], %[off])") \
106 __ASM_EMIT("add $0x04, %[off]") \
107 __ASM_EMIT("dec %[count]") \
108 __ASM_EMIT("jge 9b") \
109 __ASM_EMIT("10:")
110
111 void fmadd3_fma3(float *dst, const float *a, const float *b, size_t count)
112 {
113 IF_ARCH_X86(size_t off);
114 ARCH_X86_ASM
115 (
116 FMADDSUB_VV_CORE("dst", "dst", "a", "b", "vfmadd231")
117 : [off] "=&r" (off), [count] "+r" (count)
118 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b)
119 : "cc", "memory",
120 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
121 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
122 );
123 }
124
125 void fmsub3_fma3(float *dst, const float *a, const float *b, size_t count)
126 {
127 IF_ARCH_X86(size_t off);
128 ARCH_X86_ASM
129 (
130 FMADDSUB_VV_CORE("dst", "dst", "a", "b", "vfnmadd231")
131 : [off] "=&r" (off), [count] "+r" (count)
132 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b)
133 : "cc", "memory",
134 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
135 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
136 );
137 }
138
139 void fmrsub3_fma3(float *dst, const float *a, const float *b, size_t count)
140 {
141 IF_ARCH_X86(size_t off);
142 ARCH_X86_ASM
143 (
144 FMADDSUB_VV_CORE("dst", "dst", "a", "b", "vfmsub231")
145 : [off] "=&r" (off), [count] "+r" (count)
146 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b)
147 : "cc", "memory",
148 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
149 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
150 );
151 }
152
153 void fmadd4_fma3(float *dst, const float *a, const float *b, const float *c, size_t count)
154 {
155 IF_ARCH_X86(size_t off);
156 ARCH_X86_ASM
157 (
158 FMADDSUB_VV_CORE("dst", "a", "b", "c", "vfmadd231")
159 : [off] "=&r" (off), [count] "+r" (count)
160 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b), [c] "r" (c)
161 : "cc", "memory",
162 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
163 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
164 );
165 }
166
167 void fmsub4_fma3(float *dst, const float *a, const float *b, const float *c, size_t count)
168 {
169 IF_ARCH_X86(size_t off);
170 ARCH_X86_ASM
171 (
172 FMADDSUB_VV_CORE("dst", "a", "b", "c", "vfnmadd231")
173 : [off] "=&r" (off), [count] "+r" (count)
174 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b), [c] "r" (c)
175 : "cc", "memory",
176 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
177 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
178 );
179 }
180
181 void fmrsub4_fma3(float *dst, const float *a, const float *b, const float *c, size_t count)
182 {
183 IF_ARCH_X86(size_t off);
184 ARCH_X86_ASM
185 (
186 FMADDSUB_VV_CORE("dst", "a", "b", "c", "vfmsub231")
187 : [off] "=&r" (off), [count] "+r" (count)
188 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b), [c] "r" (c)
189 : "cc", "memory",
190 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
191 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
192 );
193 }
194
195 #undef FMADDSUB_VV_CORE
196
197 #define FMOP_VV_CORE(DST, A, B, C, OP, SEL) \
198 __ASM_EMIT("xor %[off], %[off]") \
199 __ASM_EMIT("sub $32, %[count]") \
200 __ASM_EMIT("jb 2f") \
201 /* 32x blocks */ \
202 __ASM_EMIT("1:") \
203 __ASM_EMIT("vmovups 0x000(%[" B "], %[off]), %%ymm4") \
204 __ASM_EMIT("vmovups 0x020(%[" B "], %[off]), %%ymm5") \
205 __ASM_EMIT("vmovups 0x040(%[" B "], %[off]), %%ymm6") \
206 __ASM_EMIT("vmovups 0x060(%[" B "], %[off]), %%ymm7") \
207 __ASM_EMIT("vmovups 0x000(%[" A "], %[off]), %%ymm0") \
208 __ASM_EMIT("vmovups 0x020(%[" A "], %[off]), %%ymm1") \
209 __ASM_EMIT("vmovups 0x040(%[" A "], %[off]), %%ymm2") \
210 __ASM_EMIT("vmovups 0x060(%[" A "], %[off]), %%ymm3") \
211 __ASM_EMIT("vmulps 0x000(%[" C "], %[off]), %%ymm4, %%ymm4") \
212 __ASM_EMIT("vmulps 0x020(%[" C "], %[off]), %%ymm5, %%ymm5") \
213 __ASM_EMIT("vmulps 0x040(%[" C "], %[off]), %%ymm6, %%ymm6") \
214 __ASM_EMIT("vmulps 0x060(%[" C "], %[off]), %%ymm7, %%ymm7") \
215 __ASM_EMIT(OP "ps " SEL("%%ymm4", "%%ymm0") ", " SEL("%%ymm0", "%%ymm4") ", %%ymm0") \
216 __ASM_EMIT(OP "ps " SEL("%%ymm5", "%%ymm1") ", " SEL("%%ymm1", "%%ymm5") ", %%ymm1") \
217 __ASM_EMIT(OP "ps " SEL("%%ymm6", "%%ymm2") ", " SEL("%%ymm2", "%%ymm6") ", %%ymm2") \
218 __ASM_EMIT(OP "ps " SEL("%%ymm7", "%%ymm3") ", " SEL("%%ymm3", "%%ymm7") ", %%ymm3") \
219 __ASM_EMIT("vmovups %%ymm0, 0x000(%[" DST "], %[off])") \
220 __ASM_EMIT("vmovups %%ymm1, 0x020(%[" DST "], %[off])") \
221 __ASM_EMIT("vmovups %%ymm2, 0x040(%[" DST "], %[off])") \
222 __ASM_EMIT("vmovups %%ymm3, 0x060(%[" DST "], %[off])") \
223 __ASM_EMIT("add $0x80, %[off]") \
224 __ASM_EMIT("sub $32, %[count]") \
225 __ASM_EMIT("jae 1b") \
226 /* 16x block */ \
227 __ASM_EMIT("2:") \
228 __ASM_EMIT("add $16, %[count]") /* 32 - 16 */ \
229 __ASM_EMIT("jl 4f") \
230 __ASM_EMIT("vmovups 0x000(%[" B "], %[off]), %%ymm4") \
231 __ASM_EMIT("vmovups 0x020(%[" B "], %[off]), %%ymm5") \
232 __ASM_EMIT("vmovups 0x000(%[" A "], %[off]), %%ymm0") \
233 __ASM_EMIT("vmovups 0x020(%[" A "], %[off]), %%ymm1") \
234 __ASM_EMIT("vmulps 0x000(%[" C "], %[off]), %%ymm4, %%ymm4") \
235 __ASM_EMIT("vmulps 0x020(%[" C "], %[off]), %%ymm5, %%ymm5") \
236 __ASM_EMIT(OP "ps " SEL("%%ymm4", "%%ymm0") ", " SEL("%%ymm0", "%%ymm4") ", %%ymm0") \
237 __ASM_EMIT(OP "ps " SEL("%%ymm5", "%%ymm1") ", " SEL("%%ymm1", "%%ymm5") ", %%ymm1") \
238 __ASM_EMIT("vmovups %%ymm0, 0x000(%[" DST "], %[off])") \
239 __ASM_EMIT("vmovups %%ymm1, 0x020(%[" DST "], %[off])") \
240 __ASM_EMIT("add $0x40, %[off]") \
241 __ASM_EMIT("sub $16, %[count]") \
242 /* 8x block */ \
243 __ASM_EMIT("4:") \
244 __ASM_EMIT("add $8, %[count]") /* 16 - 8 */ \
245 __ASM_EMIT("jl 6f") \
246 __ASM_EMIT("vmovups 0x000(%[" B "], %[off]), %%ymm4") \
247 __ASM_EMIT("vmovups 0x000(%[" A "], %[off]), %%ymm0") \
248 __ASM_EMIT("vmulps 0x000(%[" C "], %[off]), %%ymm4, %%ymm4") \
249 __ASM_EMIT(OP "ps " SEL("%%ymm4", "%%ymm0") ", " SEL("%%ymm0", "%%ymm4") ", %%ymm0") \
250 __ASM_EMIT("vmovups %%ymm0, 0x000(%[" DST "], %[off])") \
251 __ASM_EMIT("add $0x20, %[off]") \
252 __ASM_EMIT("sub $8, %[count]") \
253 \
254 /* 4x block */ \
255 __ASM_EMIT("6:") \
256 __ASM_EMIT("add $4, %[count]") /* 8 - 4 */ \
257 __ASM_EMIT("jl 8f") \
258 __ASM_EMIT("vmovups 0x000(%[" B "], %[off]), %%xmm4") \
259 __ASM_EMIT("vmovups 0x000(%[" A "], %[off]), %%xmm0") \
260 __ASM_EMIT("vmulps 0x000(%[" C "], %[off]), %%xmm4, %%xmm4") \
261 __ASM_EMIT(OP "ps " SEL("%%xmm4", "%%xmm0") ", " SEL("%%xmm0", "%%xmm4") ", %%xmm0") \
262 __ASM_EMIT("vmovups %%xmm0, 0x000(%[" DST "], %[off])") \
263 __ASM_EMIT("add $0x10, %[off]") \
264 __ASM_EMIT("sub $4, %[count]") \
265 \
266 /* 1x blocks */ \
267 __ASM_EMIT("8:") \
268 __ASM_EMIT("add $3, %[count]") /* 8 - 4 */ \
269 __ASM_EMIT("jl 10f") \
270 __ASM_EMIT("9:") \
271 __ASM_EMIT("vmovss 0x000(%[" B "], %[off]), %%xmm4") \
272 __ASM_EMIT("vmovss 0x000(%[" A "], %[off]), %%xmm0") \
273 __ASM_EMIT("vmulss 0x000(%[" C "], %[off]), %%xmm4, %%xmm4") \
274 __ASM_EMIT(OP "ss " SEL("%%xmm4", "%%xmm0") ", " SEL("%%xmm0", "%%xmm4") ", %%xmm0") \
275 __ASM_EMIT("vmovss %%xmm0, 0x000(%[" DST "], %[off])") \
276 __ASM_EMIT("add $0x04, %[off]") \
277 __ASM_EMIT("dec %[count]") \
278 __ASM_EMIT("jge 9b") \
279 __ASM_EMIT("10:")
280
281 void fmadd3(float *dst, const float *a, const float *b, size_t count)
282 {
283 IF_ARCH_X86(size_t off);
284 ARCH_X86_ASM
285 (
286 FMOP_VV_CORE("dst", "dst", "a", "b", "vadd", OP_DSEL)
287 : [off] "=&r" (off), [count] "+r" (count)
288 : [dst] "r"(dst), [a] "r" (a), [b] "r" (b)
289 : "cc", "memory",
290 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
291 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
292 );
293 }
294
295 void fmsub3(float *dst, const float *a, const float *b, size_t count)
296 {
297 IF_ARCH_X86(size_t off);
298 ARCH_X86_ASM
299 (
300 FMOP_VV_CORE("dst", "dst", "a", "b", "vsub", OP_DSEL)
301 : [off] "=&r" (off), [count] "+r" (count)
302 : [dst] "r"(dst), [a] "r" (a), [b] "r" (b)
303 : "cc", "memory",
304 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
305 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
306 );
307 }
308
309 void fmrsub3(float *dst, const float *a, const float *b, size_t count)
310 {
311 IF_ARCH_X86(size_t off);
312 ARCH_X86_ASM
313 (
314 FMOP_VV_CORE("dst", "dst", "a", "b", "vsub", OP_RSEL)
315 : [off] "=&r" (off), [count] "+r" (count)
316 : [dst] "r"(dst), [a] "r" (a), [b] "r" (b)
317 : "cc", "memory",
318 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
319 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
320 );
321 }
322
323 void fmmul3(float *dst, const float *a, const float *b, size_t count)
324 {
325 IF_ARCH_X86(size_t off);
326 ARCH_X86_ASM
327 (
328 FMOP_VV_CORE("dst", "dst", "a", "b", "vmul", OP_DSEL)
329 : [off] "=&r" (off), [count] "+r" (count)
330 : [dst] "r"(dst), [a] "r" (a), [b] "r" (b)
331 : "cc", "memory",
332 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
333 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
334 );
335 }
336
337 void fmdiv3(float *dst, const float *a, const float *b, size_t count)
338 {
339 IF_ARCH_X86(size_t off);
340 ARCH_X86_ASM
341 (
342 FMOP_VV_CORE("dst", "dst", "a", "b", "vdiv", OP_DSEL)
343 : [off] "=&r" (off), [count] "+r" (count)
344 : [dst] "r"(dst), [a] "r" (a), [b] "r" (b)
345 : "cc", "memory",
346 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
347 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
348 );
349 }
350
351 void fmrdiv3(float *dst, const float *a, const float *b, size_t count)
352 {
353 IF_ARCH_X86(size_t off);
354 ARCH_X86_ASM
355 (
356 FMOP_VV_CORE("dst", "dst", "a", "b", "vdiv", OP_RSEL)
357 : [off] "=&r" (off), [count] "+r" (count)
358 : [dst] "r"(dst), [a] "r" (a), [b] "r" (b)
359 : "cc", "memory",
360 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
361 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
362 );
363 }
364
365 void fmadd4(float *dst, const float *a, const float *b, const float *c, size_t count)
366 {
367 IF_ARCH_X86(size_t off);
368 ARCH_X86_ASM
369 (
370 FMOP_VV_CORE("dst", "a", "b", "c", "vadd", OP_DSEL)
371 : [off] "=&r" (off), [count] "+r" (count)
372 : [dst] "r"(dst), [a] "r" (a), [b] "r" (b), [c] "r" (c)
373 : "cc", "memory",
374 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
375 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
376 );
377 }
378
379 void fmsub4(float *dst, const float *a, const float *b, const float *c, size_t count)
380 {
381 IF_ARCH_X86(size_t off);
382 ARCH_X86_ASM
383 (
384 FMOP_VV_CORE("dst", "a", "b", "c", "vsub", OP_DSEL)
385 : [off] "=&r" (off), [count] "+r" (count)
386 : [dst] "r"(dst), [a] "r" (a), [b] "r" (b), [c] "r" (c)
387 : "cc", "memory",
388 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
389 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
390 );
391 }
392
393 void fmrsub4(float *dst, const float *a, const float *b, const float *c, size_t count)
394 {
395 IF_ARCH_X86(size_t off);
396 ARCH_X86_ASM
397 (
398 FMOP_VV_CORE("dst", "a", "b", "c", "vsub", OP_RSEL)
399 : [off] "=&r" (off), [count] "+r" (count)
400 : [dst] "r"(dst), [a] "r" (a), [b] "r" (b), [c] "r" (c)
401 : "cc", "memory",
402 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
403 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
404 );
405 }
406
407 void fmmul4(float *dst, const float *a, const float *b, const float *c, size_t count)
408 {
409 IF_ARCH_X86(size_t off);
410 ARCH_X86_ASM
411 (
412 FMOP_VV_CORE("dst", "a", "b", "c", "vmul", OP_DSEL)
413 : [off] "=&r" (off), [count] "+r" (count)
414 : [dst] "r"(dst), [a] "r" (a), [b] "r" (b), [c] "r" (c)
415 : "cc", "memory",
416 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
417 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
418 );
419 }
420
421 void fmdiv4(float *dst, const float *a, const float *b, const float *c, size_t count)
422 {
423 IF_ARCH_X86(size_t off);
424 ARCH_X86_ASM
425 (
426 FMOP_VV_CORE("dst", "a", "b", "c", "vdiv", OP_DSEL)
427 : [off] "=&r" (off), [count] "+r" (count)
428 : [dst] "r"(dst), [a] "r" (a), [b] "r" (b), [c] "r" (c)
429 : "cc", "memory",
430 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
431 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
432 );
433 }
434
435 void fmrdiv4(float *dst, const float *a, const float *b, const float *c, size_t count)
436 {
437 IF_ARCH_X86(size_t off);
438 ARCH_X86_ASM
439 (
440 FMOP_VV_CORE("dst", "a", "b", "c", "vdiv", OP_RSEL)
441 : [off] "=&r" (off), [count] "+r" (count)
442 : [dst] "r"(dst), [a] "r" (a), [b] "r" (b), [c] "r" (c)
443 : "cc", "memory",
444 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
445 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
446 );
447 }
448
449 #undef FMOP_VV_CORE
450
451 #define FMA_OFF(a, b) a
452 #define FMA_ON(a, b) b
453
454 #define FMOD_VV_CORE(DST, SRC1, SRC2, SRC3, OP_SEL, FMA_SEL) \
455 __ASM_EMIT("xor %[off], %[off]") \
456 /* x16 blocks */ \
457 __ASM_EMIT32("subl $16, %[count]") \
458 __ASM_EMIT64("sub $16, %[count]") \
459 __ASM_EMIT("jb 2f") \
460 __ASM_EMIT("1:") \
461 __ASM_EMIT("vmovups 0x00(%[" SRC3 "], %[off]), %%ymm0") \
462 __ASM_EMIT("vmovups 0x20(%[" SRC3 "], %[off]), %%ymm1") \
463 __ASM_EMIT("vmovups 0x00(%[" SRC1 "], %[off]), " OP_SEL("%%ymm2", "%%ymm6")) \
464 __ASM_EMIT("vmovups 0x20(%[" SRC1 "], %[off]), " OP_SEL("%%ymm3", "%%ymm7")) \
465 __ASM_EMIT("vmulps 0x00(%[" SRC2 "], %[off]), %%ymm0, " OP_SEL("%%ymm6", "%%ymm2")) \
466 __ASM_EMIT("vmulps 0x20(%[" SRC2 "], %[off]), %%ymm1, " OP_SEL("%%ymm7", "%%ymm3")) \
467 __ASM_EMIT("vdivps %%ymm6, %%ymm2, %%ymm4") \
468 __ASM_EMIT("vdivps %%ymm7, %%ymm3, %%ymm5") \
469 __ASM_EMIT("vcvttps2dq %%ymm4, %%ymm4") \
470 __ASM_EMIT("vcvttps2dq %%ymm5, %%ymm5") \
471 __ASM_EMIT("vcvtdq2ps %%ymm4, %%ymm4") \
472 __ASM_EMIT("vcvtdq2ps %%ymm5, %%ymm5") \
473 __ASM_EMIT(FMA_SEL("vmulps %%ymm6, %%ymm4, %%ymm4", "")) \
474 __ASM_EMIT(FMA_SEL("vmulps %%ymm7, %%ymm5, %%ymm5", "")) \
475 __ASM_EMIT(FMA_SEL("vsubps %%ymm4, %%ymm2, %%ymm2", "vfnmadd231ps %%ymm6, %%ymm4, %%ymm2")) \
476 __ASM_EMIT(FMA_SEL("vsubps %%ymm5, %%ymm3, %%ymm3", "vfnmadd231ps %%ymm7, %%ymm5, %%ymm3")) \
477 __ASM_EMIT("vmovups %%ymm2, 0x00(%[" DST "], %[off])") \
478 __ASM_EMIT("vmovups %%ymm3, 0x20(%[" DST "], %[off])") \
479 __ASM_EMIT("add $0x40, %[off]") \
480 __ASM_EMIT32("subl $16, %[count]") \
481 __ASM_EMIT64("sub $16, %[count]") \
482 __ASM_EMIT("jae 1b") \
483 __ASM_EMIT("2:") \
484 /* x8 block */ \
485 __ASM_EMIT32("addl $8, %[count]") \
486 __ASM_EMIT64("add $8, %[count]") \
487 __ASM_EMIT("jl 4f") \
488 __ASM_EMIT("vmovups 0x00(%[" SRC3 "], %[off]), %%xmm0") \
489 __ASM_EMIT("vmovups 0x10(%[" SRC3 "], %[off]), %%xmm1") \
490 __ASM_EMIT("vmovups 0x00(%[" SRC1 "], %[off]), " OP_SEL("%%xmm2", "%%xmm6")) \
491 __ASM_EMIT("vmovups 0x10(%[" SRC1 "], %[off]), " OP_SEL("%%xmm3", "%%xmm7")) \
492 __ASM_EMIT("vmulps 0x00(%[" SRC2 "], %[off]), %%xmm0, " OP_SEL("%%xmm6", "%%xmm2")) \
493 __ASM_EMIT("vmulps 0x10(%[" SRC2 "], %[off]), %%xmm1, " OP_SEL("%%xmm7", "%%xmm3")) \
494 __ASM_EMIT("vdivps %%xmm6, %%xmm2, %%xmm4") \
495 __ASM_EMIT("vdivps %%xmm7, %%xmm3, %%xmm5") \
496 __ASM_EMIT("vcvttps2dq %%xmm4, %%xmm4") \
497 __ASM_EMIT("vcvttps2dq %%xmm5, %%xmm5") \
498 __ASM_EMIT("vcvtdq2ps %%xmm4, %%xmm4") \
499 __ASM_EMIT("vcvtdq2ps %%xmm5, %%xmm5") \
500 __ASM_EMIT(FMA_SEL("vmulps %%xmm6, %%xmm4, %%xmm4", "")) \
501 __ASM_EMIT(FMA_SEL("vmulps %%xmm7, %%xmm5, %%xmm5", "")) \
502 __ASM_EMIT(FMA_SEL("vsubps %%xmm4, %%xmm2, %%xmm2", "vfnmadd231ps %%xmm6, %%xmm4, %%xmm2")) \
503 __ASM_EMIT(FMA_SEL("vsubps %%xmm5, %%xmm3, %%xmm3", "vfnmadd231ps %%xmm7, %%xmm5, %%xmm3")) \
504 __ASM_EMIT("vmovups %%xmm2, 0x00(%[" DST "], %[off])") \
505 __ASM_EMIT("vmovups %%xmm3, 0x10(%[" DST "], %[off])") \
506 __ASM_EMIT32("subl $8, %[count]") \
507 __ASM_EMIT64("sub $8, %[count]") \
508 __ASM_EMIT("add $0x20, %[off]") \
509 __ASM_EMIT("4:") \
510 /* x4 block */ \
511 __ASM_EMIT32("addl $4, %[count]") \
512 __ASM_EMIT64("add $4, %[count]") \
513 __ASM_EMIT("jl 6f") \
514 __ASM_EMIT("vmovups 0x00(%[" SRC3 "], %[off]), %%xmm0") \
515 __ASM_EMIT("vmovups 0x00(%[" SRC1 "], %[off]), " OP_SEL("%%xmm2", "%%xmm6")) \
516 __ASM_EMIT("vmulps 0x00(%[" SRC2 "], %[off]), %%xmm0, " OP_SEL("%%xmm6", "%%xmm2")) \
517 __ASM_EMIT("vdivps %%xmm6, %%xmm2, %%xmm4") \
518 __ASM_EMIT("vcvttps2dq %%xmm4, %%xmm4") \
519 __ASM_EMIT("vcvtdq2ps %%xmm4, %%xmm4") \
520 __ASM_EMIT(FMA_SEL("vmulps %%xmm6, %%xmm4, %%xmm4", "")) \
521 __ASM_EMIT(FMA_SEL("vsubps %%xmm4, %%xmm2, %%xmm2", "vfnmadd231ps %%xmm6, %%xmm4, %%xmm2")) \
522 __ASM_EMIT("vmovups %%xmm2, 0x00(%[" DST "], %[off])") \
523 __ASM_EMIT32("subl $4, %[count]") \
524 __ASM_EMIT64("sub $4, %[count]") \
525 __ASM_EMIT("add $0x10, %[off]") \
526 __ASM_EMIT("6:") \
527 /* x1 blocks */ \
528 __ASM_EMIT32("addl $3, %[count]") \
529 __ASM_EMIT64("add $3, %[count]") \
530 __ASM_EMIT("jl 8f") \
531 __ASM_EMIT("7:") \
532 __ASM_EMIT("vmovss 0x00(%[" SRC3 "], %[off]), %%xmm0") \
533 __ASM_EMIT("vmovss 0x00(%[" SRC1 "], %[off]), " OP_SEL("%%xmm2", "%%xmm6")) \
534 __ASM_EMIT("vmulss 0x00(%[" SRC2 "], %[off]), %%xmm0, " OP_SEL("%%xmm6", "%%xmm2")) \
535 __ASM_EMIT("vdivss %%xmm6, %%xmm2, %%xmm4") \
536 __ASM_EMIT("vcvttps2dq %%xmm4, %%xmm4") \
537 __ASM_EMIT("vcvtdq2ps %%xmm4, %%xmm4") \
538 __ASM_EMIT(FMA_SEL("vmulss %%xmm6, %%xmm4, %%xmm4", "")) \
539 __ASM_EMIT(FMA_SEL("vsubss %%xmm4, %%xmm2, %%xmm2", "vfnmadd231ss %%xmm6, %%xmm4, %%xmm2")) \
540 __ASM_EMIT("vmovss %%xmm2, 0x00(%[" DST "], %[off])") \
541 __ASM_EMIT("add $0x04, %[off]") \
542 __ASM_EMIT32("decl %[count]") \
543 __ASM_EMIT64("dec %[count]") \
544 __ASM_EMIT("jge 7b") \
545 __ASM_EMIT("8:")
546
547 void fmmod3(float *dst, const float *a, const float *b, size_t count)
548 {
549 IF_ARCH_X86(size_t off);
550 ARCH_X86_ASM
551 (
552 FMOD_VV_CORE("dst", "dst", "a", "b", OP_DSEL, FMA_OFF)
553 : [off] "=&r" (off), [count] "+r" (count)
554 : [dst] "r"(dst), [a] "r" (a), [b] "r" (b)
555 : "cc", "memory",
556 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
557 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
558 );
559 }
560
561 void fmmod3_fma3(float *dst, const float *a, const float *b, size_t count)
562 {
563 IF_ARCH_X86(size_t off);
564 ARCH_X86_ASM
565 (
566 FMOD_VV_CORE("dst", "dst", "a", "b", OP_DSEL, FMA_OFF)
567 : [off] "=&r" (off), [count] "+r" (count)
568 : [dst] "r"(dst), [a] "r" (a), [b] "r" (b)
569 : "cc", "memory",
570 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
571 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
572 );
573 }
574
575 void fmrmod3(float *dst, const float *a, const float *b, size_t count)
576 {
577 IF_ARCH_X86(size_t off);
578 ARCH_X86_ASM
579 (
580 FMOD_VV_CORE("dst", "dst", "a", "b", OP_RSEL, FMA_OFF)
581 : [off] "=&r" (off), [count] "+r" (count)
582 : [dst] "r"(dst), [a] "r" (a), [b] "r" (b)
583 : "cc", "memory",
584 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
585 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
586 );
587 }
588
589 void fmrmod3_fma3(float *dst, const float *a, const float *b, size_t count)
590 {
591 IF_ARCH_X86(size_t off);
592 ARCH_X86_ASM
593 (
594 FMOD_VV_CORE("dst", "dst", "a", "b", OP_RSEL, FMA_OFF)
595 : [off] "=&r" (off), [count] "+r" (count)
596 : [dst] "r"(dst), [a] "r" (a), [b] "r" (b)
597 : "cc", "memory",
598 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
599 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
600 );
601 }
602
603 void fmmod4(float *dst, const float *a, const float *b, const float *c, size_t count)
604 {
605 IF_ARCH_X86(size_t off);
606 ARCH_X86_ASM
607 (
608 FMOD_VV_CORE("dst", "a", "b", "c", OP_DSEL, FMA_OFF)
609 : [off] "=&r" (off), [count] __ASM_ARG_RW(count)
610 : [dst] "r"(dst), [a] "r" (a), [b] "r" (b), [c] "r" (c)
611 : "cc", "memory",
612 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
613 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
614 );
615 }
616
617 void fmmod4_fma3(float *dst, const float *a, const float *b, const float *c, size_t count)
618 {
619 IF_ARCH_X86(size_t off);
620 ARCH_X86_ASM
621 (
622 FMOD_VV_CORE("dst", "a", "b", "c", OP_DSEL, FMA_ON)
623 : [off] "=&r" (off), [count] __ASM_ARG_RW(count)
624 : [dst] "r"(dst), [a] "r" (a), [b] "r" (b), [c] "r" (c)
625 : "cc", "memory",
626 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
627 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
628 );
629 }
630
631 void fmrmod4(float *dst, const float *a, const float *b, const float *c, size_t count)
632 {
633 IF_ARCH_X86(size_t off);
634 ARCH_X86_ASM
635 (
636 FMOD_VV_CORE("dst", "a", "b", "c", OP_RSEL, FMA_OFF)
637 : [off] "=&r" (off), [count] __ASM_ARG_RW(count)
638 : [dst] "r"(dst), [a] "r" (a), [b] "r" (b), [c] "r" (c)
639 : "cc", "memory",
640 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
641 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
642 );
643 }
644
645 void fmrmod4_fma3(float *dst, const float *a, const float *b, const float *c, size_t count)
646 {
647 IF_ARCH_X86(size_t off);
648 ARCH_X86_ASM
649 (
650 FMOD_VV_CORE("dst", "a", "b", "c", OP_RSEL, FMA_ON)
651 : [off] "=&r" (off), [count] __ASM_ARG_RW(count)
652 : [dst] "r"(dst), [a] "r" (a), [b] "r" (b), [c] "r" (c)
653 : "cc", "memory",
654 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
655 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
656 );
657 }
658
659 #undef FMOD_VV_CORE
660
661 #undef FMA_OFF
662 #undef FMA_ON
663
664 #undef OP_DSEL
665 #undef OP_RSEL
666 }
667 }
668
669 #endif /* PRIVATE_DSP_ARCH_X86_AVX_PMATH_FMOP_VV_H_ */