Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/x86/avx/pmath/op_vv.h

raw
1 /*
2 * Copyright (C) 2020 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2020 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 31 мар. 2020 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_X86_AVX_PMATH_OP_VV_H_
23 #define PRIVATE_DSP_ARCH_X86_AVX_PMATH_OP_VV_H_
24
25 #ifndef PRIVATE_DSP_ARCH_X86_AVX_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_X86_AVX_IMPL */
28
29 namespace lsp
30 {
31 namespace avx
32 {
33 #define OP_VV_CORE(DST, SRC1, SRC2, OP) \
34 __ASM_EMIT("xor %[off], %[off]") \
35 __ASM_EMIT("sub $64, %[count]") \
36 __ASM_EMIT("jb 2f") \
37 /* 64x blocks */ \
38 __ASM_EMIT("1:") \
39 __ASM_EMIT("vmovups 0x000(%[" SRC1 "], %[off]), %%ymm0") \
40 __ASM_EMIT("vmovups 0x020(%[" SRC1 "], %[off]), %%ymm1") \
41 __ASM_EMIT("vmovups 0x040(%[" SRC1 "], %[off]), %%ymm2") \
42 __ASM_EMIT("vmovups 0x060(%[" SRC1 "], %[off]), %%ymm3") \
43 __ASM_EMIT("vmovups 0x080(%[" SRC1 "], %[off]), %%ymm4") \
44 __ASM_EMIT("vmovups 0x0a0(%[" SRC1 "], %[off]), %%ymm5") \
45 __ASM_EMIT("vmovups 0x0c0(%[" SRC1 "], %[off]), %%ymm6") \
46 __ASM_EMIT("vmovups 0x0e0(%[" SRC1 "], %[off]), %%ymm7") \
47 __ASM_EMIT(OP "ps 0x000(%[" SRC2 "], %[off]), %%ymm0, %%ymm0") \
48 __ASM_EMIT(OP "ps 0x020(%[" SRC2 "], %[off]), %%ymm1, %%ymm1") \
49 __ASM_EMIT(OP "ps 0x040(%[" SRC2 "], %[off]), %%ymm2, %%ymm2") \
50 __ASM_EMIT(OP "ps 0x060(%[" SRC2 "], %[off]), %%ymm3, %%ymm3") \
51 __ASM_EMIT(OP "ps 0x080(%[" SRC2 "], %[off]), %%ymm4, %%ymm4") \
52 __ASM_EMIT(OP "ps 0x0a0(%[" SRC2 "], %[off]), %%ymm5, %%ymm5") \
53 __ASM_EMIT(OP "ps 0x0c0(%[" SRC2 "], %[off]), %%ymm6, %%ymm6") \
54 __ASM_EMIT(OP "ps 0x0e0(%[" SRC2 "], %[off]), %%ymm7, %%ymm7") \
55 __ASM_EMIT("vmovups %%ymm0, 0x000(%[" DST "], %[off])") \
56 __ASM_EMIT("vmovups %%ymm1, 0x020(%[" DST "], %[off])") \
57 __ASM_EMIT("vmovups %%ymm2, 0x040(%[" DST "], %[off])") \
58 __ASM_EMIT("vmovups %%ymm3, 0x060(%[" DST "], %[off])") \
59 __ASM_EMIT("vmovups %%ymm4, 0x080(%[" DST "], %[off])") \
60 __ASM_EMIT("vmovups %%ymm5, 0x0a0(%[" DST "], %[off])") \
61 __ASM_EMIT("vmovups %%ymm6, 0x0c0(%[" DST "], %[off])") \
62 __ASM_EMIT("vmovups %%ymm7, 0x0e0(%[" DST "], %[off])") \
63 __ASM_EMIT("add $0x100, %[off]") \
64 __ASM_EMIT("sub $64, %[count]") \
65 __ASM_EMIT("jae 1b") \
66 /* 32x block */ \
67 __ASM_EMIT("2:") \
68 __ASM_EMIT("add $32, %[count]") \
69 __ASM_EMIT("jl 4f") \
70 __ASM_EMIT("vmovups 0x000(%[" SRC1 "], %[off]), %%ymm0") \
71 __ASM_EMIT("vmovups 0x020(%[" SRC1 "], %[off]), %%ymm1") \
72 __ASM_EMIT("vmovups 0x040(%[" SRC1 "], %[off]), %%ymm2") \
73 __ASM_EMIT("vmovups 0x060(%[" SRC1 "], %[off]), %%ymm3") \
74 __ASM_EMIT(OP "ps 0x000(%[" SRC2 "], %[off]), %%ymm0, %%ymm0") \
75 __ASM_EMIT(OP "ps 0x020(%[" SRC2 "], %[off]), %%ymm1, %%ymm1") \
76 __ASM_EMIT(OP "ps 0x040(%[" SRC2 "], %[off]), %%ymm2, %%ymm2") \
77 __ASM_EMIT(OP "ps 0x060(%[" SRC2 "], %[off]), %%ymm3, %%ymm3") \
78 __ASM_EMIT("vmovups %%ymm0, 0x000(%[" DST "], %[off])") \
79 __ASM_EMIT("vmovups %%ymm1, 0x020(%[" DST "], %[off])") \
80 __ASM_EMIT("vmovups %%ymm2, 0x040(%[" DST "], %[off])") \
81 __ASM_EMIT("vmovups %%ymm3, 0x060(%[" DST "], %[off])") \
82 __ASM_EMIT("add $0x80, %[off]") \
83 __ASM_EMIT("sub $32, %[count]") \
84 /* 16x block */ \
85 __ASM_EMIT("4:") \
86 __ASM_EMIT("add $16, %[count]") \
87 __ASM_EMIT("jl 6f") \
88 __ASM_EMIT("vmovups 0x000(%[" SRC1 "], %[off]), %%ymm0") \
89 __ASM_EMIT("vmovups 0x020(%[" SRC1 "], %[off]), %%ymm1") \
90 __ASM_EMIT(OP "ps 0x000(%[" SRC2 "], %[off]), %%ymm0, %%ymm0") \
91 __ASM_EMIT(OP "ps 0x020(%[" SRC2 "], %[off]), %%ymm1, %%ymm1") \
92 __ASM_EMIT("vmovups %%ymm0, 0x000(%[" DST "], %[off])") \
93 __ASM_EMIT("vmovups %%ymm1, 0x020(%[" DST "], %[off])") \
94 __ASM_EMIT("add $0x40, %[off]") \
95 __ASM_EMIT("sub $16, %[count]") \
96 /* 8x block */ \
97 __ASM_EMIT("6:") \
98 __ASM_EMIT("add $8, %[count]") \
99 __ASM_EMIT("jl 8f") \
100 __ASM_EMIT("vmovups 0x000(%[" SRC1 "], %[off]), %%ymm0") \
101 __ASM_EMIT(OP "ps 0x000(%[" SRC2 "], %[off]), %%ymm0, %%ymm0") \
102 __ASM_EMIT("vmovups %%ymm0, 0x000(%[" DST "], %[off])") \
103 __ASM_EMIT("add $0x20, %[off]") \
104 __ASM_EMIT("sub $8, %[count]") \
105 /* 4x block */ \
106 __ASM_EMIT("8:") \
107 __ASM_EMIT("add $4, %[count]") \
108 __ASM_EMIT("jl 10f") \
109 __ASM_EMIT("vmovups 0x000(%[" SRC1 "], %[off]), %%xmm0") \
110 __ASM_EMIT(OP "ps 0x000(%[" SRC2 "], %[off]), %%xmm0, %%xmm0") \
111 __ASM_EMIT("vmovups %%xmm0, 0x000(%[" DST "], %[off])") \
112 __ASM_EMIT("add $0x10, %[off]") \
113 __ASM_EMIT("sub $4, %[count]") \
114 \
115 /* 1x blocks */ \
116 __ASM_EMIT("10:") \
117 __ASM_EMIT("add $3, %[count]") /* 8 - 4 */ \
118 __ASM_EMIT("jl 12f") \
119 __ASM_EMIT("11:") \
120 __ASM_EMIT("vmovss 0x000(%[" SRC1 "], %[off]), %%xmm0") \
121 __ASM_EMIT(OP "ss 0x000(%[" SRC2 "], %[off]), %%xmm0, %%xmm0") \
122 __ASM_EMIT("vmovss %%xmm0, 0x000(%[" DST "], %[off])") \
123 __ASM_EMIT("add $0x04, %[off]") \
124 __ASM_EMIT("dec %[count]") \
125 __ASM_EMIT("jge 11b") \
126 __ASM_EMIT("12:")
127
128 void add2(float *dst, const float *src, size_t count)
129 {
130 IF_ARCH_X86(size_t off);
131 ARCH_X86_ASM
132 (
133 OP_VV_CORE("dst", "dst", "src", "vadd")
134 : [off] "=&r" (off), [count] "+r" (count)
135 : [dst] "r" (dst), [src] "r" (src)
136 : "%xmm0", "%xmm1", "%xmm2", "%xmm3",
137 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
138 );
139 }
140
141 void sub2(float *dst, const float *src, size_t count)
142 {
143 IF_ARCH_X86(size_t off);
144 ARCH_X86_ASM
145 (
146 OP_VV_CORE("dst", "dst", "src", "vsub")
147 : [off] "=&r" (off), [count] "+r" (count)
148 : [dst] "r" (dst), [src] "r" (src)
149 : "%xmm0", "%xmm1", "%xmm2", "%xmm3",
150 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
151 );
152 }
153
154 void rsub2(float *dst, const float *src, size_t count)
155 {
156 IF_ARCH_X86(size_t off);
157 ARCH_X86_ASM
158 (
159 OP_VV_CORE("dst", "src", "dst", "vsub")
160 : [off] "=&r" (off), [count] "+r" (count)
161 : [dst] "r" (dst), [src] "r" (src)
162 : "%xmm0", "%xmm1", "%xmm2", "%xmm3",
163 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
164 );
165 }
166
167 void mul2(float *dst, const float *src, size_t count)
168 {
169 IF_ARCH_X86(size_t off);
170 ARCH_X86_ASM
171 (
172 OP_VV_CORE("dst", "dst", "src", "vmul")
173 : [off] "=&r" (off), [count] "+r" (count)
174 : [dst] "r" (dst), [src] "r" (src)
175 : "%xmm0", "%xmm1", "%xmm2", "%xmm3",
176 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
177 );
178 }
179
180 void div2(float *dst, const float *src, size_t count)
181 {
182 IF_ARCH_X86(size_t off);
183 ARCH_X86_ASM
184 (
185 OP_VV_CORE("dst", "dst", "src", "vdiv")
186 : [off] "=&r" (off), [count] "+r" (count)
187 : [dst] "r" (dst), [src] "r" (src)
188 : "%xmm0", "%xmm1", "%xmm2", "%xmm3",
189 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
190 );
191 }
192
193 void rdiv2(float *dst, const float *src, size_t count)
194 {
195 IF_ARCH_X86(size_t off);
196 ARCH_X86_ASM
197 (
198 OP_VV_CORE("dst", "src", "dst", "vdiv")
199 : [off] "=&r" (off), [count] "+r" (count)
200 : [dst] "r" (dst), [src] "r" (src)
201 : "%xmm0", "%xmm1", "%xmm2", "%xmm3",
202 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
203 );
204 }
205
206 void add3(float *dst, const float *src1, const float *src2, size_t count)
207 {
208 IF_ARCH_X86(size_t off);
209 ARCH_X86_ASM
210 (
211 OP_VV_CORE("dst", "src1", "src2", "vadd")
212 : [off] "=&r" (off), [count] "+r" (count)
213 : [dst] "r" (dst), [src1] "r" (src1), [src2] "r" (src2)
214 : "%xmm0", "%xmm1", "%xmm2", "%xmm3",
215 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
216 );
217 }
218
219 void sub3(float *dst, const float *src1, const float *src2, size_t count)
220 {
221 IF_ARCH_X86(size_t off);
222 ARCH_X86_ASM
223 (
224 OP_VV_CORE("dst", "src1", "src2", "vsub")
225 : [off] "=&r" (off), [count] "+r" (count)
226 : [dst] "r" (dst), [src1] "r" (src1), [src2] "r" (src2)
227 : "%xmm0", "%xmm1", "%xmm2", "%xmm3",
228 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
229 );
230 }
231
232 void mul3(float *dst, const float *src1, const float *src2, size_t count)
233 {
234 IF_ARCH_X86(size_t off);
235 ARCH_X86_ASM
236 (
237 OP_VV_CORE("dst", "src1", "src2", "vmul")
238 : [off] "=&r" (off), [count] "+r" (count)
239 : [dst] "r" (dst), [src1] "r" (src1), [src2] "r" (src2)
240 : "%xmm0", "%xmm1", "%xmm2", "%xmm3",
241 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
242 );
243 }
244
245 void div3(float *dst, const float *src1, const float *src2, size_t count)
246 {
247 IF_ARCH_X86(size_t off);
248 ARCH_X86_ASM
249 (
250 OP_VV_CORE("dst", "src1", "src2", "vdiv")
251 : [off] "=&r" (off), [count] "+r" (count)
252 : [dst] "r" (dst), [src1] "r" (src1), [src2] "r" (src2)
253 : "%xmm0", "%xmm1", "%xmm2", "%xmm3",
254 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
255 );
256 }
257
258 #undef OP_VV_CORE
259
260 #define FMA_OFF(a, b) a
261 #define FMA_ON(a, b) b
262
263 #define FMOD_VV_CORE(DST, SRC1, SRC2, FMA_SEL) \
264 __ASM_EMIT("xor %[off], %[off]") \
265 /* x16 blocks */ \
266 __ASM_EMIT("sub $16, %[count]") \
267 __ASM_EMIT("jb 2f") \
268 __ASM_EMIT("1:") \
269 __ASM_EMIT("vmovups 0x00(%[" SRC1 "], %[off]), %%ymm2") \
270 __ASM_EMIT("vmovups 0x20(%[" SRC1 "], %[off]), %%ymm3") \
271 __ASM_EMIT("vmovups 0x00(%[" SRC2 "], %[off]), %%ymm0") \
272 __ASM_EMIT("vmovups 0x20(%[" SRC2 "], %[off]), %%ymm1") \
273 __ASM_EMIT("vdivps %%ymm0, %%ymm2, %%ymm4") \
274 __ASM_EMIT("vdivps %%ymm1, %%ymm3, %%ymm5") \
275 __ASM_EMIT("vcvttps2dq %%ymm4, %%ymm4") \
276 __ASM_EMIT("vcvttps2dq %%ymm5, %%ymm5") \
277 __ASM_EMIT("vcvtdq2ps %%ymm4, %%ymm4") \
278 __ASM_EMIT("vcvtdq2ps %%ymm5, %%ymm5") \
279 __ASM_EMIT(FMA_SEL("vmulps %%ymm0, %%ymm4, %%ymm4", "")) \
280 __ASM_EMIT(FMA_SEL("vmulps %%ymm1, %%ymm5, %%ymm5", "")) \
281 __ASM_EMIT(FMA_SEL("vsubps %%ymm4, %%ymm2, %%ymm2", "vfnmadd231ps %%ymm0, %%ymm4, %%ymm2")) \
282 __ASM_EMIT(FMA_SEL("vsubps %%ymm5, %%ymm3, %%ymm3", "vfnmadd231ps %%ymm1, %%ymm5, %%ymm3")) \
283 __ASM_EMIT("vmovups %%ymm2, 0x00(%[" DST "], %[off])") \
284 __ASM_EMIT("vmovups %%ymm3, 0x20(%[" DST "], %[off])") \
285 __ASM_EMIT("add $0x40, %[off]") \
286 __ASM_EMIT("sub $16, %[count]") \
287 __ASM_EMIT("jae 1b") \
288 __ASM_EMIT("2:") \
289 /* x8 block */ \
290 __ASM_EMIT("add $8, %[count]") \
291 __ASM_EMIT("jl 4f") \
292 __ASM_EMIT("vmovups 0x00(%[" SRC1 "], %[off]), %%xmm2") \
293 __ASM_EMIT("vmovups 0x10(%[" SRC1 "], %[off]), %%xmm3") \
294 __ASM_EMIT("vmovups 0x00(%[" SRC2 "], %[off]), %%xmm0") \
295 __ASM_EMIT("vmovups 0x10(%[" SRC2 "], %[off]), %%xmm1") \
296 __ASM_EMIT("vdivps %%xmm0, %%xmm2, %%xmm4") \
297 __ASM_EMIT("vdivps %%xmm1, %%xmm3, %%xmm5") \
298 __ASM_EMIT("vcvttps2dq %%xmm4, %%xmm4") \
299 __ASM_EMIT("vcvttps2dq %%xmm5, %%xmm5") \
300 __ASM_EMIT("vcvtdq2ps %%xmm4, %%xmm4") \
301 __ASM_EMIT("vcvtdq2ps %%xmm5, %%xmm5") \
302 __ASM_EMIT(FMA_SEL("vmulps %%xmm0, %%xmm4, %%xmm4", "")) \
303 __ASM_EMIT(FMA_SEL("vmulps %%xmm1, %%xmm5, %%xmm5", "")) \
304 __ASM_EMIT(FMA_SEL("vsubps %%xmm4, %%xmm2, %%xmm2", "vfnmadd231ps %%xmm0, %%xmm4, %%xmm2")) \
305 __ASM_EMIT(FMA_SEL("vsubps %%xmm5, %%xmm3, %%xmm3", "vfnmadd231ps %%xmm1, %%xmm5, %%xmm3")) \
306 __ASM_EMIT("vmovups %%xmm2, 0x00(%[" DST "], %[off])") \
307 __ASM_EMIT("vmovups %%xmm3, 0x10(%[" DST "], %[off])") \
308 __ASM_EMIT("sub $8, %[count]") \
309 __ASM_EMIT("add $0x20, %[off]") \
310 __ASM_EMIT("4:") \
311 /* x4 block */ \
312 __ASM_EMIT("add $4, %[count]") \
313 __ASM_EMIT("jl 6f") \
314 __ASM_EMIT("vmovups 0x00(%[" SRC1 "], %[off]), %%xmm2") \
315 __ASM_EMIT("vmovups 0x00(%[" SRC2 "], %[off]), %%xmm0") \
316 __ASM_EMIT("vdivps %%xmm0, %%xmm2, %%xmm4") \
317 __ASM_EMIT("vcvttps2dq %%xmm4, %%xmm4") \
318 __ASM_EMIT("vcvtdq2ps %%xmm4, %%xmm4") \
319 __ASM_EMIT(FMA_SEL("vmulps %%xmm0, %%xmm4, %%xmm4", "")) \
320 __ASM_EMIT(FMA_SEL("vsubps %%xmm4, %%xmm2, %%xmm2", "vfnmadd231ps %%xmm0, %%xmm4, %%xmm2")) \
321 __ASM_EMIT("vmovups %%xmm2, 0x00(%[" DST "], %[off])") \
322 __ASM_EMIT("sub $4, %[count]") \
323 __ASM_EMIT("add $0x10, %[off]") \
324 __ASM_EMIT("6:") \
325 /* x1 blocks */ \
326 __ASM_EMIT("add $3, %[count]") \
327 __ASM_EMIT("jl 8f") \
328 __ASM_EMIT("7:") \
329 __ASM_EMIT("vmovss 0x00(%[" SRC1 "], %[off]), %%xmm2") \
330 __ASM_EMIT("vmovss 0x00(%[" SRC2 "], %[off]), %%xmm0") \
331 __ASM_EMIT("vdivps %%xmm0, %%xmm2, %%xmm4") \
332 __ASM_EMIT("vcvttps2dq %%xmm4, %%xmm4") \
333 __ASM_EMIT("vcvtdq2ps %%xmm4, %%xmm4") \
334 __ASM_EMIT(FMA_SEL("vmulps %%xmm0, %%xmm4, %%xmm4", "")) \
335 __ASM_EMIT(FMA_SEL("vsubps %%xmm4, %%xmm2, %%xmm2", "vfnmadd231ps %%xmm0, %%xmm4, %%xmm2")) \
336 __ASM_EMIT("vmovss %%xmm2, 0x00(%[" DST "], %[off])") \
337 __ASM_EMIT("add $0x04, %[off]") \
338 __ASM_EMIT("dec %[count]") \
339 __ASM_EMIT("jge 7b") \
340 __ASM_EMIT("8:")
341
342 void mod2(float *dst, const float *src, size_t count)
343 {
344 IF_ARCH_X86(size_t off);
345 ARCH_X86_ASM
346 (
347 FMOD_VV_CORE("dst", "dst", "src", FMA_OFF)
348 : [off] "=&r" (off), [count] "+r" (count)
349 : [dst] "r" (dst), [src] "r" (src)
350 : "%xmm0", "%xmm1", "%xmm2", "%xmm3",
351 "%xmm4", "%xmm5"
352 );
353 }
354
355 void mod2_fma3(float *dst, const float *src, size_t count)
356 {
357 IF_ARCH_X86(size_t off);
358 ARCH_X86_ASM
359 (
360 FMOD_VV_CORE("dst", "dst", "src", FMA_ON)
361 : [off] "=&r" (off), [count] "+r" (count)
362 : [dst] "r" (dst), [src] "r" (src)
363 : "%xmm0", "%xmm1", "%xmm2", "%xmm3",
364 "%xmm4", "%xmm5"
365 );
366 }
367
368 void rmod2(float *dst, const float *src, size_t count)
369 {
370 IF_ARCH_X86(size_t off);
371 ARCH_X86_ASM
372 (
373 FMOD_VV_CORE("dst", "src", "dst", FMA_OFF)
374 : [off] "=&r" (off), [count] "+r" (count)
375 : [dst] "r" (dst), [src] "r" (src)
376 : "%xmm0", "%xmm1", "%xmm2", "%xmm3",
377 "%xmm4", "%xmm5"
378 );
379 }
380
381 void rmod2_fma3(float *dst, const float *src, size_t count)
382 {
383 IF_ARCH_X86(size_t off);
384 ARCH_X86_ASM
385 (
386 FMOD_VV_CORE("dst", "src", "dst", FMA_ON)
387 : [off] "=&r" (off), [count] "+r" (count)
388 : [dst] "r" (dst), [src] "r" (src)
389 : "%xmm0", "%xmm1", "%xmm2", "%xmm3",
390 "%xmm4", "%xmm5"
391 );
392 }
393
394 void mod3(float *dst, const float *src1, const float *src2, size_t count)
395 {
396 IF_ARCH_X86(size_t off);
397 ARCH_X86_ASM
398 (
399 FMOD_VV_CORE("dst", "src1", "src2", FMA_OFF)
400 : [off] "=&r" (off), [count] "+r" (count)
401 : [dst] "r" (dst), [src1] "r" (src1), [src2] "r" (src2)
402 : "%xmm0", "%xmm1", "%xmm2", "%xmm3",
403 "%xmm4", "%xmm5"
404 );
405 }
406
407 void mod3_fma3(float *dst, const float *src1, const float *src2, size_t count)
408 {
409 IF_ARCH_X86(size_t off);
410 ARCH_X86_ASM
411 (
412 FMOD_VV_CORE("dst", "src1", "src2", FMA_ON)
413 : [off] "=&r" (off), [count] "+r" (count)
414 : [dst] "r" (dst), [src1] "r" (src1), [src2] "r" (src2)
415 : "%xmm0", "%xmm1", "%xmm2", "%xmm3",
416 "%xmm4", "%xmm5"
417 );
418 }
419
420 #undef FMOD_VV_CORE
421 #undef FMA_ON
422 #undef FMA_OFF
423 }
424 }
425
426 #endif /* PRIVATE_DSP_ARCH_X86_AVX_PMATH_OP_VV_H_ */