Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/x86/sse/pmath/fmop_vv.h

raw
1 /*
2 * Copyright (C) 2020 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2020 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 31 мар. 2020 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_X86_SSE_PMATH_FMOP_VV_H_
23 #define PRIVATE_DSP_ARCH_X86_SSE_PMATH_FMOP_VV_H_
24
25 #ifndef PRIVATE_DSP_ARCH_X86_SSE_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_X86_SSE_IMPL */
28
29 namespace lsp
30 {
31 namespace sse
32 {
33 #define OP_DSEL(a, b) a
34 #define OP_RSEL(a, b) b
35
36 #define FMOP_VV_CORE(DST, A, B, C, OP, SEL) \
37 __ASM_EMIT("xor %[off], %[off]") \
38 __ASM_EMIT("sub $8, %[count]") \
39 __ASM_EMIT("jb 2f") \
40 /* 8x blocks */ \
41 __ASM_EMIT("1:") \
42 __ASM_EMIT("movups 0x00(%[" A "], %[off]), %%xmm0") \
43 __ASM_EMIT("movups 0x10(%[" A "], %[off]), %%xmm1") \
44 __ASM_EMIT("movups 0x00(%[" B "], %[off]), %%xmm2") \
45 __ASM_EMIT("movups 0x10(%[" B "], %[off]), %%xmm3") \
46 __ASM_EMIT("movups 0x00(%[" C "], %[off]), %%xmm4") \
47 __ASM_EMIT("movups 0x10(%[" C "], %[off]), %%xmm5") \
48 __ASM_EMIT("mulps %%xmm2, %%xmm4") \
49 __ASM_EMIT("mulps %%xmm3, %%xmm5") \
50 __ASM_EMIT(OP "ps " SEL("%%xmm4", "%%xmm0") ", " SEL("%%xmm0", "%%xmm4")) \
51 __ASM_EMIT(OP "ps " SEL("%%xmm5", "%%xmm1") ", " SEL("%%xmm1", "%%xmm5")) \
52 __ASM_EMIT("movups " SEL("%%xmm0", "%%xmm4") ", 0x00(%[" DST "], %[off])") \
53 __ASM_EMIT("movups " SEL("%%xmm1", "%%xmm5") ", 0x10(%[" DST "], %[off])") \
54 __ASM_EMIT("add $0x20, %[off]") \
55 __ASM_EMIT("sub $8, %[count]") \
56 __ASM_EMIT("jae 1b") \
57 \
58 /* 4x block */ \
59 __ASM_EMIT("2:") \
60 __ASM_EMIT("add $4, %[count]") /* 8 - 4 */ \
61 __ASM_EMIT("jl 4f") \
62 __ASM_EMIT("movups 0x00(%[" A "], %[off]), %%xmm0") \
63 __ASM_EMIT("movups 0x00(%[" B "], %[off]), %%xmm2") \
64 __ASM_EMIT("movups 0x00(%[" C "], %[off]), %%xmm4") \
65 __ASM_EMIT("mulps %%xmm2, %%xmm4") \
66 __ASM_EMIT(OP "ps " SEL("%%xmm4", "%%xmm0") ", " SEL("%%xmm0", "%%xmm4")) \
67 __ASM_EMIT("movups " SEL("%%xmm0", "%%xmm4") ", 0x00(%[" DST "], %[off])") \
68 __ASM_EMIT("add $0x10, %[off]") \
69 __ASM_EMIT("sub $4, %[count]") \
70 /* 1x blocks */ \
71 __ASM_EMIT("4:") \
72 __ASM_EMIT("add $3, %[count]") /* 4 - 1 */ \
73 __ASM_EMIT("jl 6f") \
74 __ASM_EMIT("5:") \
75 __ASM_EMIT("movss 0x00(%[" A "], %[off]), %%xmm0") \
76 __ASM_EMIT("movss 0x00(%[" B "], %[off]), %%xmm2") \
77 __ASM_EMIT("movss 0x00(%[" C "], %[off]), %%xmm4") \
78 __ASM_EMIT("mulss %%xmm2, %%xmm4") \
79 __ASM_EMIT(OP "ss " SEL("%%xmm4", "%%xmm0") ", " SEL("%%xmm0", "%%xmm4")) \
80 __ASM_EMIT("movss " SEL("%%xmm0", "%%xmm4") ", 0x00(%[" DST "], %[off])") \
81 __ASM_EMIT("add $0x04, %[off]") \
82 __ASM_EMIT("dec %[count]") \
83 __ASM_EMIT("jge 5b") \
84 __ASM_EMIT("6:")
85
86 void fmadd3(float *dst, const float *a, const float *b, size_t count)
87 {
88 IF_ARCH_X86(size_t off);
89 ARCH_X86_ASM
90 (
91 FMOP_VV_CORE("dst", "dst", "a", "b", "add", OP_DSEL)
92 : [off] "=&r" (off), [count] "+r" (count)
93 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b)
94 : "cc", "memory",
95 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
96 "%xmm4", "%xmm5"
97 );
98 }
99
100 void fmsub3(float *dst, const float *a, const float *b, size_t count)
101 {
102 IF_ARCH_X86(size_t off);
103 ARCH_X86_ASM
104 (
105 FMOP_VV_CORE("dst", "dst", "a", "b", "sub", OP_DSEL)
106 : [off] "=&r" (off), [count] "+r" (count)
107 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b)
108 : "cc", "memory",
109 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
110 "%xmm4", "%xmm5"
111 );
112 }
113
114 void fmrsub3(float *dst, const float *a, const float *b, size_t count)
115 {
116 IF_ARCH_X86(size_t off);
117 ARCH_X86_ASM
118 (
119 FMOP_VV_CORE("dst", "dst", "a", "b", "sub", OP_RSEL)
120 : [off] "=&r" (off), [count] "+r" (count)
121 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b)
122 : "cc", "memory",
123 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
124 "%xmm4", "%xmm5"
125 );
126 }
127
128 void fmmul3(float *dst, const float *a, const float *b, size_t count)
129 {
130 IF_ARCH_X86(size_t off);
131 ARCH_X86_ASM
132 (
133 FMOP_VV_CORE("dst", "dst", "a", "b", "mul", OP_DSEL)
134 : [off] "=&r" (off), [count] "+r" (count)
135 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b)
136 : "cc", "memory",
137 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
138 "%xmm4", "%xmm5"
139 );
140 }
141
142 void fmdiv3(float *dst, const float *a, const float *b, size_t count)
143 {
144 IF_ARCH_X86(size_t off);
145 ARCH_X86_ASM
146 (
147 FMOP_VV_CORE("dst", "dst", "a", "b", "div", OP_DSEL)
148 : [off] "=&r" (off), [count] "+r" (count)
149 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b)
150 : "cc", "memory",
151 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
152 "%xmm4", "%xmm5"
153 );
154 }
155
156 void fmrdiv3(float *dst, const float *a, const float *b, size_t count)
157 {
158 IF_ARCH_X86(size_t off);
159 ARCH_X86_ASM
160 (
161 FMOP_VV_CORE("dst", "dst", "a", "b", "div", OP_RSEL)
162 : [off] "=&r" (off), [count] "+r" (count)
163 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b)
164 : "cc", "memory",
165 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
166 "%xmm4", "%xmm5"
167 );
168 }
169
170 void fmadd4(float *dst, const float *a, const float *b, const float *c, size_t count)
171 {
172 IF_ARCH_X86(size_t off);
173 ARCH_X86_ASM
174 (
175 FMOP_VV_CORE("dst", "a", "b", "c", "add", OP_DSEL)
176 : [off] "=&r" (off), [count] "+r" (count)
177 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b), [c] "r" (c)
178 : "cc", "memory",
179 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
180 "%xmm4", "%xmm5"
181 );
182 }
183
184 void fmsub4(float *dst, const float *a, const float *b, const float *c, size_t count)
185 {
186 IF_ARCH_X86(size_t off);
187 ARCH_X86_ASM
188 (
189 FMOP_VV_CORE("dst", "a", "b", "c", "sub", OP_DSEL)
190 : [off] "=&r" (off), [count] "+r" (count)
191 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b), [c] "r" (c)
192 : "cc", "memory",
193 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
194 "%xmm4", "%xmm5"
195 );
196 }
197
198 void fmrsub4(float *dst, const float *a, const float *b, const float *c, size_t count)
199 {
200 IF_ARCH_X86(size_t off);
201 ARCH_X86_ASM
202 (
203 FMOP_VV_CORE("dst", "a", "b", "c", "sub", OP_RSEL)
204 : [off] "=&r" (off), [count] "+r" (count)
205 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b), [c] "r" (c)
206 : "cc", "memory",
207 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
208 "%xmm4", "%xmm5"
209 );
210 }
211
212 void fmmul4(float *dst, const float *a, const float *b, const float *c, size_t count)
213 {
214 IF_ARCH_X86(size_t off);
215 ARCH_X86_ASM
216 (
217 FMOP_VV_CORE("dst", "a", "b", "c", "mul", OP_DSEL)
218 : [off] "=&r" (off), [count] "+r" (count)
219 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b), [c] "r" (c)
220 : "cc", "memory",
221 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
222 "%xmm4", "%xmm5"
223 );
224 }
225
226 void fmdiv4(float *dst, const float *a, const float *b, const float *c, size_t count)
227 {
228 IF_ARCH_X86(size_t off);
229 ARCH_X86_ASM
230 (
231 FMOP_VV_CORE("dst", "a", "b", "c", "div", OP_DSEL)
232 : [off] "=&r" (off), [count] "+r" (count)
233 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b), [c] "r" (c)
234 : "cc", "memory",
235 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
236 "%xmm4", "%xmm5"
237 );
238 }
239
240 void fmrdiv4(float *dst, const float *a, const float *b, const float *c, size_t count)
241 {
242 IF_ARCH_X86(size_t off);
243 ARCH_X86_ASM
244 (
245 FMOP_VV_CORE("dst", "a", "b", "c", "div", OP_RSEL)
246 : [off] "=&r" (off), [count] "+r" (count)
247 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b), [c] "r" (c)
248 : "cc", "memory",
249 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
250 "%xmm4", "%xmm5"
251 );
252 }
253
254 #undef FMOP_K4_CORE
255
256 #undef OP_DSEL
257 #undef OP_RSEL
258 }
259 }
260
261 #endif /* PRIVATE_DSP_ARCH_X86_SSE_PMATH_FMOP_VV_H_ */