Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/x86/sse/pmath/fmop_kx.h

raw
1 /*
2 * Copyright (C) 2020 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2020 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 31 мар. 2020 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_X86_SSE_PMATH_FMOP_KX_H_
23 #define PRIVATE_DSP_ARCH_X86_SSE_PMATH_FMOP_KX_H_
24
25 #ifndef PRIVATE_DSP_ARCH_X86_SSE_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_X86_SSE_IMPL */
28
29 namespace lsp
30 {
31 namespace sse
32 {
33 #define OP_DSEL(a, b) a
34 #define OP_RSEL(a, b) b
35
36 #define FMOP_K4_CORE(DST, SRC1, SRC2, OP, SEL) \
37 __ASM_EMIT("xor %[off], %[off]") \
38 __ASM_EMIT("shufps $0x00, %%xmm0, %%xmm0") \
39 __ASM_EMIT("sub $12, %[count]") \
40 __ASM_EMIT("movaps %%xmm0, %%xmm1") \
41 __ASM_EMIT("jb 2f") \
42 /* 12x blocks */ \
43 __ASM_EMIT("1:") \
44 __ASM_EMIT("movups 0x00(%[" SRC2 "], %[off]), %%xmm5") \
45 __ASM_EMIT("movups 0x10(%[" SRC2 "], %[off]), %%xmm6") \
46 __ASM_EMIT("movups 0x20(%[" SRC2 "], %[off]), %%xmm7") \
47 __ASM_EMIT("movups 0x00(%[" SRC1 "], %[off]), %%xmm2") \
48 __ASM_EMIT("movups 0x10(%[" SRC1 "], %[off]), %%xmm3") \
49 __ASM_EMIT("movups 0x20(%[" SRC1 "], %[off]), %%xmm4") \
50 __ASM_EMIT("mulps %%xmm0, %%xmm5") \
51 __ASM_EMIT("mulps %%xmm1, %%xmm6") \
52 __ASM_EMIT("mulps %%xmm0, %%xmm7") \
53 __ASM_EMIT(OP "ps " SEL("%%xmm5", "%%xmm2") ", " SEL("%%xmm2", "%%xmm5")) \
54 __ASM_EMIT(OP "ps " SEL("%%xmm6", "%%xmm3") ", " SEL("%%xmm3", "%%xmm6")) \
55 __ASM_EMIT(OP "ps " SEL("%%xmm7", "%%xmm4") ", " SEL("%%xmm4", "%%xmm7")) \
56 __ASM_EMIT("movups " SEL("%%xmm2", "%%xmm5") ", 0x00(%[" DST "], %[off])") \
57 __ASM_EMIT("movups " SEL("%%xmm3", "%%xmm6") ", 0x10(%[" DST "], %[off])") \
58 __ASM_EMIT("movups " SEL("%%xmm4", "%%xmm7") ", 0x20(%[" DST "], %[off])") \
59 __ASM_EMIT("add $0x30, %[off]") \
60 __ASM_EMIT("sub $12, %[count]") \
61 __ASM_EMIT("jae 1b") \
62 /* 8x block */ \
63 __ASM_EMIT("2:") \
64 __ASM_EMIT("add $4, %[count]") /* 12 - 8 */ \
65 __ASM_EMIT("jl 4f") \
66 __ASM_EMIT("movups 0x00(%[" SRC2 "], %[off]), %%xmm5") \
67 __ASM_EMIT("movups 0x10(%[" SRC2 "], %[off]), %%xmm6") \
68 __ASM_EMIT("movups 0x00(%[" SRC1 "], %[off]), %%xmm2") \
69 __ASM_EMIT("movups 0x10(%[" SRC1 "], %[off]), %%xmm3") \
70 __ASM_EMIT("mulps %%xmm0, %%xmm5") \
71 __ASM_EMIT("mulps %%xmm1, %%xmm6") \
72 __ASM_EMIT(OP "ps " SEL("%%xmm5", "%%xmm2") ", " SEL("%%xmm2", "%%xmm5")) \
73 __ASM_EMIT(OP "ps " SEL("%%xmm6", "%%xmm3") ", " SEL("%%xmm3", "%%xmm6")) \
74 __ASM_EMIT("movups " SEL("%%xmm2", "%%xmm5") ", 0x00(%[" DST "], %[off])") \
75 __ASM_EMIT("movups " SEL("%%xmm3", "%%xmm6") ", 0x10(%[" DST "], %[off])") \
76 __ASM_EMIT("add $0x20, %[off]") \
77 __ASM_EMIT("sub $8, %[count]") \
78 \
79 /* 4x block */ \
80 __ASM_EMIT("4:") \
81 __ASM_EMIT("add $4, %[count]") /* 8 - 4 */ \
82 __ASM_EMIT("jl 6f") \
83 __ASM_EMIT("movups 0x00(%[" SRC2 "], %[off]), %%xmm5") \
84 __ASM_EMIT("movups 0x00(%[" SRC1 "], %[off]), %%xmm2") \
85 __ASM_EMIT("mulps %%xmm0, %%xmm5") \
86 __ASM_EMIT(OP "ps " SEL("%%xmm5", "%%xmm2") ", " SEL("%%xmm2", "%%xmm5")) \
87 __ASM_EMIT("movups " SEL("%%xmm2", "%%xmm5") ", 0x00(%[" DST "], %[off])") \
88 __ASM_EMIT("add $0x10, %[off]") \
89 __ASM_EMIT("sub $4, %[count]") \
90 /* 1x blocks */ \
91 __ASM_EMIT("6:") \
92 __ASM_EMIT("add $3, %[count]") /* 4 - 1 */ \
93 __ASM_EMIT("jl 8f") \
94 __ASM_EMIT("7:") \
95 __ASM_EMIT("movss 0x00(%[" SRC2 "], %[off]), %%xmm5") \
96 __ASM_EMIT("movss 0x00(%[" SRC1 "], %[off]), %%xmm2") \
97 __ASM_EMIT("mulss %%xmm0, %%xmm5") \
98 __ASM_EMIT(OP "ss " SEL("%%xmm5", "%%xmm2") ", " SEL("%%xmm2", "%%xmm5")) \
99 __ASM_EMIT("movss " SEL("%%xmm2", "%%xmm5") ", 0x00(%[" DST "], %[off])") \
100 __ASM_EMIT("add $0x04, %[off]") \
101 __ASM_EMIT("dec %[count]") \
102 __ASM_EMIT("jge 7b") \
103 __ASM_EMIT("8:")
104
105 void fmadd_k3(float *dst, const float *src, float k, size_t count)
106 {
107 IF_ARCH_X86(size_t off);
108 ARCH_X86_ASM
109 (
110 FMOP_K4_CORE("dst", "dst", "src", "add", OP_DSEL)
111 : [off] "=&r" (off), [count] "+r" (count),
112 [k] "+Yz"(k)
113 : [dst] "r"(dst), [src] "r"(src)
114 : "cc", "memory",
115 "%xmm1", "%xmm2", "%xmm3",
116 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
117 );
118 }
119
120 void fmsub_k3(float *dst, const float *src, float k, size_t count)
121 {
122 IF_ARCH_X86(size_t off);
123
124 ARCH_X86_ASM
125 (
126 FMOP_K4_CORE("dst", "dst", "src", "sub", OP_DSEL)
127 : [off] "=&r" (off), [count] "+r" (count),
128 [k] "+Yz"(k)
129 : [dst] "r"(dst), [src] "r"(src)
130 : "cc", "memory",
131 "%xmm1", "%xmm2", "%xmm3",
132 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
133 );
134 }
135
136 void fmrsub_k3(float *dst, const float *src, float k, size_t count)
137 {
138 IF_ARCH_X86(size_t off);
139
140 ARCH_X86_ASM
141 (
142 FMOP_K4_CORE("dst", "dst", "src", "sub", OP_RSEL)
143 : [off] "=&r" (off), [count] "+r" (count),
144 [k] "+Yz"(k)
145 : [dst] "r"(dst), [src] "r"(src)
146 : "cc", "memory",
147 "%xmm1", "%xmm2", "%xmm3",
148 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
149 );
150 }
151
152 void fmmul_k3(float *dst, const float *src, float k, size_t count)
153 {
154 IF_ARCH_X86(size_t off);
155
156 ARCH_X86_ASM
157 (
158 FMOP_K4_CORE("dst", "dst", "src", "mul", OP_DSEL)
159 : [off] "=&r" (off), [count] "+r" (count),
160 [k] "+Yz"(k)
161 : [dst] "r"(dst), [src] "r"(src)
162 : "cc", "memory",
163 "%xmm1", "%xmm2", "%xmm3",
164 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
165 );
166 }
167
168 void fmdiv_k3(float *dst, const float *src, float k, size_t count)
169 {
170 IF_ARCH_X86(size_t off);
171
172 ARCH_X86_ASM
173 (
174 FMOP_K4_CORE("dst", "dst", "src", "div", OP_DSEL)
175 : [off] "=&r" (off), [count] "+r" (count),
176 [k] "+Yz"(k)
177 : [dst] "r"(dst), [src] "r"(src)
178 : "cc", "memory",
179 "%xmm1", "%xmm2", "%xmm3",
180 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
181 );
182 }
183
184 void fmrdiv_k3(float *dst, const float *src, float k, size_t count)
185 {
186 IF_ARCH_X86(size_t off);
187
188 ARCH_X86_ASM
189 (
190 FMOP_K4_CORE("dst", "dst", "src", "div", OP_RSEL)
191 : [off] "=&r" (off), [count] "+r" (count),
192 [k] "+Yz"(k)
193 : [dst] "r"(dst), [src] "r"(src)
194 : "cc", "memory",
195 "%xmm1", "%xmm2", "%xmm3",
196 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
197 );
198 }
199
200 void fmadd_k4(float *dst, const float *src1, const float *src2, float k, size_t count)
201 {
202 IF_ARCH_X86(size_t off);
203 ARCH_X86_ASM
204 (
205 FMOP_K4_CORE("dst", "src1", "src2", "add", OP_DSEL)
206 : [count] "+r" (count), [off] "=&r" (off),
207 [k] "+Yz"(k)
208 : [dst] "r"(dst), [src1] "r"(src1), [src2] "r"(src2)
209 : "cc", "memory",
210 "%xmm1", "%xmm2", "%xmm3",
211 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
212 );
213 }
214
215 void fmsub_k4(float *dst, const float *src1, const float *src2, float k, size_t count)
216 {
217 IF_ARCH_X86(size_t off);
218 ARCH_X86_ASM
219 (
220 FMOP_K4_CORE("dst", "src1", "src2", "sub", OP_DSEL)
221 : [count] "+r" (count), [off] "=&r" (off),
222 [k] "+Yz"(k)
223 : [dst] "r"(dst), [src1] "r"(src1), [src2] "r"(src2)
224 : "cc", "memory",
225 "%xmm1", "%xmm2", "%xmm3",
226 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
227 );
228 }
229
230 void fmrsub_k4(float *dst, const float *src1, const float *src2, float k, size_t count)
231 {
232 IF_ARCH_X86(size_t off);
233 ARCH_X86_ASM
234 (
235 FMOP_K4_CORE("dst", "src1", "src2", "sub", OP_RSEL)
236 : [count] "+r" (count), [off] "=&r" (off),
237 [k] "+Yz"(k)
238 : [dst] "r"(dst), [src1] "r"(src1), [src2] "r"(src2)
239 : "cc", "memory",
240 "%xmm1", "%xmm2", "%xmm3",
241 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
242 );
243 }
244
245 void fmmul_k4(float *dst, const float *src1, const float *src2, float k, size_t count)
246 {
247 IF_ARCH_X86(size_t off);
248 ARCH_X86_ASM
249 (
250 FMOP_K4_CORE("dst", "src1", "src2", "mul", OP_DSEL)
251 : [count] "+r" (count), [off] "=&r" (off),
252 [k] "+Yz"(k)
253 : [dst] "r"(dst), [src1] "r"(src1), [src2] "r"(src2)
254 : "cc", "memory",
255 "%xmm1", "%xmm2", "%xmm3",
256 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
257 );
258 }
259
260 void fmdiv_k4(float *dst, const float *src1, const float *src2, float k, size_t count)
261 {
262 IF_ARCH_X86(size_t off);
263 ARCH_X86_ASM
264 (
265 FMOP_K4_CORE("dst", "src1", "src2", "div", OP_DSEL)
266 : [count] "+r" (count), [off] "=&r" (off),
267 [k] "+Yz"(k)
268 : [dst] "r"(dst), [src1] "r"(src1), [src2] "r"(src2)
269 : "cc", "memory",
270 "%xmm1", "%xmm2", "%xmm3",
271 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
272 );
273 }
274
275 void fmrdiv_k4(float *dst, const float *src1, const float *src2, float k, size_t count)
276 {
277 IF_ARCH_X86(size_t off);
278 ARCH_X86_ASM
279 (
280 FMOP_K4_CORE("dst", "src1", "src2", "div", OP_RSEL)
281 : [count] "+r" (count), [off] "=&r" (off),
282 [k] "+Yz"(k)
283 : [dst] "r"(dst), [src1] "r"(src1), [src2] "r"(src2)
284 : "cc", "memory",
285 "%xmm1", "%xmm2", "%xmm3",
286 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
287 );
288 }
289
290 #undef FMOP_K4_CORE
291
292 #undef OP_DSEL
293 #undef OP_RSEL
294 }
295 }
296
297 #endif /* PRIVATE_DSP_ARCH_X86_SSE_PMATH_FMOP_KX_H_ */