Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/x86/sse/pmath/op_kx.h

raw
1 /*
2 * Copyright (C) 2020 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2020 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 31 мар. 2020 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_X86_SSE_PMATH_OP_KX_H_
23 #define PRIVATE_DSP_ARCH_X86_SSE_PMATH_OP_KX_H_
24
25 #ifndef PRIVATE_DSP_ARCH_X86_SSE_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_X86_SSE_IMPL */
28
29 namespace lsp
30 {
31 namespace sse
32 {
33 #define OP_K2_CORE(DST, SRC, OP) \
34 __ASM_EMIT("xor %[off], %[off]") \
35 __ASM_EMIT("shufps $0x00, %%xmm0, %%xmm0") \
36 __ASM_EMIT("sub $24, %[count]") \
37 __ASM_EMIT("movaps %%xmm0, %%xmm1") \
38 __ASM_EMIT("jb 2f") \
39 /* 24x blocks */ \
40 __ASM_EMIT("1:") \
41 __ASM_EMIT("movups 0x00(%[" SRC "], %[off]), %%xmm2") \
42 __ASM_EMIT("movups 0x10(%[" SRC "], %[off]), %%xmm3") \
43 __ASM_EMIT("movups 0x20(%[" SRC "], %[off]), %%xmm4") \
44 __ASM_EMIT("movups 0x30(%[" SRC "], %[off]), %%xmm5") \
45 __ASM_EMIT("movups 0x40(%[" SRC "], %[off]), %%xmm6") \
46 __ASM_EMIT("movups 0x50(%[" SRC "], %[off]), %%xmm7") \
47 __ASM_EMIT(OP "ps %%xmm0, %%xmm2") \
48 __ASM_EMIT(OP "ps %%xmm1, %%xmm3") \
49 __ASM_EMIT(OP "ps %%xmm0, %%xmm4") \
50 __ASM_EMIT(OP "ps %%xmm1, %%xmm5") \
51 __ASM_EMIT(OP "ps %%xmm0, %%xmm6") \
52 __ASM_EMIT(OP "ps %%xmm1, %%xmm7") \
53 __ASM_EMIT("movups %%xmm2, 0x00(%[" DST "], %[off])") \
54 __ASM_EMIT("movups %%xmm3, 0x10(%[" DST "], %[off])") \
55 __ASM_EMIT("movups %%xmm4, 0x20(%[" DST "], %[off])") \
56 __ASM_EMIT("movups %%xmm5, 0x30(%[" DST "], %[off])") \
57 __ASM_EMIT("movups %%xmm6, 0x40(%[" DST "], %[off])") \
58 __ASM_EMIT("movups %%xmm7, 0x50(%[" DST "], %[off])") \
59 __ASM_EMIT("add $0x60, %[off]") \
60 __ASM_EMIT("sub $24, %[count]") \
61 __ASM_EMIT("jae 1b") \
62 /* 16x block */ \
63 __ASM_EMIT("2:") \
64 __ASM_EMIT("add $8, %[count]") /* 24 - 16 */ \
65 __ASM_EMIT("jl 4f") \
66 __ASM_EMIT("movups 0x00(%[" SRC "], %[off]), %%xmm2") \
67 __ASM_EMIT("movups 0x10(%[" SRC "], %[off]), %%xmm3") \
68 __ASM_EMIT("movups 0x20(%[" SRC "], %[off]), %%xmm4") \
69 __ASM_EMIT("movups 0x30(%[" SRC "], %[off]), %%xmm5") \
70 __ASM_EMIT(OP "ps %%xmm0, %%xmm2") \
71 __ASM_EMIT(OP "ps %%xmm1, %%xmm3") \
72 __ASM_EMIT(OP "ps %%xmm0, %%xmm4") \
73 __ASM_EMIT(OP "ps %%xmm1, %%xmm5") \
74 __ASM_EMIT("movups %%xmm2, 0x00(%[" DST "], %[off])") \
75 __ASM_EMIT("movups %%xmm3, 0x10(%[" DST "], %[off])") \
76 __ASM_EMIT("movups %%xmm4, 0x20(%[" DST "], %[off])") \
77 __ASM_EMIT("movups %%xmm5, 0x30(%[" DST "], %[off])") \
78 __ASM_EMIT("sub $16, %[count]") \
79 __ASM_EMIT("add $0x40, %[off]") \
80 /* 8x blocks */ \
81 __ASM_EMIT("4:") \
82 __ASM_EMIT("add $8, %[count]") /* 16 - 8 */ \
83 __ASM_EMIT("jl 6f") \
84 __ASM_EMIT("movups 0x00(%[" SRC "], %[off]), %%xmm2") \
85 __ASM_EMIT("movups 0x10(%[" SRC "], %[off]), %%xmm3") \
86 __ASM_EMIT(OP "ps %%xmm0, %%xmm2") \
87 __ASM_EMIT(OP "ps %%xmm1, %%xmm3") \
88 __ASM_EMIT("movups %%xmm2, 0x00(%[" DST "], %[off])") \
89 __ASM_EMIT("movups %%xmm3, 0x10(%[" DST "], %[off])") \
90 __ASM_EMIT("sub $8, %[count]") \
91 __ASM_EMIT("add $0x20, %[off]") \
92 /* 4x blocks */ \
93 __ASM_EMIT("6:") \
94 __ASM_EMIT("add $4, %[count]") \
95 __ASM_EMIT("jl 8f") \
96 __ASM_EMIT("movups 0x00(%[" SRC "], %[off]), %%xmm2") \
97 __ASM_EMIT(OP "ps %%xmm0, %%xmm2") \
98 __ASM_EMIT("movups %%xmm2, 0x00(%[" DST "], %[off])") \
99 __ASM_EMIT("sub $4, %[count]") \
100 __ASM_EMIT("add $0x10, %[off]") \
101 /* 1x blocks */ \
102 __ASM_EMIT("8:") \
103 __ASM_EMIT("add $3, %[count]") \
104 __ASM_EMIT("jl 10f") \
105 __ASM_EMIT("9:") \
106 __ASM_EMIT("movss 0x00(%[" SRC "], %[off]), %%xmm2") \
107 __ASM_EMIT(OP "ss %%xmm0, %%xmm2") \
108 __ASM_EMIT("movss %%xmm2, 0x00(%[" DST "], %[off])") \
109 __ASM_EMIT("add $0x04, %[off]") \
110 __ASM_EMIT("dec %[count]") \
111 __ASM_EMIT("jge 9b") \
112 __ASM_EMIT("10:")
113
114 #define ROP_K2_CORE(DST, SRC, OP) \
115 __ASM_EMIT("xor %[off], %[off]") \
116 __ASM_EMIT("shufps $0x00, %%xmm0, %%xmm0") \
117 __ASM_EMIT("sub $12, %[count]") \
118 __ASM_EMIT("jb 2f") \
119 /* 12x blocks */ \
120 __ASM_EMIT("1:") \
121 __ASM_EMIT("movups 0x00(%[" SRC "], %[off]), %%xmm4") \
122 __ASM_EMIT("movups 0x10(%[" SRC "], %[off]), %%xmm5") \
123 __ASM_EMIT("movups 0x20(%[" SRC "], %[off]), %%xmm6") \
124 __ASM_EMIT("movaps %%xmm0, %%xmm1") \
125 __ASM_EMIT("movaps %%xmm0, %%xmm2") \
126 __ASM_EMIT("movaps %%xmm1, %%xmm3") \
127 __ASM_EMIT(OP "ps %%xmm4, %%xmm1") \
128 __ASM_EMIT(OP "ps %%xmm5, %%xmm2") \
129 __ASM_EMIT(OP "ps %%xmm6, %%xmm3") \
130 __ASM_EMIT("movups %%xmm1, 0x00(%[" DST "], %[off])") \
131 __ASM_EMIT("movups %%xmm2, 0x10(%[" DST "], %[off])") \
132 __ASM_EMIT("movups %%xmm3, 0x20(%[" DST "], %[off])") \
133 __ASM_EMIT("add $0x30, %[off]") \
134 __ASM_EMIT("sub $12, %[count]") \
135 __ASM_EMIT("jae 1b") \
136 /* 8x block */ \
137 __ASM_EMIT("2:") \
138 __ASM_EMIT("add $4, %[count]") /* 12 - 8 */ \
139 __ASM_EMIT("jl 4f") \
140 __ASM_EMIT("movups 0x00(%[" SRC "], %[off]), %%xmm4") \
141 __ASM_EMIT("movups 0x10(%[" SRC "], %[off]), %%xmm5") \
142 __ASM_EMIT("movaps %%xmm0, %%xmm1") \
143 __ASM_EMIT("movaps %%xmm0, %%xmm2") \
144 __ASM_EMIT(OP "ps %%xmm4, %%xmm1") \
145 __ASM_EMIT(OP "ps %%xmm5, %%xmm2") \
146 __ASM_EMIT("movups %%xmm1, 0x00(%[" DST "], %[off])") \
147 __ASM_EMIT("movups %%xmm2, 0x10(%[" DST "], %[off])") \
148 __ASM_EMIT("sub $8, %[count]") \
149 __ASM_EMIT("add $0x20, %[off]") \
150 /* 4x blocks */ \
151 __ASM_EMIT("4:") \
152 __ASM_EMIT("add $4, %[count]") \
153 __ASM_EMIT("jl 6f") \
154 __ASM_EMIT("movups 0x00(%[" SRC "], %[off]), %%xmm4") \
155 __ASM_EMIT("movaps %%xmm0, %%xmm1") \
156 __ASM_EMIT(OP "ps %%xmm4, %%xmm1") \
157 __ASM_EMIT("movups %%xmm1, 0x00(%[" DST "], %[off])") \
158 __ASM_EMIT("sub $4, %[count]") \
159 __ASM_EMIT("add $0x10, %[off]") \
160 /* 1x blocks */ \
161 __ASM_EMIT("6:") \
162 __ASM_EMIT("add $3, %[count]") \
163 __ASM_EMIT("jl 8f") \
164 __ASM_EMIT("7:") \
165 __ASM_EMIT("movss 0x00(%[" SRC "], %[off]), %%xmm4") \
166 __ASM_EMIT("movaps %%xmm0, %%xmm1") \
167 __ASM_EMIT(OP "ss %%xmm4, %%xmm1") \
168 __ASM_EMIT("movss %%xmm1, 0x00(%[" DST "], %[off])") \
169 __ASM_EMIT("add $0x04, %[off]") \
170 __ASM_EMIT("dec %[count]") \
171 __ASM_EMIT("jge 7b") \
172 __ASM_EMIT("8:")
173
174 void add_k2(float *dst, float k, size_t count)
175 {
176 IF_ARCH_X86(size_t off);
177 ARCH_X86_ASM
178 (
179 OP_K2_CORE("dst", "dst", "add")
180 : [count] "+r" (count), [off] "=&r" (off),
181 [k] "+Yz"(k)
182 : [dst] "r" (dst)
183 : "cc", "memory",
184 "%xmm1", "%xmm2", "%xmm3",
185 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
186 );
187 }
188
189 void sub_k2(float *dst, float k, size_t count)
190 {
191 IF_ARCH_X86(size_t off);
192 ARCH_X86_ASM
193 (
194 OP_K2_CORE("dst", "dst", "sub")
195 : [count] "+r" (count), [off] "=&r" (off),
196 [k] "+Yz"(k)
197 : [dst] "r" (dst)
198 : "cc", "memory",
199 "%xmm1", "%xmm2", "%xmm3",
200 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
201 );
202 }
203
204 void mul_k2(float *dst, float k, size_t count)
205 {
206 IF_ARCH_X86(size_t off);
207 ARCH_X86_ASM
208 (
209 OP_K2_CORE("dst", "dst", "mul")
210 : [count] "+r" (count), [off] "=&r" (off),
211 [k] "+Yz"(k)
212 : [dst] "r" (dst)
213 : "cc", "memory",
214 "%xmm1", "%xmm2", "%xmm3",
215 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
216 );
217 }
218
219 void div_k2(float *dst, float k, size_t count)
220 {
221 mul_k2(dst, 1.0f / k, count);
222 }
223
224 void rsub_k2(float *dst, float k, size_t count)
225 {
226 IF_ARCH_X86(size_t off);
227 ARCH_X86_ASM
228 (
229 ROP_K2_CORE("dst", "dst", "sub")
230 : [count] "+r" (count), [off] "=&r" (off),
231 [k] "+Yz"(k)
232 : [dst] "r" (dst)
233 : "cc", "memory",
234 "%xmm1", "%xmm2", "%xmm3",
235 "%xmm4", "%xmm5", "%xmm6"
236 );
237 }
238
239 void rdiv_k2(float *dst, float k, size_t count)
240 {
241 IF_ARCH_X86(size_t off);
242 ARCH_X86_ASM
243 (
244 ROP_K2_CORE("dst", "dst", "div")
245 : [count] "+r" (count), [off] "=&r" (off),
246 [k] "+Yz"(k)
247 : [dst] "r" (dst)
248 : "cc", "memory",
249 "%xmm1", "%xmm2", "%xmm3",
250 "%xmm4", "%xmm5", "%xmm6"
251 );
252 }
253
254 void add_k3(float *dst, const float *src, float k, size_t count)
255 {
256 IF_ARCH_X86(size_t off);
257 ARCH_X86_ASM
258 (
259 OP_K2_CORE("dst", "src", "add")
260 : [count] "+r" (count), [off] "=&r" (off),
261 [k] "+Yz"(k)
262 : [dst] "r"(dst), [src] "r"(src)
263 : "cc", "memory",
264 "%xmm1", "%xmm2", "%xmm3",
265 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
266 );
267 }
268
269 void sub_k3(float *dst, const float *src, float k, size_t count)
270 {
271 IF_ARCH_X86(size_t off);
272 ARCH_X86_ASM
273 (
274 OP_K2_CORE("dst", "src", "sub")
275 : [count] "+r" (count), [off] "=&r" (off),
276 [k] "+Yz"(k)
277 : [dst] "r"(dst), [src] "r"(src)
278 : "cc", "memory",
279 "%xmm1", "%xmm2", "%xmm3",
280 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
281 );
282 }
283
284 void mul_k3(float *dst, const float *src, float k, size_t count)
285 {
286 IF_ARCH_X86(size_t off);
287 ARCH_X86_ASM
288 (
289 OP_K2_CORE("dst", "src", "mul")
290 : [count] "+r" (count), [off] "=&r" (off),
291 [k] "+Yz"(k)
292 : [dst] "r"(dst), [src] "r"(src)
293 : "cc", "memory",
294 "%xmm1", "%xmm2", "%xmm3",
295 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
296 );
297 }
298
299 void div_k3(float *dst, const float *src, float k, size_t count)
300 {
301 mul_k3(dst, src, 1.0f / k, count);
302 }
303
304 void rsub_k3(float *dst, const float *src, float k, size_t count)
305 {
306 IF_ARCH_X86(size_t off);
307 ARCH_X86_ASM
308 (
309 ROP_K2_CORE("dst", "src", "sub")
310 : [count] "+r" (count), [off] "=&r" (off),
311 [k] "+Yz"(k)
312 : [dst] "r"(dst), [src] "r"(src)
313 : "cc", "memory",
314 "%xmm1", "%xmm2", "%xmm3",
315 "%xmm4", "%xmm5", "%xmm6"
316 );
317 }
318
319 void rdiv_k3(float *dst, const float *src, float k, size_t count)
320 {
321 IF_ARCH_X86(size_t off);
322 ARCH_X86_ASM
323 (
324 ROP_K2_CORE("dst", "src", "div")
325 : [count] "+r" (count), [off] "=&r" (off),
326 [k] "+Yz"(k)
327 : [dst] "r"(dst), [src] "r"(src)
328 : "cc", "memory",
329 "%xmm1", "%xmm2", "%xmm3",
330 "%xmm4", "%xmm5", "%xmm6"
331 );
332 }
333
334 #undef ROP_K2_CORE
335 #undef OP_K2_CORE
336 }
337 }
338
339 #endif /* PRIVATE_DSP_ARCH_X86_SSE_PMATH_OP_KX_H_ */