Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/x86/sse/pmath/minmax.h

raw
1 /*
2 * Copyright (C) 2020 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2020 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 31 мар. 2020 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_X86_SSE_PMATH_MINMAX_H_
23 #define PRIVATE_DSP_ARCH_X86_SSE_PMATH_MINMAX_H_
24
25 #ifndef PRIVATE_DSP_ARCH_X86_SSE_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_X86_SSE_IMPL */
28
29 namespace lsp
30 {
31 namespace sse
32 {
33 IF_ARCH_X86(
34 static const uint32_t minmax_abs[] __lsp_aligned16 =
35 {
36 LSP_DSP_VEC4(0x7fffffff)
37 };
38 )
39
40 #define MINMAX_CORE(DST, A, B, OP) \
41 __ASM_EMIT("xor %[off], %[off]") \
42 __ASM_EMIT("sub $16, %[count]") \
43 __ASM_EMIT("jb 2f") \
44 /* 16x blocks */ \
45 __ASM_EMIT("1:") \
46 __ASM_EMIT("movups 0x00(%[" A "], %[off]), %%xmm0") \
47 __ASM_EMIT("movups 0x10(%[" A "], %[off]), %%xmm1") \
48 __ASM_EMIT("movups 0x20(%[" A "], %[off]), %%xmm2") \
49 __ASM_EMIT("movups 0x30(%[" A "], %[off]), %%xmm3") \
50 __ASM_EMIT("movups 0x00(%[" B "], %[off]), %%xmm4") \
51 __ASM_EMIT("movups 0x10(%[" B "], %[off]), %%xmm5") \
52 __ASM_EMIT("movups 0x20(%[" B "], %[off]), %%xmm6") \
53 __ASM_EMIT("movups 0x30(%[" B "], %[off]), %%xmm7") \
54 __ASM_EMIT(OP " %%xmm4, %%xmm0") \
55 __ASM_EMIT(OP " %%xmm5, %%xmm1") \
56 __ASM_EMIT(OP " %%xmm6, %%xmm2") \
57 __ASM_EMIT(OP " %%xmm7, %%xmm3") \
58 __ASM_EMIT("movups %%xmm0, 0x00(%[" DST "], %[off])") \
59 __ASM_EMIT("movups %%xmm1, 0x10(%[" DST "], %[off])") \
60 __ASM_EMIT("movups %%xmm2, 0x20(%[" DST "], %[off])") \
61 __ASM_EMIT("movups %%xmm3, 0x30(%[" DST "], %[off])") \
62 __ASM_EMIT("add $0x40, %[off]") \
63 __ASM_EMIT("sub $16, %[count]") \
64 __ASM_EMIT("jae 1b") \
65 \
66 /* 8x block */ \
67 __ASM_EMIT("2:") \
68 __ASM_EMIT("add $8, %[count]") \
69 __ASM_EMIT("jl 4f") \
70 __ASM_EMIT("movups 0x00(%[" A "], %[off]), %%xmm0") \
71 __ASM_EMIT("movups 0x10(%[" A "], %[off]), %%xmm1") \
72 __ASM_EMIT("movups 0x00(%[" B "], %[off]), %%xmm4") \
73 __ASM_EMIT("movups 0x10(%[" B "], %[off]), %%xmm5") \
74 __ASM_EMIT(OP " %%xmm4, %%xmm0") \
75 __ASM_EMIT(OP " %%xmm5, %%xmm1") \
76 __ASM_EMIT("movups %%xmm0, 0x00(%[" DST "], %[off])") \
77 __ASM_EMIT("movups %%xmm1, 0x10(%[" DST "], %[off])") \
78 __ASM_EMIT("add $0x20, %[off]") \
79 __ASM_EMIT("sub $8, %[count]") \
80 /* 4x block */ \
81 __ASM_EMIT("4:") \
82 __ASM_EMIT("add $4, %[count]") \
83 __ASM_EMIT("jl 6f") \
84 __ASM_EMIT("movups 0x00(%[" A "], %[off]), %%xmm0") \
85 __ASM_EMIT("movups 0x00(%[" B "], %[off]), %%xmm4") \
86 __ASM_EMIT(OP " %%xmm4, %%xmm0") \
87 __ASM_EMIT("movups %%xmm0, 0x00(%[" DST "], %[off])") \
88 __ASM_EMIT("add $0x10, %[off]") \
89 __ASM_EMIT("sub $4, %[count]") \
90 /* 1x blocks */ \
91 __ASM_EMIT("6:") \
92 __ASM_EMIT("add $3, %[count]") \
93 __ASM_EMIT("jl 8f") \
94 __ASM_EMIT("7:") \
95 __ASM_EMIT("movss 0x00(%[" A "], %[off]), %%xmm0") \
96 __ASM_EMIT("movss 0x00(%[" B "], %[off]), %%xmm4") \
97 __ASM_EMIT(OP " %%xmm4, %%xmm0") \
98 __ASM_EMIT("movss %%xmm0, 0x00(%[" DST "], %[off])") \
99 __ASM_EMIT("add $0x04, %[off]") \
100 __ASM_EMIT("dec %[count]") \
101 __ASM_EMIT("jge 7b") \
102 __ASM_EMIT("8:")
103
104 void pmin2(float *dst, const float *src, size_t count)
105 {
106 IF_ARCH_X86(size_t off);
107 ARCH_X86_ASM
108 (
109 MINMAX_CORE("dst", "dst", "src", "minps")
110 : [off] "=&r" (off), [count] "+r" (count)
111 : [dst] "r" (dst), [src] "r" (src)
112 : "cc", "memory",
113 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
114 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
115 );
116 }
117
118 void pmin3(float *dst, const float *a, const float *b, size_t count)
119 {
120 IF_ARCH_X86(size_t off);
121 ARCH_X86_ASM
122 (
123 MINMAX_CORE("dst", "a", "b", "minps")
124 : [off] "=&r" (off), [count] "+r" (count)
125 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b)
126 : "cc", "memory",
127 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
128 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
129 );
130 }
131
132 void pmax2(float *dst, const float *src, size_t count)
133 {
134 IF_ARCH_X86(size_t off);
135 ARCH_X86_ASM
136 (
137 MINMAX_CORE("dst", "dst", "src", "maxps")
138 : [off] "=&r" (off), [count] "+r" (count)
139 : [dst] "r" (dst), [src] "r" (src)
140 : "cc", "memory",
141 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
142 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
143 );
144 }
145
146 void pmax3(float *dst, const float *a, const float *b, size_t count)
147 {
148 IF_ARCH_X86(size_t off);
149 ARCH_X86_ASM
150 (
151 MINMAX_CORE("dst", "a", "b", "maxps")
152 : [off] "=&r" (off), [count] "+r" (count)
153 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b)
154 : "cc", "memory",
155 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
156 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
157 );
158 }
159
160 #undef MINMAX_CORE
161
162 #define SIGN_MINMAX_CORE(DST, A, B, CMP) \
163 __ASM_EMIT("movaps %[X_SIGN], %%xmm6") \
164 __ASM_EMIT("xor %[off], %[off]") \
165 __ASM_EMIT("movaps %%xmm6, %%xmm7") \
166 __ASM_EMIT("sub $4, %[count]") \
167 __ASM_EMIT("jb 2f") \
168 /* 4x blocks */ \
169 __ASM_EMIT("1:") \
170 __ASM_EMIT("movups 0x00(%[" A "], %[off]), %%xmm0") \
171 __ASM_EMIT("movups 0x00(%[" B "], %[off]), %%xmm1") \
172 __ASM_EMIT("movaps %%xmm0, %%xmm2") \
173 __ASM_EMIT("movaps %%xmm1, %%xmm3") \
174 __ASM_EMIT("andps %%xmm6, %%xmm2") /* xmm2 = abs(a) */ \
175 __ASM_EMIT("andps %%xmm7, %%xmm3") /* xmm3 = abs(b) */\
176 __ASM_EMIT("cmpps " CMP ", %%xmm3, %%xmm2") /* xmm2 = abs(a) <=> abs(b) */ \
177 __ASM_EMIT("andps %%xmm2, %%xmm0") /* xmm0 = a & (abs(a) <=> abs(b)) */ \
178 __ASM_EMIT("andnps %%xmm1, %%xmm2") /* xmm2 = b & ~(abs(a) <=> abs(b)) */ \
179 __ASM_EMIT("orps %%xmm2, %%xmm0") /* xmm0 = (a & (abs(a) <=> abs(b))) | (b & ~(abs(a) <=> abs(b))) */ \
180 __ASM_EMIT("movups %%xmm0, 0x00(%[" DST "], %[off])") \
181 __ASM_EMIT("add $0x10, %[off]") \
182 __ASM_EMIT("sub $4, %[count]") \
183 __ASM_EMIT("jae 1b") \
184 \
185 /* 1x blocks */ \
186 __ASM_EMIT("2:") \
187 __ASM_EMIT("add $3, %[count]") \
188 __ASM_EMIT("jl 4f") \
189 __ASM_EMIT("3:") \
190 __ASM_EMIT("movss 0x00(%[" A "], %[off]), %%xmm0") \
191 __ASM_EMIT("movss 0x00(%[" B "], %[off]), %%xmm1") \
192 __ASM_EMIT("movaps %%xmm0, %%xmm2") \
193 __ASM_EMIT("movaps %%xmm1, %%xmm3") \
194 __ASM_EMIT("andps %%xmm6, %%xmm2") /* xmm2 = abs(a) */ \
195 __ASM_EMIT("andps %%xmm7, %%xmm3") /* xmm3 = abs(b) */\
196 __ASM_EMIT("cmpps " CMP ", %%xmm3, %%xmm2") /* xmm2 = abs(a) <=> abs(b) */ \
197 __ASM_EMIT("andps %%xmm2, %%xmm0") /* xmm0 = a & (abs(a) <=> abs(b)) */ \
198 __ASM_EMIT("andnps %%xmm1, %%xmm2") /* xmm2 = b & ~(abs(a) <=> abs(b)) */ \
199 __ASM_EMIT("orps %%xmm2, %%xmm0") /* xmm0 = (a & (abs(a) <=> abs(b))) | (b & ~(abs(a) <=> abs(b))) */ \
200 __ASM_EMIT("movss %%xmm0, 0x00(%[" DST "], %[off])") \
201 __ASM_EMIT("add $0x04, %[off]") \
202 __ASM_EMIT("dec %[count]") \
203 __ASM_EMIT("jge 3b") \
204 __ASM_EMIT("4:")
205
206 void psmin2(float *dst, const float *src, size_t count)
207 {
208 IF_ARCH_X86(size_t off);
209 ARCH_X86_ASM
210 (
211 SIGN_MINMAX_CORE("dst", "dst", "src", "$1")
212 : [off] "=&r" (off), [count] "+r" (count)
213 : [dst] "r" (dst), [src] "r" (src),
214 [X_SIGN] "m" (minmax_abs)
215 : "cc", "memory",
216 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
217 "%xmm6", "%xmm7"
218 );
219 }
220
221 void psmin3(float *dst, const float *a, const float *b, size_t count)
222 {
223 IF_ARCH_X86(size_t off);
224 ARCH_X86_ASM
225 (
226 SIGN_MINMAX_CORE("dst", "a", "b", "$1")
227 : [off] "=&r" (off), [count] "+r" (count)
228 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b),
229 [X_SIGN] "m" (minmax_abs)
230 : "cc", "memory",
231 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
232 "%xmm6", "%xmm7"
233 );
234 }
235
236 void psmax2(float *dst, const float *src, size_t count)
237 {
238 IF_ARCH_X86(size_t off);
239 ARCH_X86_ASM
240 (
241 SIGN_MINMAX_CORE("dst", "dst", "src", "$6")
242 : [off] "=&r" (off), [count] "+r" (count)
243 : [dst] "r" (dst), [src] "r" (src),
244 [X_SIGN] "m" (minmax_abs)
245 : "cc", "memory",
246 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
247 "%xmm6", "%xmm7"
248 );
249 }
250
251 void psmax3(float *dst, const float *a, const float *b, size_t count)
252 {
253 IF_ARCH_X86(size_t off);
254 ARCH_X86_ASM
255 (
256 SIGN_MINMAX_CORE("dst", "a", "b", "$6")
257 : [off] "=&r" (off), [count] "+r" (count)
258 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b),
259 [X_SIGN] "m" (minmax_abs)
260 : "cc", "memory",
261 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
262 "%xmm6", "%xmm7"
263 );
264 }
265
266 #undef SIGN_MINMAX_CORE
267
268 #define ABS_MINMAX_CORE(DST, A, B, OP) \
269 __ASM_EMIT("movaps %[X_SIGN], %%xmm6") \
270 __ASM_EMIT("xor %[off], %[off]") \
271 __ASM_EMIT("movaps %%xmm6, %%xmm7") \
272 __ASM_EMIT("sub $12, %[count]") \
273 __ASM_EMIT("jb 2f") \
274 /* 12x blocks */ \
275 __ASM_EMIT("1:") \
276 __ASM_EMIT("movups 0x00(%[" A "], %[off]), %%xmm0") \
277 __ASM_EMIT("movups 0x10(%[" A "], %[off]), %%xmm1") \
278 __ASM_EMIT("movups 0x20(%[" A "], %[off]), %%xmm2") \
279 __ASM_EMIT("movups 0x00(%[" B "], %[off]), %%xmm3") \
280 __ASM_EMIT("movups 0x10(%[" B "], %[off]), %%xmm4") \
281 __ASM_EMIT("movups 0x20(%[" B "], %[off]), %%xmm5") \
282 __ASM_EMIT("andps %%xmm6, %%xmm0") \
283 __ASM_EMIT("andps %%xmm7, %%xmm1") \
284 __ASM_EMIT("andps %%xmm6, %%xmm2") \
285 __ASM_EMIT("andps %%xmm7, %%xmm3") \
286 __ASM_EMIT("andps %%xmm6, %%xmm4") \
287 __ASM_EMIT("andps %%xmm7, %%xmm5") \
288 __ASM_EMIT(OP " %%xmm3, %%xmm0") \
289 __ASM_EMIT(OP " %%xmm4, %%xmm1") \
290 __ASM_EMIT(OP " %%xmm5, %%xmm2") \
291 __ASM_EMIT("movups %%xmm0, 0x00(%[" DST "], %[off])") \
292 __ASM_EMIT("movups %%xmm1, 0x10(%[" DST "], %[off])") \
293 __ASM_EMIT("movups %%xmm2, 0x20(%[" DST "], %[off])") \
294 __ASM_EMIT("add $0x30, %[off]") \
295 __ASM_EMIT("sub $12, %[count]") \
296 __ASM_EMIT("jae 1b") \
297 \
298 /* 8x block */ \
299 __ASM_EMIT("2:") \
300 __ASM_EMIT("add $4, %[count]") \
301 __ASM_EMIT("jl 4f") \
302 __ASM_EMIT("movups 0x00(%[" A "], %[off]), %%xmm0") \
303 __ASM_EMIT("movups 0x10(%[" A "], %[off]), %%xmm1") \
304 __ASM_EMIT("movups 0x00(%[" B "], %[off]), %%xmm3") \
305 __ASM_EMIT("movups 0x10(%[" B "], %[off]), %%xmm4") \
306 __ASM_EMIT("andps %%xmm6, %%xmm0") \
307 __ASM_EMIT("andps %%xmm7, %%xmm1") \
308 __ASM_EMIT("andps %%xmm7, %%xmm3") \
309 __ASM_EMIT("andps %%xmm6, %%xmm4") \
310 __ASM_EMIT(OP " %%xmm3, %%xmm0") \
311 __ASM_EMIT(OP " %%xmm4, %%xmm1") \
312 __ASM_EMIT("movups %%xmm0, 0x00(%[" DST "], %[off])") \
313 __ASM_EMIT("movups %%xmm1, 0x10(%[" DST "], %[off])") \
314 __ASM_EMIT("add $0x20, %[off]") \
315 __ASM_EMIT("sub $8, %[count]") \
316 /* 4x block */ \
317 __ASM_EMIT("4:") \
318 __ASM_EMIT("add $4, %[count]") \
319 __ASM_EMIT("jl 6f") \
320 __ASM_EMIT("movups 0x00(%[" A "], %[off]), %%xmm0") \
321 __ASM_EMIT("movups 0x00(%[" B "], %[off]), %%xmm3") \
322 __ASM_EMIT("andps %%xmm6, %%xmm0") \
323 __ASM_EMIT("andps %%xmm7, %%xmm3") \
324 __ASM_EMIT(OP " %%xmm3, %%xmm0") \
325 __ASM_EMIT("movups %%xmm0, 0x00(%[" DST "], %[off])") \
326 __ASM_EMIT("add $0x10, %[off]") \
327 __ASM_EMIT("sub $4, %[count]") \
328 /* 1x blocks */ \
329 __ASM_EMIT("6:") \
330 __ASM_EMIT("add $3, %[count]") \
331 __ASM_EMIT("jl 8f") \
332 __ASM_EMIT("7:") \
333 __ASM_EMIT("movss 0x00(%[" A "], %[off]), %%xmm0") \
334 __ASM_EMIT("movss 0x00(%[" B "], %[off]), %%xmm3") \
335 __ASM_EMIT("andps %%xmm6, %%xmm0") \
336 __ASM_EMIT("andps %%xmm7, %%xmm3") \
337 __ASM_EMIT(OP " %%xmm3, %%xmm0") \
338 __ASM_EMIT("movss %%xmm0, 0x00(%[" DST "], %[off])") \
339 __ASM_EMIT("add $0x04, %[off]") \
340 __ASM_EMIT("dec %[count]") \
341 __ASM_EMIT("jge 7b") \
342 __ASM_EMIT("8:")
343
344 void pamin2(float *dst, const float *src, size_t count)
345 {
346 IF_ARCH_X86(size_t off);
347 ARCH_X86_ASM
348 (
349 ABS_MINMAX_CORE("dst", "dst", "src", "minps")
350 : [off] "=&r" (off), [count] "+r" (count)
351 : [dst] "r" (dst), [src] "r" (src),
352 [X_SIGN] "m" (minmax_abs)
353 : "cc", "memory",
354 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
355 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
356 );
357 }
358
359 void pamin3(float *dst, const float *a, const float *b, size_t count)
360 {
361 IF_ARCH_X86(size_t off);
362 ARCH_X86_ASM
363 (
364 ABS_MINMAX_CORE("dst", "a", "b", "minps")
365 : [off] "=&r" (off), [count] "+r" (count)
366 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b),
367 [X_SIGN] "m" (minmax_abs)
368 : "cc", "memory",
369 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
370 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
371 );
372 }
373
374 void pamax2(float *dst, const float *src, size_t count)
375 {
376 IF_ARCH_X86(size_t off);
377 ARCH_X86_ASM
378 (
379 ABS_MINMAX_CORE("dst", "dst", "src", "maxps")
380 : [off] "=&r" (off), [count] "+r" (count)
381 : [dst] "r" (dst), [src] "r" (src),
382 [X_SIGN] "m" (minmax_abs)
383 : "cc", "memory",
384 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
385 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
386 );
387 }
388
389 void pamax3(float *dst, const float *a, const float *b, size_t count)
390 {
391 IF_ARCH_X86(size_t off);
392 ARCH_X86_ASM
393 (
394 ABS_MINMAX_CORE("dst", "a", "b", "maxps")
395 : [off] "=&r" (off), [count] "+r" (count)
396 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b),
397 [X_SIGN] "m" (minmax_abs)
398 : "cc", "memory",
399 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
400 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
401 );
402 }
403
404 #undef ABS_MINMAX_CORE
405 }
406 }
407
408
409 #endif /* PRIVATE_DSP_ARCH_X86_SSE_PMATH_MINMAX_H_ */