Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/x86/avx/pmath/minmax.h

raw
1 /*
2 * Copyright (C) 2020 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2020 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 31 мар. 2020 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_X86_AVX_PMATH_MINMAX_H_
23 #define PRIVATE_DSP_ARCH_X86_AVX_PMATH_MINMAX_H_
24
25 #ifndef PRIVATE_DSP_ARCH_X86_AVX_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_X86_AVX_IMPL */
28
29 namespace lsp
30 {
31 namespace avx
32 {
33 IF_ARCH_X86(
34 static const uint32_t minmax_abs[] __lsp_aligned32 =
35 {
36 LSP_DSP_VEC8(0x7fffffff)
37 };
38 )
39
40 #define MINMAX_CORE(DST, A, B, OP) \
41 __ASM_EMIT("xor %[off], %[off]") \
42 __ASM_EMIT("sub $32, %[count]") \
43 __ASM_EMIT("jb 2f") \
44 /* 32x blocks */ \
45 __ASM_EMIT("1:") \
46 __ASM_EMIT("vmovups 0x00(%[" A "], %[off]), %%ymm0") \
47 __ASM_EMIT("vmovups 0x20(%[" A "], %[off]), %%ymm1") \
48 __ASM_EMIT("vmovups 0x40(%[" A "], %[off]), %%ymm2") \
49 __ASM_EMIT("vmovups 0x60(%[" A "], %[off]), %%ymm3") \
50 __ASM_EMIT(OP " 0x00(%[" B "], %[off]), %%ymm0, %%ymm0") \
51 __ASM_EMIT(OP " 0x20(%[" B "], %[off]), %%ymm1, %%ymm1") \
52 __ASM_EMIT(OP " 0x40(%[" B "], %[off]), %%ymm2, %%ymm2") \
53 __ASM_EMIT(OP " 0x60(%[" B "], %[off]), %%ymm3, %%ymm3") \
54 __ASM_EMIT("vmovups %%ymm0, 0x00(%[" DST "], %[off])") \
55 __ASM_EMIT("vmovups %%ymm1, 0x20(%[" DST "], %[off])") \
56 __ASM_EMIT("vmovups %%ymm2, 0x40(%[" DST "], %[off])") \
57 __ASM_EMIT("vmovups %%ymm3, 0x60(%[" DST "], %[off])") \
58 __ASM_EMIT("add $0x80, %[off]") \
59 __ASM_EMIT("sub $32, %[count]") \
60 __ASM_EMIT("jae 1b") \
61 /* 16x block */ \
62 __ASM_EMIT("2:") \
63 __ASM_EMIT("add $16, %[count]") \
64 __ASM_EMIT("jl 4f") \
65 __ASM_EMIT("vmovups 0x00(%[" A "], %[off]), %%ymm0") \
66 __ASM_EMIT("vmovups 0x20(%[" A "], %[off]), %%ymm1") \
67 __ASM_EMIT(OP " 0x00(%[" B "], %[off]), %%ymm0, %%ymm0") \
68 __ASM_EMIT(OP " 0x20(%[" B "], %[off]), %%ymm1, %%ymm1") \
69 __ASM_EMIT("vmovups %%ymm0, 0x00(%[" DST "], %[off])") \
70 __ASM_EMIT("vmovups %%ymm1, 0x20(%[" DST "], %[off])") \
71 __ASM_EMIT("add $0x40, %[off]") \
72 __ASM_EMIT("sub $16, %[count]") \
73 /* 8x block */ \
74 __ASM_EMIT("4:") \
75 __ASM_EMIT("add $8, %[count]") \
76 __ASM_EMIT("jl 6f") \
77 __ASM_EMIT("vmovups 0x00(%[" A "], %[off]), %%xmm0") \
78 __ASM_EMIT("vmovups 0x10(%[" A "], %[off]), %%xmm1") \
79 __ASM_EMIT(OP " 0x00(%[" B "], %[off]), %%xmm0, %%xmm0") \
80 __ASM_EMIT(OP " 0x10(%[" B "], %[off]), %%xmm1, %%xmm1") \
81 __ASM_EMIT("vmovups %%xmm0, 0x00(%[" DST "], %[off])") \
82 __ASM_EMIT("vmovups %%xmm1, 0x10(%[" DST "], %[off])") \
83 __ASM_EMIT("add $0x20, %[off]") \
84 __ASM_EMIT("sub $8, %[count]") \
85 /* 4x block */ \
86 __ASM_EMIT("6:") \
87 __ASM_EMIT("add $4, %[count]") \
88 __ASM_EMIT("jl 8f") \
89 __ASM_EMIT("vmovups 0x00(%[" A "], %[off]), %%xmm0") \
90 __ASM_EMIT(OP " 0x00(%[" B "], %[off]), %%xmm0, %%xmm0") \
91 __ASM_EMIT("vmovups %%xmm0, 0x00(%[" DST "], %[off])") \
92 __ASM_EMIT("add $0x10, %[off]") \
93 __ASM_EMIT("sub $4, %[count]") \
94 /* 1x blocks */ \
95 __ASM_EMIT("8:") \
96 __ASM_EMIT("add $3, %[count]") \
97 __ASM_EMIT("jl 10f") \
98 __ASM_EMIT("9:") \
99 __ASM_EMIT("vmovss 0x00(%[" A "], %[off]), %%xmm0") \
100 __ASM_EMIT("vmovss 0x00(%[" B "], %[off]), %%xmm1") \
101 __ASM_EMIT(OP " %%xmm1, %%xmm0, %%xmm0") \
102 __ASM_EMIT("vmovss %%xmm0, 0x00(%[" DST "], %[off])") \
103 __ASM_EMIT("add $0x04, %[off]") \
104 __ASM_EMIT("dec %[count]") \
105 __ASM_EMIT("jge 9b") \
106 __ASM_EMIT("10:")
107
108 void pmin2(float *dst, const float *src, size_t count)
109 {
110 IF_ARCH_X86(size_t off);
111 ARCH_X86_ASM
112 (
113 MINMAX_CORE("dst", "dst", "src", "vminps")
114 : [off] "=&r" (off), [count] "+r" (count)
115 : [dst] "r" (dst), [src] "r" (src)
116 : "cc", "memory",
117 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
118 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
119 );
120 }
121
122 void pmin3(float *dst, const float *a, const float *b, size_t count)
123 {
124 IF_ARCH_X86(size_t off);
125 ARCH_X86_ASM
126 (
127 MINMAX_CORE("dst", "a", "b", "vminps")
128 : [off] "=&r" (off), [count] "+r" (count)
129 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b)
130 : "cc", "memory",
131 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
132 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
133 );
134 }
135
136 void pmax2(float *dst, const float *src, size_t count)
137 {
138 IF_ARCH_X86(size_t off);
139 ARCH_X86_ASM
140 (
141 MINMAX_CORE("dst", "dst", "src", "vmaxps")
142 : [off] "=&r" (off), [count] "+r" (count)
143 : [dst] "r" (dst), [src] "r" (src)
144 : "cc", "memory",
145 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
146 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
147 );
148 }
149
150 void pmax3(float *dst, const float *a, const float *b, size_t count)
151 {
152 IF_ARCH_X86(size_t off);
153 ARCH_X86_ASM
154 (
155 MINMAX_CORE("dst", "a", "b", "vmaxps")
156 : [off] "=&r" (off), [count] "+r" (count)
157 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b)
158 : "cc", "memory",
159 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
160 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
161 );
162 }
163
164 #undef MINMAX_CORE
165
166 #define SIGN_MINMAX_CORE(DST, A, B, CMP) \
167 __ASM_EMIT("vmovaps %[X_SIGN], %%ymm6") \
168 __ASM_EMIT("xor %[off], %[off]") \
169 __ASM_EMIT("vmovaps %%ymm6, %%ymm7") \
170 __ASM_EMIT("sub $8, %[count]") \
171 __ASM_EMIT("jb 2f") \
172 /* 8x blocks */ \
173 __ASM_EMIT("1:") \
174 __ASM_EMIT("vmovups 0x00(%[" A "], %[off]), %%ymm0") \
175 __ASM_EMIT("vmovups 0x00(%[" B "], %[off]), %%ymm1") \
176 __ASM_EMIT("vandps %%ymm6, %%ymm0, %%ymm2") \
177 __ASM_EMIT("vandps %%ymm7, %%ymm1, %%ymm3") \
178 __ASM_EMIT("vcmpps " CMP ", %%ymm3, %%ymm2, %%ymm2") \
179 __ASM_EMIT("vblendvps %%ymm2, %%ymm0, %%ymm1, %%ymm0") \
180 __ASM_EMIT("vmovups %%ymm0, 0x00(%[" DST "], %[off])") \
181 __ASM_EMIT("add $0x20, %[off]") \
182 __ASM_EMIT("sub $8, %[count]") \
183 __ASM_EMIT("jae 1b") \
184 /* 4x block */ \
185 __ASM_EMIT("2:") \
186 __ASM_EMIT("add $4, %[count]") \
187 __ASM_EMIT("jl 4f") \
188 __ASM_EMIT("vmovups 0x00(%[" A "], %[off]), %%xmm0") \
189 __ASM_EMIT("vmovups 0x00(%[" B "], %[off]), %%xmm1") \
190 __ASM_EMIT("vandps %%xmm6, %%xmm0, %%xmm2") \
191 __ASM_EMIT("vandps %%xmm7, %%xmm1, %%xmm3") \
192 __ASM_EMIT("vcmpps " CMP ", %%xmm3, %%xmm2, %%xmm2") \
193 __ASM_EMIT("vblendvps %%xmm2, %%xmm0, %%xmm1, %%xmm0") \
194 __ASM_EMIT("vmovups %%xmm0, 0x00(%[" DST "], %[off])") \
195 __ASM_EMIT("add $0x10, %[off]") \
196 __ASM_EMIT("sub $4, %[count]") \
197 /* 1x blocks */ \
198 __ASM_EMIT("4:") \
199 __ASM_EMIT("add $3, %[count]") \
200 __ASM_EMIT("jl 6f") \
201 __ASM_EMIT("5:") \
202 __ASM_EMIT("vmovss 0x00(%[" A "], %[off]), %%xmm0") \
203 __ASM_EMIT("vmovss 0x00(%[" B "], %[off]), %%xmm1") \
204 __ASM_EMIT("vandps %%xmm6, %%xmm0, %%xmm2") \
205 __ASM_EMIT("vandps %%xmm7, %%xmm1, %%xmm3") \
206 __ASM_EMIT("vcmpps " CMP ", %%xmm3, %%xmm2, %%xmm2") \
207 __ASM_EMIT("vblendvps %%xmm2, %%xmm0, %%xmm1, %%xmm0") \
208 __ASM_EMIT("vmovss %%xmm0, 0x00(%[" DST "], %[off])") \
209 __ASM_EMIT("add $0x04, %[off]") \
210 __ASM_EMIT("dec %[count]") \
211 __ASM_EMIT("jge 5b") \
212 __ASM_EMIT("6:")
213
214 void psmin2(float *dst, const float *src, size_t count)
215 {
216 IF_ARCH_X86(size_t off);
217 ARCH_X86_ASM
218 (
219 SIGN_MINMAX_CORE("dst", "dst", "src", "$1")
220 : [off] "=&r" (off), [count] "+r" (count)
221 : [dst] "r" (dst), [src] "r" (src),
222 [X_SIGN] "m" (minmax_abs)
223 : "cc", "memory",
224 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
225 "%xmm6", "%xmm7"
226 );
227 }
228
229 void psmin3(float *dst, const float *a, const float *b, size_t count)
230 {
231 IF_ARCH_X86(size_t off);
232 ARCH_X86_ASM
233 (
234 SIGN_MINMAX_CORE("dst", "a", "b", "$1")
235 : [off] "=&r" (off), [count] "+r" (count)
236 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b),
237 [X_SIGN] "m" (minmax_abs)
238 : "cc", "memory",
239 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
240 "%xmm6", "%xmm7"
241 );
242 }
243
244 void psmax2(float *dst, const float *src, size_t count)
245 {
246 IF_ARCH_X86(size_t off);
247 ARCH_X86_ASM
248 (
249 SIGN_MINMAX_CORE("dst", "dst", "src", "$5")
250 : [off] "=&r" (off), [count] "+r" (count)
251 : [dst] "r" (dst), [src] "r" (src),
252 [X_SIGN] "m" (minmax_abs)
253 : "cc", "memory",
254 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
255 "%xmm6", "%xmm7"
256 );
257 }
258
259 void psmax3(float *dst, const float *a, const float *b, size_t count)
260 {
261 IF_ARCH_X86(size_t off);
262 ARCH_X86_ASM
263 (
264 SIGN_MINMAX_CORE("dst", "a", "b", "$5")
265 : [off] "=&r" (off), [count] "+r" (count)
266 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b),
267 [X_SIGN] "m" (minmax_abs)
268 : "cc", "memory",
269 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
270 "%xmm6", "%xmm7"
271 );
272 }
273
274 #undef SIGN_MINMAX_CORE
275
276 #define ABS_MINMAX_CORE(DST, A, B, OP) \
277 __ASM_EMIT("vmovaps %[X_SIGN], %%ymm6") \
278 __ASM_EMIT("xor %[off], %[off]") \
279 __ASM_EMIT("vmovaps %%ymm6, %%ymm7") \
280 __ASM_EMIT("sub $24, %[count]") \
281 __ASM_EMIT("jb 2f") \
282 /* 24x blocks */ \
283 __ASM_EMIT("1:") \
284 __ASM_EMIT("vandps 0x00(%[" A "], %[off]), %%ymm6, %%ymm0") \
285 __ASM_EMIT("vandps 0x20(%[" A "], %[off]), %%ymm7, %%ymm1") \
286 __ASM_EMIT("vandps 0x40(%[" A "], %[off]), %%ymm6, %%ymm2") \
287 __ASM_EMIT("vandps 0x00(%[" B "], %[off]), %%ymm7, %%ymm3") \
288 __ASM_EMIT("vandps 0x20(%[" B "], %[off]), %%ymm6, %%ymm4") \
289 __ASM_EMIT("vandps 0x40(%[" B "], %[off]), %%ymm7, %%ymm5") \
290 __ASM_EMIT(OP " %%ymm3, %%ymm0, %%ymm0") \
291 __ASM_EMIT(OP " %%ymm4, %%ymm1, %%ymm1") \
292 __ASM_EMIT(OP " %%ymm5, %%ymm2, %%ymm2") \
293 __ASM_EMIT("vmovups %%ymm0, 0x00(%[" DST "], %[off])") \
294 __ASM_EMIT("vmovups %%ymm1, 0x20(%[" DST "], %[off])") \
295 __ASM_EMIT("vmovups %%ymm2, 0x40(%[" DST "], %[off])") \
296 __ASM_EMIT("add $0x60, %[off]") \
297 __ASM_EMIT("sub $24, %[count]") \
298 __ASM_EMIT("jae 1b") \
299 /* 16x block */ \
300 __ASM_EMIT("2:") \
301 __ASM_EMIT("add $8, %[count]") \
302 __ASM_EMIT("jl 4f") \
303 __ASM_EMIT("vandps 0x00(%[" A "], %[off]), %%ymm6, %%ymm0") \
304 __ASM_EMIT("vandps 0x20(%[" A "], %[off]), %%ymm7, %%ymm1") \
305 __ASM_EMIT("vandps 0x00(%[" B "], %[off]), %%ymm6, %%ymm3") \
306 __ASM_EMIT("vandps 0x20(%[" B "], %[off]), %%ymm7, %%ymm4") \
307 __ASM_EMIT(OP " %%ymm3, %%ymm0, %%ymm0") \
308 __ASM_EMIT(OP " %%ymm4, %%ymm1, %%ymm1") \
309 __ASM_EMIT("vmovups %%ymm0, 0x00(%[" DST "], %[off])") \
310 __ASM_EMIT("vmovups %%ymm1, 0x20(%[" DST "], %[off])") \
311 __ASM_EMIT("add $0x40, %[off]") \
312 __ASM_EMIT("sub $16, %[count]") \
313 /* 8x block */ \
314 __ASM_EMIT("4:") \
315 __ASM_EMIT("add $8, %[count]") \
316 __ASM_EMIT("jl 6f") \
317 __ASM_EMIT("vandps 0x00(%[" A "], %[off]), %%xmm6, %%xmm0") \
318 __ASM_EMIT("vandps 0x10(%[" A "], %[off]), %%xmm7, %%xmm1") \
319 __ASM_EMIT("vandps 0x00(%[" B "], %[off]), %%xmm6, %%xmm3") \
320 __ASM_EMIT("vandps 0x10(%[" B "], %[off]), %%xmm7, %%xmm4") \
321 __ASM_EMIT(OP " %%xmm3, %%xmm0, %%xmm0") \
322 __ASM_EMIT(OP " %%xmm4, %%xmm1, %%xmm1") \
323 __ASM_EMIT("vmovups %%xmm0, 0x00(%[" DST "], %[off])") \
324 __ASM_EMIT("vmovups %%xmm1, 0x10(%[" DST "], %[off])") \
325 __ASM_EMIT("add $0x20, %[off]") \
326 __ASM_EMIT("sub $8, %[count]") \
327 /* 4x block */ \
328 __ASM_EMIT("6:") \
329 __ASM_EMIT("add $4, %[count]") \
330 __ASM_EMIT("jl 8f") \
331 __ASM_EMIT("vandps 0x00(%[" A "], %[off]), %%xmm6, %%xmm0") \
332 __ASM_EMIT("vandps 0x00(%[" B "], %[off]), %%xmm7, %%xmm3") \
333 __ASM_EMIT(OP " %%xmm3, %%xmm0, %%xmm0") \
334 __ASM_EMIT("vmovups %%xmm0, 0x00(%[" DST "], %[off])") \
335 __ASM_EMIT("add $0x10, %[off]") \
336 __ASM_EMIT("sub $4, %[count]") \
337 /* 1x blocks */ \
338 __ASM_EMIT("8:") \
339 __ASM_EMIT("add $3, %[count]") \
340 __ASM_EMIT("jl 10f") \
341 __ASM_EMIT("9:") \
342 __ASM_EMIT("vmovss 0x00(%[" A "], %[off]), %%xmm0") \
343 __ASM_EMIT("vmovss 0x00(%[" B "], %[off]), %%xmm3") \
344 __ASM_EMIT("vandps %%xmm0, %%xmm6, %%xmm0") \
345 __ASM_EMIT("vandps %%xmm3, %%xmm7, %%xmm3") \
346 __ASM_EMIT(OP " %%xmm3, %%xmm0, %%xmm0") \
347 __ASM_EMIT("vmovss %%xmm0, 0x00(%[" DST "], %[off])") \
348 __ASM_EMIT("add $0x04, %[off]") \
349 __ASM_EMIT("dec %[count]") \
350 __ASM_EMIT("jge 9b") \
351 __ASM_EMIT("10:")
352
353 void pamin2(float *dst, const float *src, size_t count)
354 {
355 IF_ARCH_X86(size_t off);
356 ARCH_X86_ASM
357 (
358 ABS_MINMAX_CORE("dst", "dst", "src", "vminps")
359 : [off] "=&r" (off), [count] "+r" (count)
360 : [dst] "r" (dst), [src] "r" (src),
361 [X_SIGN] "m" (minmax_abs)
362 : "cc", "memory",
363 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
364 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
365 );
366 }
367
368 void pamin3(float *dst, const float *a, const float *b, size_t count)
369 {
370 IF_ARCH_X86(size_t off);
371 ARCH_X86_ASM
372 (
373 ABS_MINMAX_CORE("dst", "a", "b", "vminps")
374 : [off] "=&r" (off), [count] "+r" (count)
375 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b),
376 [X_SIGN] "m" (minmax_abs)
377 : "cc", "memory",
378 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
379 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
380 );
381 }
382
383 void pamax2(float *dst, const float *src, size_t count)
384 {
385 IF_ARCH_X86(size_t off);
386 ARCH_X86_ASM
387 (
388 ABS_MINMAX_CORE("dst", "dst", "src", "vmaxps")
389 : [off] "=&r" (off), [count] "+r" (count)
390 : [dst] "r" (dst), [src] "r" (src),
391 [X_SIGN] "m" (minmax_abs)
392 : "cc", "memory",
393 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
394 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
395 );
396 }
397
398 void pamax3(float *dst, const float *a, const float *b, size_t count)
399 {
400 IF_ARCH_X86(size_t off);
401 ARCH_X86_ASM
402 (
403 ABS_MINMAX_CORE("dst", "a", "b", "vmaxps")
404 : [off] "=&r" (off), [count] "+r" (count)
405 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b),
406 [X_SIGN] "m" (minmax_abs)
407 : "cc", "memory",
408 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
409 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
410 );
411 }
412
413 #undef ABS_MINMAX_CORE
414 }
415 }
416
417
418 #endif /* PRIVATE_DSP_ARCH_X86_AVX_PMATH_MINMAX_H_ */