Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/x86/avx512/pmath/ms_minmax.h

raw
1 /*
2 * Copyright (C) 2026 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2026 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 29 мар. 2026 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_X86_AVX512_PMATH_MS_MINMAX_H_
23 #define PRIVATE_DSP_ARCH_X86_AVX512_PMATH_MS_MINMAX_H_
24
25 #ifndef PRIVATE_DSP_ARCH_X86_AVX512_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_X86_AVX512_IMPL */
28
29 namespace lsp
30 {
31 namespace avx512
32 {
33 IF_ARCH_X86(
34 static const uint32_t ms_minmax_abs[] __lsp_aligned64 =
35 {
36 LSP_DSP_VEC16(0x7fffffff),
37 LSP_DSP_VEC16(0x3f000000) // 0.5f
38 };
39 )
40
41 #define MS_MUL(x) __ASM_EMIT(x)
42 #define MS_NOMUL(x)
43
44 #define MS_MINMAX_CORE(DST, A, B, OP, IF_MUL) \
45 __ASM_EMIT("xor %[off], %[off]") \
46 __ASM_EMIT("sub $32, %[count]") \
47 IF_MUL("vmovaps 0x40 + %[CC], %%zmm6") /* zmm6 = 0.5f */ \
48 __ASM_EMIT("jb 2f") \
49 /* 32x blocks */ \
50 __ASM_EMIT("1:") \
51 __ASM_EMIT("vmovups 0x00(%[" A "], %[off]), %%zmm0") /* zmm0 = l */ \
52 __ASM_EMIT("vmovups 0x40(%[" A "], %[off]), %%zmm1") \
53 __ASM_EMIT("vmovups 0x00(%[" B "], %[off]), %%zmm4") /* zmm4 = r */ \
54 __ASM_EMIT("vmovups 0x40(%[" B "], %[off]), %%zmm5") \
55 __ASM_EMIT("vsubps %%zmm4, %%zmm0, %%zmm2") /* zmm2 = s = l - r */ \
56 __ASM_EMIT("vsubps %%zmm5, %%zmm1, %%zmm3") \
57 __ASM_EMIT("vaddps %%zmm4, %%zmm0, %%zmm0") /* zmm0 = m = l + r */ \
58 __ASM_EMIT("vaddps %%zmm5, %%zmm1, %%zmm1") \
59 __ASM_EMIT(OP " %%zmm2, %%zmm0, %%zmm0") /* zmm0 = O = OP(m, s) */ \
60 __ASM_EMIT(OP " %%zmm3, %%zmm1, %%zmm1") \
61 IF_MUL("vmulps %%zmm6, %%zmm0, %%zmm0") /* zmm0 = O * 0.5f */ \
62 IF_MUL("vmulps %%zmm6, %%zmm1, %%zmm1") \
63 __ASM_EMIT("vmovups %%zmm0, 0x00(%[" DST "], %[off])") \
64 __ASM_EMIT("vmovups %%zmm1, 0x40(%[" DST "], %[off])") \
65 __ASM_EMIT("add $0x80, %[off]") \
66 __ASM_EMIT("sub $32, %[count]") \
67 __ASM_EMIT("jae 1b") \
68 /* 16x block */ \
69 __ASM_EMIT("2:") \
70 __ASM_EMIT("add $16, %[count]") \
71 __ASM_EMIT("jl 4f") \
72 __ASM_EMIT("vmovups 0x00(%[" A "], %[off]), %%zmm0") /* zmm0 = l */ \
73 __ASM_EMIT("vmovups 0x00(%[" B "], %[off]), %%zmm4") /* zmm4 = r */ \
74 __ASM_EMIT("vsubps %%zmm4, %%zmm0, %%zmm2") /* zmm2 = s = l - r */ \
75 __ASM_EMIT("vaddps %%zmm4, %%zmm0, %%zmm0") /* zmm0 = m = l + r */ \
76 __ASM_EMIT(OP " %%zmm2, %%zmm0, %%zmm0") /* zmm0 = O = OP(m, s) */ \
77 IF_MUL("vmulps %%zmm6, %%zmm0, %%zmm0") /* zmm0 = O * 0.5f */ \
78 __ASM_EMIT("vmovups %%zmm0, 0x00(%[" DST "], %[off])") \
79 __ASM_EMIT("sub $16, %[count]") \
80 __ASM_EMIT("add $0x40, %[off]") \
81 /* 8x block */ \
82 __ASM_EMIT("4:") \
83 __ASM_EMIT("add $8, %[count]") \
84 __ASM_EMIT("jl 6f") \
85 __ASM_EMIT("vmovups 0x00(%[" A "], %[off]), %%ymm0") /* ymm0 = l */ \
86 __ASM_EMIT("vmovups 0x00(%[" B "], %[off]), %%ymm4") /* ymm4 = r */ \
87 __ASM_EMIT("vsubps %%ymm4, %%ymm0, %%ymm2") /* ymm2 = s = l - r */ \
88 __ASM_EMIT("vaddps %%ymm4, %%ymm0, %%ymm0") /* ymm0 = m = l + r */ \
89 __ASM_EMIT(OP " %%ymm2, %%ymm0, %%ymm0") /* ymm0 = O = OP(m, s) */ \
90 IF_MUL("vmulps %%ymm6, %%ymm0, %%ymm0") /* ymm0 = O * 0.5f */ \
91 __ASM_EMIT("vmovups %%ymm0, 0x00(%[" DST "], %[off])") \
92 __ASM_EMIT("sub $8, %[count]") \
93 __ASM_EMIT("add $0x20, %[off]") \
94 /* 4x block */ \
95 __ASM_EMIT("6:") \
96 __ASM_EMIT("add $4, %[count]") \
97 __ASM_EMIT("jl 8f") \
98 __ASM_EMIT("vmovups 0x00(%[" A "], %[off]), %%xmm0") /* xmm0 = l */ \
99 __ASM_EMIT("vmovups 0x00(%[" B "], %[off]), %%xmm4") /* xmm4 = r */ \
100 __ASM_EMIT("vsubps %%xmm4, %%xmm0, %%xmm2") /* xmm2 = s = l - r */ \
101 __ASM_EMIT("vaddps %%xmm4, %%xmm0, %%xmm0") /* xmm0 = m = l + r */ \
102 __ASM_EMIT(OP " %%xmm2, %%xmm0, %%xmm0") /* xmm0 = O = OP(m, s) */ \
103 IF_MUL("vmulps %%xmm6, %%xmm0, %%xmm0") /* xmm0 = O * 0.5f */ \
104 __ASM_EMIT("vmovups %%xmm0, 0x00(%[" DST "], %[off])") \
105 __ASM_EMIT("sub $4, %[count]") \
106 __ASM_EMIT("add $0x10, %[off]") \
107 /* 1x blocks */ \
108 __ASM_EMIT("8:") \
109 __ASM_EMIT("add $3, %[count]") \
110 __ASM_EMIT("jl 10f") \
111 __ASM_EMIT("9:") \
112 __ASM_EMIT("vmovss 0x00(%[" A "], %[off]), %%xmm0") /* xmm0 = l */ \
113 __ASM_EMIT("vmovss 0x00(%[" B "], %[off]), %%xmm4") /* xmm4 = r */ \
114 __ASM_EMIT("vsubss %%xmm4, %%xmm0, %%xmm2") /* xmm2 = s = l - r */ \
115 __ASM_EMIT("vaddss %%xmm4, %%xmm0, %%xmm0") /* xmm0 = m = l + r */ \
116 __ASM_EMIT(OP " %%xmm2, %%xmm0, %%xmm0") /* xmm0 = O = OP(m, s) */ \
117 IF_MUL("vmulss %%xmm6, %%xmm0, %%xmm0") /* xmm0 = O * 0.5f */ \
118 __ASM_EMIT("vmovss %%xmm0, 0x00(%[" DST "], %[off])") \
119 __ASM_EMIT("add $0x04, %[off]") \
120 __ASM_EMIT("dec %[count]") \
121 __ASM_EMIT("jge 9b") \
122 __ASM_EMIT("10:")
123
124 void ms_pmin2(float *dst, const float *src, size_t count)
125 {
126 IF_ARCH_X86(size_t off);
127 ARCH_X86_ASM
128 (
129 MS_MINMAX_CORE("dst", "dst", "src", "vminps", MS_MUL)
130 : [off] "=&r" (off), [count] "+r" (count)
131 : [dst] "r" (dst), [src] "r" (src),
132 [CC] "o" (ms_minmax_abs)
133 : "cc", "memory",
134 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
135 "%xmm4", "%xmm5", "%xmm6"
136 );
137 }
138
139 void ms_pmin3(float *dst, const float *a, const float *b, size_t count)
140 {
141 IF_ARCH_X86(size_t off);
142 ARCH_X86_ASM
143 (
144 MS_MINMAX_CORE("dst", "a", "b", "vminps", MS_MUL)
145 : [off] "=&r" (off), [count] "+r" (count)
146 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b),
147 [CC] "o" (ms_minmax_abs)
148 : "cc", "memory",
149 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
150 "%xmm4", "%xmm5", "%xmm6"
151 );
152 }
153
154 void ms_pmax2(float *dst, const float *src, size_t count)
155 {
156 IF_ARCH_X86(size_t off);
157 ARCH_X86_ASM
158 (
159 MS_MINMAX_CORE("dst", "dst", "src", "vmaxps", MS_MUL)
160 : [off] "=&r" (off), [count] "+r" (count)
161 : [dst] "r" (dst), [src] "r" (src),
162 [CC] "o" (ms_minmax_abs)
163 : "cc", "memory",
164 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
165 "%xmm4", "%xmm5", "%xmm6"
166 );
167 }
168
169 void ms_pmax3(float *dst, const float *a, const float *b, size_t count)
170 {
171 IF_ARCH_X86(size_t off);
172 ARCH_X86_ASM
173 (
174 MS_MINMAX_CORE("dst", "a", "b", "vmaxps", MS_MUL)
175 : [off] "=&r" (off), [count] "+r" (count)
176 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b),
177 [CC] "o" (ms_minmax_abs)
178 : "cc", "memory",
179 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
180 "%xmm4", "%xmm5", "%xmm6"
181 );
182 }
183
184 void lr_pmin2(float *dst, const float *src, size_t count)
185 {
186 IF_ARCH_X86(size_t off);
187 ARCH_X86_ASM
188 (
189 MS_MINMAX_CORE("dst", "dst", "src", "vminps", MS_NOMUL)
190 : [off] "=&r" (off), [count] "+r" (count)
191 : [dst] "r" (dst), [src] "r" (src),
192 [CC] "o" (ms_minmax_abs)
193 : "cc", "memory",
194 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
195 "%xmm4", "%xmm5"
196 );
197 }
198
199 void lr_pmin3(float *dst, const float *a, const float *b, size_t count)
200 {
201 IF_ARCH_X86(size_t off);
202 ARCH_X86_ASM
203 (
204 MS_MINMAX_CORE("dst", "a", "b", "vminps", MS_NOMUL)
205 : [off] "=&r" (off), [count] "+r" (count)
206 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b),
207 [CC] "o" (ms_minmax_abs)
208 : "cc", "memory",
209 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
210 "%xmm4", "%xmm5"
211 );
212 }
213
214 void lr_pmax2(float *dst, const float *src, size_t count)
215 {
216 IF_ARCH_X86(size_t off);
217 ARCH_X86_ASM
218 (
219 MS_MINMAX_CORE("dst", "dst", "src", "vmaxps", MS_NOMUL)
220 : [off] "=&r" (off), [count] "+r" (count)
221 : [dst] "r" (dst), [src] "r" (src),
222 [CC] "o" (ms_minmax_abs)
223 : "cc", "memory",
224 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
225 "%xmm4", "%xmm5"
226 );
227 }
228
229 void lr_pmax3(float *dst, const float *a, const float *b, size_t count)
230 {
231 IF_ARCH_X86(size_t off);
232 ARCH_X86_ASM
233 (
234 MS_MINMAX_CORE("dst", "a", "b", "vmaxps", MS_NOMUL)
235 : [off] "=&r" (off), [count] "+r" (count)
236 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b),
237 [CC] "o" (ms_minmax_abs)
238 : "cc", "memory",
239 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
240 "%xmm4", "%xmm5"
241 );
242 }
243
244 #define MS_SIGN_MINMAX_CORE(DST, A, B, CMP, IF_MUL) \
245 __ASM_EMIT("xor %[off], %[off]") \
246 __ASM_EMIT("sub $32, %[count]") \
247 __ASM_EMIT("vmovaps 0x00 + %[CC], %%zmm6") /* zmm6 = MASK */ \
248 IF_MUL("vmovaps 0x40 + %[CC], %%zmm7") /* zmm7 = 0.5f */ \
249 __ASM_EMIT("jb 2f") \
250 /* 32x blocks */ \
251 __ASM_EMIT("1:") \
252 __ASM_EMIT("vmovups 0x00(%[" A "], %[off]), %%zmm0") /* zmm0 = l */ \
253 __ASM_EMIT("vmovups 0x40(%[" A "], %[off]), %%zmm1") \
254 __ASM_EMIT("vmovups 0x00(%[" B "], %[off]), %%zmm4") /* zmm4 = r */ \
255 __ASM_EMIT("vmovups 0x40(%[" B "], %[off]), %%zmm5") \
256 __ASM_EMIT("vsubps %%zmm4, %%zmm0, %%zmm2") /* zmm2 = s = l - r */ \
257 __ASM_EMIT("vsubps %%zmm5, %%zmm1, %%zmm3") \
258 __ASM_EMIT("vaddps %%zmm4, %%zmm0, %%zmm0") /* zmm0 = m = l + r */ \
259 __ASM_EMIT("vaddps %%zmm5, %%zmm1, %%zmm1") \
260 __ASM_EMIT("vandps %%zmm6, %%zmm2, %%zmm5") /* zmm5 = fabsf(s) */ \
261 __ASM_EMIT("vandps %%zmm6, %%zmm0, %%zmm4") /* zmm4 = fabsf(m) */ \
262 __ASM_EMIT("vcmpps " CMP ", %%zmm5, %%zmm4, %%k4") /* k4 = OP(fabsf(m), fabsf(s)) */ \
263 __ASM_EMIT("vandps %%zmm6, %%zmm3, %%zmm5") \
264 __ASM_EMIT("vandps %%zmm6, %%zmm1, %%zmm4") \
265 __ASM_EMIT("vcmpps " CMP ", %%zmm5, %%zmm4, %%k5") \
266 __ASM_EMIT("vmovaps %%zmm0, %%zmm2 %{%%k4%}") /* zmm2 = O = OP(fabsf(m), fabsf(s)) ? m : s */ \
267 __ASM_EMIT("vmovaps %%zmm1, %%zmm3 %{%%k5%}") \
268 IF_MUL("vmulps %%zmm7, %%zmm2, %%zmm2") /* zmm0 = O * 0.5f */ \
269 IF_MUL("vmulps %%zmm7, %%zmm3, %%zmm3") \
270 __ASM_EMIT("vmovups %%zmm2, 0x00(%[" DST "], %[off])") \
271 __ASM_EMIT("vmovups %%zmm3, 0x40(%[" DST "], %[off])") \
272 __ASM_EMIT("add $0x80, %[off]") \
273 __ASM_EMIT("sub $32, %[count]") \
274 __ASM_EMIT("jae 1b") \
275 /* 16x block */ \
276 __ASM_EMIT("2:") \
277 __ASM_EMIT("add $16, %[count]") \
278 __ASM_EMIT("jl 4f") \
279 __ASM_EMIT("vmovups 0x00(%[" A "], %[off]), %%zmm0") /* zmm0 = l */ \
280 __ASM_EMIT("vmovups 0x00(%[" B "], %[off]), %%zmm4") /* zmm4 = r */ \
281 __ASM_EMIT("vsubps %%zmm4, %%zmm0, %%zmm2") /* zmm2 = s = l - r */ \
282 __ASM_EMIT("vaddps %%zmm4, %%zmm0, %%zmm0") /* zmm0 = m = l + r */ \
283 __ASM_EMIT("vandps %%zmm6, %%zmm2, %%zmm5") /* zmm5 = fabsf(s) */ \
284 __ASM_EMIT("vandps %%zmm6, %%zmm0, %%zmm4") /* zmm4 = fabsf(m) */ \
285 __ASM_EMIT("vcmpps " CMP ", %%zmm5, %%zmm4, %%k4") /* k4 = OP(fabsf(m), fabsf(s)) */ \
286 __ASM_EMIT("vmovaps %%zmm0, %%zmm2 %{%%k4%}") /* zmm2 = O = OP(fabsf(m), fabsf(s)) ? m : s */ \
287 IF_MUL("vmulps %%zmm7, %%zmm2, %%zmm2") /* zmm0 = O * 0.5f */ \
288 __ASM_EMIT("vmovups %%zmm2, 0x00(%[" DST "], %[off])") \
289 __ASM_EMIT("sub $16, %[count]") \
290 __ASM_EMIT("add $0x40, %[off]") \
291 /* 8x block */ \
292 __ASM_EMIT("4:") \
293 __ASM_EMIT("add $8, %[count]") \
294 __ASM_EMIT("jl 6f") \
295 __ASM_EMIT("vmovups 0x00(%[" A "], %[off]), %%ymm0") /* ymm0 = l */ \
296 __ASM_EMIT("vmovups 0x00(%[" B "], %[off]), %%ymm4") /* ymm4 = r */ \
297 __ASM_EMIT("vsubps %%ymm4, %%ymm0, %%ymm2") /* ymm2 = s = l - r */ \
298 __ASM_EMIT("vaddps %%ymm4, %%ymm0, %%ymm0") /* ymm0 = m = l + r */ \
299 __ASM_EMIT("vandps %%ymm6, %%ymm2, %%ymm1") /* ymm1 = fabsf(s) */ \
300 __ASM_EMIT("vandps %%ymm6, %%ymm0, %%ymm3") /* ymm3 = fabsf(m) */ \
301 __ASM_EMIT("vcmpps " CMP ", %%ymm1, %%ymm3, %%ymm3") /* ymm4 = OP(fabsf(m), fabsf(s)) */ \
302 __ASM_EMIT("vblendvps %%ymm3, %%ymm0, %%ymm2, %%ymm0") /* ymm0 = O = OP(fabsf(m), fabsf(s)) ? m : s */ \
303 IF_MUL("vmulps %%ymm7, %%ymm0, %%ymm0") /* ymm0 = O * 0.5f */ \
304 __ASM_EMIT("vmovups %%ymm0, 0x00(%[" DST "], %[off])") \
305 __ASM_EMIT("sub $8, %[count]") \
306 __ASM_EMIT("add $0x20, %[off]") \
307 /* 4x block */ \
308 __ASM_EMIT("6:") \
309 __ASM_EMIT("add $4, %[count]") \
310 __ASM_EMIT("jl 8f") \
311 __ASM_EMIT("vmovups 0x00(%[" A "], %[off]), %%xmm0") /* xmm0 = l */ \
312 __ASM_EMIT("vmovups 0x00(%[" B "], %[off]), %%xmm4") /* xmm4 = r */ \
313 __ASM_EMIT("vsubps %%xmm4, %%xmm0, %%xmm2") /* xmm2 = s = l - r */ \
314 __ASM_EMIT("vaddps %%xmm4, %%xmm0, %%xmm0") /* xmm0 = m = l + r */ \
315 __ASM_EMIT("vandps %%xmm6, %%xmm2, %%xmm1") /* xmm1 = fabsf(s) */ \
316 __ASM_EMIT("vandps %%xmm6, %%xmm0, %%xmm3") /* xmm3 = fabsf(m) */ \
317 __ASM_EMIT("vcmpps " CMP ", %%xmm1, %%xmm3, %%xmm3") /* xmm4 = OP(fabsf(m), fabsf(s)) */ \
318 __ASM_EMIT("vblendvps %%xmm3, %%xmm0, %%xmm2, %%xmm0") /* xmm0 = O = OP(fabsf(m), fabsf(s)) ? m : s */ \
319 IF_MUL("vmulps %%xmm7, %%xmm0, %%xmm0") /* xmm0 = O * 0.5f */ \
320 __ASM_EMIT("vmovups %%xmm0, 0x00(%[" DST "], %[off])") \
321 __ASM_EMIT("sub $4, %[count]") \
322 __ASM_EMIT("add $0x10, %[off]") \
323 /* 1x blocks */ \
324 __ASM_EMIT("8:") \
325 __ASM_EMIT("add $3, %[count]") \
326 __ASM_EMIT("jl 10f") \
327 __ASM_EMIT("9:") \
328 __ASM_EMIT("vmovss 0x00(%[" A "], %[off]), %%xmm0") /* xmm0 = l */ \
329 __ASM_EMIT("vmovss 0x00(%[" B "], %[off]), %%xmm4") /* xmm4 = r */ \
330 __ASM_EMIT("vsubss %%xmm4, %%xmm0, %%xmm2") /* xmm2 = s = l - r */ \
331 __ASM_EMIT("vaddss %%xmm4, %%xmm0, %%xmm0") /* xmm0 = m = l + r */ \
332 __ASM_EMIT("vandps %%xmm6, %%xmm2, %%xmm1") /* xmm1 = fabsf(s) */ \
333 __ASM_EMIT("vandps %%xmm6, %%xmm0, %%xmm3") /* xmm3 = fabsf(m) */ \
334 __ASM_EMIT("vcmpps " CMP ", %%xmm1, %%xmm3, %%xmm3") /* xmm4 = OP(fabsf(m), fabsf(s)) */ \
335 __ASM_EMIT("vblendvps %%xmm3, %%xmm0, %%xmm2, %%xmm0") /* xmm0 = O = OP(fabsf(m), fabsf(s)) ? m : s */ \
336 IF_MUL("vmulss %%xmm7, %%xmm0, %%xmm0") /* xmm0 = O * 0.5f */ \
337 __ASM_EMIT("vmovss %%xmm0, 0x00(%[" DST "], %[off])") \
338 __ASM_EMIT("add $0x04, %[off]") \
339 __ASM_EMIT("dec %[count]") \
340 __ASM_EMIT("jge 9b") \
341 __ASM_EMIT("10:")
342
343 void ms_psmin2(float *dst, const float *src, size_t count)
344 {
345 IF_ARCH_X86(size_t off);
346 ARCH_X86_ASM
347 (
348 MS_SIGN_MINMAX_CORE("dst", "dst", "src", "$1", MS_MUL)
349 : [off] "=&r" (off), [count] "+r" (count)
350 : [dst] "r" (dst), [src] "r" (src),
351 [CC] "o" (ms_minmax_abs)
352 : "cc", "memory",
353 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
354 "%xmm4", "%xmm5", "%xmm6", "%xmm7",
355 "%k4", "%k5"
356 );
357 }
358
359 void ms_psmin3(float *dst, const float *a, const float *b, size_t count)
360 {
361 IF_ARCH_X86(size_t off);
362 ARCH_X86_ASM
363 (
364 MS_SIGN_MINMAX_CORE("dst", "a", "b", "$1", MS_MUL)
365 : [off] "=&r" (off), [count] "+r" (count)
366 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b),
367 [CC] "o" (ms_minmax_abs)
368 : "cc", "memory",
369 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
370 "%xmm4", "%xmm5", "%xmm6", "%xmm7",
371 "%k4", "%k5"
372 );
373 }
374
375 void ms_psmax2(float *dst, const float *src, size_t count)
376 {
377 IF_ARCH_X86(size_t off);
378 ARCH_X86_ASM
379 (
380 MS_SIGN_MINMAX_CORE("dst", "dst", "src", "$5", MS_MUL)
381 : [off] "=&r" (off), [count] "+r" (count)
382 : [dst] "r" (dst), [src] "r" (src),
383 [CC] "o" (ms_minmax_abs)
384 : "cc", "memory",
385 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
386 "%xmm4", "%xmm5", "%xmm6", "%xmm7",
387 "%k4", "%k5"
388 );
389 }
390
391 void ms_psmax3(float *dst, const float *a, const float *b, size_t count)
392 {
393 IF_ARCH_X86(size_t off);
394 ARCH_X86_ASM
395 (
396 MS_SIGN_MINMAX_CORE("dst", "a", "b", "$5", MS_MUL)
397 : [off] "=&r" (off), [count] "+r" (count)
398 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b),
399 [CC] "o" (ms_minmax_abs)
400 : "cc", "memory",
401 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
402 "%xmm4", "%xmm5", "%xmm6", "%xmm7",
403 "%k4", "%k5"
404 );
405 }
406
407 void lr_psmin2(float *dst, const float *src, size_t count)
408 {
409 IF_ARCH_X86(size_t off);
410 ARCH_X86_ASM
411 (
412 MS_SIGN_MINMAX_CORE("dst", "dst", "src", "$1", MS_NOMUL)
413 : [off] "=&r" (off), [count] "+r" (count)
414 : [dst] "r" (dst), [src] "r" (src),
415 [CC] "o" (ms_minmax_abs)
416 : "cc", "memory",
417 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
418 "%xmm4", "%xmm5", "%xmm6",
419 "%k4", "%k5"
420 );
421 }
422
423 void lr_psmin3(float *dst, const float *a, const float *b, size_t count)
424 {
425 IF_ARCH_X86(size_t off);
426 ARCH_X86_ASM
427 (
428 MS_SIGN_MINMAX_CORE("dst", "a", "b", "$1", MS_NOMUL)
429 : [off] "=&r" (off), [count] "+r" (count)
430 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b),
431 [CC] "o" (ms_minmax_abs)
432 : "cc", "memory",
433 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
434 "%xmm4", "%xmm5", "%xmm6",
435 "%k4", "%k5"
436 );
437 }
438
439 void lr_psmax2(float *dst, const float *src, size_t count)
440 {
441 IF_ARCH_X86(size_t off);
442 ARCH_X86_ASM
443 (
444 MS_SIGN_MINMAX_CORE("dst", "dst", "src", "$5", MS_NOMUL)
445 : [off] "=&r" (off), [count] "+r" (count)
446 : [dst] "r" (dst), [src] "r" (src),
447 [CC] "o" (ms_minmax_abs)
448 : "cc", "memory",
449 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
450 "%xmm4", "%xmm5", "%xmm6",
451 "%k4", "%k5"
452 );
453 }
454
455 void lr_psmax3(float *dst, const float *a, const float *b, size_t count)
456 {
457 IF_ARCH_X86(size_t off);
458 ARCH_X86_ASM
459 (
460 MS_SIGN_MINMAX_CORE("dst", "a", "b", "$5", MS_NOMUL)
461 : [off] "=&r" (off), [count] "+r" (count)
462 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b),
463 [CC] "o" (ms_minmax_abs)
464 : "cc", "memory",
465 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
466 "%xmm4", "%xmm5", "%xmm6",
467 "%k4", "%k5"
468 );
469 }
470
471 #undef MS_SIGN_MINMAX_CORE
472
473 #define MS_ABS_MINMAX_CORE(DST, A, B, OP, IF_MUL) \
474 __ASM_EMIT("xor %[off], %[off]") \
475 __ASM_EMIT("sub $32, %[count]") \
476 __ASM_EMIT("vmovaps 0x00 + %[CC], %%zmm6") /* ymm6 = MASK */ \
477 IF_MUL("vmovaps 0x40 + %[CC], %%zmm7") /* ymm7 = 0.5f */ \
478 __ASM_EMIT("jb 2f") \
479 /* 32x blocks */ \
480 __ASM_EMIT("1:") \
481 __ASM_EMIT("vmovups 0x00(%[" A "], %[off]), %%zmm0") /* zmm0 = l */ \
482 __ASM_EMIT("vmovups 0x40(%[" A "], %[off]), %%zmm1") \
483 __ASM_EMIT("vmovups 0x00(%[" B "], %[off]), %%zmm4") /* zmm4 = r */ \
484 __ASM_EMIT("vmovups 0x40(%[" B "], %[off]), %%zmm5") \
485 __ASM_EMIT("vsubps %%zmm4, %%zmm0, %%zmm2") /* zmm2 = s = l - r */ \
486 __ASM_EMIT("vsubps %%zmm5, %%zmm1, %%zmm3") \
487 __ASM_EMIT("vaddps %%zmm4, %%zmm0, %%zmm0") /* zmm0 = m = l + r */ \
488 __ASM_EMIT("vaddps %%zmm5, %%zmm1, %%zmm1") \
489 __ASM_EMIT("vandps %%zmm6, %%zmm2, %%zmm2") /* zmm2 = fabsf(s) */ \
490 __ASM_EMIT("vandps %%zmm6, %%zmm3, %%zmm3") \
491 __ASM_EMIT("vandps %%zmm6, %%zmm0, %%zmm0") /* zmm0 = fabsf(m) */ \
492 __ASM_EMIT("vandps %%zmm6, %%zmm1, %%zmm1") \
493 __ASM_EMIT(OP " %%zmm2, %%zmm0, %%zmm0") /* zmm0 = O = OP(fabsf(m), fabsf(s)) */ \
494 __ASM_EMIT(OP " %%zmm3, %%zmm1, %%zmm1") \
495 IF_MUL("vmulps %%zmm7, %%zmm0, %%zmm0") /* zmm0 = O * 0.5f */ \
496 IF_MUL("vmulps %%zmm7, %%zmm1, %%zmm1") \
497 __ASM_EMIT("vmovups %%zmm0, 0x00(%[" DST "], %[off])") \
498 __ASM_EMIT("vmovups %%zmm1, 0x40(%[" DST "], %[off])") \
499 __ASM_EMIT("add $0x80, %[off]") \
500 __ASM_EMIT("sub $32, %[count]") \
501 __ASM_EMIT("jae 1b") \
502 /* 16x blocks */ \
503 __ASM_EMIT("2:") \
504 __ASM_EMIT("add $16, %[count]") \
505 __ASM_EMIT("jl 4f") \
506 __ASM_EMIT("vmovups 0x00(%[" A "], %[off]), %%zmm0") /* zmm0 = l */ \
507 __ASM_EMIT("vmovups 0x00(%[" B "], %[off]), %%zmm4") /* zmm4 = r */ \
508 __ASM_EMIT("vsubps %%zmm4, %%zmm0, %%zmm2") /* zmm2 = s = l - r */ \
509 __ASM_EMIT("vaddps %%zmm4, %%zmm0, %%zmm0") /* zmm0 = m = l + r */ \
510 __ASM_EMIT("vandps %%zmm6, %%zmm2, %%zmm2") /* zmm2 = fabsf(s) */ \
511 __ASM_EMIT("vandps %%zmm6, %%zmm0, %%zmm0") /* zmm0 = fabsf(m) */ \
512 __ASM_EMIT(OP " %%zmm2, %%zmm0, %%zmm0") /* zmm0 = O = OP(fabsf(m), fabsf(s)) */ \
513 IF_MUL("vmulps %%zmm7, %%zmm0, %%zmm0") /* zmm0 = O * 0.5f */ \
514 __ASM_EMIT("vmovups %%zmm0, 0x00(%[" DST "], %[off])") \
515 __ASM_EMIT("add $0x40, %[off]") \
516 __ASM_EMIT("sub $16, %[count]") \
517 __ASM_EMIT("jae 1b") \
518 /* 8x block */ \
519 __ASM_EMIT("4:") \
520 __ASM_EMIT("add $8, %[count]") \
521 __ASM_EMIT("jl 6f") \
522 __ASM_EMIT("vmovups 0x00(%[" A "], %[off]), %%ymm0") /* ymm0 = l */ \
523 __ASM_EMIT("vmovups 0x00(%[" B "], %[off]), %%ymm4") /* ymm4 = r */ \
524 __ASM_EMIT("vsubps %%ymm4, %%ymm0, %%ymm2") /* ymm2 = s = l - r */ \
525 __ASM_EMIT("vaddps %%ymm4, %%ymm0, %%ymm0") /* ymm0 = m = l + r */ \
526 __ASM_EMIT("vandps %%ymm6, %%ymm2, %%ymm2") /* ymm2 = fabsf(s) */ \
527 __ASM_EMIT("vandps %%ymm6, %%ymm0, %%ymm0") /* ymm0 = fabsf(m) */ \
528 __ASM_EMIT(OP " %%ymm2, %%ymm0, %%ymm0") /* ymm0 = O = OP(fabsf(m), fabsf(s)) */ \
529 IF_MUL("vmulps %%ymm7, %%ymm0, %%ymm0") /* ymm0 = O * 0.5f */ \
530 __ASM_EMIT("vmovups %%ymm0, 0x00(%[" DST "], %[off])") \
531 __ASM_EMIT("sub $8, %[count]") \
532 __ASM_EMIT("add $0x20, %[off]") \
533 /* 4x block */ \
534 __ASM_EMIT("6:") \
535 __ASM_EMIT("add $4, %[count]") \
536 __ASM_EMIT("jl 8f") \
537 __ASM_EMIT("vmovups 0x00(%[" A "], %[off]), %%xmm0") /* xmm0 = l */ \
538 __ASM_EMIT("vmovups 0x00(%[" B "], %[off]), %%xmm4") /* xmm4 = r */ \
539 __ASM_EMIT("vsubps %%xmm4, %%xmm0, %%xmm2") /* xmm2 = s = l - r */ \
540 __ASM_EMIT("vaddps %%xmm4, %%xmm0, %%xmm0") /* xmm0 = m = l + r */ \
541 __ASM_EMIT("vandps %%xmm6, %%xmm2, %%xmm2") /* xmm2 = fabsf(s) */ \
542 __ASM_EMIT("vandps %%xmm6, %%xmm0, %%xmm0") /* xmm0 = fabsf(m) */ \
543 __ASM_EMIT(OP " %%xmm2, %%xmm0, %%xmm0") /* xmm0 = O = OP(fabsf(m), fabsf(s)) */ \
544 IF_MUL("vmulps %%xmm7, %%xmm0, %%xmm0") /* xmm0 = O * 0.5f */ \
545 __ASM_EMIT("vmovups %%xmm0, 0x00(%[" DST "], %[off])") \
546 __ASM_EMIT("sub $4, %[count]") \
547 __ASM_EMIT("add $0x10, %[off]") \
548 /* 1x blocks */ \
549 __ASM_EMIT("8:") \
550 __ASM_EMIT("add $3, %[count]") \
551 __ASM_EMIT("jl 10f") \
552 __ASM_EMIT("9:") \
553 __ASM_EMIT("vmovss 0x00(%[" A "], %[off]), %%xmm0") /* xmm0 = l */ \
554 __ASM_EMIT("vmovss 0x00(%[" B "], %[off]), %%xmm4") /* xmm4 = r */ \
555 __ASM_EMIT("vsubss %%xmm4, %%xmm0, %%xmm2") /* xmm2 = s = l - r */ \
556 __ASM_EMIT("vaddss %%xmm4, %%xmm0, %%xmm0") /* xmm0 = m = l + r */ \
557 __ASM_EMIT("vandps %%xmm6, %%xmm2, %%xmm2") /* xmm2 = fabsf(s) */ \
558 __ASM_EMIT("vandps %%xmm6, %%xmm0, %%xmm0") /* xmm0 = fabsf(m) */ \
559 __ASM_EMIT(OP " %%xmm2, %%xmm0, %%xmm0") /* xmm0 = O = OP(fabsf(m), fabsf(s)) */ \
560 IF_MUL("vmulss %%xmm7, %%xmm0, %%xmm0") /* xmm0 = O * 0.5f */ \
561 __ASM_EMIT("vmovss %%xmm0, 0x00(%[" DST "], %[off])") \
562 __ASM_EMIT("add $0x04, %[off]") \
563 __ASM_EMIT("dec %[count]") \
564 __ASM_EMIT("jge 9b") \
565 __ASM_EMIT("10:")
566
567 void ms_pamin2(float *dst, const float *src, size_t count)
568 {
569 IF_ARCH_X86(size_t off);
570 ARCH_X86_ASM
571 (
572 MS_ABS_MINMAX_CORE("dst", "dst", "src", "vminps", MS_MUL)
573 : [off] "=&r" (off), [count] "+r" (count)
574 : [dst] "r" (dst), [src] "r" (src),
575 [CC] "o" (ms_minmax_abs)
576 : "cc", "memory",
577 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
578 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
579 );
580 }
581
582 void ms_pamin3(float *dst, const float *a, const float *b, size_t count)
583 {
584 IF_ARCH_X86(size_t off);
585 ARCH_X86_ASM
586 (
587 MS_ABS_MINMAX_CORE("dst", "a", "b", "vminps", MS_MUL)
588 : [off] "=&r" (off), [count] "+r" (count)
589 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b),
590 [CC] "o" (ms_minmax_abs)
591 : "cc", "memory",
592 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
593 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
594 );
595 }
596
597 void ms_pamax2(float *dst, const float *src, size_t count)
598 {
599 IF_ARCH_X86(size_t off);
600 ARCH_X86_ASM
601 (
602 MS_ABS_MINMAX_CORE("dst", "dst", "src", "vmaxps", MS_MUL)
603 : [off] "=&r" (off), [count] "+r" (count)
604 : [dst] "r" (dst), [src] "r" (src),
605 [CC] "o" (ms_minmax_abs)
606 : "cc", "memory",
607 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
608 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
609 );
610 }
611
612 void ms_pamax3(float *dst, const float *a, const float *b, size_t count)
613 {
614 IF_ARCH_X86(size_t off);
615 ARCH_X86_ASM
616 (
617 MS_ABS_MINMAX_CORE("dst", "a", "b", "vmaxps", MS_MUL)
618 : [off] "=&r" (off), [count] "+r" (count)
619 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b),
620 [CC] "o" (ms_minmax_abs)
621 : "cc", "memory",
622 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
623 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
624 );
625 }
626
627 void lr_pamin2(float *dst, const float *src, size_t count)
628 {
629 IF_ARCH_X86(size_t off);
630 ARCH_X86_ASM
631 (
632 MS_ABS_MINMAX_CORE("dst", "dst", "src", "vminps", MS_NOMUL)
633 : [off] "=&r" (off), [count] "+r" (count)
634 : [dst] "r" (dst), [src] "r" (src),
635 [CC] "o" (ms_minmax_abs)
636 : "cc", "memory",
637 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
638 "%xmm4", "%xmm5", "%xmm6"
639 );
640 }
641
642 void lr_pamin3(float *dst, const float *a, const float *b, size_t count)
643 {
644 IF_ARCH_X86(size_t off);
645 ARCH_X86_ASM
646 (
647 MS_ABS_MINMAX_CORE("dst", "a", "b", "vminps", MS_NOMUL)
648 : [off] "=&r" (off), [count] "+r" (count)
649 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b),
650 [CC] "o" (ms_minmax_abs)
651 : "cc", "memory",
652 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
653 "%xmm4", "%xmm5", "%xmm6"
654 );
655 }
656
657 void lr_pamax2(float *dst, const float *src, size_t count)
658 {
659 IF_ARCH_X86(size_t off);
660 ARCH_X86_ASM
661 (
662 MS_ABS_MINMAX_CORE("dst", "dst", "src", "vmaxps", MS_NOMUL)
663 : [off] "=&r" (off), [count] "+r" (count)
664 : [dst] "r" (dst), [src] "r" (src),
665 [CC] "o" (ms_minmax_abs)
666 : "cc", "memory",
667 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
668 "%xmm4", "%xmm5", "%xmm6"
669 );
670 }
671
672 void lr_pamax3(float *dst, const float *a, const float *b, size_t count)
673 {
674 IF_ARCH_X86(size_t off);
675 ARCH_X86_ASM
676 (
677 MS_ABS_MINMAX_CORE("dst", "a", "b", "vmaxps", MS_NOMUL)
678 : [off] "=&r" (off), [count] "+r" (count)
679 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b),
680 [CC] "o" (ms_minmax_abs)
681 : "cc", "memory",
682 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
683 "%xmm4", "%xmm5", "%xmm6"
684 );
685 }
686
687 #undef MS_ABS_MINMAX_CORE
688
689 #undef MS_MUL
690 #undef MS_NOMUL
691
692 } /* namespace avx512 */
693 } /* namespace lsp */
694
695
696
697
698 #endif /* PRIVATE_DSP_ARCH_X86_AVX512_PMATH_MS_MINMAX_H_ */