Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/x86/avx/pmath/ms_minmax.h

raw
1 /*
2 * Copyright (C) 2026 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2026 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 28 мар. 2026 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_X86_AVX_PMATH_MS_MINMAX_H_
23 #define PRIVATE_DSP_ARCH_X86_AVX_PMATH_MS_MINMAX_H_
24
25 #ifndef PRIVATE_DSP_ARCH_X86_AVX_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_X86_AVX_IMPL */
28
29 namespace lsp
30 {
31 namespace avx
32 {
33 IF_ARCH_X86(
34 static const uint32_t ms_minmax_abs[] __lsp_aligned32 =
35 {
36 LSP_DSP_VEC8(0x7fffffff),
37 LSP_DSP_VEC8(0x3f000000) // 0.5f
38 };
39 )
40
41 #define MS_MUL(x) __ASM_EMIT(x)
42 #define MS_NOMUL(x)
43
44 #define MS_MINMAX_CORE(DST, A, B, OP, IF_MUL) \
45 __ASM_EMIT("xor %[off], %[off]") \
46 __ASM_EMIT("sub $16, %[count]") \
47 IF_MUL("vmovaps 0x20 + %[CC], %%ymm6") /* ymm6 = 0.5f */ \
48 __ASM_EMIT("jb 2f") \
49 /* 16x blocks */ \
50 __ASM_EMIT("1:") \
51 __ASM_EMIT("vmovups 0x00(%[" A "], %[off]), %%ymm0") /* ymm0 = l */ \
52 __ASM_EMIT("vmovups 0x20(%[" A "], %[off]), %%ymm1") \
53 __ASM_EMIT("vmovups 0x00(%[" B "], %[off]), %%ymm4") /* ymm4 = r */ \
54 __ASM_EMIT("vmovups 0x20(%[" B "], %[off]), %%ymm5") \
55 __ASM_EMIT("vsubps %%ymm4, %%ymm0, %%ymm2") /* ymm2 = s = l - r */ \
56 __ASM_EMIT("vsubps %%ymm5, %%ymm1, %%ymm3") \
57 __ASM_EMIT("vaddps %%ymm4, %%ymm0, %%ymm0") /* ymm0 = m = l + r */ \
58 __ASM_EMIT("vaddps %%ymm5, %%ymm1, %%ymm1") \
59 __ASM_EMIT(OP " %%ymm2, %%ymm0, %%ymm0") /* ymm0 = O = OP(m, s) */ \
60 __ASM_EMIT(OP " %%ymm3, %%ymm1, %%ymm1") \
61 IF_MUL("vmulps %%ymm6, %%ymm0, %%ymm0") /* ymm0 = O * 0.5f */ \
62 IF_MUL("vmulps %%ymm6, %%ymm1, %%ymm1") \
63 __ASM_EMIT("vmovups %%ymm0, 0x00(%[" DST "], %[off])") \
64 __ASM_EMIT("vmovups %%ymm1, 0x20(%[" DST "], %[off])") \
65 __ASM_EMIT("add $0x40, %[off]") \
66 __ASM_EMIT("sub $16, %[count]") \
67 __ASM_EMIT("jae 1b") \
68 /* 8x block */ \
69 __ASM_EMIT("2:") \
70 __ASM_EMIT("add $8, %[count]") \
71 __ASM_EMIT("jl 4f") \
72 __ASM_EMIT("vmovups 0x00(%[" A "], %[off]), %%ymm0") /* ymm0 = l */ \
73 __ASM_EMIT("vmovups 0x00(%[" B "], %[off]), %%ymm4") /* ymm4 = r */ \
74 __ASM_EMIT("vsubps %%ymm4, %%ymm0, %%ymm2") /* ymm2 = s = l - r */ \
75 __ASM_EMIT("vaddps %%ymm4, %%ymm0, %%ymm0") /* ymm0 = m = l + r */ \
76 __ASM_EMIT(OP " %%ymm2, %%ymm0, %%ymm0") /* ymm0 = O = OP(m, s) */ \
77 IF_MUL("vmulps %%ymm6, %%ymm0, %%ymm0") /* ymm0 = O * 0.5f */ \
78 __ASM_EMIT("vmovups %%ymm0, 0x00(%[" DST "], %[off])") \
79 __ASM_EMIT("sub $8, %[count]") \
80 __ASM_EMIT("add $0x20, %[off]") \
81 /* 4x block */ \
82 __ASM_EMIT("4:") \
83 __ASM_EMIT("add $4, %[count]") \
84 __ASM_EMIT("jl 6f") \
85 __ASM_EMIT("vmovups 0x00(%[" A "], %[off]), %%xmm0") /* xmm0 = l */ \
86 __ASM_EMIT("vmovups 0x00(%[" B "], %[off]), %%xmm4") /* xmm4 = r */ \
87 __ASM_EMIT("vsubps %%xmm4, %%xmm0, %%xmm2") /* xmm2 = s = l - r */ \
88 __ASM_EMIT("vaddps %%xmm4, %%xmm0, %%xmm0") /* xmm0 = m = l + r */ \
89 __ASM_EMIT(OP " %%xmm2, %%xmm0, %%xmm0") /* xmm0 = O = OP(m, s) */ \
90 IF_MUL("vmulps %%xmm6, %%xmm0, %%xmm0") /* xmm0 = O * 0.5f */ \
91 __ASM_EMIT("vmovups %%xmm0, 0x00(%[" DST "], %[off])") \
92 __ASM_EMIT("sub $4, %[count]") \
93 __ASM_EMIT("add $0x10, %[off]") \
94 /* 1x blocks */ \
95 __ASM_EMIT("6:") \
96 __ASM_EMIT("add $3, %[count]") \
97 __ASM_EMIT("jl 8f") \
98 __ASM_EMIT("7:") \
99 __ASM_EMIT("vmovss 0x00(%[" A "], %[off]), %%xmm0") /* xmm0 = l */ \
100 __ASM_EMIT("vmovss 0x00(%[" B "], %[off]), %%xmm4") /* xmm4 = r */ \
101 __ASM_EMIT("vsubss %%xmm4, %%xmm0, %%xmm2") /* xmm2 = s = l - r */ \
102 __ASM_EMIT("vaddss %%xmm4, %%xmm0, %%xmm0") /* xmm0 = m = l + r */ \
103 __ASM_EMIT(OP " %%xmm2, %%xmm0, %%xmm0") /* xmm0 = O = OP(m, s) */ \
104 IF_MUL("vmulss %%xmm6, %%xmm0, %%xmm0") /* xmm0 = O * 0.5f */ \
105 __ASM_EMIT("vmovss %%xmm0, 0x00(%[" DST "], %[off])") \
106 __ASM_EMIT("add $0x04, %[off]") \
107 __ASM_EMIT("dec %[count]") \
108 __ASM_EMIT("jge 7b") \
109 __ASM_EMIT("8:")
110
111 void ms_pmin2(float *dst, const float *src, size_t count)
112 {
113 IF_ARCH_X86(size_t off);
114 ARCH_X86_ASM
115 (
116 MS_MINMAX_CORE("dst", "dst", "src", "vminps", MS_MUL)
117 : [off] "=&r" (off), [count] "+r" (count)
118 : [dst] "r" (dst), [src] "r" (src),
119 [CC] "o" (ms_minmax_abs)
120 : "cc", "memory",
121 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
122 "%xmm4", "%xmm5", "%xmm6"
123 );
124 }
125
126 void ms_pmin3(float *dst, const float *a, const float *b, size_t count)
127 {
128 IF_ARCH_X86(size_t off);
129 ARCH_X86_ASM
130 (
131 MS_MINMAX_CORE("dst", "a", "b", "vminps", MS_MUL)
132 : [off] "=&r" (off), [count] "+r" (count)
133 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b),
134 [CC] "o" (ms_minmax_abs)
135 : "cc", "memory",
136 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
137 "%xmm4", "%xmm5", "%xmm6"
138 );
139 }
140
141 void ms_pmax2(float *dst, const float *src, size_t count)
142 {
143 IF_ARCH_X86(size_t off);
144 ARCH_X86_ASM
145 (
146 MS_MINMAX_CORE("dst", "dst", "src", "vmaxps", MS_MUL)
147 : [off] "=&r" (off), [count] "+r" (count)
148 : [dst] "r" (dst), [src] "r" (src),
149 [CC] "o" (ms_minmax_abs)
150 : "cc", "memory",
151 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
152 "%xmm4", "%xmm5", "%xmm6"
153 );
154 }
155
156 void ms_pmax3(float *dst, const float *a, const float *b, size_t count)
157 {
158 IF_ARCH_X86(size_t off);
159 ARCH_X86_ASM
160 (
161 MS_MINMAX_CORE("dst", "a", "b", "vmaxps", MS_MUL)
162 : [off] "=&r" (off), [count] "+r" (count)
163 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b),
164 [CC] "o" (ms_minmax_abs)
165 : "cc", "memory",
166 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
167 "%xmm4", "%xmm5", "%xmm6"
168 );
169 }
170
171 void lr_pmin2(float *dst, const float *src, size_t count)
172 {
173 IF_ARCH_X86(size_t off);
174 ARCH_X86_ASM
175 (
176 MS_MINMAX_CORE("dst", "dst", "src", "vminps", MS_NOMUL)
177 : [off] "=&r" (off), [count] "+r" (count)
178 : [dst] "r" (dst), [src] "r" (src),
179 [CC] "o" (ms_minmax_abs)
180 : "cc", "memory",
181 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
182 "%xmm4", "%xmm5", "%xmm6"
183 );
184 }
185
186 void lr_pmin3(float *dst, const float *a, const float *b, size_t count)
187 {
188 IF_ARCH_X86(size_t off);
189 ARCH_X86_ASM
190 (
191 MS_MINMAX_CORE("dst", "a", "b", "vminps", MS_NOMUL)
192 : [off] "=&r" (off), [count] "+r" (count)
193 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b),
194 [CC] "o" (ms_minmax_abs)
195 : "cc", "memory",
196 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
197 "%xmm4", "%xmm5", "%xmm6"
198 );
199 }
200
201 void lr_pmax2(float *dst, const float *src, size_t count)
202 {
203 IF_ARCH_X86(size_t off);
204 ARCH_X86_ASM
205 (
206 MS_MINMAX_CORE("dst", "dst", "src", "vmaxps", MS_NOMUL)
207 : [off] "=&r" (off), [count] "+r" (count)
208 : [dst] "r" (dst), [src] "r" (src),
209 [CC] "o" (ms_minmax_abs)
210 : "cc", "memory",
211 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
212 "%xmm4", "%xmm5", "%xmm6"
213 );
214 }
215
216 void lr_pmax3(float *dst, const float *a, const float *b, size_t count)
217 {
218 IF_ARCH_X86(size_t off);
219 ARCH_X86_ASM
220 (
221 MS_MINMAX_CORE("dst", "a", "b", "vmaxps", MS_NOMUL)
222 : [off] "=&r" (off), [count] "+r" (count)
223 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b),
224 [CC] "o" (ms_minmax_abs)
225 : "cc", "memory",
226 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
227 "%xmm4", "%xmm5", "%xmm6"
228 );
229 }
230
231 #undef MS_MINMAX_CORE
232
233 #define MS_SIGN_MINMAX_CORE(DST, A, B, CMP, IF_MUL) \
234 __ASM_EMIT("xor %[off], %[off]") \
235 __ASM_EMIT("sub $16, %[count]") \
236 __ASM_EMIT("vmovaps 0x00 + %[CC], %%ymm6") /* ymm6 = MASK */ \
237 IF_MUL("vmovaps 0x20 + %[CC], %%ymm7") /* ymm7 = 0.5f */ \
238 __ASM_EMIT("jb 2f") \
239 /* 16x blocks */ \
240 __ASM_EMIT("1:") \
241 __ASM_EMIT("vmovups 0x00(%[" A "], %[off]), %%ymm0") /* ymm0 = l */ \
242 __ASM_EMIT("vmovups 0x20(%[" A "], %[off]), %%ymm1") \
243 __ASM_EMIT("vmovups 0x00(%[" B "], %[off]), %%ymm4") /* ymm4 = r */ \
244 __ASM_EMIT("vmovups 0x20(%[" B "], %[off]), %%ymm5") \
245 __ASM_EMIT("vsubps %%ymm4, %%ymm0, %%ymm2") /* ymm2 = s = l - r */ \
246 __ASM_EMIT("vsubps %%ymm5, %%ymm1, %%ymm3") \
247 __ASM_EMIT("vaddps %%ymm4, %%ymm0, %%ymm0") /* ymm0 = m = l + r */ \
248 __ASM_EMIT("vaddps %%ymm5, %%ymm1, %%ymm1") \
249 __ASM_EMIT("vandps %%ymm6, %%ymm2, %%ymm4") /* ymm4 = fabsf(s) */ \
250 __ASM_EMIT("vandps %%ymm6, %%ymm0, %%ymm5") /* ymm5 = fabsf(m) */ \
251 __ASM_EMIT("vcmpps " CMP ", %%ymm4, %%ymm5, %%ymm5") /* ymm4 = OP(fabsf(m), fabsf(s)) */ \
252 __ASM_EMIT("vblendvps %%ymm5, %%ymm0, %%ymm2, %%ymm0") /* ymm0 = O = OP(fabsf(m), fabsf(s)) ? m : s */ \
253 __ASM_EMIT("vandps %%ymm6, %%ymm3, %%ymm4") \
254 __ASM_EMIT("vandps %%ymm6, %%ymm1, %%ymm5") \
255 __ASM_EMIT("vcmpps " CMP ", %%ymm4, %%ymm5, %%ymm5") \
256 __ASM_EMIT("vblendvps %%ymm5, %%ymm1, %%ymm3, %%ymm1") \
257 IF_MUL("vmulps %%ymm7, %%ymm0, %%ymm0") /* ymm0 = O * 0.5f */ \
258 IF_MUL("vmulps %%ymm7, %%ymm1, %%ymm1") \
259 __ASM_EMIT("vmovups %%ymm0, 0x00(%[" DST "], %[off])") \
260 __ASM_EMIT("vmovups %%ymm1, 0x20(%[" DST "], %[off])") \
261 __ASM_EMIT("add $0x40, %[off]") \
262 __ASM_EMIT("sub $16, %[count]") \
263 __ASM_EMIT("jae 1b") \
264 /* 8x block */ \
265 __ASM_EMIT("2:") \
266 __ASM_EMIT("add $8, %[count]") \
267 __ASM_EMIT("jl 4f") \
268 __ASM_EMIT("vmovups 0x00(%[" A "], %[off]), %%ymm0") /* ymm0 = l */ \
269 __ASM_EMIT("vmovups 0x00(%[" B "], %[off]), %%ymm4") /* ymm4 = r */ \
270 __ASM_EMIT("vsubps %%ymm4, %%ymm0, %%ymm2") /* ymm2 = s = l - r */ \
271 __ASM_EMIT("vaddps %%ymm4, %%ymm0, %%ymm0") /* ymm0 = m = l + r */ \
272 __ASM_EMIT("vandps %%ymm6, %%ymm2, %%ymm4") /* ymm4 = fabsf(s) */ \
273 __ASM_EMIT("vandps %%ymm6, %%ymm0, %%ymm5") /* ymm5 = fabsf(m) */ \
274 __ASM_EMIT("vcmpps " CMP ", %%ymm4, %%ymm5, %%ymm5") /* ymm4 = OP(fabsf(m), fabsf(s)) */ \
275 __ASM_EMIT("vblendvps %%ymm5, %%ymm0, %%ymm2, %%ymm0") /* ymm0 = O = OP(fabsf(m), fabsf(s)) ? m : s */ \
276 IF_MUL("vmulps %%ymm7, %%ymm0, %%ymm0") /* ymm0 = O * 0.5f */ \
277 __ASM_EMIT("vmovups %%ymm0, 0x00(%[" DST "], %[off])") \
278 __ASM_EMIT("sub $8, %[count]") \
279 __ASM_EMIT("add $0x20, %[off]") \
280 /* 4x block */ \
281 __ASM_EMIT("4:") \
282 __ASM_EMIT("add $4, %[count]") \
283 __ASM_EMIT("jl 6f") \
284 __ASM_EMIT("vmovups 0x00(%[" A "], %[off]), %%xmm0") /* xmm0 = l */ \
285 __ASM_EMIT("vmovups 0x00(%[" B "], %[off]), %%xmm4") /* xmm4 = r */ \
286 __ASM_EMIT("vsubps %%xmm4, %%xmm0, %%xmm2") /* xmm2 = s = l - r */ \
287 __ASM_EMIT("vaddps %%xmm4, %%xmm0, %%xmm0") /* xmm0 = m = l + r */ \
288 __ASM_EMIT("vandps %%xmm6, %%xmm2, %%xmm4") /* xmm4 = fabsf(s) */ \
289 __ASM_EMIT("vandps %%xmm6, %%xmm0, %%xmm5") /* xmm5 = fabsf(m) */ \
290 __ASM_EMIT("vcmpps " CMP ", %%xmm4, %%xmm5, %%xmm5") /* xmm4 = OP(fabsf(m), fabsf(s)) */ \
291 __ASM_EMIT("vblendvps %%xmm5, %%xmm0, %%xmm2, %%xmm0") /* xmm0 = O = OP(fabsf(m), fabsf(s)) ? m : s */ \
292 IF_MUL("vmulps %%xmm7, %%xmm0, %%xmm0") /* xmm0 = O * 0.5f */ \
293 __ASM_EMIT("vmovups %%xmm0, 0x00(%[" DST "], %[off])") \
294 __ASM_EMIT("sub $4, %[count]") \
295 __ASM_EMIT("add $0x10, %[off]") \
296 /* 1x blocks */ \
297 __ASM_EMIT("6:") \
298 __ASM_EMIT("add $3, %[count]") \
299 __ASM_EMIT("jl 8f") \
300 __ASM_EMIT("7:") \
301 __ASM_EMIT("vmovss 0x00(%[" A "], %[off]), %%xmm0") /* xmm0 = l */ \
302 __ASM_EMIT("vmovss 0x00(%[" B "], %[off]), %%xmm4") /* xmm4 = r */ \
303 __ASM_EMIT("vsubss %%xmm4, %%xmm0, %%xmm2") /* xmm2 = s = l - r */ \
304 __ASM_EMIT("vaddss %%xmm4, %%xmm0, %%xmm0") /* xmm0 = m = l + r */ \
305 __ASM_EMIT("vandps %%xmm6, %%xmm2, %%xmm4") /* xmm4 = fabsf(s) */ \
306 __ASM_EMIT("vandps %%xmm6, %%xmm0, %%xmm5") /* xmm5 = fabsf(m) */ \
307 __ASM_EMIT("vcmpps " CMP ", %%xmm4, %%xmm5, %%xmm5") /* xmm4 = OP(fabsf(m), fabsf(s)) */ \
308 __ASM_EMIT("vblendvps %%xmm5, %%xmm0, %%xmm2, %%xmm0") /* xmm0 = O = OP(fabsf(m), fabsf(s)) ? m : s */ \
309 IF_MUL("vmulss %%xmm7, %%xmm0, %%xmm0") /* xmm0 = O * 0.5f */ \
310 __ASM_EMIT("vmovss %%xmm0, 0x00(%[" DST "], %[off])") \
311 __ASM_EMIT("add $0x04, %[off]") \
312 __ASM_EMIT("dec %[count]") \
313 __ASM_EMIT("jge 7b") \
314 __ASM_EMIT("8:")
315
316 void ms_psmin2(float *dst, const float *src, size_t count)
317 {
318 IF_ARCH_X86(size_t off);
319 ARCH_X86_ASM
320 (
321 MS_SIGN_MINMAX_CORE("dst", "dst", "src", "$1", MS_MUL)
322 : [off] "=&r" (off), [count] "+r" (count)
323 : [dst] "r" (dst), [src] "r" (src),
324 [CC] "o" (ms_minmax_abs)
325 : "cc", "memory",
326 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
327 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
328 );
329 }
330
331 void ms_psmin3(float *dst, const float *a, const float *b, size_t count)
332 {
333 IF_ARCH_X86(size_t off);
334 ARCH_X86_ASM
335 (
336 MS_SIGN_MINMAX_CORE("dst", "a", "b", "$1", MS_MUL)
337 : [off] "=&r" (off), [count] "+r" (count)
338 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b),
339 [CC] "o" (ms_minmax_abs)
340 : "cc", "memory",
341 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
342 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
343 );
344 }
345
346 void ms_psmax2(float *dst, const float *src, size_t count)
347 {
348 IF_ARCH_X86(size_t off);
349 ARCH_X86_ASM
350 (
351 MS_SIGN_MINMAX_CORE("dst", "dst", "src", "$5", MS_MUL)
352 : [off] "=&r" (off), [count] "+r" (count)
353 : [dst] "r" (dst), [src] "r" (src),
354 [CC] "o" (ms_minmax_abs)
355 : "cc", "memory",
356 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
357 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
358 );
359 }
360
361 void ms_psmax3(float *dst, const float *a, const float *b, size_t count)
362 {
363 IF_ARCH_X86(size_t off);
364 ARCH_X86_ASM
365 (
366 MS_SIGN_MINMAX_CORE("dst", "a", "b", "$5", MS_MUL)
367 : [off] "=&r" (off), [count] "+r" (count)
368 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b),
369 [CC] "o" (ms_minmax_abs)
370 : "cc", "memory",
371 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
372 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
373 );
374 }
375
376 void lr_psmin2(float *dst, const float *src, size_t count)
377 {
378 IF_ARCH_X86(size_t off);
379 ARCH_X86_ASM
380 (
381 MS_SIGN_MINMAX_CORE("dst", "dst", "src", "$1", MS_NOMUL)
382 : [off] "=&r" (off), [count] "+r" (count)
383 : [dst] "r" (dst), [src] "r" (src),
384 [CC] "o" (ms_minmax_abs)
385 : "cc", "memory",
386 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
387 "%xmm4", "%xmm5", "%xmm6"
388 );
389 }
390
391 void lr_psmin3(float *dst, const float *a, const float *b, size_t count)
392 {
393 IF_ARCH_X86(size_t off);
394 ARCH_X86_ASM
395 (
396 MS_SIGN_MINMAX_CORE("dst", "a", "b", "$1", MS_NOMUL)
397 : [off] "=&r" (off), [count] "+r" (count)
398 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b),
399 [CC] "o" (ms_minmax_abs)
400 : "cc", "memory",
401 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
402 "%xmm4", "%xmm5", "%xmm6"
403 );
404 }
405
406 void lr_psmax2(float *dst, const float *src, size_t count)
407 {
408 IF_ARCH_X86(size_t off);
409 ARCH_X86_ASM
410 (
411 MS_SIGN_MINMAX_CORE("dst", "dst", "src", "$5", MS_NOMUL)
412 : [off] "=&r" (off), [count] "+r" (count)
413 : [dst] "r" (dst), [src] "r" (src),
414 [CC] "o" (ms_minmax_abs)
415 : "cc", "memory",
416 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
417 "%xmm4", "%xmm5", "%xmm6"
418 );
419 }
420
421 void lr_psmax3(float *dst, const float *a, const float *b, size_t count)
422 {
423 IF_ARCH_X86(size_t off);
424 ARCH_X86_ASM
425 (
426 MS_SIGN_MINMAX_CORE("dst", "a", "b", "$5", MS_NOMUL)
427 : [off] "=&r" (off), [count] "+r" (count)
428 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b),
429 [CC] "o" (ms_minmax_abs)
430 : "cc", "memory",
431 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
432 "%xmm4", "%xmm5", "%xmm6"
433 );
434 }
435
436 #undef MS_SIGN_MINMAX_CORE
437
438 #define MS_ABS_MINMAX_CORE(DST, A, B, OP, IF_MUL) \
439 __ASM_EMIT("xor %[off], %[off]") \
440 __ASM_EMIT("sub $16, %[count]") \
441 __ASM_EMIT("vmovaps 0x00 + %[CC], %%ymm6") /* ymm6 = MASK */ \
442 IF_MUL("vmovaps 0x20 + %[CC], %%ymm7") /* ymm7 = 0.5f */ \
443 __ASM_EMIT("jb 2f") \
444 /* 16x blocks */ \
445 __ASM_EMIT("1:") \
446 __ASM_EMIT("vmovups 0x00(%[" A "], %[off]), %%ymm0") /* ymm0 = l */ \
447 __ASM_EMIT("vmovups 0x20(%[" A "], %[off]), %%ymm1") \
448 __ASM_EMIT("vmovups 0x00(%[" B "], %[off]), %%ymm4") /* ymm4 = r */ \
449 __ASM_EMIT("vmovups 0x20(%[" B "], %[off]), %%ymm5") \
450 __ASM_EMIT("vsubps %%ymm4, %%ymm0, %%ymm2") /* ymm2 = s = l - r */ \
451 __ASM_EMIT("vsubps %%ymm5, %%ymm1, %%ymm3") \
452 __ASM_EMIT("vaddps %%ymm4, %%ymm0, %%ymm0") /* ymm0 = m = l + r */ \
453 __ASM_EMIT("vaddps %%ymm5, %%ymm1, %%ymm1") \
454 __ASM_EMIT("vandps %%ymm6, %%ymm2, %%ymm2") /* ymm2 = fabsf(s) */ \
455 __ASM_EMIT("vandps %%ymm6, %%ymm3, %%ymm3") \
456 __ASM_EMIT("vandps %%ymm6, %%ymm0, %%ymm0") /* ymm0 = fabsf(m) */ \
457 __ASM_EMIT("vandps %%ymm6, %%ymm1, %%ymm1") \
458 __ASM_EMIT(OP " %%ymm2, %%ymm0, %%ymm0") /* ymm0 = O = OP(fabsf(m), fabsf(s)) */ \
459 __ASM_EMIT(OP " %%ymm3, %%ymm1, %%ymm1") \
460 IF_MUL("vmulps %%ymm7, %%ymm0, %%ymm0") /* ymm0 = O * 0.5f */ \
461 IF_MUL("vmulps %%ymm7, %%ymm1, %%ymm1") \
462 __ASM_EMIT("vmovups %%ymm0, 0x00(%[" DST "], %[off])") \
463 __ASM_EMIT("vmovups %%ymm1, 0x20(%[" DST "], %[off])") \
464 __ASM_EMIT("add $0x40, %[off]") \
465 __ASM_EMIT("sub $16, %[count]") \
466 __ASM_EMIT("jae 1b") \
467 /* 8x block */ \
468 __ASM_EMIT("2:") \
469 __ASM_EMIT("add $8, %[count]") \
470 __ASM_EMIT("jl 4f") \
471 __ASM_EMIT("vmovups 0x00(%[" A "], %[off]), %%ymm0") /* ymm0 = l */ \
472 __ASM_EMIT("vmovups 0x00(%[" B "], %[off]), %%ymm4") /* ymm4 = r */ \
473 __ASM_EMIT("vsubps %%ymm4, %%ymm0, %%ymm2") /* ymm2 = s = l - r */ \
474 __ASM_EMIT("vaddps %%ymm4, %%ymm0, %%ymm0") /* ymm0 = m = l + r */ \
475 __ASM_EMIT("vandps %%ymm6, %%ymm2, %%ymm2") /* ymm2 = fabsf(s) */ \
476 __ASM_EMIT("vandps %%ymm6, %%ymm0, %%ymm0") /* ymm0 = fabsf(m) */ \
477 __ASM_EMIT(OP " %%ymm2, %%ymm0, %%ymm0") /* ymm0 = O = OP(fabsf(m), fabsf(s)) */ \
478 IF_MUL("vmulps %%ymm7, %%ymm0, %%ymm0") /* ymm0 = O * 0.5f */ \
479 __ASM_EMIT("vmovups %%ymm0, 0x00(%[" DST "], %[off])") \
480 __ASM_EMIT("sub $8, %[count]") \
481 __ASM_EMIT("add $0x20, %[off]") \
482 /* 4x block */ \
483 __ASM_EMIT("4:") \
484 __ASM_EMIT("add $4, %[count]") \
485 __ASM_EMIT("jl 6f") \
486 __ASM_EMIT("vmovups 0x00(%[" A "], %[off]), %%xmm0") /* xmm0 = l */ \
487 __ASM_EMIT("vmovups 0x00(%[" B "], %[off]), %%xmm4") /* xmm4 = r */ \
488 __ASM_EMIT("vsubps %%xmm4, %%xmm0, %%xmm2") /* xmm2 = s = l - r */ \
489 __ASM_EMIT("vaddps %%xmm4, %%xmm0, %%xmm0") /* xmm0 = m = l + r */ \
490 __ASM_EMIT("vandps %%xmm6, %%xmm2, %%xmm2") /* xmm2 = fabsf(s) */ \
491 __ASM_EMIT("vandps %%xmm6, %%xmm0, %%xmm0") /* xmm0 = fabsf(m) */ \
492 __ASM_EMIT(OP " %%xmm2, %%xmm0, %%xmm0") /* xmm0 = O = OP(fabsf(m), fabsf(s)) */ \
493 IF_MUL("vmulps %%xmm7, %%xmm0, %%xmm0") /* xmm0 = O * 0.5f */ \
494 __ASM_EMIT("vmovups %%xmm0, 0x00(%[" DST "], %[off])") \
495 __ASM_EMIT("sub $4, %[count]") \
496 __ASM_EMIT("add $0x10, %[off]") \
497 /* 1x blocks */ \
498 __ASM_EMIT("6:") \
499 __ASM_EMIT("add $3, %[count]") \
500 __ASM_EMIT("jl 8f") \
501 __ASM_EMIT("7:") \
502 __ASM_EMIT("vmovss 0x00(%[" A "], %[off]), %%xmm0") /* xmm0 = l */ \
503 __ASM_EMIT("vmovss 0x00(%[" B "], %[off]), %%xmm4") /* xmm4 = r */ \
504 __ASM_EMIT("vsubss %%xmm4, %%xmm0, %%xmm2") /* xmm2 = s = l - r */ \
505 __ASM_EMIT("vaddss %%xmm4, %%xmm0, %%xmm0") /* xmm0 = m = l + r */ \
506 __ASM_EMIT("vandps %%xmm6, %%xmm2, %%xmm2") /* xmm2 = fabsf(s) */ \
507 __ASM_EMIT("vandps %%xmm6, %%xmm0, %%xmm0") /* xmm0 = fabsf(m) */ \
508 __ASM_EMIT(OP " %%xmm2, %%xmm0, %%xmm0") /* xmm0 = O = OP(fabsf(m), fabsf(s)) */ \
509 IF_MUL("vmulss %%xmm7, %%xmm0, %%xmm0") /* xmm0 = O * 0.5f */ \
510 __ASM_EMIT("vmovss %%xmm0, 0x00(%[" DST "], %[off])") \
511 __ASM_EMIT("add $0x04, %[off]") \
512 __ASM_EMIT("dec %[count]") \
513 __ASM_EMIT("jge 7b") \
514 __ASM_EMIT("8:")
515
516 void ms_pamin2(float *dst, const float *src, size_t count)
517 {
518 IF_ARCH_X86(size_t off);
519 ARCH_X86_ASM
520 (
521 MS_ABS_MINMAX_CORE("dst", "dst", "src", "vminps", MS_MUL)
522 : [off] "=&r" (off), [count] "+r" (count)
523 : [dst] "r" (dst), [src] "r" (src),
524 [CC] "o" (ms_minmax_abs)
525 : "cc", "memory",
526 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
527 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
528 );
529 }
530
531 void ms_pamin3(float *dst, const float *a, const float *b, size_t count)
532 {
533 IF_ARCH_X86(size_t off);
534 ARCH_X86_ASM
535 (
536 MS_ABS_MINMAX_CORE("dst", "a", "b", "vminps", MS_MUL)
537 : [off] "=&r" (off), [count] "+r" (count)
538 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b),
539 [CC] "o" (ms_minmax_abs)
540 : "cc", "memory",
541 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
542 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
543 );
544 }
545
546 void ms_pamax2(float *dst, const float *src, size_t count)
547 {
548 IF_ARCH_X86(size_t off);
549 ARCH_X86_ASM
550 (
551 MS_ABS_MINMAX_CORE("dst", "dst", "src", "vmaxps", MS_MUL)
552 : [off] "=&r" (off), [count] "+r" (count)
553 : [dst] "r" (dst), [src] "r" (src),
554 [CC] "o" (ms_minmax_abs)
555 : "cc", "memory",
556 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
557 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
558 );
559 }
560
561 void ms_pamax3(float *dst, const float *a, const float *b, size_t count)
562 {
563 IF_ARCH_X86(size_t off);
564 ARCH_X86_ASM
565 (
566 MS_ABS_MINMAX_CORE("dst", "a", "b", "vmaxps", MS_MUL)
567 : [off] "=&r" (off), [count] "+r" (count)
568 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b),
569 [CC] "o" (ms_minmax_abs)
570 : "cc", "memory",
571 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
572 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
573 );
574 }
575
576 void lr_pamin2(float *dst, const float *src, size_t count)
577 {
578 IF_ARCH_X86(size_t off);
579 ARCH_X86_ASM
580 (
581 MS_ABS_MINMAX_CORE("dst", "dst", "src", "vminps", MS_NOMUL)
582 : [off] "=&r" (off), [count] "+r" (count)
583 : [dst] "r" (dst), [src] "r" (src),
584 [CC] "o" (ms_minmax_abs)
585 : "cc", "memory",
586 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
587 "%xmm4", "%xmm5", "%xmm6"
588 );
589 }
590
591 void lr_pamin3(float *dst, const float *a, const float *b, size_t count)
592 {
593 IF_ARCH_X86(size_t off);
594 ARCH_X86_ASM
595 (
596 MS_ABS_MINMAX_CORE("dst", "a", "b", "vminps", MS_NOMUL)
597 : [off] "=&r" (off), [count] "+r" (count)
598 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b),
599 [CC] "o" (ms_minmax_abs)
600 : "cc", "memory",
601 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
602 "%xmm4", "%xmm5", "%xmm6"
603 );
604 }
605
606 void lr_pamax2(float *dst, const float *src, size_t count)
607 {
608 IF_ARCH_X86(size_t off);
609 ARCH_X86_ASM
610 (
611 MS_ABS_MINMAX_CORE("dst", "dst", "src", "vmaxps", MS_NOMUL)
612 : [off] "=&r" (off), [count] "+r" (count)
613 : [dst] "r" (dst), [src] "r" (src),
614 [CC] "o" (ms_minmax_abs)
615 : "cc", "memory",
616 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
617 "%xmm4", "%xmm5", "%xmm6"
618 );
619 }
620
621 void lr_pamax3(float *dst, const float *a, const float *b, size_t count)
622 {
623 IF_ARCH_X86(size_t off);
624 ARCH_X86_ASM
625 (
626 MS_ABS_MINMAX_CORE("dst", "a", "b", "vmaxps", MS_NOMUL)
627 : [off] "=&r" (off), [count] "+r" (count)
628 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b),
629 [CC] "o" (ms_minmax_abs)
630 : "cc", "memory",
631 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
632 "%xmm4", "%xmm5", "%xmm6"
633 );
634 }
635
636 #undef MS_ABS_MINMAX_CORE
637
638 #undef MS_MUL
639 #undef MS_NOMUL
640
641 } /* namespace avx */
642 } /* namespace lsp */
643
644
645 #endif /* PRIVATE_DSP_ARCH_X86_AVX_PMATH_MS_MINMAX_H_ */