Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/x86/sse/pmath/ms_minmax.h

raw
1 /*
2 * Copyright (C) 2026 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2026 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 28 мар. 2026 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_X86_SSE_PMATH_MS_MINMAX_H_
23 #define PRIVATE_DSP_ARCH_X86_SSE_PMATH_MS_MINMAX_H_
24
25 #ifndef PRIVATE_DSP_ARCH_X86_SSE_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_X86_SSE_IMPL */
28
29 namespace lsp
30 {
31 namespace sse
32 {
33 IF_ARCH_X86(
34 static const uint32_t ms_minmax_abs[] __lsp_aligned16 =
35 {
36 LSP_DSP_VEC4(0x7fffffff),
37 LSP_DSP_VEC4(0x3f000000) // 0.5f
38 };
39 )
40
41 #define MS_MUL(x) __ASM_EMIT(x)
42 #define MS_NOMUL(x)
43
44 #define MS_MINMAX_CORE(DST, A, B, OP, IF_MUL) \
45 __ASM_EMIT("xor %[off], %[off]") \
46 __ASM_EMIT("sub $8, %[count]") \
47 IF_MUL("movaps 0x10 + %[CC], %%xmm6") /* xmm6 = 0.5f */ \
48 __ASM_EMIT("jb 2f") \
49 /* 8x blocks */ \
50 __ASM_EMIT("1:") \
51 __ASM_EMIT("movups 0x00(%[" A "], %[off]), %%xmm0") /* xmm0 = l */ \
52 __ASM_EMIT("movups 0x10(%[" A "], %[off]), %%xmm1") \
53 __ASM_EMIT("movups 0x00(%[" B "], %[off]), %%xmm4") /* xmm4 = r */ \
54 __ASM_EMIT("movups 0x10(%[" B "], %[off]), %%xmm5") \
55 __ASM_EMIT("movaps %%xmm0, %%xmm2") /* xmm2 = l */ \
56 __ASM_EMIT("movaps %%xmm1, %%xmm3") \
57 __ASM_EMIT("addps %%xmm4, %%xmm0") /* xmm0 = m = l + r */ \
58 __ASM_EMIT("addps %%xmm5, %%xmm1") \
59 __ASM_EMIT("subps %%xmm4, %%xmm2") /* xmm2 = s = l - r */ \
60 __ASM_EMIT("subps %%xmm5, %%xmm3") \
61 __ASM_EMIT(OP " %%xmm2, %%xmm0") /* xmm0 = OP(m, s) */ \
62 __ASM_EMIT(OP " %%xmm3, %%xmm1") \
63 IF_MUL("mulps %%xmm6, %%xmm0") /* xmm0 = OP(m, s) * 0.5f */ \
64 IF_MUL("mulps %%xmm6, %%xmm1") \
65 __ASM_EMIT("movups %%xmm0, 0x00(%[" DST "], %[off])") \
66 __ASM_EMIT("movups %%xmm1, 0x10(%[" DST "], %[off])") \
67 __ASM_EMIT("add $0x20, %[off]") \
68 __ASM_EMIT("sub $8, %[count]") \
69 __ASM_EMIT("jae 1b") \
70 /* 4x block */ \
71 __ASM_EMIT("2:") \
72 __ASM_EMIT("add $4, %[count]") \
73 __ASM_EMIT("jl 4f") \
74 __ASM_EMIT("movups 0x00(%[" A "], %[off]), %%xmm0") /* xmm0 = l */ \
75 __ASM_EMIT("movups 0x00(%[" B "], %[off]), %%xmm4") /* xmm4 = r */ \
76 __ASM_EMIT("movaps %%xmm0, %%xmm2") /* xmm2 = l */ \
77 __ASM_EMIT("addps %%xmm4, %%xmm0") /* xmm0 = m = l + r */ \
78 __ASM_EMIT("subps %%xmm4, %%xmm2") /* xmm2 = s = l - r */ \
79 __ASM_EMIT(OP " %%xmm2, %%xmm0") /* xmm0 = OP(m, s) */ \
80 IF_MUL("mulps %%xmm6, %%xmm0") /* xmm0 = OP(m, s) * 0.5f */ \
81 __ASM_EMIT("movups %%xmm0, 0x00(%[" DST "], %[off])") \
82 __ASM_EMIT("sub $4, %[count]") \
83 __ASM_EMIT("add $0x10, %[off]") \
84 /* 1x blocks */ \
85 __ASM_EMIT("4:") \
86 __ASM_EMIT("add $3, %[count]") \
87 __ASM_EMIT("jl 6f") \
88 __ASM_EMIT("5:") \
89 __ASM_EMIT("movss 0x00(%[" A "], %[off]), %%xmm0") /* xmm0 = l */ \
90 __ASM_EMIT("movss 0x00(%[" B "], %[off]), %%xmm4") /* xmm4 = r */ \
91 __ASM_EMIT("movaps %%xmm0, %%xmm2") /* xmm2 = l */ \
92 __ASM_EMIT("addss %%xmm4, %%xmm0") /* xmm0 = m = l + r */ \
93 __ASM_EMIT("subss %%xmm4, %%xmm2") /* xmm2 = s = l - r */ \
94 __ASM_EMIT(OP " %%xmm2, %%xmm0") /* xmm0 = OP(m, s) */ \
95 IF_MUL("mulss %%xmm6, %%xmm0") /* xmm0 = OP(m, s) * 0.5f */ \
96 __ASM_EMIT("movss %%xmm0, 0x00(%[" DST "], %[off])") \
97 __ASM_EMIT("add $0x04, %[off]") \
98 __ASM_EMIT("dec %[count]") \
99 __ASM_EMIT("jge 5b") \
100 __ASM_EMIT("6:")
101
102 void ms_pmin2(float *dst, const float *src, size_t count)
103 {
104 IF_ARCH_X86(size_t off);
105 ARCH_X86_ASM
106 (
107 MS_MINMAX_CORE("dst", "dst", "src", "minps", MS_MUL)
108 : [off] "=&r" (off), [count] "+r" (count)
109 : [dst] "r" (dst), [src] "r" (src),
110 [CC] "o" (ms_minmax_abs)
111 : "cc", "memory",
112 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
113 "%xmm4", "%xmm5", "%xmm6"
114 );
115 }
116
117 void ms_pmin3(float *dst, const float *a, const float *b, size_t count)
118 {
119 IF_ARCH_X86(size_t off);
120 ARCH_X86_ASM
121 (
122 MS_MINMAX_CORE("dst", "a", "b", "minps", MS_MUL)
123 : [off] "=&r" (off), [count] "+r" (count)
124 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b),
125 [CC] "o" (ms_minmax_abs)
126 : "cc", "memory",
127 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
128 "%xmm4", "%xmm5", "%xmm6"
129 );
130 }
131
132 void ms_pmax2(float *dst, const float *src, size_t count)
133 {
134 IF_ARCH_X86(size_t off);
135 ARCH_X86_ASM
136 (
137 MS_MINMAX_CORE("dst", "dst", "src", "maxps", MS_MUL)
138 : [off] "=&r" (off), [count] "+r" (count)
139 : [dst] "r" (dst), [src] "r" (src),
140 [CC] "o" (ms_minmax_abs)
141 : "cc", "memory",
142 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
143 "%xmm4", "%xmm5", "%xmm6"
144 );
145 }
146
147 void ms_pmax3(float *dst, const float *a, const float *b, size_t count)
148 {
149 IF_ARCH_X86(size_t off);
150 ARCH_X86_ASM
151 (
152 MS_MINMAX_CORE("dst", "a", "b", "maxps", MS_MUL)
153 : [off] "=&r" (off), [count] "+r" (count)
154 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b),
155 [CC] "o" (ms_minmax_abs)
156 : "cc", "memory",
157 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
158 "%xmm4", "%xmm5", "%xmm6"
159 );
160 }
161
162 void lr_pmin2(float *dst, const float *src, size_t count)
163 {
164 IF_ARCH_X86(size_t off);
165 ARCH_X86_ASM
166 (
167 MS_MINMAX_CORE("dst", "dst", "src", "minps", MS_NOMUL)
168 : [off] "=&r" (off), [count] "+r" (count)
169 : [dst] "r" (dst), [src] "r" (src),
170 [CC] "o" (ms_minmax_abs)
171 : "cc", "memory",
172 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
173 "%xmm4", "%xmm5"
174 );
175 }
176
177 void lr_pmin3(float *dst, const float *a, const float *b, size_t count)
178 {
179 IF_ARCH_X86(size_t off);
180 ARCH_X86_ASM
181 (
182 MS_MINMAX_CORE("dst", "a", "b", "minps", MS_NOMUL)
183 : [off] "=&r" (off), [count] "+r" (count)
184 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b),
185 [CC] "o" (ms_minmax_abs)
186 : "cc", "memory",
187 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
188 "%xmm4", "%xmm5"
189 );
190 }
191
192 void lr_pmax2(float *dst, const float *src, size_t count)
193 {
194 IF_ARCH_X86(size_t off);
195 ARCH_X86_ASM
196 (
197 MS_MINMAX_CORE("dst", "dst", "src", "maxps", MS_NOMUL)
198 : [off] "=&r" (off), [count] "+r" (count)
199 : [dst] "r" (dst), [src] "r" (src),
200 [CC] "o" (ms_minmax_abs)
201 : "cc", "memory",
202 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
203 "%xmm4", "%xmm5"
204 );
205 }
206
207 void lr_pmax3(float *dst, const float *a, const float *b, size_t count)
208 {
209 IF_ARCH_X86(size_t off);
210 ARCH_X86_ASM
211 (
212 MS_MINMAX_CORE("dst", "a", "b", "maxps", MS_NOMUL)
213 : [off] "=&r" (off), [count] "+r" (count)
214 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b),
215 [CC] "o" (ms_minmax_abs)
216 : "cc", "memory",
217 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
218 "%xmm4", "%xmm5"
219 );
220 }
221
222 #undef MS_MINMAX_CORE
223
224
225 #define MS_SIGN_MINMAX_CORE(DST, A, B, CMP, IF_MUL) \
226 __ASM_EMIT("xor %[off], %[off]") \
227 __ASM_EMIT("sub $4, %[count]") \
228 __ASM_EMIT("movaps 0x00 + %[CC], %%xmm6") /* xmm6 = MASK */ \
229 IF_MUL("movaps 0x10 + %[CC], %%xmm7") /* xmm7 = 0.5f */ \
230 __ASM_EMIT("jb 2f") \
231 /* 4x blocks */ \
232 __ASM_EMIT("1:") \
233 __ASM_EMIT("movups 0x00(%[" A "], %[off]), %%xmm0") /* xmm0 = l */ \
234 __ASM_EMIT("movups 0x00(%[" B "], %[off]), %%xmm2") /* xmm2 = r */ \
235 __ASM_EMIT("movaps %%xmm0, %%xmm1") /* xmm1 = l */ \
236 __ASM_EMIT("addps %%xmm2, %%xmm0") /* xmm0 = m = l + r */ \
237 __ASM_EMIT("subps %%xmm2, %%xmm1") /* xmm1 = s = l - r */ \
238 __ASM_EMIT("movaps %%xmm0, %%xmm2") /* xmm2 = m */ \
239 __ASM_EMIT("movaps %%xmm1, %%xmm3") /* xmm3 = s */ \
240 __ASM_EMIT("andps %%xmm6, %%xmm2") /* xmm2 = fabsf(m) */ \
241 __ASM_EMIT("andps %%xmm6, %%xmm3") /* xmm3 = fabsf(s) */ \
242 __ASM_EMIT("cmpps " CMP ", %%xmm3, %%xmm2") /* xmm2 = fabsf(m) <=> fabsf(s) */ \
243 __ASM_EMIT("andps %%xmm2, %%xmm0") /* xmm0 = m & (abs(m) <=> abs(s)) */ \
244 __ASM_EMIT("andnps %%xmm1, %%xmm2") /* xmm2 = s & ~(abs(m) <=> abs(s)) */ \
245 __ASM_EMIT("orps %%xmm2, %%xmm0") /* xmm0 = O = (m & (abs(m) <=> abs(s))) | (s & ~(abs(m) <=> abs(s))) */ \
246 IF_MUL("mulps %%xmm7, %%xmm0") /* xmm0 = O + 0.5f */ \
247 __ASM_EMIT("movups %%xmm0, 0x00(%[" DST "], %[off])") \
248 __ASM_EMIT("add $0x10, %[off]") \
249 __ASM_EMIT("sub $4, %[count]") \
250 __ASM_EMIT("jae 1b") \
251 /* 1x blocks */ \
252 __ASM_EMIT("2:") \
253 __ASM_EMIT("add $3, %[count]") \
254 __ASM_EMIT("jl 4f") \
255 __ASM_EMIT("3:") \
256 __ASM_EMIT("movss 0x00(%[" A "], %[off]), %%xmm0") /* xmm0 = l */ \
257 __ASM_EMIT("movss 0x00(%[" B "], %[off]), %%xmm2") /* xmm2 = r */ \
258 __ASM_EMIT("movaps %%xmm0, %%xmm1") /* xmm1 = l */ \
259 __ASM_EMIT("addss %%xmm2, %%xmm0") /* xmm0 = m = l + r */ \
260 __ASM_EMIT("subss %%xmm2, %%xmm1") /* xmm1 = s = l - r */ \
261 __ASM_EMIT("movaps %%xmm0, %%xmm2") /* xmm2 = m */ \
262 __ASM_EMIT("movaps %%xmm1, %%xmm3") /* xmm3 = s */ \
263 __ASM_EMIT("andps %%xmm6, %%xmm2") /* xmm2 = fabsf(m) */ \
264 __ASM_EMIT("andps %%xmm6, %%xmm3") /* xmm3 = fabsf(s) */ \
265 __ASM_EMIT("cmpps " CMP ", %%xmm3, %%xmm2") /* xmm2 = fabsf(m) <=> fabsf(s) */ \
266 __ASM_EMIT("andps %%xmm2, %%xmm0") /* xmm0 = m & (abs(m) <=> abs(s)) */ \
267 __ASM_EMIT("andnps %%xmm1, %%xmm2") /* xmm2 = s & ~(abs(m) <=> abs(s)) */ \
268 __ASM_EMIT("orps %%xmm2, %%xmm0") /* xmm0 = O = (m & (abs(m) <=> abs(s))) | (s & ~(abs(m) <=> abs(s))) */ \
269 IF_MUL("mulss %%xmm7, %%xmm0") /* xmm0 = O + 0.5f */ \
270 __ASM_EMIT("movss %%xmm0, 0x00(%[" DST "], %[off])") \
271 __ASM_EMIT("add $0x04, %[off]") \
272 __ASM_EMIT("dec %[count]") \
273 __ASM_EMIT("jge 3b") \
274 __ASM_EMIT("4:")
275
276 void ms_psmin2(float *dst, const float *src, size_t count)
277 {
278 IF_ARCH_X86(size_t off);
279 ARCH_X86_ASM
280 (
281 MS_SIGN_MINMAX_CORE("dst", "dst", "src", "$1", MS_MUL)
282 : [off] "=&r" (off), [count] "+r" (count)
283 : [dst] "r" (dst), [src] "r" (src),
284 [CC] "o" (ms_minmax_abs)
285 : "cc", "memory",
286 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
287 "%xmm6", "%xmm7"
288 );
289 }
290
291 void ms_psmin3(float *dst, const float *a, const float *b, size_t count)
292 {
293 IF_ARCH_X86(size_t off);
294 ARCH_X86_ASM
295 (
296 MS_SIGN_MINMAX_CORE("dst", "a", "b", "$1", MS_MUL)
297 : [off] "=&r" (off), [count] "+r" (count)
298 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b),
299 [CC] "o" (ms_minmax_abs)
300 : "cc", "memory",
301 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
302 "%xmm6", "%xmm7"
303 );
304 }
305
306 void ms_psmax2(float *dst, const float *src, size_t count)
307 {
308 IF_ARCH_X86(size_t off);
309 ARCH_X86_ASM
310 (
311 MS_SIGN_MINMAX_CORE("dst", "dst", "src", "$6", MS_MUL)
312 : [off] "=&r" (off), [count] "+r" (count)
313 : [dst] "r" (dst), [src] "r" (src),
314 [CC] "o" (ms_minmax_abs)
315 : "cc", "memory",
316 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
317 "%xmm6", "%xmm7"
318 );
319 }
320
321 void ms_psmax3(float *dst, const float *a, const float *b, size_t count)
322 {
323 IF_ARCH_X86(size_t off);
324 ARCH_X86_ASM
325 (
326 MS_SIGN_MINMAX_CORE("dst", "a", "b", "$6", MS_MUL)
327 : [off] "=&r" (off), [count] "+r" (count)
328 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b),
329 [CC] "o" (ms_minmax_abs)
330 : "cc", "memory",
331 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
332 "%xmm6", "%xmm7"
333 );
334 }
335
336 void lr_psmin2(float *dst, const float *src, size_t count)
337 {
338 IF_ARCH_X86(size_t off);
339 ARCH_X86_ASM
340 (
341 MS_SIGN_MINMAX_CORE("dst", "dst", "src", "$1", MS_NOMUL)
342 : [off] "=&r" (off), [count] "+r" (count)
343 : [dst] "r" (dst), [src] "r" (src),
344 [CC] "o" (ms_minmax_abs)
345 : "cc", "memory",
346 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
347 "%xmm6"
348 );
349 }
350
351 void lr_psmin3(float *dst, const float *a, const float *b, size_t count)
352 {
353 IF_ARCH_X86(size_t off);
354 ARCH_X86_ASM
355 (
356 MS_SIGN_MINMAX_CORE("dst", "a", "b", "$1", MS_NOMUL)
357 : [off] "=&r" (off), [count] "+r" (count)
358 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b),
359 [CC] "o" (ms_minmax_abs)
360 : "cc", "memory",
361 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
362 "%xmm6"
363 );
364 }
365
366 void lr_psmax2(float *dst, const float *src, size_t count)
367 {
368 IF_ARCH_X86(size_t off);
369 ARCH_X86_ASM
370 (
371 MS_SIGN_MINMAX_CORE("dst", "dst", "src", "$6", MS_NOMUL)
372 : [off] "=&r" (off), [count] "+r" (count)
373 : [dst] "r" (dst), [src] "r" (src),
374 [CC] "o" (ms_minmax_abs)
375 : "cc", "memory",
376 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
377 "%xmm6"
378 );
379 }
380
381 void lr_psmax3(float *dst, const float *a, const float *b, size_t count)
382 {
383 IF_ARCH_X86(size_t off);
384 ARCH_X86_ASM
385 (
386 MS_SIGN_MINMAX_CORE("dst", "a", "b", "$6", MS_NOMUL)
387 : [off] "=&r" (off), [count] "+r" (count)
388 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b),
389 [CC] "o" (ms_minmax_abs)
390 : "cc", "memory",
391 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
392 "%xmm6"
393 );
394 }
395 #undef MS_SIGN_MINMAX_CORE
396
397 #define MS_ABS_MINMAX_CORE(DST, A, B, OP, IF_MUL) \
398 __ASM_EMIT("xor %[off], %[off]") \
399 __ASM_EMIT("sub $8, %[count]") \
400 __ASM_EMIT("movaps 0x00 + %[CC], %%xmm6") /* xmm6 = MASK */ \
401 IF_MUL("movaps 0x10 + %[CC], %%xmm7") /* xmm7 = 0.5f */ \
402 __ASM_EMIT("jb 2f") \
403 /* 8x blocks */ \
404 __ASM_EMIT("1:") \
405 __ASM_EMIT("movups 0x00(%[" A "], %[off]), %%xmm0") /* xmm0 = l */ \
406 __ASM_EMIT("movups 0x10(%[" A "], %[off]), %%xmm1") \
407 __ASM_EMIT("movups 0x00(%[" B "], %[off]), %%xmm4") /* xmm4 = r */ \
408 __ASM_EMIT("movups 0x10(%[" B "], %[off]), %%xmm5") \
409 __ASM_EMIT("movaps %%xmm0, %%xmm2") /* xmm2 = l */ \
410 __ASM_EMIT("movaps %%xmm1, %%xmm3") \
411 __ASM_EMIT("addps %%xmm4, %%xmm0") /* xmm0 = m = l + r */ \
412 __ASM_EMIT("addps %%xmm5, %%xmm1") \
413 __ASM_EMIT("subps %%xmm4, %%xmm2") /* xmm2 = s = l - r */ \
414 __ASM_EMIT("subps %%xmm5, %%xmm3") \
415 __ASM_EMIT("andps %%xmm6, %%xmm0") /* xmm0 = fabsf(m) */ \
416 __ASM_EMIT("andps %%xmm6, %%xmm1") \
417 __ASM_EMIT("andps %%xmm6, %%xmm2") /* xmm2 = fabsf(s) */ \
418 __ASM_EMIT("andps %%xmm6, %%xmm3") \
419 __ASM_EMIT(OP " %%xmm2, %%xmm0") /* xmm0 = OP(m, s) */ \
420 __ASM_EMIT(OP " %%xmm3, %%xmm1") \
421 IF_MUL("mulps %%xmm7, %%xmm0") /* xmm0 = OP(m, s) * 0.5f */ \
422 IF_MUL("mulps %%xmm7, %%xmm1") \
423 __ASM_EMIT("movups %%xmm0, 0x00(%[" DST "], %[off])") \
424 __ASM_EMIT("movups %%xmm1, 0x10(%[" DST "], %[off])") \
425 __ASM_EMIT("add $0x20, %[off]") \
426 __ASM_EMIT("sub $8, %[count]") \
427 __ASM_EMIT("jae 1b") \
428 /* 4x block */ \
429 __ASM_EMIT("2:") \
430 __ASM_EMIT("add $4, %[count]") \
431 __ASM_EMIT("jl 4f") \
432 __ASM_EMIT("movups 0x00(%[" A "], %[off]), %%xmm0") /* xmm0 = l */ \
433 __ASM_EMIT("movups 0x00(%[" B "], %[off]), %%xmm4") /* xmm4 = r */ \
434 __ASM_EMIT("movaps %%xmm0, %%xmm2") /* xmm2 = l */ \
435 __ASM_EMIT("addps %%xmm4, %%xmm0") /* xmm0 = m = l + r */ \
436 __ASM_EMIT("subps %%xmm4, %%xmm2") /* xmm2 = s = l - r */ \
437 __ASM_EMIT("andps %%xmm6, %%xmm0") /* xmm0 = fabsf(m) */ \
438 __ASM_EMIT("andps %%xmm6, %%xmm2") /* xmm2 = fabsf(s) */ \
439 __ASM_EMIT(OP " %%xmm2, %%xmm0") /* xmm0 = OP(m, s) */ \
440 IF_MUL("mulps %%xmm7, %%xmm0") /* xmm0 = OP(m, s) * 0.5f */ \
441 __ASM_EMIT("movups %%xmm0, 0x00(%[" DST "], %[off])") \
442 __ASM_EMIT("sub $4, %[count]") \
443 __ASM_EMIT("add $0x10, %[off]") \
444 /* 1x blocks */ \
445 __ASM_EMIT("4:") \
446 __ASM_EMIT("add $3, %[count]") \
447 __ASM_EMIT("jl 6f") \
448 __ASM_EMIT("5:") \
449 __ASM_EMIT("movss 0x00(%[" A "], %[off]), %%xmm0") /* xmm0 = l */ \
450 __ASM_EMIT("movss 0x00(%[" B "], %[off]), %%xmm4") /* xmm4 = r */ \
451 __ASM_EMIT("movaps %%xmm0, %%xmm2") /* xmm2 = l */ \
452 __ASM_EMIT("addss %%xmm4, %%xmm0") /* xmm0 = m = l + r */ \
453 __ASM_EMIT("subss %%xmm4, %%xmm2") /* xmm2 = s = l - r */ \
454 __ASM_EMIT("andps %%xmm6, %%xmm0") /* xmm0 = fabsf(m) */ \
455 __ASM_EMIT("andps %%xmm6, %%xmm2") /* xmm2 = fabsf(s) */ \
456 __ASM_EMIT(OP " %%xmm2, %%xmm0") /* xmm0 = OP(m, s) */ \
457 IF_MUL("mulss %%xmm7, %%xmm0") /* xmm0 = OP(m, s) * 0.5f */ \
458 __ASM_EMIT("movss %%xmm0, 0x00(%[" DST "], %[off])") \
459 __ASM_EMIT("add $0x04, %[off]") \
460 __ASM_EMIT("dec %[count]") \
461 __ASM_EMIT("jge 5b") \
462 __ASM_EMIT("6:")
463
464 void ms_pamin2(float *dst, const float *src, size_t count)
465 {
466 IF_ARCH_X86(size_t off);
467 ARCH_X86_ASM
468 (
469 MS_ABS_MINMAX_CORE("dst", "dst", "src", "minps", MS_MUL)
470 : [off] "=&r" (off), [count] "+r" (count)
471 : [dst] "r" (dst), [src] "r" (src),
472 [CC] "o" (ms_minmax_abs)
473 : "cc", "memory",
474 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
475 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
476 );
477 }
478
479 void ms_pamin3(float *dst, const float *a, const float *b, size_t count)
480 {
481 IF_ARCH_X86(size_t off);
482 ARCH_X86_ASM
483 (
484 MS_ABS_MINMAX_CORE("dst", "a", "b", "minps", MS_MUL)
485 : [off] "=&r" (off), [count] "+r" (count)
486 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b),
487 [CC] "o" (ms_minmax_abs)
488 : "cc", "memory",
489 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
490 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
491 );
492 }
493
494 void ms_pamax2(float *dst, const float *src, size_t count)
495 {
496 IF_ARCH_X86(size_t off);
497 ARCH_X86_ASM
498 (
499 MS_ABS_MINMAX_CORE("dst", "dst", "src", "maxps", MS_MUL)
500 : [off] "=&r" (off), [count] "+r" (count)
501 : [dst] "r" (dst), [src] "r" (src),
502 [CC] "o" (ms_minmax_abs)
503 : "cc", "memory",
504 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
505 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
506 );
507 }
508
509 void ms_pamax3(float *dst, const float *a, const float *b, size_t count)
510 {
511 IF_ARCH_X86(size_t off);
512 ARCH_X86_ASM
513 (
514 MS_ABS_MINMAX_CORE("dst", "a", "b", "maxps", MS_MUL)
515 : [off] "=&r" (off), [count] "+r" (count)
516 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b),
517 [CC] "o" (ms_minmax_abs)
518 : "cc", "memory",
519 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
520 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
521 );
522 }
523
524
525 void lr_pamin2(float *dst, const float *src, size_t count)
526 {
527 IF_ARCH_X86(size_t off);
528 ARCH_X86_ASM
529 (
530 MS_ABS_MINMAX_CORE("dst", "dst", "src", "minps", MS_NOMUL)
531 : [off] "=&r" (off), [count] "+r" (count)
532 : [dst] "r" (dst), [src] "r" (src),
533 [CC] "o" (ms_minmax_abs)
534 : "cc", "memory",
535 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
536 "%xmm4", "%xmm5", "%xmm6"
537 );
538 }
539
540 void lr_pamin3(float *dst, const float *a, const float *b, size_t count)
541 {
542 IF_ARCH_X86(size_t off);
543 ARCH_X86_ASM
544 (
545 MS_ABS_MINMAX_CORE("dst", "a", "b", "minps", MS_NOMUL)
546 : [off] "=&r" (off), [count] "+r" (count)
547 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b),
548 [CC] "o" (ms_minmax_abs)
549 : "cc", "memory",
550 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
551 "%xmm4", "%xmm5", "%xmm6"
552 );
553 }
554
555 void lr_pamax2(float *dst, const float *src, size_t count)
556 {
557 IF_ARCH_X86(size_t off);
558 ARCH_X86_ASM
559 (
560 MS_ABS_MINMAX_CORE("dst", "dst", "src", "maxps", MS_NOMUL)
561 : [off] "=&r" (off), [count] "+r" (count)
562 : [dst] "r" (dst), [src] "r" (src),
563 [CC] "o" (ms_minmax_abs)
564 : "cc", "memory",
565 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
566 "%xmm4", "%xmm5", "%xmm6"
567 );
568 }
569
570 void lr_pamax3(float *dst, const float *a, const float *b, size_t count)
571 {
572 IF_ARCH_X86(size_t off);
573 ARCH_X86_ASM
574 (
575 MS_ABS_MINMAX_CORE("dst", "a", "b", "maxps", MS_NOMUL)
576 : [off] "=&r" (off), [count] "+r" (count)
577 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b),
578 [CC] "o" (ms_minmax_abs)
579 : "cc", "memory",
580 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
581 "%xmm4", "%xmm5", "%xmm6"
582 );
583 }
584
585 #undef MS_ABS_MINMAX_CORE
586
587 #undef MS_MUL
588 #undef MS_NOMUL
589
590 } /* namespace sse */
591 } /* namespace lsp */
592
593
594 #endif /* PRIVATE_DSP_ARCH_X86_SSE_PMATH_MS_MINMAX_H_ */