Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/arm/neon-d32/pmath/ms_minmax.h

raw
1 /*
2 * Copyright (C) 2026 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2026 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 29 мар. 2026 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_ARM_NEON_D32_PMATH_MS_MINMAX_H_
23 #define PRIVATE_DSP_ARCH_ARM_NEON_D32_PMATH_MS_MINMAX_H_
24
25 #ifndef PRIVATE_DSP_ARCH_ARM_NEON_D32_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_ARM_NEON_D32_IMPL */
28
29 namespace lsp
30 {
31 namespace neon_d32
32 {
33 IF_ARCH_ARM(
34 static const float ms_minmax_abs[] __lsp_aligned16 =
35 {
36 LSP_DSP_VEC4(0.5f) // 0.5f
37 };
38 )
39
40 #define MS_MUL(x) __ASM_EMIT(x)
41 #define MS_NOMUL(x)
42
43 #define MS_MINMAX_CORE(DST, A, B, OP, INCA, IF_MUL) \
44 __ASM_EMIT("subs %[count], #16") \
45 IF_MUL("vldm %[CC], {q12}") /* q12 = 0.5f */ \
46 __ASM_EMIT("blo 2f") \
47 /* 16x blocks */ \
48 __ASM_EMIT("1:") \
49 __ASM_EMIT("vldm %[" A "]" INCA ", {q0-q3}") /* q0 = l */ \
50 __ASM_EMIT("vldm %[" B "]!, {q8-q11}") /* q8 = r */ \
51 __ASM_EMIT("vsub.f32 q4, q0, q8") /* q4 = s = l - r */ \
52 __ASM_EMIT("vsub.f32 q5, q1, q9") \
53 __ASM_EMIT("vsub.f32 q6, q2, q10") \
54 __ASM_EMIT("vsub.f32 q7, q3, q11") \
55 __ASM_EMIT("vadd.f32 q0, q0, q8") /* q0 = m = l + r */ \
56 __ASM_EMIT("vadd.f32 q1, q1, q9") \
57 __ASM_EMIT("vadd.f32 q2, q2, q10") \
58 __ASM_EMIT("vadd.f32 q3, q3, q11") \
59 __ASM_EMIT(OP ".f32 q0, q0, q4") /* q0 = O = OP(m, s) */ \
60 __ASM_EMIT(OP ".f32 q1, q1, q5") \
61 __ASM_EMIT(OP ".f32 q2, q2, q6") \
62 __ASM_EMIT(OP ".f32 q3, q3, q7") \
63 IF_MUL("vmul.f32 q0, q0, q12") /* q0 = O * 0.5f */ \
64 IF_MUL("vmul.f32 q1, q1, q12") \
65 IF_MUL("vmul.f32 q2, q2, q12") \
66 IF_MUL("vmul.f32 q3, q3, q12") \
67 __ASM_EMIT("subs %[count], #16") \
68 __ASM_EMIT("vstm %[" DST "]!, {q0-q3}") \
69 __ASM_EMIT("bhs 1b") \
70 /* 8x block */ \
71 __ASM_EMIT("2:") \
72 __ASM_EMIT("adds %[count], #8") \
73 __ASM_EMIT("blt 4f") \
74 __ASM_EMIT("vldm %[" A "]" INCA ", {q0-q1}") /* q0 = l */ \
75 __ASM_EMIT("vldm %[" B "]!, {q8-q9}") /* q8 = r */ \
76 __ASM_EMIT("vsub.f32 q4, q0, q8") /* q4 = s = l - r */ \
77 __ASM_EMIT("vsub.f32 q5, q1, q9") \
78 __ASM_EMIT("vadd.f32 q0, q0, q8") /* q0 = m = l + r */ \
79 __ASM_EMIT("vadd.f32 q1, q1, q9") \
80 __ASM_EMIT(OP ".f32 q0, q0, q4") /* q0 = O = OP(m, s) */ \
81 __ASM_EMIT(OP ".f32 q1, q1, q5") \
82 IF_MUL("vmul.f32 q0, q0, q12") /* q0 = O * 0.5f */ \
83 IF_MUL("vmul.f32 q1, q1, q12") \
84 __ASM_EMIT("sub %[count], #8") \
85 __ASM_EMIT("vstm %[" DST "]!, {q0-q1}") \
86 /* 4x block */ \
87 __ASM_EMIT("4:") \
88 __ASM_EMIT("adds %[count], #4") \
89 __ASM_EMIT("blt 6f") \
90 __ASM_EMIT("vldm %[" A "]" INCA ", {q0}") /* q0 = l */ \
91 __ASM_EMIT("vldm %[" B "]!, {q8}") /* q8 = r */ \
92 __ASM_EMIT("vsub.f32 q4, q0, q8") /* q4 = s = l - r */ \
93 __ASM_EMIT("vadd.f32 q0, q0, q8") /* q0 = m = l + r */ \
94 __ASM_EMIT(OP ".f32 q0, q0, q4") /* q0 = O = OP(m, s) */ \
95 IF_MUL("vmul.f32 q0, q0, q12") /* q0 = O * 0.5f */ \
96 __ASM_EMIT("sub %[count], #4") \
97 __ASM_EMIT("vstm %[" DST "]!, {q0}") \
98 /* 1x block */ \
99 __ASM_EMIT("6:") \
100 __ASM_EMIT("adds %[count], #3") \
101 __ASM_EMIT("blt 8f") \
102 __ASM_EMIT("7:") \
103 __ASM_EMIT("vld1.32 {d0[], d1[]}, [%[" A "]]" INCA) \
104 __ASM_EMIT("vld1.32 {d16[], d17[]}, [%[" B "]]!") \
105 __ASM_EMIT("vsub.f32 q4, q0, q8") /* q4 = s = l - r */ \
106 __ASM_EMIT("vadd.f32 q0, q0, q8") /* q0 = m = l + r */ \
107 __ASM_EMIT(OP ".f32 q0, q0, q4") /* q0 = O = OP(m, s) */ \
108 IF_MUL("vmul.f32 q0, q0, q12") /* q0 = O * 0.5f */ \
109 __ASM_EMIT("subs %[count], #1") \
110 __ASM_EMIT("vst1.32 {d0[0]}, [%[" DST "]]!") \
111 __ASM_EMIT("bge 7b") \
112 __ASM_EMIT("8:")
113
114 void ms_pmin2(float *dst, const float *src, size_t count)
115 {
116 ARCH_ARM_ASM
117 (
118 MS_MINMAX_CORE("dst", "dst", "src", "vmin", "", MS_MUL)
119 : [dst] "+r" (dst), [src] "+r" (src),
120 [count] "+r" (count)
121 : [CC] "r" (&ms_minmax_abs[0])
122 : "cc", "memory",
123 "q0", "q1", "q2", "q3",
124 "q4", "q5", "q6", "q7",
125 "q8", "q9", "q10", "q11",
126 "q12"
127 );
128 }
129
130 void ms_pmin3(float *dst, const float *a, const float *b, size_t count)
131 {
132 ARCH_ARM_ASM
133 (
134 MS_MINMAX_CORE("dst", "a", "b", "vmin", "!", MS_MUL)
135 : [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b),
136 [count] "+r" (count)
137 : [CC] "r" (&ms_minmax_abs[0])
138 : "cc", "memory",
139 "q0", "q1", "q2", "q3",
140 "q4", "q5", "q6", "q7",
141 "q8", "q9", "q10", "q11",
142 "q12"
143 );
144 }
145
146 void ms_pmax2(float *dst, const float *src, size_t count)
147 {
148 ARCH_ARM_ASM
149 (
150 MS_MINMAX_CORE("dst", "dst", "src", "vmax", "", MS_MUL)
151 : [dst] "+r" (dst), [src] "+r" (src),
152 [count] "+r" (count)
153 : [CC] "r" (&ms_minmax_abs[0])
154 : "cc", "memory",
155 "q0", "q1", "q2", "q3",
156 "q4", "q5", "q6", "q7",
157 "q8", "q9", "q10", "q11",
158 "q12"
159 );
160 }
161
162 void ms_pmax3(float *dst, const float *a, const float *b, size_t count)
163 {
164 ARCH_ARM_ASM
165 (
166 MS_MINMAX_CORE("dst", "a", "b", "vmax", "!", MS_MUL)
167 : [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b),
168 [count] "+r" (count)
169 : [CC] "r" (&ms_minmax_abs[0])
170 : "cc", "memory",
171 "q0", "q1", "q2", "q3",
172 "q4", "q5", "q6", "q7",
173 "q8", "q9", "q10", "q11",
174 "q12"
175 );
176 }
177
178
179 void lr_pmin2(float *dst, const float *src, size_t count)
180 {
181 ARCH_ARM_ASM
182 (
183 MS_MINMAX_CORE("dst", "dst", "src", "vmin", "", MS_NOMUL)
184 : [dst] "+r" (dst), [src] "+r" (src),
185 [count] "+r" (count)
186 :
187 : "cc", "memory",
188 "q0", "q1", "q2", "q3",
189 "q4", "q5", "q6", "q7",
190 "q8", "q9", "q10", "q11"
191 );
192 }
193
194 void lr_pmin3(float *dst, const float *a, const float *b, size_t count)
195 {
196 ARCH_ARM_ASM
197 (
198 MS_MINMAX_CORE("dst", "a", "b", "vmin", "!", MS_NOMUL)
199 : [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b),
200 [count] "+r" (count)
201 :
202 : "cc", "memory",
203 "q0", "q1", "q2", "q3",
204 "q4", "q5", "q6", "q7",
205 "q8", "q9", "q10", "q11"
206 );
207 }
208
209 void lr_pmax2(float *dst, const float *src, size_t count)
210 {
211 ARCH_ARM_ASM
212 (
213 MS_MINMAX_CORE("dst", "dst", "src", "vmax", "", MS_NOMUL)
214 : [dst] "+r" (dst), [src] "+r" (src),
215 [count] "+r" (count)
216 :
217 : "cc", "memory",
218 "q0", "q1", "q2", "q3",
219 "q4", "q5", "q6", "q7",
220 "q8", "q9", "q10", "q11"
221 );
222 }
223
224 void lr_pmax3(float *dst, const float *a, const float *b, size_t count)
225 {
226 ARCH_ARM_ASM
227 (
228 MS_MINMAX_CORE("dst", "a", "b", "vmax", "!", MS_NOMUL)
229 : [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b),
230 [count] "+r" (count)
231 :
232 : "cc", "memory",
233 "q0", "q1", "q2", "q3",
234 "q4", "q5", "q6", "q7",
235 "q8", "q9", "q10", "q11"
236 );
237 }
238
239 #undef MS_MINMAX_CORE
240
241 #define MS_SIGN_MINMAX_CORE(DST, A, B, OP, INCA, IF_MUL) \
242 __ASM_EMIT("subs %[count], #16") \
243 IF_MUL("vldm %[CC], {q12}") /* q12 = 0.5f */ \
244 __ASM_EMIT("blo 2f") \
245 /* 16x blocks */ \
246 __ASM_EMIT("1:") \
247 __ASM_EMIT("vldm %[" A "]" INCA ", {q0-q3}") /* q0 = m */ \
248 __ASM_EMIT("vldm %[" B "]!, {q8-q11}") /* q8 = s */ \
249 __ASM_EMIT("vsub.f32 q4, q0, q8") /* q4 = s = l - r */ \
250 __ASM_EMIT("vsub.f32 q5, q1, q9") \
251 __ASM_EMIT("vsub.f32 q6, q2, q10") \
252 __ASM_EMIT("vsub.f32 q7, q3, q11") \
253 __ASM_EMIT("vadd.f32 q0, q0, q8") /* q0 = m = l + r */ \
254 __ASM_EMIT("vadd.f32 q1, q1, q9") \
255 __ASM_EMIT("vadd.f32 q2, q2, q10") \
256 __ASM_EMIT("vadd.f32 q3, q3, q11") \
257 __ASM_EMIT("vacgt.f32 q8, q4, q0") /* q8 = fabsf(s) > fabsf(m) */ \
258 __ASM_EMIT("vacgt.f32 q9, q5, q1") \
259 __ASM_EMIT("vacgt.f32 q10, q6, q2") \
260 __ASM_EMIT("vacgt.f32 q11, q7, q3") \
261 __ASM_EMIT(OP " q0, q4, q8") /* q0 = O */ \
262 __ASM_EMIT(OP " q1, q5, q9") \
263 __ASM_EMIT(OP " q2, q6, q10") \
264 __ASM_EMIT(OP " q3, q7, q11") \
265 IF_MUL("vmul.f32 q0, q0, q12") /* q0 = O * 0.5f */ \
266 IF_MUL("vmul.f32 q1, q1, q12") \
267 IF_MUL("vmul.f32 q2, q2, q12") \
268 IF_MUL("vmul.f32 q3, q3, q12") \
269 __ASM_EMIT("subs %[count], #16") \
270 __ASM_EMIT("vstm %[" DST "]!, {q0-q3}") \
271 __ASM_EMIT("bhs 1b") \
272 /* 8x block */ \
273 __ASM_EMIT("2:") \
274 __ASM_EMIT("adds %[count], #8") \
275 __ASM_EMIT("blt 4f") \
276 __ASM_EMIT("vldm %[" A "]" INCA ", {q0-q1}") \
277 __ASM_EMIT("vldm %[" B "]!, {q8-q9}") \
278 __ASM_EMIT("vsub.f32 q4, q0, q8") /* q4 = s = l - r */ \
279 __ASM_EMIT("vsub.f32 q5, q1, q9") \
280 __ASM_EMIT("vadd.f32 q0, q0, q8") /* q0 = m = l + r */ \
281 __ASM_EMIT("vadd.f32 q1, q1, q9") \
282 __ASM_EMIT("vacgt.f32 q8, q4, q0") /* q8 = fabsf(s) > fabsf(m) */ \
283 __ASM_EMIT("vacgt.f32 q9, q5, q1") \
284 __ASM_EMIT(OP " q0, q4, q8") /* q0 = O */ \
285 __ASM_EMIT(OP " q1, q5, q9") \
286 IF_MUL("vmul.f32 q0, q0, q12") /* q0 = O * 0.5f */ \
287 IF_MUL("vmul.f32 q1, q1, q12") \
288 __ASM_EMIT("sub %[count], #8") \
289 __ASM_EMIT("vstm %[" DST "]!, {q0-q1}") \
290 /* 4x block */ \
291 __ASM_EMIT("4:") \
292 __ASM_EMIT("adds %[count], #4") \
293 __ASM_EMIT("blt 6f") \
294 __ASM_EMIT("vldm %[" A "]" INCA ", {q0}") \
295 __ASM_EMIT("vldm %[" B "]!, {q8}") \
296 __ASM_EMIT("vsub.f32 q4, q0, q8") /* q4 = s = l - r */ \
297 __ASM_EMIT("vadd.f32 q0, q0, q8") /* q0 = m = l + r */ \
298 __ASM_EMIT("vacgt.f32 q8, q4, q0") /* q8 = fabsf(s) > fabsf(m) */ \
299 __ASM_EMIT(OP " q0, q4, q8") /* q0 = O */ \
300 IF_MUL("vmul.f32 q0, q0, q12") /* q0 = O * 0.5f */ \
301 __ASM_EMIT("sub %[count], #4") \
302 __ASM_EMIT("vstm %[" DST "]!, {q0}") \
303 /* 1x block */ \
304 __ASM_EMIT("6:") \
305 __ASM_EMIT("adds %[count], #3") \
306 __ASM_EMIT("blt 8f") \
307 __ASM_EMIT("7:") \
308 __ASM_EMIT("vld1.32 {d0[], d1[]}, [%[" A "]]" INCA) \
309 __ASM_EMIT("vld1.32 {d16[], d17[]}, [%[" B "]]!") \
310 __ASM_EMIT("vsub.f32 q4, q0, q8") /* q4 = s = l - r */ \
311 __ASM_EMIT("vadd.f32 q0, q0, q8") /* q0 = m = l + r */ \
312 __ASM_EMIT("vacgt.f32 q8, q4, q0") /* q8 = fabsf(s) > fabsf(m) */ \
313 __ASM_EMIT(OP " q0, q4, q8") /* q0 = O */ \
314 IF_MUL("vmul.f32 q0, q0, q12") /* q0 = O * 0.5f */ \
315 __ASM_EMIT("subs %[count], #1") \
316 __ASM_EMIT("vst1.32 {d0[0]}, [%[" DST "]]!") \
317 __ASM_EMIT("bge 7b") \
318 __ASM_EMIT("8:")
319
320 void ms_psmin2(float *dst, const float *src, size_t count)
321 {
322 ARCH_ARM_ASM
323 (
324 MS_SIGN_MINMAX_CORE("dst", "dst", "src", "vbif", "", MS_MUL)
325 : [dst] "+r" (dst), [src] "+r" (src),
326 [count] "+r" (count)
327 : [CC] "r" (&ms_minmax_abs[0])
328 : "cc", "memory",
329 "q0", "q1", "q2", "q3",
330 "q4", "q5", "q6", "q7",
331 "q8", "q9", "q10", "q11",
332 "q12"
333 );
334 }
335
336 void ms_psmin3(float *dst, const float *a, const float *b, size_t count)
337 {
338 ARCH_ARM_ASM
339 (
340 MS_SIGN_MINMAX_CORE("dst", "a", "b", "vbif", "!", MS_MUL)
341 : [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b),
342 [count] "+r" (count)
343 : [CC] "r" (&ms_minmax_abs[0])
344 : "cc", "memory",
345 "q0", "q1", "q2", "q3",
346 "q4", "q5", "q6", "q7",
347 "q8", "q9", "q10", "q11",
348 "q12"
349 );
350 }
351
352 void ms_psmax2(float *dst, const float *src, size_t count)
353 {
354 ARCH_ARM_ASM
355 (
356 MS_SIGN_MINMAX_CORE("dst", "dst", "src", "vbit", "", MS_MUL)
357 : [dst] "+r" (dst), [src] "+r" (src),
358 [count] "+r" (count)
359 : [CC] "r" (&ms_minmax_abs[0])
360 : "cc", "memory",
361 "q0", "q1", "q2", "q3",
362 "q4", "q5", "q6", "q7",
363 "q8", "q9", "q10", "q11",
364 "q12"
365 );
366 }
367
368 void ms_psmax3(float *dst, const float *a, const float *b, size_t count)
369 {
370 ARCH_ARM_ASM
371 (
372 MS_SIGN_MINMAX_CORE("dst", "a", "b", "vbit", "!", MS_MUL)
373 : [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b),
374 [count] "+r" (count)
375 : [CC] "r" (&ms_minmax_abs[0])
376 : "cc", "memory",
377 "q0", "q1", "q2", "q3",
378 "q4", "q5", "q6", "q7",
379 "q8", "q9", "q10", "q11",
380 "q12"
381 );
382 }
383
384 void lr_psmin2(float *dst, const float *src, size_t count)
385 {
386 ARCH_ARM_ASM
387 (
388 MS_SIGN_MINMAX_CORE("dst", "dst", "src", "vbif", "", MS_NOMUL)
389 : [dst] "+r" (dst), [src] "+r" (src),
390 [count] "+r" (count)
391 :
392 : "cc", "memory",
393 "q0", "q1", "q2", "q3",
394 "q4", "q5", "q6", "q7",
395 "q8", "q9", "q10", "q11"
396 );
397 }
398
399 void lr_psmin3(float *dst, const float *a, const float *b, size_t count)
400 {
401 ARCH_ARM_ASM
402 (
403 MS_SIGN_MINMAX_CORE("dst", "a", "b", "vbif", "!", MS_NOMUL)
404 : [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b),
405 [count] "+r" (count)
406 :
407 : "cc", "memory",
408 "q0", "q1", "q2", "q3",
409 "q4", "q5", "q6", "q7",
410 "q8", "q9", "q10", "q11"
411 );
412 }
413
414 void lr_psmax2(float *dst, const float *src, size_t count)
415 {
416 ARCH_ARM_ASM
417 (
418 MS_SIGN_MINMAX_CORE("dst", "dst", "src", "vbit", "", MS_NOMUL)
419 : [dst] "+r" (dst), [src] "+r" (src),
420 [count] "+r" (count)
421 :
422 : "cc", "memory",
423 "q0", "q1", "q2", "q3",
424 "q4", "q5", "q6", "q7",
425 "q8", "q9", "q10", "q11"
426 );
427 }
428
429 void lr_psmax3(float *dst, const float *a, const float *b, size_t count)
430 {
431 ARCH_ARM_ASM
432 (
433 MS_SIGN_MINMAX_CORE("dst", "a", "b", "vbit", "!", MS_NOMUL)
434 : [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b),
435 [count] "+r" (count)
436 :
437 : "cc", "memory",
438 "q0", "q1", "q2", "q3",
439 "q4", "q5", "q6", "q7",
440 "q8", "q9", "q10", "q11"
441 );
442 }
443
444 #undef MS_SIGN_MINMAX_CORE
445
446 #define MS_ABS_MINMAX_CORE(DST, A, B, OP, INCA, IF_MUL) \
447 __ASM_EMIT("subs %[count], #16") \
448 IF_MUL("vldm %[CC], {q12}") /* q12 = 0.5f */ \
449 __ASM_EMIT("blo 2f") \
450 /* 16x blocks */ \
451 __ASM_EMIT("1:") \
452 __ASM_EMIT("vldm %[" A "]" INCA ", {q0-q3}") \
453 __ASM_EMIT("vldm %[" B "]!, {q8-q11}") \
454 __ASM_EMIT("vsub.f32 q4, q0, q8") /* q4 = s = l - r */ \
455 __ASM_EMIT("vsub.f32 q5, q1, q9") \
456 __ASM_EMIT("vsub.f32 q6, q2, q10") \
457 __ASM_EMIT("vsub.f32 q7, q3, q11") \
458 __ASM_EMIT("vadd.f32 q0, q0, q8") /* q0 = m = l + r */ \
459 __ASM_EMIT("vadd.f32 q1, q1, q9") \
460 __ASM_EMIT("vadd.f32 q2, q2, q10") \
461 __ASM_EMIT("vadd.f32 q3, q3, q11") \
462 __ASM_EMIT("vabs.f32 q4, q4") /* q4 = fabsf(s) */ \
463 __ASM_EMIT("vabs.f32 q5, q5") \
464 __ASM_EMIT("vabs.f32 q6, q6") \
465 __ASM_EMIT("vabs.f32 q7, q7") \
466 __ASM_EMIT("vabs.f32 q0, q0") /* q0 = fabsf(m) */ \
467 __ASM_EMIT("vabs.f32 q1, q1") \
468 __ASM_EMIT("vabs.f32 q2, q2") \
469 __ASM_EMIT("vabs.f32 q3, q3") \
470 __ASM_EMIT(OP ".f32 q0, q0, q4") /* q0 = O = OP(fabsf(m), fabsf(s)) */ \
471 __ASM_EMIT(OP ".f32 q1, q1, q5") \
472 __ASM_EMIT(OP ".f32 q2, q2, q6") \
473 __ASM_EMIT(OP ".f32 q3, q3, q7") \
474 IF_MUL("vmul.f32 q0, q0, q12") /* q0 = O * 0.5f */ \
475 IF_MUL("vmul.f32 q1, q1, q12") \
476 IF_MUL("vmul.f32 q2, q2, q12") \
477 IF_MUL("vmul.f32 q3, q3, q12") \
478 __ASM_EMIT("subs %[count], #16") \
479 __ASM_EMIT("vstm %[" DST "]!, {q0-q3}") \
480 __ASM_EMIT("bhs 1b") \
481 /* 8x block */ \
482 __ASM_EMIT("2:") \
483 __ASM_EMIT("adds %[count], #8") \
484 __ASM_EMIT("blt 4f") \
485 __ASM_EMIT("vldm %[" A "]" INCA ", {q0-q1}") \
486 __ASM_EMIT("vldm %[" B "]!, {q8-q9}") \
487 __ASM_EMIT("vsub.f32 q4, q0, q8") /* q4 = s = l - r */ \
488 __ASM_EMIT("vsub.f32 q5, q1, q9") \
489 __ASM_EMIT("vadd.f32 q0, q0, q8") /* q0 = m = l + r */ \
490 __ASM_EMIT("vadd.f32 q1, q1, q9") \
491 __ASM_EMIT("vabs.f32 q4, q4") /* q4 = fabsf(s) */ \
492 __ASM_EMIT("vabs.f32 q5, q5") \
493 __ASM_EMIT("vabs.f32 q0, q0") /* q0 = fabsf(m) */ \
494 __ASM_EMIT("vabs.f32 q1, q1") \
495 __ASM_EMIT(OP ".f32 q0, q0, q4") /* q0 = O = OP(fabsf(m), fabsf(s)) */ \
496 __ASM_EMIT(OP ".f32 q1, q1, q5") \
497 IF_MUL("vmul.f32 q0, q0, q12") /* q0 = O * 0.5f */ \
498 IF_MUL("vmul.f32 q1, q1, q12") \
499 __ASM_EMIT("sub %[count], #8") \
500 __ASM_EMIT("vstm %[" DST "]!, {q0-q1}") \
501 /* 4x block */ \
502 __ASM_EMIT("4:") \
503 __ASM_EMIT("adds %[count], #4") \
504 __ASM_EMIT("blt 6f") \
505 __ASM_EMIT("vldm %[" B "]!, {q8}") \
506 __ASM_EMIT("vldm %[" A "]" INCA ", {q0}") \
507 __ASM_EMIT("vsub.f32 q4, q0, q8") /* q4 = s = l - r */ \
508 __ASM_EMIT("vadd.f32 q0, q0, q8") /* q0 = m = l + r */ \
509 __ASM_EMIT("vabs.f32 q4, q4") /* q4 = fabsf(s) */ \
510 __ASM_EMIT("vabs.f32 q0, q0") /* q0 = fabsf(m) */ \
511 __ASM_EMIT(OP ".f32 q0, q0, q4") /* q0 = O = OP(fabsf(m), fabsf(s)) */ \
512 IF_MUL("vmul.f32 q0, q0, q12") /* q0 = O * 0.5f */ \
513 __ASM_EMIT("sub %[count], #4") \
514 __ASM_EMIT("vstm %[" DST "]!, {q0}") \
515 /* 1x block */ \
516 __ASM_EMIT("6:") \
517 __ASM_EMIT("adds %[count], #3") \
518 __ASM_EMIT("blt 8f") \
519 __ASM_EMIT("7:") \
520 __ASM_EMIT("vld1.32 {d0[], d1[]}, [%[" A "]]" INCA) \
521 __ASM_EMIT("vld1.32 {d16[], d17[]}, [%[" B "]]!") \
522 __ASM_EMIT("vsub.f32 q4, q0, q8") /* q4 = s = l - r */ \
523 __ASM_EMIT("vadd.f32 q0, q0, q8") /* q0 = m = l + r */ \
524 __ASM_EMIT("vabs.f32 q4, q4") /* q4 = fabsf(s) */ \
525 __ASM_EMIT("vabs.f32 q0, q0") /* q0 = fabsf(m) */ \
526 __ASM_EMIT(OP ".f32 q0, q0, q4") /* q0 = O = OP(fabsf(m), fabsf(s)) */ \
527 IF_MUL("vmul.f32 q0, q0, q12") /* q0 = O * 0.5f */ \
528 __ASM_EMIT("subs %[count], #1") \
529 __ASM_EMIT("vst1.32 {d0[0]}, [%[" DST "]]!") \
530 __ASM_EMIT("bge 7b") \
531 __ASM_EMIT("8:")
532
533 void ms_pamin2(float *dst, const float *src, size_t count)
534 {
535 ARCH_ARM_ASM
536 (
537 MS_ABS_MINMAX_CORE("dst", "dst", "src", "vmin", "", MS_MUL)
538 : [dst] "+r" (dst), [src] "+r" (src),
539 [count] "+r" (count)
540 : [CC] "r" (&ms_minmax_abs[0])
541 : "cc", "memory",
542 "q0", "q1", "q2", "q3",
543 "q4", "q5", "q6", "q7",
544 "q8", "q9", "q10", "q11",
545 "q12"
546 );
547 }
548
549 void ms_pamin3(float *dst, const float *a, const float *b, size_t count)
550 {
551 ARCH_ARM_ASM
552 (
553 MS_ABS_MINMAX_CORE("dst", "a", "b", "vmin", "!", MS_MUL)
554 : [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b),
555 [count] "+r" (count)
556 : [CC] "r" (&ms_minmax_abs[0])
557 : "cc", "memory",
558 "q0", "q1", "q2", "q3",
559 "q4", "q5", "q6", "q7",
560 "q8", "q9", "q10", "q11",
561 "q12"
562 );
563 }
564
565 void ms_pamax2(float *dst, const float *src, size_t count)
566 {
567 ARCH_ARM_ASM
568 (
569 MS_ABS_MINMAX_CORE("dst", "dst", "src", "vmax", "", MS_MUL)
570 : [dst] "+r" (dst), [src] "+r" (src),
571 [count] "+r" (count)
572 : [CC] "r" (&ms_minmax_abs[0])
573 : "cc", "memory",
574 "q0", "q1", "q2", "q3",
575 "q4", "q5", "q6", "q7",
576 "q8", "q9", "q10", "q11",
577 "q12"
578 );
579 }
580
581 void ms_pamax3(float *dst, const float *a, const float *b, size_t count)
582 {
583 ARCH_ARM_ASM
584 (
585 MS_ABS_MINMAX_CORE("dst", "a", "b", "vmax", "!", MS_MUL)
586 : [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b),
587 [count] "+r" (count)
588 : [CC] "r" (&ms_minmax_abs[0])
589 : "cc", "memory",
590 "q0", "q1", "q2", "q3",
591 "q4", "q5", "q6", "q7",
592 "q8", "q9", "q10", "q11",
593 "q12"
594 );
595 }
596
597 void lr_pamin2(float *dst, const float *src, size_t count)
598 {
599 ARCH_ARM_ASM
600 (
601 MS_ABS_MINMAX_CORE("dst", "dst", "src", "vmin", "", MS_NOMUL)
602 : [dst] "+r" (dst), [src] "+r" (src),
603 [count] "+r" (count)
604 :
605 : "cc", "memory",
606 "q0", "q1", "q2", "q3",
607 "q4", "q5", "q6", "q7",
608 "q8", "q9", "q10", "q11"
609 );
610 }
611
612 void lr_pamin3(float *dst, const float *a, const float *b, size_t count)
613 {
614 ARCH_ARM_ASM
615 (
616 MS_ABS_MINMAX_CORE("dst", "a", "b", "vmin", "!", MS_NOMUL)
617 : [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b),
618 [count] "+r" (count)
619 :
620 : "cc", "memory",
621 "q0", "q1", "q2", "q3",
622 "q4", "q5", "q6", "q7",
623 "q8", "q9", "q10", "q11"
624 );
625 }
626
627 void lr_pamax2(float *dst, const float *src, size_t count)
628 {
629 ARCH_ARM_ASM
630 (
631 MS_ABS_MINMAX_CORE("dst", "dst", "src", "vmax", "", MS_NOMUL)
632 : [dst] "+r" (dst), [src] "+r" (src),
633 [count] "+r" (count)
634 :
635 : "cc", "memory",
636 "q0", "q1", "q2", "q3",
637 "q4", "q5", "q6", "q7",
638 "q8", "q9", "q10", "q11"
639 );
640 }
641
642 void lr_pamax3(float *dst, const float *a, const float *b, size_t count)
643 {
644 ARCH_ARM_ASM
645 (
646 MS_ABS_MINMAX_CORE("dst", "a", "b", "vmax", "!", MS_NOMUL)
647 : [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b),
648 [count] "+r" (count)
649 :
650 : "cc", "memory",
651 "q0", "q1", "q2", "q3",
652 "q4", "q5", "q6", "q7",
653 "q8", "q9", "q10", "q11"
654 );
655 }
656
657 #undef MS_ABS_MINMAX_CORE
658
659 #undef MS_MUL
660 #undef MS_NOMUL
661 } /* namespace neon_d32 */
662 } /* namespace lsp */
663
664
665 #endif /* PRIVATE_DSP_ARCH_ARM_NEON_D32_PMATH_MS_MINMAX_H_ */