Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/aarch64/asimd/pmath/ms_minmax.h

raw
1 /*
2 * Copyright (C) 2026 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2026 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 29 мар. 2026 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_AARCH64_ASIMD_PMATH_MS_MINMAX_H_
23 #define PRIVATE_DSP_ARCH_AARCH64_ASIMD_PMATH_MS_MINMAX_H_
24
25 #ifndef PRIVATE_DSP_ARCH_AARCH64_ASIMD_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_AARCH64_ASIMD_IMPL */
28
29 namespace lsp
30 {
31 namespace asimd
32 {
33 IF_ARCH_AARCH64(
34 static const float ms_minmax_abs[] __lsp_aligned16 =
35 {
36 LSP_DSP_VEC4(0.5f) // 0.5f
37 };
38 )
39
40 #define MS_MUL(x) __ASM_EMIT(x)
41 #define MS_NOMUL(x)
42
43 #define INC_ON(cmd) cmd
44 #define INC_OFF(cmd)
45
46 #define MS_MINMAX_CORE(DST, A, B, OP, INCA, IF_MUL) \
47 __ASM_EMIT("subs %[count], %[count], #32") \
48 IF_MUL("ldr q31, [%[CC], #0x00]") /* v31 = 0.5f */ \
49 __ASM_EMIT("b.lo 2f") \
50 /* 32x blocks */ \
51 __ASM_EMIT("1:") \
52 __ASM_EMIT("ldp q0, q1, [%[" A "], #0x00]") /* v0 = l */ \
53 __ASM_EMIT("ldp q2, q3, [%[" A "], #0x20]") \
54 __ASM_EMIT("ldp q4, q5, [%[" A "], #0x40]") \
55 __ASM_EMIT("ldp q6, q7, [%[" A "], #0x60]") \
56 __ASM_EMIT("ldp q16, q17, [%[" B "], #0x00]") /* v12 = r */ \
57 __ASM_EMIT("ldp q18, q19, [%[" B "], #0x20]") \
58 __ASM_EMIT("ldp q20, q21, [%[" B "], #0x40]") \
59 __ASM_EMIT("ldp q22, q23, [%[" B "], #0x60]") \
60 __ASM_EMIT("fsub v8.4s, v0.4s, v16.4s") /* v8 = s = l - r */ \
61 __ASM_EMIT("fsub v9.4s, v1.4s, v17.4s") \
62 __ASM_EMIT("fsub v10.4s, v2.4s, v18.4s") \
63 __ASM_EMIT("fsub v11.4s, v3.4s, v19.4s") \
64 __ASM_EMIT("fsub v12.4s, v4.4s, v20.4s") \
65 __ASM_EMIT("fsub v13.4s, v5.4s, v21.4s") \
66 __ASM_EMIT("fsub v14.4s, v6.4s, v22.4s") \
67 __ASM_EMIT("fsub v15.4s, v7.4s, v23.4s") \
68 __ASM_EMIT("fadd v0.4s, v0.4s, v16.4s") /* v0 = m = l + r */ \
69 __ASM_EMIT("fadd v1.4s, v1.4s, v17.4s") \
70 __ASM_EMIT("fadd v2.4s, v2.4s, v18.4s") \
71 __ASM_EMIT("fadd v3.4s, v3.4s, v19.4s") \
72 __ASM_EMIT("fadd v4.4s, v4.4s, v20.4s") \
73 __ASM_EMIT("fadd v5.4s, v5.4s, v21.4s") \
74 __ASM_EMIT("fadd v6.4s, v6.4s, v22.4s") \
75 __ASM_EMIT("fadd v7.4s, v7.4s, v23.4s") \
76 __ASM_EMIT(OP " v0.4s, v0.4s, v8.4s") /* v0 = O = OP(m, s) */ \
77 __ASM_EMIT(OP " v1.4s, v1.4s, v9.4s") \
78 __ASM_EMIT(OP " v2.4s, v2.4s, v10.4s") \
79 __ASM_EMIT(OP " v3.4s, v3.4s, v11.4s") \
80 __ASM_EMIT(OP " v4.4s, v4.4s, v12.4s") \
81 __ASM_EMIT(OP " v5.4s, v5.4s, v13.4s") \
82 __ASM_EMIT(OP " v6.4s, v6.4s, v14.4s") \
83 __ASM_EMIT(OP " v7.4s, v7.4s, v15.4s") \
84 IF_MUL("fmul v0.4s, v0.4s, v31.4s") /* v0 = O * 0.5f */ \
85 IF_MUL("fmul v1.4s, v1.4s, v31.4s") \
86 IF_MUL("fmul v2.4s, v2.4s, v31.4s") \
87 IF_MUL("fmul v3.4s, v3.4s, v31.4s") \
88 IF_MUL("fmul v4.4s, v4.4s, v31.4s") \
89 IF_MUL("fmul v5.4s, v5.4s, v31.4s") \
90 IF_MUL("fmul v6.4s, v6.4s, v31.4s") \
91 IF_MUL("fmul v7.4s, v7.4s, v31.4s") \
92 __ASM_EMIT("subs %[count], %[count], #32") \
93 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
94 __ASM_EMIT("stp q2, q3, [%[" DST "], #0x20]") \
95 __ASM_EMIT("stp q4, q5, [%[" DST "], #0x40]") \
96 __ASM_EMIT("stp q6, q7, [%[" DST "], #0x60]") \
97 __ASM_EMIT(INCA("add %[" A "], %[" A "], #0x80")) \
98 __ASM_EMIT("add %[" B "], %[" B "], #0x80") \
99 __ASM_EMIT("add %[" DST "], %[" DST "], #0x80") \
100 __ASM_EMIT("b.hs 1b") \
101 /* 16x block */ \
102 __ASM_EMIT("2:") \
103 __ASM_EMIT("adds %[count], %[count], #16") \
104 __ASM_EMIT("b.lt 4f") \
105 __ASM_EMIT("ldp q0, q1, [%[" A "], #0x00]") /* v0 = l */ \
106 __ASM_EMIT("ldp q2, q3, [%[" A "], #0x20]") \
107 __ASM_EMIT("ldp q16, q17, [%[" B "], #0x00]") /* v12 = r */ \
108 __ASM_EMIT("ldp q18, q19, [%[" B "], #0x20]") \
109 __ASM_EMIT("fsub v8.4s, v0.4s, v16.4s") /* v8 = s = l - r */ \
110 __ASM_EMIT("fsub v9.4s, v1.4s, v17.4s") \
111 __ASM_EMIT("fsub v10.4s, v2.4s, v18.4s") \
112 __ASM_EMIT("fsub v11.4s, v3.4s, v19.4s") \
113 __ASM_EMIT("fadd v0.4s, v0.4s, v16.4s") /* v0 = m = l + r */ \
114 __ASM_EMIT("fadd v1.4s, v1.4s, v17.4s") \
115 __ASM_EMIT("fadd v2.4s, v2.4s, v18.4s") \
116 __ASM_EMIT("fadd v3.4s, v3.4s, v19.4s") \
117 __ASM_EMIT(OP " v0.4s, v0.4s, v8.4s") /* v0 = O = OP(m, s) */ \
118 __ASM_EMIT(OP " v1.4s, v1.4s, v9.4s") \
119 __ASM_EMIT(OP " v2.4s, v2.4s, v10.4s") \
120 __ASM_EMIT(OP " v3.4s, v3.4s, v11.4s") \
121 IF_MUL("fmul v0.4s, v0.4s, v31.4s") /* v0 = O * 0.5f */ \
122 IF_MUL("fmul v1.4s, v1.4s, v31.4s") \
123 IF_MUL("fmul v2.4s, v2.4s, v31.4s") \
124 IF_MUL("fmul v3.4s, v3.4s, v31.4s") \
125 __ASM_EMIT("sub %[count], %[count], #16") \
126 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
127 __ASM_EMIT("stp q2, q3, [%[" DST "], #0x20]") \
128 __ASM_EMIT(INCA("add %[" A "], %[" A "], #0x40")) \
129 __ASM_EMIT("add %[" B "], %[" B "], #0x40") \
130 __ASM_EMIT("add %[" DST "], %[" DST "], #0x40") \
131 /* 8x block */ \
132 __ASM_EMIT("4:") \
133 __ASM_EMIT("adds %[count], %[count], #8") \
134 __ASM_EMIT("b.lt 6f") \
135 __ASM_EMIT("ldp q0, q1, [%[" A "], #0x00]") /* v0 = l */ \
136 __ASM_EMIT("ldp q16, q17, [%[" B "], #0x00]") /* v12 = r */ \
137 __ASM_EMIT("fsub v8.4s, v0.4s, v16.4s") /* v8 = s = l - r */ \
138 __ASM_EMIT("fsub v9.4s, v1.4s, v17.4s") \
139 __ASM_EMIT("fadd v0.4s, v0.4s, v16.4s") /* v0 = m = l + r */ \
140 __ASM_EMIT("fadd v1.4s, v1.4s, v17.4s") \
141 __ASM_EMIT(OP " v0.4s, v0.4s, v8.4s") /* v0 = O = OP(m, s) */ \
142 __ASM_EMIT(OP " v1.4s, v1.4s, v9.4s") \
143 IF_MUL("fmul v0.4s, v0.4s, v31.4s") /* v0 = O * 0.5f */ \
144 IF_MUL("fmul v1.4s, v1.4s, v31.4s") \
145 __ASM_EMIT("sub %[count], %[count], #8") \
146 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
147 __ASM_EMIT(INCA("add %[" A "], %[" A "], #0x20")) \
148 __ASM_EMIT("add %[" B "], %[" B "], #0x20") \
149 __ASM_EMIT("add %[" DST "], %[" DST "], #0x20") \
150 /* 4x block */ \
151 __ASM_EMIT("6:") \
152 __ASM_EMIT("adds %[count], %[count], #4") \
153 __ASM_EMIT("b.lt 8f") \
154 __ASM_EMIT("ldr q0, [%[" A "], #0x00]") /* v0 = l */ \
155 __ASM_EMIT("ldr q16, [%[" B "], #0x00]") /* v12 = r */ \
156 __ASM_EMIT("fsub v8.4s, v0.4s, v16.4s") /* v8 = s = l - r */ \
157 __ASM_EMIT("fadd v0.4s, v0.4s, v16.4s") /* v0 = m = l + r */ \
158 __ASM_EMIT(OP " v0.4s, v0.4s, v8.4s") /* v0 = O = OP(m, s) */ \
159 IF_MUL("fmul v0.4s, v0.4s, v31.4s") /* v0 = O * 0.5f */ \
160 __ASM_EMIT("sub %[count], %[count], #4") \
161 __ASM_EMIT("str q0, [%[" DST "], #0x00]") \
162 __ASM_EMIT(INCA("add %[" A "], %[" A "], #0x10")) \
163 __ASM_EMIT("add %[" B "], %[" B "], #0x10") \
164 __ASM_EMIT("add %[" DST "], %[" DST "], #0x10") \
165 /* 1x block */ \
166 __ASM_EMIT("8:") \
167 __ASM_EMIT("adds %[count], %[count], #3") /* 4 - 3 */ \
168 __ASM_EMIT("b.lt 10f") \
169 __ASM_EMIT("9:") \
170 __ASM_EMIT("ld1r {v0.4s}, [%[" A "]]") \
171 __ASM_EMIT("ld1r {v16.4s}, [%[" B "]]") \
172 __ASM_EMIT("fsub v8.4s, v0.4s, v16.4s") /* v8 = s = l - r */ \
173 __ASM_EMIT("fadd v0.4s, v0.4s, v16.4s") /* v0 = m = l + r */ \
174 __ASM_EMIT(OP " v0.4s, v0.4s, v8.4s") /* v0 = O = OP(m, s) */ \
175 IF_MUL("fmul v0.4s, v0.4s, v31.4s") /* v0 = O * 0.5f */ \
176 __ASM_EMIT("subs %[count], %[count], #1") \
177 __ASM_EMIT("st1 {v0.s}[0], [%[" DST "]]") \
178 __ASM_EMIT(INCA("add %[" A "], %[" A "], #0x04")) \
179 __ASM_EMIT("add %[" B "], %[" B "], #0x04") \
180 __ASM_EMIT("add %[" DST "], %[" DST "], #0x04") \
181 __ASM_EMIT("b.ge 9b") \
182 __ASM_EMIT("10:")
183
184 void ms_pmin2(float *dst, const float *src, size_t count)
185 {
186 ARCH_AARCH64_ASM
187 (
188 MS_MINMAX_CORE("dst", "dst", "src", "fmin", INC_OFF, MS_MUL)
189 : [dst] "+r" (dst), [src] "+r" (src),
190 [count] "+r" (count)
191 : [CC] "r" (&ms_minmax_abs[0])
192 : "cc", "memory",
193 "v0", "v1", "v2", "v3",
194 "v4", "v5", "v6", "v7",
195 "v8", "v9", "v10", "v11",
196 "v12", "v13", "v14", "v15",
197 "v16", "v17", "v18", "v19",
198 "v20", "v21", "v22", "v23",
199 "v31"
200 );
201 }
202
203 void ms_pmin3(float *dst, const float *a, const float *b, size_t count)
204 {
205 ARCH_AARCH64_ASM
206 (
207 MS_MINMAX_CORE("dst", "a", "b", "fmin", INC_ON, MS_MUL)
208 : [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b),
209 [count] "+r" (count)
210 : [CC] "r" (&ms_minmax_abs[0])
211 : "cc", "memory",
212 "v0", "v1", "v2", "v3",
213 "v4", "v5", "v6", "v7",
214 "v8", "v9", "v10", "v11",
215 "v12", "v13", "v14", "v15",
216 "v16", "v17", "v18", "v19",
217 "v20", "v21", "v22", "v23",
218 "v31"
219 );
220 }
221
222 void ms_pmax2(float *dst, const float *src, size_t count)
223 {
224 ARCH_AARCH64_ASM
225 (
226 MS_MINMAX_CORE("dst", "dst", "src", "fmax", INC_OFF, MS_MUL)
227 : [dst] "+r" (dst), [src] "+r" (src),
228 [count] "+r" (count)
229 : [CC] "r" (&ms_minmax_abs[0])
230 : "cc", "memory",
231 "v0", "v1", "v2", "v3",
232 "v4", "v5", "v6", "v7",
233 "v8", "v9", "v10", "v11",
234 "v12", "v13", "v14", "v15",
235 "v16", "v17", "v18", "v19",
236 "v20", "v21", "v22", "v23",
237 "v31"
238 );
239 }
240
241 void ms_pmax3(float *dst, const float *a, const float *b, size_t count)
242 {
243 ARCH_AARCH64_ASM
244 (
245 MS_MINMAX_CORE("dst", "a", "b", "fmax", INC_ON, MS_MUL)
246 : [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b),
247 [count] "+r" (count)
248 : [CC] "r" (&ms_minmax_abs[0])
249 : "cc", "memory",
250 "v0", "v1", "v2", "v3",
251 "v4", "v5", "v6", "v7",
252 "v8", "v9", "v10", "v11",
253 "v12", "v13", "v14", "v15",
254 "v16", "v17", "v18", "v19",
255 "v20", "v21", "v22", "v23",
256 "v31"
257 );
258 }
259
260 void lr_pmin2(float *dst, const float *src, size_t count)
261 {
262 ARCH_AARCH64_ASM
263 (
264 MS_MINMAX_CORE("dst", "dst", "src", "fmin", INC_OFF, MS_NOMUL)
265 : [dst] "+r" (dst), [src] "+r" (src),
266 [count] "+r" (count)
267 :
268 : "cc", "memory",
269 "v0", "v1", "v2", "v3",
270 "v4", "v5", "v6", "v7",
271 "v8", "v9", "v10", "v11",
272 "v12", "v13", "v14", "v15",
273 "v16", "v17", "v18", "v19",
274 "v20", "v21", "v22", "v23"
275 );
276 }
277
278 void lr_pmin3(float *dst, const float *a, const float *b, size_t count)
279 {
280 ARCH_AARCH64_ASM
281 (
282 MS_MINMAX_CORE("dst", "a", "b", "fmin", INC_ON, MS_NOMUL)
283 : [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b),
284 [count] "+r" (count)
285 :
286 : "cc", "memory",
287 "v0", "v1", "v2", "v3",
288 "v4", "v5", "v6", "v7",
289 "v8", "v9", "v10", "v11",
290 "v12", "v13", "v14", "v15",
291 "v16", "v17", "v18", "v19",
292 "v20", "v21", "v22", "v23"
293 );
294 }
295
296 void lr_pmax2(float *dst, const float *src, size_t count)
297 {
298 ARCH_AARCH64_ASM
299 (
300 MS_MINMAX_CORE("dst", "dst", "src", "fmax", INC_OFF, MS_NOMUL)
301 : [dst] "+r" (dst), [src] "+r" (src),
302 [count] "+r" (count)
303 :
304 : "cc", "memory",
305 "v0", "v1", "v2", "v3",
306 "v4", "v5", "v6", "v7",
307 "v8", "v9", "v10", "v11",
308 "v12", "v13", "v14", "v15",
309 "v16", "v17", "v18", "v19",
310 "v20", "v21", "v22", "v23"
311 );
312 }
313
314 void lr_pmax3(float *dst, const float *a, const float *b, size_t count)
315 {
316 ARCH_AARCH64_ASM
317 (
318 MS_MINMAX_CORE("dst", "a", "b", "fmax", INC_ON, MS_NOMUL)
319 : [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b),
320 [count] "+r" (count)
321 :
322 : "cc", "memory",
323 "v0", "v1", "v2", "v3",
324 "v4", "v5", "v6", "v7",
325 "v8", "v9", "v10", "v11",
326 "v12", "v13", "v14", "v15",
327 "v16", "v17", "v18", "v19",
328 "v20", "v21", "v22", "v23"
329 );
330 }
331
332 #undef MS_MINMAX_CORE
333
334 #define MS_SIGN_MINMAX_CORE(DST, A, B, OP, INCA, IF_MUL) \
335 __ASM_EMIT("subs %[count], %[count], #32") \
336 IF_MUL("ldr q31, [%[CC], #0x00]") /* v31 = 0.5f */ \
337 __ASM_EMIT("b.lo 2f") \
338 /* 32x blocks */ \
339 __ASM_EMIT("1:") \
340 __ASM_EMIT("ldp q0, q1, [%[" A "], #0x00]") /* v0 = l */ \
341 __ASM_EMIT("ldp q2, q3, [%[" A "], #0x20]") \
342 __ASM_EMIT("ldp q4, q5, [%[" A "], #0x40]") \
343 __ASM_EMIT("ldp q6, q7, [%[" A "], #0x60]") \
344 __ASM_EMIT("ldp q16, q17, [%[" B "], #0x00]") /* v12 = r */ \
345 __ASM_EMIT("ldp q18, q19, [%[" B "], #0x20]") \
346 __ASM_EMIT("ldp q20, q21, [%[" B "], #0x40]") \
347 __ASM_EMIT("ldp q22, q23, [%[" B "], #0x60]") \
348 __ASM_EMIT("fsub v8.4s, v0.4s, v16.4s") /* v8 = s = l - r */ \
349 __ASM_EMIT("fsub v9.4s, v1.4s, v17.4s") \
350 __ASM_EMIT("fsub v10.4s, v2.4s, v18.4s") \
351 __ASM_EMIT("fsub v11.4s, v3.4s, v19.4s") \
352 __ASM_EMIT("fsub v12.4s, v4.4s, v20.4s") \
353 __ASM_EMIT("fsub v13.4s, v5.4s, v21.4s") \
354 __ASM_EMIT("fsub v14.4s, v6.4s, v22.4s") \
355 __ASM_EMIT("fsub v15.4s, v7.4s, v23.4s") \
356 __ASM_EMIT("fadd v0.4s, v0.4s, v16.4s") /* v0 = m = l + r */ \
357 __ASM_EMIT("fadd v1.4s, v1.4s, v17.4s") \
358 __ASM_EMIT("fadd v2.4s, v2.4s, v18.4s") \
359 __ASM_EMIT("fadd v3.4s, v3.4s, v19.4s") \
360 __ASM_EMIT("fadd v4.4s, v4.4s, v20.4s") \
361 __ASM_EMIT("fadd v5.4s, v5.4s, v21.4s") \
362 __ASM_EMIT("fadd v6.4s, v6.4s, v22.4s") \
363 __ASM_EMIT("fadd v7.4s, v7.4s, v23.4s") \
364 __ASM_EMIT("facgt v16.4s, v8.4s, v0.4s") /* v16 = fabsf(s) > fabsf(m) */ \
365 __ASM_EMIT("facgt v17.4s, v9.4s, v1.4s") \
366 __ASM_EMIT("facgt v18.4s, v10.4s, v2.4s") \
367 __ASM_EMIT("facgt v19.4s, v11.4s, v3.4s") \
368 __ASM_EMIT("facgt v20.4s, v12.4s, v4.4s") \
369 __ASM_EMIT("facgt v21.4s, v13.4s, v5.4s") \
370 __ASM_EMIT("facgt v22.4s, v14.4s, v6.4s") \
371 __ASM_EMIT("facgt v23.4s, v15.4s, v7.4s") \
372 __ASM_EMIT(OP " v0.16b, v8.16b, v16.16b") /* v0 = O = OP(fabsf(s) > fabsf(m)) ? s : m */ \
373 __ASM_EMIT(OP " v1.16b, v9.16b, v17.16b") \
374 __ASM_EMIT(OP " v2.16b, v10.16b, v18.16b") \
375 __ASM_EMIT(OP " v3.16b, v11.16b, v19.16b") \
376 __ASM_EMIT(OP " v4.16b, v12.16b, v20.16b") \
377 __ASM_EMIT(OP " v5.16b, v13.16b, v21.16b") \
378 __ASM_EMIT(OP " v6.16b, v14.16b, v22.16b") \
379 __ASM_EMIT(OP " v7.16b, v15.16b, v23.16b") \
380 IF_MUL("fmul v0.4s, v0.4s, v31.4s") /* v0 = O * 0.5f */ \
381 IF_MUL("fmul v1.4s, v1.4s, v31.4s") \
382 IF_MUL("fmul v2.4s, v2.4s, v31.4s") \
383 IF_MUL("fmul v3.4s, v3.4s, v31.4s") \
384 IF_MUL("fmul v4.4s, v4.4s, v31.4s") \
385 IF_MUL("fmul v5.4s, v5.4s, v31.4s") \
386 IF_MUL("fmul v6.4s, v6.4s, v31.4s") \
387 IF_MUL("fmul v7.4s, v7.4s, v31.4s") \
388 __ASM_EMIT("subs %[count], %[count], #32") \
389 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
390 __ASM_EMIT("stp q2, q3, [%[" DST "], #0x20]") \
391 __ASM_EMIT("stp q4, q5, [%[" DST "], #0x40]") \
392 __ASM_EMIT("stp q6, q7, [%[" DST "], #0x60]") \
393 __ASM_EMIT(INCA("add %[" A "], %[" A "], #0x80")) \
394 __ASM_EMIT("add %[" B "], %[" B "], #0x80") \
395 __ASM_EMIT("add %[" DST "], %[" DST "], #0x80") \
396 __ASM_EMIT("b.hs 1b") \
397 /* 16x block */ \
398 __ASM_EMIT("2:") \
399 __ASM_EMIT("adds %[count], %[count], #16") \
400 __ASM_EMIT("b.lt 4f") \
401 __ASM_EMIT("ldp q0, q1, [%[" A "], #0x00]") /* v0 = l */ \
402 __ASM_EMIT("ldp q2, q3, [%[" A "], #0x20]") \
403 __ASM_EMIT("ldp q16, q17, [%[" B "], #0x00]") /* v12 = r */ \
404 __ASM_EMIT("ldp q18, q19, [%[" B "], #0x20]") \
405 __ASM_EMIT("fsub v8.4s, v0.4s, v16.4s") /* v8 = s = l - r */ \
406 __ASM_EMIT("fsub v9.4s, v1.4s, v17.4s") \
407 __ASM_EMIT("fsub v10.4s, v2.4s, v18.4s") \
408 __ASM_EMIT("fsub v11.4s, v3.4s, v19.4s") \
409 __ASM_EMIT("fadd v0.4s, v0.4s, v16.4s") /* v0 = m = l + r */ \
410 __ASM_EMIT("fadd v1.4s, v1.4s, v17.4s") \
411 __ASM_EMIT("fadd v2.4s, v2.4s, v18.4s") \
412 __ASM_EMIT("fadd v3.4s, v3.4s, v19.4s") \
413 __ASM_EMIT("facgt v16.4s, v8.4s, v0.4s") /* v16 = fabsf(s) > fabsf(m) */ \
414 __ASM_EMIT("facgt v17.4s, v9.4s, v1.4s") \
415 __ASM_EMIT("facgt v18.4s, v10.4s, v2.4s") \
416 __ASM_EMIT("facgt v19.4s, v11.4s, v3.4s") \
417 __ASM_EMIT(OP " v0.16b, v8.16b, v16.16b") /* v0 = O = OP(fabsf(s) > fabsf(m)) ? s : m */ \
418 __ASM_EMIT(OP " v1.16b, v9.16b, v17.16b") \
419 __ASM_EMIT(OP " v2.16b, v10.16b, v18.16b") \
420 __ASM_EMIT(OP " v3.16b, v11.16b, v19.16b") \
421 IF_MUL("fmul v0.4s, v0.4s, v31.4s") /* v0 = O * 0.5f */ \
422 IF_MUL("fmul v1.4s, v1.4s, v31.4s") \
423 IF_MUL("fmul v2.4s, v2.4s, v31.4s") \
424 IF_MUL("fmul v3.4s, v3.4s, v31.4s") \
425 __ASM_EMIT("sub %[count], %[count], #16") \
426 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
427 __ASM_EMIT("stp q2, q3, [%[" DST "], #0x20]") \
428 __ASM_EMIT(INCA("add %[" A "], %[" A "], #0x40")) \
429 __ASM_EMIT("add %[" B "], %[" B "], #0x40") \
430 __ASM_EMIT("add %[" DST "], %[" DST "], #0x40") \
431 /* 8x block */ \
432 __ASM_EMIT("4:") \
433 __ASM_EMIT("adds %[count], %[count], #8") \
434 __ASM_EMIT("b.lt 6f") \
435 __ASM_EMIT("ldp q0, q1, [%[" A "], #0x00]") /* v0 = l */ \
436 __ASM_EMIT("ldp q16, q17, [%[" B "], #0x00]") /* v12 = r */ \
437 __ASM_EMIT("fsub v8.4s, v0.4s, v16.4s") /* v8 = s = l - r */ \
438 __ASM_EMIT("fsub v9.4s, v1.4s, v17.4s") \
439 __ASM_EMIT("fadd v0.4s, v0.4s, v16.4s") /* v0 = m = l + r */ \
440 __ASM_EMIT("fadd v1.4s, v1.4s, v17.4s") \
441 __ASM_EMIT("facgt v16.4s, v8.4s, v0.4s") /* v16 = fabsf(s) > fabsf(m) */ \
442 __ASM_EMIT("facgt v17.4s, v9.4s, v1.4s") \
443 __ASM_EMIT(OP " v0.16b, v8.16b, v16.16b") /* v0 = O = OP(fabsf(s) > fabsf(m)) ? s : m */ \
444 __ASM_EMIT(OP " v1.16b, v9.16b, v17.16b") \
445 IF_MUL("fmul v0.4s, v0.4s, v31.4s") /* v0 = O * 0.5f */ \
446 IF_MUL("fmul v1.4s, v1.4s, v31.4s") \
447 __ASM_EMIT("sub %[count], %[count], #8") \
448 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
449 __ASM_EMIT(INCA("add %[" A "], %[" A "], #0x20")) \
450 __ASM_EMIT("add %[" B "], %[" B "], #0x20") \
451 __ASM_EMIT("add %[" DST "], %[" DST "], #0x20") \
452 /* 4x block */ \
453 __ASM_EMIT("6:") \
454 __ASM_EMIT("adds %[count], %[count], #4") /* 8 - 4 */ \
455 __ASM_EMIT("b.lt 8f") \
456 __ASM_EMIT("ldr q0, [%[" A "], #0x00]") /* v0 = l */ \
457 __ASM_EMIT("ldr q16, [%[" B "], #0x00]") /* v12 = r */ \
458 __ASM_EMIT("fsub v8.4s, v0.4s, v16.4s") /* v8 = s = l - r */ \
459 __ASM_EMIT("fadd v0.4s, v0.4s, v16.4s") /* v0 = m = l + r */ \
460 __ASM_EMIT("facgt v16.4s, v8.4s, v0.4s") /* v16 = fabsf(s) > fabsf(m) */ \
461 __ASM_EMIT(OP " v0.16b, v8.16b, v16.16b") /* v0 = O = OP(fabsf(s) > fabsf(m)) ? s : m */ \
462 IF_MUL("fmul v0.4s, v0.4s, v31.4s") /* v0 = O * 0.5f */ \
463 __ASM_EMIT("sub %[count], %[count], #4") \
464 __ASM_EMIT("str q0, [%[" DST "], #0x00]") \
465 __ASM_EMIT(INCA("add %[" A "], %[" A "], #0x10")) \
466 __ASM_EMIT("add %[" B "], %[" B "], #0x10") \
467 __ASM_EMIT("add %[" DST "], %[" DST "], #0x10") \
468 /* 1x block */ \
469 __ASM_EMIT("8:") \
470 __ASM_EMIT("adds %[count], %[count], #3") /* 4 - 3 */ \
471 __ASM_EMIT("b.lt 10f") \
472 __ASM_EMIT("9:") \
473 __ASM_EMIT("ld1r {v0.4s}, [%[" A "]]") \
474 __ASM_EMIT("ld1r {v16.4s}, [%[" B "]]") \
475 __ASM_EMIT("fsub v8.4s, v0.4s, v16.4s") /* v8 = s = l - r */ \
476 __ASM_EMIT("fadd v0.4s, v0.4s, v16.4s") /* v0 = m = l + r */ \
477 __ASM_EMIT("facgt v16.4s, v8.4s, v0.4s") /* v16 = fabsf(s) > fabsf(m) */ \
478 __ASM_EMIT(OP " v0.16b, v8.16b, v16.16b") /* v0 = O = OP(fabsf(s) > fabsf(m)) ? s : m */ \
479 IF_MUL("fmul v0.4s, v0.4s, v31.4s") /* v0 = O * 0.5f */ \
480 __ASM_EMIT("subs %[count], %[count], #1") \
481 __ASM_EMIT("st1 {v0.s}[0], [%[" DST "]]") \
482 __ASM_EMIT(INCA("add %[" A "], %[" A "], #0x04")) \
483 __ASM_EMIT("add %[" B "], %[" B "], #0x04") \
484 __ASM_EMIT("add %[" DST "], %[" DST "], #0x04") \
485 __ASM_EMIT("b.ge 9b") \
486 __ASM_EMIT("10:")
487
488 void ms_psmin2(float *dst, const float *src, size_t count)
489 {
490 ARCH_AARCH64_ASM
491 (
492 MS_SIGN_MINMAX_CORE("dst", "dst", "src", "bif", INC_OFF, MS_MUL)
493 : [dst] "+r" (dst), [src] "+r" (src),
494 [count] "+r" (count)
495 : [CC] "r" (&ms_minmax_abs[0])
496 : "cc", "memory",
497 "v0", "v1", "v2", "v3",
498 "v4", "v5", "v6", "v7",
499 "v8", "v9", "v10", "v11",
500 "v12", "v13", "v14", "v15",
501 "v16", "v17", "v18", "v19",
502 "v20", "v21", "v22", "v23",
503 "v31"
504 );
505 }
506
507 void ms_psmin3(float *dst, const float *a, const float *b, size_t count)
508 {
509 ARCH_AARCH64_ASM
510 (
511 MS_SIGN_MINMAX_CORE("dst", "a", "b", "bif", INC_ON, MS_MUL)
512 : [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b),
513 [count] "+r" (count)
514 : [CC] "r" (&ms_minmax_abs[0])
515 : "cc", "memory",
516 "v0", "v1", "v2", "v3",
517 "v4", "v5", "v6", "v7",
518 "v8", "v9", "v10", "v11",
519 "v12", "v13", "v14", "v15",
520 "v16", "v17", "v18", "v19",
521 "v20", "v21", "v22", "v23",
522 "v31"
523 );
524 }
525
526 void ms_psmax2(float *dst, const float *src, size_t count)
527 {
528 ARCH_AARCH64_ASM
529 (
530 MS_SIGN_MINMAX_CORE("dst", "dst", "src", "bit", INC_OFF, MS_MUL)
531 : [dst] "+r" (dst), [src] "+r" (src),
532 [count] "+r" (count)
533 : [CC] "r" (&ms_minmax_abs[0])
534 : "cc", "memory",
535 "v0", "v1", "v2", "v3",
536 "v4", "v5", "v6", "v7",
537 "v8", "v9", "v10", "v11",
538 "v12", "v13", "v14", "v15",
539 "v16", "v17", "v18", "v19",
540 "v20", "v21", "v22", "v23",
541 "v31"
542 );
543 }
544
545 void ms_psmax3(float *dst, const float *a, const float *b, size_t count)
546 {
547 ARCH_AARCH64_ASM
548 (
549 MS_SIGN_MINMAX_CORE("dst", "a", "b", "bit", INC_ON, MS_MUL)
550 : [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b),
551 [count] "+r" (count)
552 : [CC] "r" (&ms_minmax_abs[0])
553 : "cc", "memory",
554 "v0", "v1", "v2", "v3",
555 "v4", "v5", "v6", "v7",
556 "v8", "v9", "v10", "v11",
557 "v12", "v13", "v14", "v15",
558 "v16", "v17", "v18", "v19",
559 "v20", "v21", "v22", "v23",
560 "v31"
561 );
562 }
563
564 void lr_psmin2(float *dst, const float *src, size_t count)
565 {
566 ARCH_AARCH64_ASM
567 (
568 MS_SIGN_MINMAX_CORE("dst", "dst", "src", "bif", INC_OFF, MS_NOMUL)
569 : [dst] "+r" (dst), [src] "+r" (src),
570 [count] "+r" (count)
571 :
572 : "cc", "memory",
573 "v0", "v1", "v2", "v3",
574 "v4", "v5", "v6", "v7",
575 "v8", "v9", "v10", "v11",
576 "v12", "v13", "v14", "v15",
577 "v16", "v17", "v18", "v19",
578 "v20", "v21", "v22", "v23"
579 );
580 }
581
582 void lr_psmin3(float *dst, const float *a, const float *b, size_t count)
583 {
584 ARCH_AARCH64_ASM
585 (
586 MS_SIGN_MINMAX_CORE("dst", "a", "b", "bif", INC_ON, MS_NOMUL)
587 : [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b),
588 [count] "+r" (count)
589 :
590 : "cc", "memory",
591 "v0", "v1", "v2", "v3",
592 "v4", "v5", "v6", "v7",
593 "v8", "v9", "v10", "v11",
594 "v12", "v13", "v14", "v15",
595 "v16", "v17", "v18", "v19",
596 "v20", "v21", "v22", "v23"
597 );
598 }
599
600 void lr_psmax2(float *dst, const float *src, size_t count)
601 {
602 ARCH_AARCH64_ASM
603 (
604 MS_SIGN_MINMAX_CORE("dst", "dst", "src", "bit", INC_OFF, MS_NOMUL)
605 : [dst] "+r" (dst), [src] "+r" (src),
606 [count] "+r" (count)
607 :
608 : "cc", "memory",
609 "v0", "v1", "v2", "v3",
610 "v4", "v5", "v6", "v7",
611 "v8", "v9", "v10", "v11",
612 "v12", "v13", "v14", "v15",
613 "v16", "v17", "v18", "v19",
614 "v20", "v21", "v22", "v23"
615 );
616 }
617
618 void lr_psmax3(float *dst, const float *a, const float *b, size_t count)
619 {
620 ARCH_AARCH64_ASM
621 (
622 MS_SIGN_MINMAX_CORE("dst", "a", "b", "bit", INC_ON, MS_NOMUL)
623 : [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b),
624 [count] "+r" (count)
625 :
626 : "cc", "memory",
627 "v0", "v1", "v2", "v3",
628 "v4", "v5", "v6", "v7",
629 "v8", "v9", "v10", "v11",
630 "v12", "v13", "v14", "v15",
631 "v16", "v17", "v18", "v19",
632 "v20", "v21", "v22", "v23"
633 );
634 }
635
636 #undef MS_SIGN_MINMAX_CORE
637
638 #define MM_ABS_MINMAX_CORE(DST, A, B, OP, INCA, IF_MUL) \
639 __ASM_EMIT("subs %[count], %[count], #32") \
640 IF_MUL("ldr q31, [%[CC], #0x00]") /* v31 = 0.5f */ \
641 __ASM_EMIT("b.lo 2f") \
642 /* 32x blocks */ \
643 __ASM_EMIT("1:") \
644 __ASM_EMIT("ldp q0, q1, [%[" A "], #0x00]") /* v0 = l */ \
645 __ASM_EMIT("ldp q2, q3, [%[" A "], #0x20]") \
646 __ASM_EMIT("ldp q4, q5, [%[" A "], #0x40]") \
647 __ASM_EMIT("ldp q6, q7, [%[" A "], #0x60]") \
648 __ASM_EMIT("ldp q16, q17, [%[" B "], #0x00]") /* v12 = r */ \
649 __ASM_EMIT("ldp q18, q19, [%[" B "], #0x20]") \
650 __ASM_EMIT("ldp q20, q21, [%[" B "], #0x40]") \
651 __ASM_EMIT("ldp q22, q23, [%[" B "], #0x60]") \
652 __ASM_EMIT("fsub v8.4s, v0.4s, v16.4s") /* v8 = s = l - r */ \
653 __ASM_EMIT("fsub v9.4s, v1.4s, v17.4s") \
654 __ASM_EMIT("fsub v10.4s, v2.4s, v18.4s") \
655 __ASM_EMIT("fsub v11.4s, v3.4s, v19.4s") \
656 __ASM_EMIT("fsub v12.4s, v4.4s, v20.4s") \
657 __ASM_EMIT("fsub v13.4s, v5.4s, v21.4s") \
658 __ASM_EMIT("fsub v14.4s, v6.4s, v22.4s") \
659 __ASM_EMIT("fsub v15.4s, v7.4s, v23.4s") \
660 __ASM_EMIT("fadd v0.4s, v0.4s, v16.4s") /* v0 = m = l + r */ \
661 __ASM_EMIT("fadd v1.4s, v1.4s, v17.4s") \
662 __ASM_EMIT("fadd v2.4s, v2.4s, v18.4s") \
663 __ASM_EMIT("fadd v3.4s, v3.4s, v19.4s") \
664 __ASM_EMIT("fadd v4.4s, v4.4s, v20.4s") \
665 __ASM_EMIT("fadd v5.4s, v5.4s, v21.4s") \
666 __ASM_EMIT("fadd v6.4s, v6.4s, v22.4s") \
667 __ASM_EMIT("fadd v7.4s, v7.4s, v23.4s") \
668 __ASM_EMIT("fabs v8.4s, v8.4s") /* v8 = fabsf(s) */ \
669 __ASM_EMIT("fabs v9.4s, v9.4s") \
670 __ASM_EMIT("fabs v10.4s, v10.4s") \
671 __ASM_EMIT("fabs v11.4s, v11.4s") \
672 __ASM_EMIT("fabs v12.4s, v12.4s") \
673 __ASM_EMIT("fabs v13.4s, v13.4s") \
674 __ASM_EMIT("fabs v14.4s, v14.4s") \
675 __ASM_EMIT("fabs v15.4s, v15.4s") \
676 __ASM_EMIT("fabs v0.4s, v0.4s") /* v0 = fabsf(m) */ \
677 __ASM_EMIT("fabs v1.4s, v1.4s") \
678 __ASM_EMIT("fabs v2.4s, v2.4s") \
679 __ASM_EMIT("fabs v3.4s, v3.4s") \
680 __ASM_EMIT("fabs v4.4s, v4.4s") \
681 __ASM_EMIT("fabs v5.4s, v5.4s") \
682 __ASM_EMIT("fabs v6.4s, v6.4s") \
683 __ASM_EMIT("fabs v7.4s, v7.4s") \
684 __ASM_EMIT(OP " v0.4s, v0.4s, v8.4s") /* v0 = O = OP(fabsf(m), fabsf(s)) */ \
685 __ASM_EMIT(OP " v1.4s, v1.4s, v9.4s") \
686 __ASM_EMIT(OP " v2.4s, v2.4s, v10.4s") \
687 __ASM_EMIT(OP " v3.4s, v3.4s, v11.4s") \
688 __ASM_EMIT(OP " v4.4s, v4.4s, v12.4s") \
689 __ASM_EMIT(OP " v5.4s, v5.4s, v13.4s") \
690 __ASM_EMIT(OP " v6.4s, v6.4s, v14.4s") \
691 __ASM_EMIT(OP " v7.4s, v7.4s, v15.4s") \
692 IF_MUL("fmul v0.4s, v0.4s, v31.4s") /* v0 = O * 0.5f */ \
693 IF_MUL("fmul v1.4s, v1.4s, v31.4s") \
694 IF_MUL("fmul v2.4s, v2.4s, v31.4s") \
695 IF_MUL("fmul v3.4s, v3.4s, v31.4s") \
696 IF_MUL("fmul v4.4s, v4.4s, v31.4s") \
697 IF_MUL("fmul v5.4s, v5.4s, v31.4s") \
698 IF_MUL("fmul v6.4s, v6.4s, v31.4s") \
699 IF_MUL("fmul v7.4s, v7.4s, v31.4s") \
700 __ASM_EMIT("subs %[count], %[count], #32") \
701 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
702 __ASM_EMIT("stp q2, q3, [%[" DST "], #0x20]") \
703 __ASM_EMIT("stp q4, q5, [%[" DST "], #0x40]") \
704 __ASM_EMIT("stp q6, q7, [%[" DST "], #0x60]") \
705 __ASM_EMIT(INCA("add %[" A "], %[" A "], #0x80")) \
706 __ASM_EMIT("add %[" B "], %[" B "], #0x80") \
707 __ASM_EMIT("add %[" DST "], %[" DST "], #0x80") \
708 __ASM_EMIT("b.hs 1b") \
709 /* 16x block */ \
710 __ASM_EMIT("2:") \
711 __ASM_EMIT("adds %[count], %[count], #16") \
712 __ASM_EMIT("b.lt 4f") \
713 __ASM_EMIT("ldp q0, q1, [%[" A "], #0x00]") /* v0 = l */ \
714 __ASM_EMIT("ldp q2, q3, [%[" A "], #0x20]") \
715 __ASM_EMIT("ldp q16, q17, [%[" B "], #0x00]") /* v12 = r */ \
716 __ASM_EMIT("ldp q18, q19, [%[" B "], #0x20]") \
717 __ASM_EMIT("fsub v8.4s, v0.4s, v16.4s") /* v8 = s = l - r */ \
718 __ASM_EMIT("fsub v9.4s, v1.4s, v17.4s") \
719 __ASM_EMIT("fsub v10.4s, v2.4s, v18.4s") \
720 __ASM_EMIT("fsub v11.4s, v3.4s, v19.4s") \
721 __ASM_EMIT("fadd v0.4s, v0.4s, v16.4s") /* v0 = m = l + r */ \
722 __ASM_EMIT("fadd v1.4s, v1.4s, v17.4s") \
723 __ASM_EMIT("fadd v2.4s, v2.4s, v18.4s") \
724 __ASM_EMIT("fadd v3.4s, v3.4s, v19.4s") \
725 __ASM_EMIT("fabs v8.4s, v8.4s") /* v8 = fabsf(s) */ \
726 __ASM_EMIT("fabs v9.4s, v9.4s") \
727 __ASM_EMIT("fabs v10.4s, v10.4s") \
728 __ASM_EMIT("fabs v11.4s, v11.4s") \
729 __ASM_EMIT("fabs v0.4s, v0.4s") /* v0 = fabsf(m) */ \
730 __ASM_EMIT("fabs v1.4s, v1.4s") \
731 __ASM_EMIT("fabs v2.4s, v2.4s") \
732 __ASM_EMIT("fabs v3.4s, v3.4s") \
733 __ASM_EMIT(OP " v0.4s, v0.4s, v8.4s") /* v0 = O = OP(fabsf(m), fabsf(s)) */ \
734 __ASM_EMIT(OP " v1.4s, v1.4s, v9.4s") \
735 __ASM_EMIT(OP " v2.4s, v2.4s, v10.4s") \
736 __ASM_EMIT(OP " v3.4s, v3.4s, v11.4s") \
737 IF_MUL("fmul v0.4s, v0.4s, v31.4s") /* v0 = O * 0.5f */ \
738 IF_MUL("fmul v1.4s, v1.4s, v31.4s") \
739 IF_MUL("fmul v2.4s, v2.4s, v31.4s") \
740 IF_MUL("fmul v3.4s, v3.4s, v31.4s") \
741 __ASM_EMIT("sub %[count], %[count], #16") \
742 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
743 __ASM_EMIT("stp q2, q3, [%[" DST "], #0x20]") \
744 __ASM_EMIT(INCA("add %[" A "], %[" A "], #0x40")) \
745 __ASM_EMIT("add %[" B "], %[" B "], #0x40") \
746 __ASM_EMIT("add %[" DST "], %[" DST "], #0x40") \
747 /* 8x block */ \
748 __ASM_EMIT("4:") \
749 __ASM_EMIT("adds %[count], %[count], #8") \
750 __ASM_EMIT("b.lt 6f") \
751 __ASM_EMIT("ldp q0, q1, [%[" A "], #0x00]") /* v0 = l */ \
752 __ASM_EMIT("ldp q16, q17, [%[" B "], #0x00]") /* v12 = r */ \
753 __ASM_EMIT("fsub v8.4s, v0.4s, v16.4s") /* v8 = s = l - r */ \
754 __ASM_EMIT("fsub v9.4s, v1.4s, v17.4s") \
755 __ASM_EMIT("fadd v0.4s, v0.4s, v16.4s") /* v0 = m = l + r */ \
756 __ASM_EMIT("fadd v1.4s, v1.4s, v17.4s") \
757 __ASM_EMIT("fabs v8.4s, v8.4s") /* v8 = fabsf(s) */ \
758 __ASM_EMIT("fabs v9.4s, v9.4s") \
759 __ASM_EMIT("fabs v0.4s, v0.4s") /* v0 = fabsf(m) */ \
760 __ASM_EMIT("fabs v1.4s, v1.4s") \
761 __ASM_EMIT(OP " v0.4s, v0.4s, v8.4s") /* v0 = O = OP(fabsf(m), fabsf(s)) */ \
762 __ASM_EMIT(OP " v1.4s, v1.4s, v9.4s") \
763 IF_MUL("fmul v0.4s, v0.4s, v31.4s") /* v0 = O * 0.5f */ \
764 IF_MUL("fmul v1.4s, v1.4s, v31.4s") \
765 __ASM_EMIT("sub %[count], %[count], #8") \
766 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
767 __ASM_EMIT(INCA("add %[" A "], %[" A "], #0x20")) \
768 __ASM_EMIT("add %[" B "], %[" B "], #0x20") \
769 __ASM_EMIT("add %[" DST "], %[" DST "], #0x20") \
770 /* 4x block */ \
771 __ASM_EMIT("6:") \
772 __ASM_EMIT("adds %[count], %[count], #4") /* 8 - 4 */ \
773 __ASM_EMIT("b.lt 8f") \
774 __ASM_EMIT("ldr q0, [%[" A "], #0x00]") /* v0 = l */ \
775 __ASM_EMIT("ldr q16, [%[" B "], #0x00]") /* v12 = r */ \
776 __ASM_EMIT("fsub v8.4s, v0.4s, v16.4s") /* v8 = s = l - r */ \
777 __ASM_EMIT("fadd v0.4s, v0.4s, v16.4s") /* v0 = m = l + r */ \
778 __ASM_EMIT("fabs v8.4s, v8.4s") /* v8 = fabsf(s) */ \
779 __ASM_EMIT("fabs v0.4s, v0.4s") /* v0 = fabsf(m) */ \
780 __ASM_EMIT(OP " v0.4s, v0.4s, v8.4s") /* v0 = O = OP(fabsf(m), fabsf(s)) */ \
781 IF_MUL("fmul v0.4s, v0.4s, v31.4s") /* v0 = O * 0.5f */ \
782 __ASM_EMIT("sub %[count], %[count], #4") \
783 __ASM_EMIT("str q0, [%[" DST "], #0x00]") \
784 __ASM_EMIT(INCA("add %[" A "], %[" A "], #0x10")) \
785 __ASM_EMIT("add %[" B "], %[" B "], #0x10") \
786 __ASM_EMIT("add %[" DST "], %[" DST "], #0x10") \
787 /* 1x block */ \
788 __ASM_EMIT("8:") \
789 __ASM_EMIT("adds %[count], %[count], #3") /* 4 - 3 */ \
790 __ASM_EMIT("b.lt 10f") \
791 __ASM_EMIT("9:") \
792 __ASM_EMIT("ld1r {v0.4s}, [%[" A "]]") \
793 __ASM_EMIT("ld1r {v16.4s}, [%[" B "]]") \
794 __ASM_EMIT("fsub v8.4s, v0.4s, v16.4s") /* v8 = s = l - r */ \
795 __ASM_EMIT("fadd v0.4s, v0.4s, v16.4s") /* v0 = m = l + r */ \
796 __ASM_EMIT("fabs v8.4s, v8.4s") /* v8 = fabsf(s) */ \
797 __ASM_EMIT("fabs v0.4s, v0.4s") /* v0 = fabsf(m) */ \
798 __ASM_EMIT(OP " v0.4s, v0.4s, v8.4s") /* v0 = O = OP(fabsf(m), fabsf(s)) */ \
799 IF_MUL("fmul v0.4s, v0.4s, v31.4s") /* v0 = O * 0.5f */ \
800 __ASM_EMIT("subs %[count], %[count], #1") \
801 __ASM_EMIT("st1 {v0.s}[0], [%[" DST "]]") \
802 __ASM_EMIT(INCA("add %[" A "], %[" A "], #0x04")) \
803 __ASM_EMIT("add %[" B "], %[" B "], #0x04") \
804 __ASM_EMIT("add %[" DST "], %[" DST "], #0x04") \
805 __ASM_EMIT("b.ge 9b") \
806 __ASM_EMIT("10:")
807
808 void ms_pamin2(float *dst, const float *src, size_t count)
809 {
810 ARCH_AARCH64_ASM
811 (
812 MM_ABS_MINMAX_CORE("dst", "dst", "src", "fmin", INC_OFF, MS_MUL)
813 : [dst] "+r" (dst), [src] "+r" (src),
814 [count] "+r" (count)
815 : [CC] "r" (&ms_minmax_abs[0])
816 : "cc", "memory",
817 "v0", "v1", "v2", "v3",
818 "v4", "v5", "v6", "v7",
819 "v8", "v9", "v10", "v11",
820 "v12", "v13", "v14", "v15",
821 "v16", "v17", "v18", "v19",
822 "v20", "v21", "v22", "v23",
823 "v31"
824 );
825 }
826
827 void ms_pamin3(float *dst, const float *a, const float *b, size_t count)
828 {
829 ARCH_AARCH64_ASM
830 (
831 MM_ABS_MINMAX_CORE("dst", "a", "b", "fmin", INC_ON, MS_MUL)
832 : [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b),
833 [count] "+r" (count)
834 : [CC] "r" (&ms_minmax_abs[0])
835 : "cc", "memory",
836 "v0", "v1", "v2", "v3",
837 "v4", "v5", "v6", "v7",
838 "v8", "v9", "v10", "v11",
839 "v12", "v13", "v14", "v15",
840 "v16", "v17", "v18", "v19",
841 "v20", "v21", "v22", "v23",
842 "v31"
843 );
844 }
845
846 void ms_pamax2(float *dst, const float *src, size_t count)
847 {
848 ARCH_AARCH64_ASM
849 (
850 MM_ABS_MINMAX_CORE("dst", "dst", "src", "fmax", INC_OFF, MS_MUL)
851 : [dst] "+r" (dst), [src] "+r" (src),
852 [count] "+r" (count)
853 : [CC] "r" (&ms_minmax_abs[0])
854 : "cc", "memory",
855 "v0", "v1", "v2", "v3",
856 "v4", "v5", "v6", "v7",
857 "v8", "v9", "v10", "v11",
858 "v12", "v13", "v14", "v15",
859 "v16", "v17", "v18", "v19",
860 "v20", "v21", "v22", "v23",
861 "v31"
862 );
863 }
864
865 void ms_pamax3(float *dst, const float *a, const float *b, size_t count)
866 {
867 ARCH_AARCH64_ASM
868 (
869 MM_ABS_MINMAX_CORE("dst", "a", "b", "fmax", INC_ON, MS_MUL)
870 : [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b),
871 [count] "+r" (count)
872 : [CC] "r" (&ms_minmax_abs[0])
873 : "cc", "memory",
874 "v0", "v1", "v2", "v3",
875 "v4", "v5", "v6", "v7",
876 "v8", "v9", "v10", "v11",
877 "v12", "v13", "v14", "v15",
878 "v16", "v17", "v18", "v19",
879 "v20", "v21", "v22", "v23",
880 "v31"
881 );
882 }
883
884 void lr_pamin2(float *dst, const float *src, size_t count)
885 {
886 ARCH_AARCH64_ASM
887 (
888 MM_ABS_MINMAX_CORE("dst", "dst", "src", "fmin", INC_OFF, MS_NOMUL)
889 : [dst] "+r" (dst), [src] "+r" (src),
890 [count] "+r" (count)
891 : [CC] "r" (&ms_minmax_abs[0])
892 : "cc", "memory",
893 "v0", "v1", "v2", "v3",
894 "v4", "v5", "v6", "v7",
895 "v8", "v9", "v10", "v11",
896 "v12", "v13", "v14", "v15",
897 "v16", "v17", "v18", "v19",
898 "v20", "v21", "v22", "v23",
899 "v31"
900 );
901 }
902
903 void lr_pamin3(float *dst, const float *a, const float *b, size_t count)
904 {
905 ARCH_AARCH64_ASM
906 (
907 MM_ABS_MINMAX_CORE("dst", "a", "b", "fmin", INC_ON, MS_NOMUL)
908 : [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b),
909 [count] "+r" (count)
910 :
911 : "cc", "memory",
912 "v0", "v1", "v2", "v3",
913 "v4", "v5", "v6", "v7",
914 "v8", "v9", "v10", "v11",
915 "v12", "v13", "v14", "v15",
916 "v16", "v17", "v18", "v19",
917 "v20", "v21", "v22", "v23"
918 );
919 }
920
921 void lr_pamax2(float *dst, const float *src, size_t count)
922 {
923 ARCH_AARCH64_ASM
924 (
925 MM_ABS_MINMAX_CORE("dst", "dst", "src", "fmax", INC_OFF, MS_NOMUL)
926 : [dst] "+r" (dst), [src] "+r" (src),
927 [count] "+r" (count)
928 :
929 : "cc", "memory",
930 "v0", "v1", "v2", "v3",
931 "v4", "v5", "v6", "v7",
932 "v8", "v9", "v10", "v11",
933 "v12", "v13", "v14", "v15",
934 "v16", "v17", "v18", "v19",
935 "v20", "v21", "v22", "v23"
936 );
937 }
938
939 void lr_pamax3(float *dst, const float *a, const float *b, size_t count)
940 {
941 ARCH_AARCH64_ASM
942 (
943 MM_ABS_MINMAX_CORE("dst", "a", "b", "fmax", INC_ON, MS_NOMUL)
944 : [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b),
945 [count] "+r" (count)
946 :
947 : "cc", "memory",
948 "v0", "v1", "v2", "v3",
949 "v4", "v5", "v6", "v7",
950 "v8", "v9", "v10", "v11",
951 "v12", "v13", "v14", "v15",
952 "v16", "v17", "v18", "v19",
953 "v20", "v21", "v22", "v23"
954 );
955 }
956
957 #undef MM_ABS_MINMAX_CORE
958
959 #undef INC_ON
960 #undef INC_OFF
961
962 #undef MS_MUL
963 #undef MS_NOMUL
964 } /* namespace asimd */
965 } /* namespace lsp */
966
967
968
969
970 #endif /* PRIVATE_DSP_ARCH_AARCH64_ASIMD_PMATH_MS_MINMAX_H_ */