Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/aarch64/asimd/pmath/minmax.h

raw
1 /*
2 * Copyright (C) 2020 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2020 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 31 мар. 2020 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_AARCH64_ASIMD_PMATH_MINMAX_H_
23 #define PRIVATE_DSP_ARCH_AARCH64_ASIMD_PMATH_MINMAX_H_
24
25 #ifndef PRIVATE_DSP_ARCH_AARCH64_ASIMD_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_AARCH64_ASIMD_IMPL */
28
29 namespace lsp
30 {
31 namespace asimd
32 {
33 #define INC_ON(cmd) cmd
34 #define INC_OFF(cmd)
35
36 #define MINMAX_CORE(DST, A, B, OP, INCA) \
37 __ASM_EMIT("subs %[count], %[count], #32") \
38 __ASM_EMIT("b.lo 2f") \
39 /* 32x blocks */ \
40 __ASM_EMIT("1:") \
41 __ASM_EMIT("ldp q8, q9, [%[" B "], #0x00]") \
42 __ASM_EMIT("ldp q10, q11, [%[" B "], #0x20]") \
43 __ASM_EMIT("ldp q12, q13, [%[" B "], #0x40]") \
44 __ASM_EMIT("ldp q14, q15, [%[" B "], #0x60]") \
45 __ASM_EMIT("ldp q0, q1, [%[" A "], #0x00]") \
46 __ASM_EMIT("ldp q2, q3, [%[" A "], #0x20]") \
47 __ASM_EMIT("ldp q4, q5, [%[" A "], #0x40]") \
48 __ASM_EMIT("ldp q6, q7, [%[" A "], #0x60]") \
49 __ASM_EMIT(OP " v0.4s, v0.4s, v8.4s") \
50 __ASM_EMIT(OP " v1.4s, v1.4s, v9.4s") \
51 __ASM_EMIT(OP " v2.4s, v2.4s, v10.4s") \
52 __ASM_EMIT(OP " v3.4s, v3.4s, v11.4s") \
53 __ASM_EMIT(OP " v4.4s, v4.4s, v12.4s") \
54 __ASM_EMIT(OP " v5.4s, v5.4s, v13.4s") \
55 __ASM_EMIT(OP " v6.4s, v6.4s, v14.4s") \
56 __ASM_EMIT(OP " v7.4s, v7.4s, v15.4s") \
57 __ASM_EMIT("subs %[count], %[count], #32") \
58 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
59 __ASM_EMIT("stp q2, q3, [%[" DST "], #0x20]") \
60 __ASM_EMIT("stp q4, q5, [%[" DST "], #0x40]") \
61 __ASM_EMIT("stp q6, q7, [%[" DST "], #0x60]") \
62 __ASM_EMIT(INCA("add %[" A "], %[" A "], #0x80")) \
63 __ASM_EMIT("add %[" B "], %[" B "], #0x80") \
64 __ASM_EMIT("add %[" DST "], %[" DST "], #0x80") \
65 __ASM_EMIT("b.hs 1b") \
66 /* 16x block */ \
67 __ASM_EMIT("2:") \
68 __ASM_EMIT("adds %[count], %[count], #16") /* 32 - 16 */ \
69 __ASM_EMIT("b.lt 4f") \
70 __ASM_EMIT("ldp q8, q9, [%[" B "], #0x00]") \
71 __ASM_EMIT("ldp q10, q11, [%[" B "], #0x20]") \
72 __ASM_EMIT("ldp q0, q1, [%[" A "], #0x00]") \
73 __ASM_EMIT("ldp q2, q3, [%[" A "], #0x20]") \
74 __ASM_EMIT(OP " v0.4s, v0.4s, v8.4s") \
75 __ASM_EMIT(OP " v1.4s, v1.4s, v9.4s") \
76 __ASM_EMIT(OP " v2.4s, v2.4s, v10.4s") \
77 __ASM_EMIT(OP " v3.4s, v3.4s, v11.4s") \
78 __ASM_EMIT("sub %[count], %[count], #16") \
79 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
80 __ASM_EMIT("stp q2, q3, [%[" DST "], #0x20]") \
81 __ASM_EMIT(INCA("add %[" A "], %[" A "], #0x40")) \
82 __ASM_EMIT("add %[" B "], %[" B "], #0x40") \
83 __ASM_EMIT("add %[" DST "], %[" DST "], #0x40") \
84 /* 8x block */ \
85 __ASM_EMIT("4:") \
86 __ASM_EMIT("adds %[count], %[count], #8") /* 16 - 8 */ \
87 __ASM_EMIT("b.lt 6f") \
88 __ASM_EMIT("ldp q8, q9, [%[" B "], #0x00]") \
89 __ASM_EMIT("ldp q0, q1, [%[" A "], #0x00]") \
90 __ASM_EMIT(OP " v0.4s, v0.4s, v8.4s") \
91 __ASM_EMIT(OP " v1.4s, v1.4s, v9.4s") \
92 __ASM_EMIT("sub %[count], %[count], #8") \
93 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
94 __ASM_EMIT(INCA("add %[" A "], %[" A "], #0x20")) \
95 __ASM_EMIT("add %[" B "], %[" B "], #0x20") \
96 __ASM_EMIT("add %[" DST "], %[" DST "], #0x20") \
97 /* 4x block */ \
98 __ASM_EMIT("6:") \
99 __ASM_EMIT("adds %[count], %[count], #4") /* 8 - 4 */ \
100 __ASM_EMIT("b.lt 8f") \
101 __ASM_EMIT("ldr q8, [%[" B "], #0x00]") \
102 __ASM_EMIT("ldr q0, [%[" A "], #0x00]") \
103 __ASM_EMIT(OP " v0.4s, v0.4s, v8.4s") \
104 __ASM_EMIT("sub %[count], %[count], #4") \
105 __ASM_EMIT("str q0, [%[" DST "], #0x00]") \
106 __ASM_EMIT(INCA("add %[" A "], %[" A "], #0x10")) \
107 __ASM_EMIT("add %[" B "], %[" B "], #0x10") \
108 __ASM_EMIT("add %[" DST "], %[" DST "], #0x10") \
109 /* 1x block */ \
110 __ASM_EMIT("8:") \
111 __ASM_EMIT("adds %[count], %[count], #3") /* 4 - 3 */ \
112 __ASM_EMIT("b.lt 10f") \
113 __ASM_EMIT("9:") \
114 __ASM_EMIT("ld1r {v8.4s}, [%[" B "]]") \
115 __ASM_EMIT("ld1r {v0.4s}, [%[" A "]]") \
116 __ASM_EMIT(OP " v0.4s, v0.4s, v8.4s") \
117 __ASM_EMIT("subs %[count], %[count], #1") \
118 __ASM_EMIT("st1 {v0.s}[0], [%[" DST "]]") \
119 __ASM_EMIT(INCA("add %[" A "], %[" A "], #0x04")) \
120 __ASM_EMIT("add %[" B "], %[" B "], #0x04") \
121 __ASM_EMIT("add %[" DST "], %[" DST "], #0x04") \
122 __ASM_EMIT("b.ge 9b") \
123 __ASM_EMIT("10:")
124
125 void pmin2(float *dst, const float *src, size_t count)
126 {
127 ARCH_AARCH64_ASM
128 (
129 MINMAX_CORE("dst", "dst", "src", "fmin", INC_OFF)
130 : [dst] "+r" (dst), [src] "+r" (src),
131 [count] "+r" (count)
132 :
133 : "cc", "memory",
134 "v0", "v1", "v2", "v3",
135 "v4", "v5", "v6", "v7",
136 "v8", "v9", "v10", "v11",
137 "v12", "v13", "v14", "v15"
138 );
139 }
140
141 void pmin3(float *dst, const float *a, const float *b, size_t count)
142 {
143 ARCH_AARCH64_ASM
144 (
145 MINMAX_CORE("dst", "a", "b", "fmin", INC_ON)
146 : [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b),
147 [count] "+r" (count)
148 :
149 : "cc", "memory",
150 "v0", "v1", "v2", "v3",
151 "v4", "v5", "v6", "v7",
152 "v8", "v9", "v10", "v11",
153 "v12", "v13", "v14", "v15"
154 );
155 }
156
157 void pmax2(float *dst, const float *src, size_t count)
158 {
159 ARCH_AARCH64_ASM
160 (
161 MINMAX_CORE("dst", "dst", "src", "fmax", INC_OFF)
162 : [dst] "+r" (dst), [src] "+r" (src),
163 [count] "+r" (count)
164 :
165 : "cc", "memory",
166 "v0", "v1", "v2", "v3",
167 "v4", "v5", "v6", "v7",
168 "v8", "v9", "v10", "v11",
169 "v12", "v13", "v14", "v15"
170 );
171 }
172
173 void pmax3(float *dst, const float *a, const float *b, size_t count)
174 {
175 ARCH_AARCH64_ASM
176 (
177 MINMAX_CORE("dst", "a", "b", "fmax", INC_ON)
178 : [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b),
179 [count] "+r" (count)
180 :
181 : "cc", "memory",
182 "v0", "v1", "v2", "v3",
183 "v4", "v5", "v6", "v7",
184 "v8", "v9", "v10", "v11",
185 "v12", "v13", "v14", "v15"
186 );
187 }
188
189 #undef MINMAX_CORE
190
191 #define SIGN_MINMAX_CORE(DST, A, B, OP, INCA) \
192 __ASM_EMIT("subs %[count], %[count], #16") \
193 __ASM_EMIT("b.lo 2f") \
194 /* 16x blocks */ \
195 __ASM_EMIT("1:") \
196 __ASM_EMIT("ldp q4, q5, [%[" B "], #0x00]") \
197 __ASM_EMIT("ldp q6, q7, [%[" B "], #0x20]") \
198 __ASM_EMIT("ldp q0, q1, [%[" A "], #0x00]") \
199 __ASM_EMIT("ldp q2, q3, [%[" A "], #0x20]") \
200 __ASM_EMIT("facgt v8.4s, v4.4s, v0.4s") \
201 __ASM_EMIT("facgt v9.4s, v5.4s, v1.4s") \
202 __ASM_EMIT("facgt v10.4s, v6.4s, v2.4s") \
203 __ASM_EMIT("facgt v11.4s, v7.4s, v3.4s") \
204 __ASM_EMIT(OP " v0.16b, v4.16b, v8.16b") \
205 __ASM_EMIT(OP " v1.16b, v5.16b, v9.16b") \
206 __ASM_EMIT(OP " v2.16b, v6.16b, v10.16b") \
207 __ASM_EMIT(OP " v3.16b, v7.16b, v11.16b") \
208 __ASM_EMIT("subs %[count], %[count], #16") \
209 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
210 __ASM_EMIT("stp q2, q3, [%[" DST "], #0x20]") \
211 __ASM_EMIT(INCA("add %[" A "], %[" A "], #0x40")) \
212 __ASM_EMIT("add %[" B "], %[" B "], #0x40") \
213 __ASM_EMIT("add %[" DST "], %[" DST "], #0x40") \
214 __ASM_EMIT("b.hs 1b") \
215 /* 8x block */ \
216 __ASM_EMIT("2:") \
217 __ASM_EMIT("adds %[count], %[count], #8") /* 16 - 8 */ \
218 __ASM_EMIT("b.lt 4f") \
219 __ASM_EMIT("ldp q4, q5, [%[" B "], #0x00]") \
220 __ASM_EMIT("ldp q0, q1, [%[" A "], #0x00]") \
221 __ASM_EMIT("facgt v8.4s, v4.4s, v0.4s") \
222 __ASM_EMIT("facgt v9.4s, v5.4s, v1.4s") \
223 __ASM_EMIT(OP " v0.16b, v4.16b, v8.16b") \
224 __ASM_EMIT(OP " v1.16b, v5.16b, v9.16b") \
225 __ASM_EMIT("sub %[count], %[count], #8") \
226 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
227 __ASM_EMIT(INCA("add %[" A "], %[" A "], #0x20")) \
228 __ASM_EMIT("add %[" B "], %[" B "], #0x20") \
229 __ASM_EMIT("add %[" DST "], %[" DST "], #0x20") \
230 /* 4x block */ \
231 __ASM_EMIT("4:") \
232 __ASM_EMIT("adds %[count], %[count], #4") /* 8 - 4 */ \
233 __ASM_EMIT("b.lt 6f") \
234 __ASM_EMIT("ldr q4, [%[" B "], #0x00]") \
235 __ASM_EMIT("ldr q0, [%[" A "], #0x00]") \
236 __ASM_EMIT("facgt v8.4s, v4.4s, v0.4s") \
237 __ASM_EMIT(OP " v0.16b, v4.16b, v8.16b") \
238 __ASM_EMIT("sub %[count], %[count], #4") \
239 __ASM_EMIT("str q0, [%[" DST "], #0x00]") \
240 __ASM_EMIT(INCA("add %[" A "], %[" A "], #0x10")) \
241 __ASM_EMIT("add %[" B "], %[" B "], #0x10") \
242 __ASM_EMIT("add %[" DST "], %[" DST "], #0x10") \
243 /* 1x block */ \
244 __ASM_EMIT("6:") \
245 __ASM_EMIT("adds %[count], %[count], #3") /* 4 - 3 */ \
246 __ASM_EMIT("b.lt 8f") \
247 __ASM_EMIT("7:") \
248 __ASM_EMIT("ld1r {v4.4s}, [%[" B "]]") \
249 __ASM_EMIT("ld1r {v0.4s}, [%[" A "]]") \
250 __ASM_EMIT("facgt v8.4s, v4.4s, v0.4s") \
251 __ASM_EMIT(OP " v0.16b, v4.16b, v8.16b") \
252 __ASM_EMIT("subs %[count], %[count], #1") \
253 __ASM_EMIT("st1 {v0.s}[0], [%[" DST "]]") \
254 __ASM_EMIT(INCA("add %[" A "], %[" A "], #0x04")) \
255 __ASM_EMIT("add %[" B "], %[" B "], #0x04") \
256 __ASM_EMIT("add %[" DST "], %[" DST "], #0x04") \
257 __ASM_EMIT("b.ge 7b") \
258 __ASM_EMIT("8:")
259
260 void psmin2(float *dst, const float *src, size_t count)
261 {
262 ARCH_AARCH64_ASM
263 (
264 SIGN_MINMAX_CORE("dst", "dst", "src", "bif", INC_OFF)
265 : [dst] "+r" (dst), [src] "+r" (src),
266 [count] "+r" (count)
267 :
268 : "cc", "memory",
269 "v0", "v1", "v2", "v3",
270 "v4", "v5", "v6", "v7",
271 "v8", "v9", "v10", "v11",
272 "v12", "v13", "v14", "v15"
273 );
274 }
275
276 void psmin3(float *dst, const float *a, const float *b, size_t count)
277 {
278 ARCH_AARCH64_ASM
279 (
280 SIGN_MINMAX_CORE("dst", "a", "b", "bif", INC_ON)
281 : [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b),
282 [count] "+r" (count)
283 :
284 : "cc", "memory",
285 "v0", "v1", "v2", "v3",
286 "v4", "v5", "v6", "v7",
287 "v8", "v9", "v10", "v11",
288 "v12", "v13", "v14", "v15"
289 );
290 }
291
292 void psmax2(float *dst, const float *src, size_t count)
293 {
294 ARCH_AARCH64_ASM
295 (
296 SIGN_MINMAX_CORE("dst", "dst", "src", "bit", INC_OFF)
297 : [dst] "+r" (dst), [src] "+r" (src),
298 [count] "+r" (count)
299 :
300 : "cc", "memory",
301 "v0", "v1", "v2", "v3",
302 "v4", "v5", "v6", "v7",
303 "v8", "v9", "v10", "v11",
304 "v12", "v13", "v14", "v15"
305 );
306 }
307
308 void psmax3(float *dst, const float *a, const float *b, size_t count)
309 {
310 ARCH_AARCH64_ASM
311 (
312 SIGN_MINMAX_CORE("dst", "a", "b", "bit", INC_ON)
313 : [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b),
314 [count] "+r" (count)
315 :
316 : "cc", "memory",
317 "v0", "v1", "v2", "v3",
318 "v4", "v5", "v6", "v7",
319 "v8", "v9", "v10", "v11",
320 "v12", "v13", "v14", "v15"
321 );
322 }
323
324 #undef SIGN_MINMAX_CORE
325
326 #define ABS_MINMAX_CORE(DST, A, B, OP, INCA) \
327 __ASM_EMIT("subs %[count], %[count], #32") \
328 __ASM_EMIT("b.lo 2f") \
329 /* 32x blocks */ \
330 __ASM_EMIT("1:") \
331 __ASM_EMIT("ldp q8, q9, [%[" B "], #0x00]") \
332 __ASM_EMIT("ldp q10, q11, [%[" B "], #0x20]") \
333 __ASM_EMIT("ldp q12, q13, [%[" B "], #0x40]") \
334 __ASM_EMIT("ldp q14, q15, [%[" B "], #0x60]") \
335 __ASM_EMIT("ldp q0, q1, [%[" A "], #0x00]") \
336 __ASM_EMIT("ldp q2, q3, [%[" A "], #0x20]") \
337 __ASM_EMIT("ldp q4, q5, [%[" A "], #0x40]") \
338 __ASM_EMIT("ldp q6, q7, [%[" A "], #0x60]") \
339 __ASM_EMIT("fabs v0.4s, v0.4s") \
340 __ASM_EMIT("fabs v1.4s, v1.4s") \
341 __ASM_EMIT("fabs v2.4s, v2.4s") \
342 __ASM_EMIT("fabs v3.4s, v3.4s") \
343 __ASM_EMIT("fabs v4.4s, v4.4s") \
344 __ASM_EMIT("fabs v5.4s, v5.4s") \
345 __ASM_EMIT("fabs v6.4s, v6.4s") \
346 __ASM_EMIT("fabs v7.4s, v7.4s") \
347 __ASM_EMIT("fabs v8.4s, v8.4s") \
348 __ASM_EMIT("fabs v9.4s, v9.4s") \
349 __ASM_EMIT("fabs v10.4s, v10.4s") \
350 __ASM_EMIT("fabs v11.4s, v11.4s") \
351 __ASM_EMIT("fabs v12.4s, v12.4s") \
352 __ASM_EMIT("fabs v13.4s, v13.4s") \
353 __ASM_EMIT("fabs v14.4s, v14.4s") \
354 __ASM_EMIT("fabs v15.4s, v15.4s") \
355 __ASM_EMIT(OP " v0.4s, v0.4s, v8.4s") \
356 __ASM_EMIT(OP " v1.4s, v1.4s, v9.4s") \
357 __ASM_EMIT(OP " v2.4s, v2.4s, v10.4s") \
358 __ASM_EMIT(OP " v3.4s, v3.4s, v11.4s") \
359 __ASM_EMIT(OP " v4.4s, v4.4s, v12.4s") \
360 __ASM_EMIT(OP " v5.4s, v5.4s, v13.4s") \
361 __ASM_EMIT(OP " v6.4s, v6.4s, v14.4s") \
362 __ASM_EMIT(OP " v7.4s, v7.4s, v15.4s") \
363 __ASM_EMIT("subs %[count], %[count], #32") \
364 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
365 __ASM_EMIT("stp q2, q3, [%[" DST "], #0x20]") \
366 __ASM_EMIT("stp q4, q5, [%[" DST "], #0x40]") \
367 __ASM_EMIT("stp q6, q7, [%[" DST "], #0x60]") \
368 __ASM_EMIT(INCA("add %[" A "], %[" A "], #0x80")) \
369 __ASM_EMIT("add %[" B "], %[" B "], #0x80") \
370 __ASM_EMIT("add %[" DST "], %[" DST "], #0x80") \
371 __ASM_EMIT("b.hs 1b") \
372 /* 16x block */ \
373 __ASM_EMIT("2:") \
374 __ASM_EMIT("adds %[count], %[count], #16") /* 32 - 16 */ \
375 __ASM_EMIT("b.lt 4f") \
376 __ASM_EMIT("ldp q8, q9, [%[" B "], #0x00]") \
377 __ASM_EMIT("ldp q10, q11, [%[" B "], #0x20]") \
378 __ASM_EMIT("ldp q0, q1, [%[" A "], #0x00]") \
379 __ASM_EMIT("ldp q2, q3, [%[" A "], #0x20]") \
380 __ASM_EMIT("fabs v0.4s, v0.4s") \
381 __ASM_EMIT("fabs v1.4s, v1.4s") \
382 __ASM_EMIT("fabs v2.4s, v2.4s") \
383 __ASM_EMIT("fabs v3.4s, v3.4s") \
384 __ASM_EMIT("fabs v8.4s, v8.4s") \
385 __ASM_EMIT("fabs v9.4s, v9.4s") \
386 __ASM_EMIT("fabs v10.4s, v10.4s") \
387 __ASM_EMIT("fabs v11.4s, v11.4s") \
388 __ASM_EMIT(OP " v0.4s, v0.4s, v8.4s") \
389 __ASM_EMIT(OP " v1.4s, v1.4s, v9.4s") \
390 __ASM_EMIT(OP " v2.4s, v2.4s, v10.4s") \
391 __ASM_EMIT(OP " v3.4s, v3.4s, v11.4s") \
392 __ASM_EMIT("sub %[count], %[count], #16") \
393 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
394 __ASM_EMIT("stp q2, q3, [%[" DST "], #0x20]") \
395 __ASM_EMIT(INCA("add %[" A "], %[" A "], #0x40")) \
396 __ASM_EMIT("add %[" B "], %[" B "], #0x40") \
397 __ASM_EMIT("add %[" DST "], %[" DST "], #0x40") \
398 /* 8x block */ \
399 __ASM_EMIT("4:") \
400 __ASM_EMIT("adds %[count], %[count], #8") /* 16 - 8 */ \
401 __ASM_EMIT("b.lt 6f") \
402 __ASM_EMIT("ldp q8, q9, [%[" B "], #0x00]") \
403 __ASM_EMIT("ldp q0, q1, [%[" A "], #0x00]") \
404 __ASM_EMIT("fabs v0.4s, v0.4s") \
405 __ASM_EMIT("fabs v1.4s, v1.4s") \
406 __ASM_EMIT("fabs v8.4s, v8.4s") \
407 __ASM_EMIT("fabs v9.4s, v9.4s") \
408 __ASM_EMIT(OP " v0.4s, v0.4s, v8.4s") \
409 __ASM_EMIT(OP " v1.4s, v1.4s, v9.4s") \
410 __ASM_EMIT("sub %[count], %[count], #8") \
411 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
412 __ASM_EMIT(INCA("add %[" A "], %[" A "], #0x20")) \
413 __ASM_EMIT("add %[" B "], %[" B "], #0x20") \
414 __ASM_EMIT("add %[" DST "], %[" DST "], #0x20") \
415 /* 4x block */ \
416 __ASM_EMIT("6:") \
417 __ASM_EMIT("adds %[count], %[count], #4") /* 8 - 4 */ \
418 __ASM_EMIT("b.lt 8f") \
419 __ASM_EMIT("ldr q8, [%[" B "], #0x00]") \
420 __ASM_EMIT("ldr q0, [%[" A "], #0x00]") \
421 __ASM_EMIT("fabs v0.4s, v0.4s") \
422 __ASM_EMIT("fabs v8.4s, v8.4s") \
423 __ASM_EMIT(OP " v0.4s, v0.4s, v8.4s") \
424 __ASM_EMIT("sub %[count], %[count], #4") \
425 __ASM_EMIT("str q0, [%[" DST "], #0x00]") \
426 __ASM_EMIT(INCA("add %[" A "], %[" A "], #0x10")) \
427 __ASM_EMIT("add %[" B "], %[" B "], #0x10") \
428 __ASM_EMIT("add %[" DST "], %[" DST "], #0x10") \
429 /* 1x block */ \
430 __ASM_EMIT("8:") \
431 __ASM_EMIT("adds %[count], %[count], #3") /* 4 - 3 */ \
432 __ASM_EMIT("b.lt 10f") \
433 __ASM_EMIT("9:") \
434 __ASM_EMIT("ld1r {v8.4s}, [%[" B "]]") \
435 __ASM_EMIT("ld1r {v0.4s}, [%[" A "]]") \
436 __ASM_EMIT("fabs v0.4s, v0.4s") \
437 __ASM_EMIT("fabs v8.4s, v8.4s") \
438 __ASM_EMIT(OP " v0.4s, v0.4s, v8.4s") \
439 __ASM_EMIT("subs %[count], %[count], #1") \
440 __ASM_EMIT("st1 {v0.s}[0], [%[" DST "]]") \
441 __ASM_EMIT(INCA("add %[" A "], %[" A "], #0x04")) \
442 __ASM_EMIT("add %[" B "], %[" B "], #0x04") \
443 __ASM_EMIT("add %[" DST "], %[" DST "], #0x04") \
444 __ASM_EMIT("b.ge 9b") \
445 __ASM_EMIT("10:")
446
447 void pamin2(float *dst, const float *src, size_t count)
448 {
449 ARCH_AARCH64_ASM
450 (
451 ABS_MINMAX_CORE("dst", "dst", "src", "fmin", INC_OFF)
452 : [dst] "+r" (dst), [src] "+r" (src),
453 [count] "+r" (count)
454 :
455 : "cc", "memory",
456 "v0", "v1", "v2", "v3",
457 "v4", "v5", "v6", "v7",
458 "v8", "v9", "v10", "v11",
459 "v12", "v13", "v14", "v15"
460 );
461 }
462
463 void pamin3(float *dst, const float *a, const float *b, size_t count)
464 {
465 ARCH_AARCH64_ASM
466 (
467 ABS_MINMAX_CORE("dst", "a", "b", "fmin", INC_ON)
468 : [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b),
469 [count] "+r" (count)
470 :
471 : "cc", "memory",
472 "v0", "v1", "v2", "v3",
473 "v4", "v5", "v6", "v7",
474 "v8", "v9", "v10", "v11",
475 "v12", "v13", "v14", "v15"
476 );
477 }
478
479 void pamax2(float *dst, const float *src, size_t count)
480 {
481 ARCH_AARCH64_ASM
482 (
483 ABS_MINMAX_CORE("dst", "dst", "src", "fmax", INC_OFF)
484 : [dst] "+r" (dst), [src] "+r" (src),
485 [count] "+r" (count)
486 :
487 : "cc", "memory",
488 "v0", "v1", "v2", "v3",
489 "v4", "v5", "v6", "v7",
490 "v8", "v9", "v10", "v11",
491 "v12", "v13", "v14", "v15"
492 );
493 }
494
495 void pamax3(float *dst, const float *a, const float *b, size_t count)
496 {
497 ARCH_AARCH64_ASM
498 (
499 ABS_MINMAX_CORE("dst", "a", "b", "fmax", INC_ON)
500 : [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b),
501 [count] "+r" (count)
502 :
503 : "cc", "memory",
504 "v0", "v1", "v2", "v3",
505 "v4", "v5", "v6", "v7",
506 "v8", "v9", "v10", "v11",
507 "v12", "v13", "v14", "v15"
508 );
509 }
510
511 #undef ABS_MINMAX_CORE
512
513 #undef INC_ON
514 #undef INC_OFF
515 }
516 }
517
518
519
520 #endif /* PRIVATE_DSP_ARCH_AARCH64_ASIMD_PMATH_MINMAX_H_ */