Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/aarch64/asimd/pmath/abs_vv.h

raw
1 /*
2 * Copyright (C) 2024 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2024 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 31 мар. 2020 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_AARCH64_ASIMD_PMATH_ABS_VV_H_
23 #define PRIVATE_DSP_ARCH_AARCH64_ASIMD_PMATH_ABS_VV_H_
24
25 #ifndef PRIVATE_DSP_ARCH_AARCH64_ASIMD_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_AARCH64_ASIMD_IMPL */
28
29 namespace lsp
30 {
31 namespace asimd
32 {
33 void abs1(float *dst, size_t count)
34 {
35 ARCH_AARCH64_ASM
36 (
37 /* 32x block */
38 __ASM_EMIT("subs %[count], %[count], #32")
39 __ASM_EMIT("b.lt 2f")
40 __ASM_EMIT("1:")
41 __ASM_EMIT("ldp q0, q1, [%[dst], #0x00]")
42 __ASM_EMIT("ldp q2, q3, [%[dst], #0x20]")
43 __ASM_EMIT("ldp q4, q5, [%[dst], #0x40]")
44 __ASM_EMIT("ldp q6, q7, [%[dst], #0x60]")
45 __ASM_EMIT("fabs v0.4s, v0.4s")
46 __ASM_EMIT("fabs v1.4s, v1.4s")
47 __ASM_EMIT("fabs v2.4s, v2.4s")
48 __ASM_EMIT("fabs v3.4s, v3.4s")
49 __ASM_EMIT("fabs v4.4s, v4.4s")
50 __ASM_EMIT("fabs v5.4s, v5.4s")
51 __ASM_EMIT("fabs v6.4s, v6.4s")
52 __ASM_EMIT("fabs v7.4s, v7.4s")
53 __ASM_EMIT("fabs v8.4s, v8.4s")
54 __ASM_EMIT("stp q0, q1, [%[dst], #0x00]")
55 __ASM_EMIT("stp q2, q3, [%[dst], #0x20]")
56 __ASM_EMIT("stp q4, q5, [%[dst], #0x40]")
57 __ASM_EMIT("stp q6, q7, [%[dst], #0x60]")
58
59 __ASM_EMIT("subs %[count], %[count], #32")
60 __ASM_EMIT("add %[dst], %[dst], #0x80")
61 __ASM_EMIT("b.hs 1b")
62
63 /* 16x block */
64 __ASM_EMIT("2:")
65 __ASM_EMIT("adds %[count], %[count], #16") // 32-16
66 __ASM_EMIT("b.lt 4f")
67
68 __ASM_EMIT("ldp q0, q1, [%[dst], #0x00]")
69 __ASM_EMIT("ldp q2, q3, [%[dst], #0x20]")
70 __ASM_EMIT("fabs v0.4s, v0.4s")
71 __ASM_EMIT("fabs v1.4s, v1.4s")
72 __ASM_EMIT("fabs v2.4s, v2.4s")
73 __ASM_EMIT("fabs v3.4s, v3.4s")
74 __ASM_EMIT("stp q0, q1, [%[dst], #0x00]")
75 __ASM_EMIT("stp q2, q3, [%[dst], #0x20]")
76
77 __ASM_EMIT("sub %[count], %[count], #16")
78 __ASM_EMIT("add %[dst], %[dst], #0x40")
79
80 /* 8x block */
81 __ASM_EMIT("4:")
82 __ASM_EMIT("adds %[count], %[count], #8") // 16-8
83 __ASM_EMIT("b.lt 6f")
84 __ASM_EMIT("ldp q0, q1, [%[dst], #0x00]")
85 __ASM_EMIT("fabs v0.4s, v0.4s")
86 __ASM_EMIT("fabs v1.4s, v1.4s")
87 __ASM_EMIT("stp q0, q1, [%[dst], #0x00]")
88 __ASM_EMIT("sub %[count], %[count], #8")
89 __ASM_EMIT("add %[dst], %[dst], #0x20")
90
91 /* 4x block */
92 __ASM_EMIT("6:")
93 __ASM_EMIT("adds %[count], %[count], #4") // 8-4
94 __ASM_EMIT("b.lt 8f")
95 __ASM_EMIT("ldr q0, [%[dst], #0x000]")
96 __ASM_EMIT("fabs v0.4s, v0.4s")
97 __ASM_EMIT("str q0, [%[dst], #0x000]")
98 __ASM_EMIT("sub %[count], %[count], #4")
99 __ASM_EMIT("add %[dst], %[dst], #0x10")
100
101 /* 1x blocks */
102 __ASM_EMIT("8:")
103 __ASM_EMIT("adds %[count], %[count], #3") // 4-1
104 __ASM_EMIT("b.lt 10f")
105 __ASM_EMIT("13:")
106 __ASM_EMIT("ld1r {v0.4s}, [%[dst]]")
107 __ASM_EMIT("fabs v0.4s, v0.4s")
108 __ASM_EMIT("subs %[count], %[count], #1")
109 __ASM_EMIT("st1 {v0.s}[0], [%[dst]]")
110 __ASM_EMIT("add %[dst], %[dst], #0x04")
111 __ASM_EMIT("b.ge 13b")
112
113 /* End of copy */
114 __ASM_EMIT("10:")
115
116 : [dst] "+r" (dst),
117 [count] "+r" (count)
118 :
119 : "cc", "memory",
120 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7"
121 );
122 }
123
124 void abs2(float *dst, const float *src, size_t count)
125 {
126 ARCH_AARCH64_ASM
127 (
128 /* 32x block */
129 __ASM_EMIT("subs %[count], %[count], #32")
130 __ASM_EMIT("b.lt 2f")
131 __ASM_EMIT("1:")
132 __ASM_EMIT("ldp q0, q1, [%[src], #0x00]")
133 __ASM_EMIT("ldp q2, q3, [%[src], #0x20]")
134 __ASM_EMIT("ldp q4, q5, [%[src], #0x40]")
135 __ASM_EMIT("ldp q6, q7, [%[src], #0x60]")
136 __ASM_EMIT("fabs v0.4s, v0.4s")
137 __ASM_EMIT("fabs v1.4s, v1.4s")
138 __ASM_EMIT("fabs v2.4s, v2.4s")
139 __ASM_EMIT("fabs v3.4s, v3.4s")
140 __ASM_EMIT("fabs v4.4s, v4.4s")
141 __ASM_EMIT("fabs v5.4s, v5.4s")
142 __ASM_EMIT("fabs v6.4s, v6.4s")
143 __ASM_EMIT("fabs v7.4s, v7.4s")
144 __ASM_EMIT("fabs v8.4s, v8.4s")
145 __ASM_EMIT("stp q0, q1, [%[dst], #0x00]")
146 __ASM_EMIT("stp q2, q3, [%[dst], #0x20]")
147 __ASM_EMIT("stp q4, q5, [%[dst], #0x40]")
148 __ASM_EMIT("stp q6, q7, [%[dst], #0x60]")
149
150 __ASM_EMIT("subs %[count], %[count], #32")
151 __ASM_EMIT("add %[src], %[src], #0x80")
152 __ASM_EMIT("add %[dst], %[dst], #0x80")
153 __ASM_EMIT("b.hs 1b")
154
155 /* 16x block */
156 __ASM_EMIT("2:")
157 __ASM_EMIT("adds %[count], %[count], #16") // 32-16
158 __ASM_EMIT("b.lt 4f")
159
160 __ASM_EMIT("ldp q0, q1, [%[src], #0x00]")
161 __ASM_EMIT("ldp q2, q3, [%[src], #0x20]")
162 __ASM_EMIT("fabs v0.4s, v0.4s")
163 __ASM_EMIT("fabs v1.4s, v1.4s")
164 __ASM_EMIT("fabs v2.4s, v2.4s")
165 __ASM_EMIT("fabs v3.4s, v3.4s")
166 __ASM_EMIT("stp q0, q1, [%[dst], #0x00]")
167 __ASM_EMIT("stp q2, q3, [%[dst], #0x20]")
168
169 __ASM_EMIT("sub %[count], %[count], #16")
170 __ASM_EMIT("add %[src], %[src], #0x40")
171 __ASM_EMIT("add %[dst], %[dst], #0x40")
172
173 /* 8x block */
174 __ASM_EMIT("4:")
175 __ASM_EMIT("adds %[count], %[count], #8") // 16-8
176 __ASM_EMIT("b.lt 6f")
177 __ASM_EMIT("ldp q0, q1, [%[src], #0x00]")
178 __ASM_EMIT("fabs v0.4s, v0.4s")
179 __ASM_EMIT("fabs v1.4s, v1.4s")
180 __ASM_EMIT("stp q0, q1, [%[dst], #0x00]")
181 __ASM_EMIT("sub %[count], %[count], #8")
182 __ASM_EMIT("add %[src], %[src], #0x20")
183 __ASM_EMIT("add %[dst], %[dst], #0x20")
184
185 /* 4x block */
186 __ASM_EMIT("6:")
187 __ASM_EMIT("adds %[count], %[count], #4") // 8-4
188 __ASM_EMIT("b.lt 8f")
189 __ASM_EMIT("ldr q0, [%[src], #0x000]")
190 __ASM_EMIT("fabs v0.4s, v0.4s")
191 __ASM_EMIT("str q0, [%[dst], #0x000]")
192 __ASM_EMIT("sub %[count], %[count], #4")
193 __ASM_EMIT("add %[src], %[src], #0x10")
194 __ASM_EMIT("add %[dst], %[dst], #0x10")
195
196 /* 1x blocks */
197 __ASM_EMIT("8:")
198 __ASM_EMIT("adds %[count], %[count], #3") // 4-1
199 __ASM_EMIT("b.lt 10f")
200 __ASM_EMIT("13:")
201 __ASM_EMIT("ld1r {v0.4s}, [%[src]]")
202 __ASM_EMIT("fabs v0.4s, v0.4s")
203 __ASM_EMIT("subs %[count], %[count], #1")
204 __ASM_EMIT("st1 {v0.s}[0], [%[dst]]")
205 __ASM_EMIT("add %[src], %[src], #0x04")
206 __ASM_EMIT("add %[dst], %[dst], #0x04")
207 __ASM_EMIT("b.ge 13b")
208
209 /* End of copy */
210 __ASM_EMIT("10:")
211
212 : [src] "+r" (src), [dst] "+r"(dst),
213 [count] "+r" (count)
214 :
215 : "cc", "memory",
216 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7"
217 );
218 }
219
220 #define OP_DSEL(a, b) a
221 #define OP_RSEL(a, b) b
222
223 #define OP_ABS_VV2_CORE(DST, SRC, OP, SEL) \
224 __ASM_EMIT("subs %[count], %[count], #32") \
225 __ASM_EMIT("blo 2f") \
226 /* 32x blocks */ \
227 __ASM_EMIT("1:") \
228 __ASM_EMIT("ldp q0, q1, [%[" SEL(DST, SRC) "], #0x00]") \
229 __ASM_EMIT("ldp q2, q3, [%[" SEL(DST, SRC) "], #0x20]") \
230 __ASM_EMIT("ldp q4, q5, [%[" SEL(DST, SRC) "], #0x40]") \
231 __ASM_EMIT("ldp q6, q7, [%[" SEL(DST, SRC) "], #0x60]") \
232 __ASM_EMIT("ldp q16, q17, [%[" SEL(SRC, DST) "], #0x00]") \
233 __ASM_EMIT("ldp q18, q19, [%[" SEL(SRC, DST) "], #0x20]") \
234 __ASM_EMIT("ldp q20, q21, [%[" SEL(SRC, DST) "], #0x40]") \
235 __ASM_EMIT("ldp q22, q23, [%[" SEL(SRC, DST) "], #0x60]") \
236 __ASM_EMIT("fabs " SEL("v16.4s, v16.4s", "v0.4s, v0.4s")) \
237 __ASM_EMIT("fabs " SEL("v17.4s, v17.4s", "v1.4s, v1.4s")) \
238 __ASM_EMIT("fabs " SEL("v18.4s, v18.4s", "v2.4s, v2.4s")) \
239 __ASM_EMIT("fabs " SEL("v19.4s, v19.4s", "v3.4s, v3.4s")) \
240 __ASM_EMIT("fabs " SEL("v20.4s, v20.4s", "v4.4s, v4.4s")) \
241 __ASM_EMIT("fabs " SEL("v21.4s, v21.4s", "v5.4s, v5.4s")) \
242 __ASM_EMIT("fabs " SEL("v22.4s, v22.4s", "v6.4s, v6.4s")) \
243 __ASM_EMIT("fabs " SEL("v23.4s, v23.4s", "v7.4s, v7.4s")) \
244 __ASM_EMIT(OP " v0.4s, v0.4s, v16.4s") \
245 __ASM_EMIT(OP " v1.4s, v1.4s, v17.4s") \
246 __ASM_EMIT(OP " v2.4s, v2.4s, v18.4s") \
247 __ASM_EMIT(OP " v3.4s, v3.4s, v19.4s") \
248 __ASM_EMIT(OP " v4.4s, v4.4s, v20.4s") \
249 __ASM_EMIT(OP " v5.4s, v5.4s, v21.4s") \
250 __ASM_EMIT(OP " v6.4s, v6.4s, v22.4s") \
251 __ASM_EMIT(OP " v7.4s, v7.4s, v23.4s") \
252 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
253 __ASM_EMIT("stp q2, q3, [%[" DST "], #0x20]") \
254 __ASM_EMIT("stp q4, q5, [%[" DST "], #0x40]") \
255 __ASM_EMIT("stp q6, q7, [%[" DST "], #0x60]") \
256 __ASM_EMIT("subs %[count], %[count], #32") \
257 __ASM_EMIT("add %[" DST "], %[" DST "], #0x80") \
258 __ASM_EMIT("add %[" SRC "], %[" SRC "], #0x80") \
259 __ASM_EMIT("bhs 1b") \
260 /* 16x block */ \
261 __ASM_EMIT("2:") \
262 __ASM_EMIT("adds %[count], %[count], #16") /* 32 - 16 */ \
263 __ASM_EMIT("blt 4f") \
264 __ASM_EMIT("ldp q0, q1, [%[" SEL(DST, SRC) "], #0x00]") \
265 __ASM_EMIT("ldp q2, q3, [%[" SEL(DST, SRC) "], #0x20]") \
266 __ASM_EMIT("ldp q16, q17, [%[" SEL(SRC, DST) "], #0x00]") \
267 __ASM_EMIT("ldp q18, q19, [%[" SEL(SRC, DST) "], #0x20]") \
268 __ASM_EMIT("fabs " SEL("v16.4s, v16.4s", "v0.4s, v0.4s")) \
269 __ASM_EMIT("fabs " SEL("v17.4s, v17.4s", "v1.4s, v1.4s")) \
270 __ASM_EMIT("fabs " SEL("v18.4s, v18.4s", "v2.4s, v2.4s")) \
271 __ASM_EMIT("fabs " SEL("v19.4s, v19.4s", "v3.4s, v3.4s")) \
272 __ASM_EMIT(OP " v0.4s, v0.4s, v16.4s") \
273 __ASM_EMIT(OP " v1.4s, v1.4s, v17.4s") \
274 __ASM_EMIT(OP " v2.4s, v2.4s, v18.4s") \
275 __ASM_EMIT(OP " v3.4s, v3.4s, v19.4s") \
276 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
277 __ASM_EMIT("stp q2, q3, [%[" DST "], #0x20]") \
278 __ASM_EMIT("sub %[count], %[count], #16") \
279 __ASM_EMIT("add %[" DST "], %[" DST "], #0x40") \
280 __ASM_EMIT("add %[" SRC "], %[" SRC "], #0x40") \
281 /* 8x block */ \
282 __ASM_EMIT("4:") \
283 __ASM_EMIT("adds %[count], %[count], #8") /* 16 - 8 */ \
284 __ASM_EMIT("blt 6f") \
285 __ASM_EMIT("ldp q0, q1, [%[" SEL(DST, SRC) "], #0x00]") \
286 __ASM_EMIT("ldp q2, q3, [%[" SEL(DST, SRC) "], #0x20]") \
287 __ASM_EMIT("ldp q16, q17, [%[" SEL(SRC, DST) "], #0x00]") \
288 __ASM_EMIT("ldp q18, q19, [%[" SEL(SRC, DST) "], #0x20]") \
289 __ASM_EMIT("fabs " SEL("v16.4s, v16.4s", "v0.4s, v0.4s")) \
290 __ASM_EMIT("fabs " SEL("v17.4s, v17.4s", "v1.4s, v1.4s")) \
291 __ASM_EMIT(OP " v0.4s, v0.4s, v16.4s") \
292 __ASM_EMIT(OP " v1.4s, v1.4s, v17.4s") \
293 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
294 __ASM_EMIT("sub %[count], %[count], #8") \
295 __ASM_EMIT("add %[" DST "], %[" DST "], #0x20") \
296 __ASM_EMIT("add %[" SRC "], %[" SRC "], #0x20") \
297 /* 4x block */ \
298 __ASM_EMIT("6:") \
299 __ASM_EMIT("adds %[count], %[count], #4") /* 8 - 4 */ \
300 __ASM_EMIT("blt 8f") \
301 __ASM_EMIT("ldr q0, [%[" SEL(DST, SRC) "], #0x00]") \
302 __ASM_EMIT("ldr q16, [%[" SEL(SRC, DST) "], #0x00]") \
303 __ASM_EMIT("fabs " SEL("v16.4s, v16.4s", "v0.4s, v0.4s")) \
304 __ASM_EMIT(OP " v0.4s, v0.4s, v16.4s") \
305 __ASM_EMIT("str q0, [%[" DST "], #0x00]") \
306 __ASM_EMIT("sub %[count], %[count], #4") \
307 __ASM_EMIT("add %[" DST "], %[" DST "], #0x10") \
308 __ASM_EMIT("add %[" SRC "], %[" SRC "], #0x10") \
309 /* 1x block */ \
310 __ASM_EMIT("8:") \
311 __ASM_EMIT("adds %[count], %[count], #3") /* 4 - 3 */ \
312 __ASM_EMIT("blt 10f") \
313 __ASM_EMIT("9:") \
314 __ASM_EMIT("ld1r {v0.4s}, [%[" SEL(DST, SRC) "]]") \
315 __ASM_EMIT("ld1r {v16.4s}, [%[" SEL(SRC, DST) "]]") \
316 __ASM_EMIT("fabs " SEL("v16.4s, v16.4s", "v0.4s, v0.4s")) \
317 __ASM_EMIT(OP " v0.4s, v0.4s, v16.4s") \
318 __ASM_EMIT("st1 {v0.s}[0], [%[" DST "]]") \
319 __ASM_EMIT("subs %[count], %[count], #1") \
320 __ASM_EMIT("add %[" DST "], %[" DST "], #0x04") \
321 __ASM_EMIT("add %[" SRC "], %[" SRC "], #0x04") \
322 __ASM_EMIT("bge 9b") \
323 __ASM_EMIT("10:")
324
325 void abs_add2(float *dst, const float *src, size_t count)
326 {
327 ARCH_AARCH64_ASM
328 (
329 OP_ABS_VV2_CORE("dst", "src", "fadd", OP_DSEL)
330 : [dst] "+r" (dst), [src] "+r" (src),
331 [count] "+r" (count)
332 :
333 : "cc", "memory",
334 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
335 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"
336 );
337 }
338
339 void abs_sub2(float *dst, const float *src, size_t count)
340 {
341 ARCH_AARCH64_ASM
342 (
343 OP_ABS_VV2_CORE("dst", "src", "fsub", OP_DSEL)
344 : [dst] "+r" (dst), [src] "+r" (src),
345 [count] "+r" (count)
346 :
347 : "cc", "memory",
348 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
349 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"
350 );
351 }
352
353 void abs_rsub2(float *dst, const float *src, size_t count)
354 {
355 ARCH_AARCH64_ASM
356 (
357 OP_ABS_VV2_CORE("dst", "src", "fsub", OP_RSEL)
358 : [dst] "+r" (dst), [src] "+r" (src),
359 [count] "+r" (count)
360 :
361 : "cc", "memory",
362 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
363 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"
364 );
365 }
366
367 void abs_mul2(float *dst, const float *src, size_t count)
368 {
369 ARCH_AARCH64_ASM
370 (
371 OP_ABS_VV2_CORE("dst", "src", "fmul", OP_DSEL)
372 : [dst] "+r" (dst), [src] "+r" (src),
373 [count] "+r" (count)
374 :
375 : "cc", "memory",
376 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
377 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"
378 );
379 }
380
381 void abs_max2(float *dst, const float *src, size_t count)
382 {
383 ARCH_AARCH64_ASM
384 (
385 OP_ABS_VV2_CORE("dst", "src", "fmax", OP_DSEL)
386 : [dst] "+r" (dst), [src] "+r" (src),
387 [count] "+r" (count)
388 :
389 : "cc", "memory",
390 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
391 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"
392 );
393 }
394
395 void abs_min2(float *dst, const float *src, size_t count)
396 {
397 ARCH_AARCH64_ASM
398 (
399 OP_ABS_VV2_CORE("dst", "src", "fmin", OP_DSEL)
400 : [dst] "+r" (dst), [src] "+r" (src),
401 [count] "+r" (count)
402 :
403 : "cc", "memory",
404 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
405 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"
406 );
407 }
408
409 #undef OP_ABS_VV2_CORE
410
411 #define OP_ABS_DIV2_CORE(DST, SRC, SEL) \
412 __ASM_EMIT("subs %[count], %[count], #16") \
413 __ASM_EMIT("blo 2f") \
414 /* 16x blocks */ \
415 __ASM_EMIT("1:") \
416 __ASM_EMIT("ldp q0, q1, [%[" SEL(SRC, DST) "], #0x00]") \
417 __ASM_EMIT("ldp q2, q3, [%[" SEL(SRC, DST) "], #0x20]") \
418 __ASM_EMIT("ldp q4, q5, [%[" SEL(DST, SRC) "], #0x00]") \
419 __ASM_EMIT("ldp q6, q7, [%[" SEL(DST, SRC) "], #0x20]") \
420 __ASM_EMIT("fabs " SEL("v0.4s, v0.4s", "v4.4s, v4.4s")) \
421 __ASM_EMIT("fabs " SEL("v1.4s, v1.4s", "v5.4s, v5.4s")) \
422 __ASM_EMIT("fabs " SEL("v2.4s, v2.4s", "v6.4s, v6.4s")) \
423 __ASM_EMIT("fabs " SEL("v3.4s, v3.4s", "v7.4s, v7.4s")) \
424 __ASM_EMIT("frecpe v16.4s, v0.4s") /* v16 = s2 */ \
425 __ASM_EMIT("frecpe v17.4s, v1.4s") \
426 __ASM_EMIT("frecpe v18.4s, v2.4s") \
427 __ASM_EMIT("frecpe v19.4s, v3.4s") \
428 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2) */ \
429 __ASM_EMIT("frecps v25.4s, v17.4s, v1.4s") \
430 __ASM_EMIT("frecps v26.4s, v18.4s, v2.4s") \
431 __ASM_EMIT("frecps v27.4s, v19.4s, v3.4s") \
432 __ASM_EMIT("fmul v16.4s, v24.4s, v16.4s") /* v16 = s2' = s2 * (2 - R*s2) */ \
433 __ASM_EMIT("fmul v17.4s, v25.4s, v17.4s") \
434 __ASM_EMIT("fmul v18.4s, v26.4s, v18.4s") \
435 __ASM_EMIT("fmul v19.4s, v27.4s, v19.4s") \
436 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2') */ \
437 __ASM_EMIT("frecps v25.4s, v17.4s, v1.4s") \
438 __ASM_EMIT("frecps v26.4s, v18.4s, v2.4s") \
439 __ASM_EMIT("frecps v27.4s, v19.4s, v3.4s") \
440 __ASM_EMIT("fmul v0.4s, v24.4s, v16.4s") /* v0 = s2" = s2' * (2 - R*s2) = 1/s2 */ \
441 __ASM_EMIT("fmul v1.4s, v25.4s, v17.4s") \
442 __ASM_EMIT("fmul v2.4s, v26.4s, v18.4s") \
443 __ASM_EMIT("fmul v3.4s, v27.4s, v19.4s") \
444 __ASM_EMIT("fmul v0.4s, v0.4s, v4.4s") /* s1 / s2 */ \
445 __ASM_EMIT("fmul v1.4s, v1.4s, v5.4s") \
446 __ASM_EMIT("fmul v2.4s, v2.4s, v6.4s") \
447 __ASM_EMIT("fmul v3.4s, v3.4s, v7.4s") \
448 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
449 __ASM_EMIT("stp q2, q3, [%[" DST "], #0x20]") \
450 __ASM_EMIT("subs %[count], %[count], #16") \
451 __ASM_EMIT("add %[" DST "], %[" DST "], #0x40") \
452 __ASM_EMIT("add %[" SRC "], %[" SRC "], #0x40") \
453 __ASM_EMIT("bhs 1b") \
454 /* 8x block */ \
455 __ASM_EMIT("2:") \
456 __ASM_EMIT("adds %[count], %[count], #8") /* 16 - 8 */ \
457 __ASM_EMIT("blt 4f") \
458 __ASM_EMIT("ldp q0, q1, [%[" SEL(SRC, DST) "], #0x00]") \
459 __ASM_EMIT("ldp q4, q5, [%[" SEL(DST, SRC) "], #0x00]") \
460 __ASM_EMIT("fabs " SEL("v0.4s, v0.4s", "v4.4s, v4.4s")) \
461 __ASM_EMIT("fabs " SEL("v1.4s, v1.4s", "v5.4s, v5.4s")) \
462 __ASM_EMIT("frecpe v16.4s, v0.4s") /* v16 = s2 */ \
463 __ASM_EMIT("frecpe v17.4s, v1.4s") \
464 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2) */ \
465 __ASM_EMIT("frecps v25.4s, v17.4s, v1.4s") \
466 __ASM_EMIT("fmul v16.4s, v24.4s, v16.4s") /* v16 = s2' = s2 * (2 - R*s2) */ \
467 __ASM_EMIT("fmul v17.4s, v25.4s, v17.4s") \
468 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2') */ \
469 __ASM_EMIT("frecps v25.4s, v17.4s, v1.4s") \
470 __ASM_EMIT("fmul v0.4s, v24.4s, v16.4s") /* v0 = s2" = s2' * (2 - R*s2) = 1/s2 */ \
471 __ASM_EMIT("fmul v1.4s, v25.4s, v17.4s") \
472 __ASM_EMIT("fmul v0.4s, v0.4s, v4.4s") /* s1 / s2 */ \
473 __ASM_EMIT("fmul v1.4s, v1.4s, v5.4s") \
474 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
475 __ASM_EMIT("sub %[count], %[count], #8") \
476 __ASM_EMIT("add %[" DST "], %[" DST "], #0x20") \
477 __ASM_EMIT("add %[" SRC "], %[" SRC "], #0x20") \
478 /* 4x block */ \
479 __ASM_EMIT("4:") \
480 __ASM_EMIT("adds %[count], %[count], #4") /* 8 - 4 */ \
481 __ASM_EMIT("blt 6f") \
482 __ASM_EMIT("ldr q0, [%[" SEL(SRC, DST) "], #0x00]") \
483 __ASM_EMIT("ldr q4, [%[" SEL(DST, SRC) "], #0x00]") \
484 __ASM_EMIT("fabs " SEL("v0.4s, v0.4s", "v4.4s, v4.4s")) \
485 __ASM_EMIT("frecpe v16.4s, v0.4s") /* v16 = s2 */ \
486 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2) */ \
487 __ASM_EMIT("fmul v16.4s, v24.4s, v16.4s") /* v16 = s2' = s2 * (2 - R*s2) */ \
488 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2') */ \
489 __ASM_EMIT("fmul v0.4s, v24.4s, v16.4s") /* v0 = s2" = s2' * (2 - R*s2) = 1/s2 */ \
490 __ASM_EMIT("fmul v0.4s, v0.4s, v4.4s") /* s1 / s2 */ \
491 __ASM_EMIT("str q0, [%[" DST "], #0x00]") \
492 __ASM_EMIT("sub %[count], %[count], #4") \
493 __ASM_EMIT("add %[" DST "], %[" DST "], #0x10") \
494 __ASM_EMIT("add %[" SRC "], %[" SRC "], #0x10") \
495 /* 1x block */ \
496 __ASM_EMIT("6:") \
497 __ASM_EMIT("adds %[count], %[count], #3") /* 4 - 3 */ \
498 __ASM_EMIT("blt 8f") \
499 __ASM_EMIT("7:") \
500 __ASM_EMIT("ld1r {v0.4s}, [%[" SEL(SRC, DST) "]]") \
501 __ASM_EMIT("ld1r {v4.4s}, [%[" SEL(DST, SRC) "]]") \
502 __ASM_EMIT("fabs " SEL("v0.4s, v0.4s", "v4.4s, v4.4s")) \
503 __ASM_EMIT("frecpe v16.4s, v0.4s") /* v16 = s2 */ \
504 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2) */ \
505 __ASM_EMIT("fmul v16.4s, v24.4s, v16.4s") /* v16 = s2' = s2 * (2 - R*s2) */ \
506 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2') */ \
507 __ASM_EMIT("fmul v0.4s, v24.4s, v16.4s") /* v0 = s2" = s2' * (2 - R*s2) = 1/s2 */ \
508 __ASM_EMIT("fmul v0.4s, v0.4s, v4.4s") /* s1 / s2 */ \
509 __ASM_EMIT("st1 {v0.s}[0], [%[" DST "]]") \
510 __ASM_EMIT("subs %[count], %[count], #1") \
511 __ASM_EMIT("add %[" DST "], %[" DST "], #0x04") \
512 __ASM_EMIT("add %[" SRC "], %[" SRC "], #0x04") \
513 __ASM_EMIT("bge 7b") \
514 __ASM_EMIT("8:")
515
516 void abs_div2(float *dst, const float *src, size_t count)
517 {
518 ARCH_AARCH64_ASM
519 (
520 OP_ABS_DIV2_CORE("dst", "src", OP_DSEL)
521 : [dst] "+r" (dst), [src] "+r" (src),
522 [count] "+r" (count)
523 :
524 : "cc", "memory",
525 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
526 "v16", "v17", "v18", "v19",
527 "v24", "v25", "v26", "v27"
528 );
529 }
530
531 void abs_rdiv2(float *dst, const float *src, size_t count)
532 {
533 ARCH_AARCH64_ASM
534 (
535 OP_ABS_DIV2_CORE("dst", "src", OP_RSEL)
536 : [dst] "+r" (dst), [src] "+r" (src),
537 [count] "+r" (count)
538 :
539 : "cc", "memory",
540 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
541 "v16", "v17", "v18", "v19",
542 "v24", "v25", "v26", "v27"
543 );
544 }
545
546 #undef OP_ABS_DIV2_CORE
547
548 #define OP_ABS_VV3_CORE(DST, SRC1, SRC2, OP, SEL) \
549 __ASM_EMIT("subs %[count], %[count], #32") \
550 __ASM_EMIT("blo 2f") \
551 /* 32x blocks */ \
552 __ASM_EMIT("1:") \
553 __ASM_EMIT("ldp q16, q17, [%[" SRC2 "], #0x00]") \
554 __ASM_EMIT("ldp q18, q19, [%[" SRC2 "], #0x20]") \
555 __ASM_EMIT("ldp q20, q21, [%[" SRC2 "], #0x40]") \
556 __ASM_EMIT("ldp q22, q23, [%[" SRC2 "], #0x60]") \
557 __ASM_EMIT("ldp q0, q1, [%[" SRC1 "], #0x00]") \
558 __ASM_EMIT("ldp q2, q3, [%[" SRC1 "], #0x20]") \
559 __ASM_EMIT("ldp q4, q5, [%[" SRC1 "], #0x40]") \
560 __ASM_EMIT("ldp q6, q7, [%[" SRC1 "], #0x60]") \
561 __ASM_EMIT("fabs v16.4s, v16.4s") \
562 __ASM_EMIT("fabs v17.4s, v17.4s") \
563 __ASM_EMIT("fabs v18.4s, v18.4s") \
564 __ASM_EMIT("fabs v19.4s, v19.4s") \
565 __ASM_EMIT("fabs v20.4s, v20.4s") \
566 __ASM_EMIT("fabs v21.4s, v21.4s") \
567 __ASM_EMIT("fabs v22.4s, v22.4s") \
568 __ASM_EMIT("fabs v23.4s, v23.4s") \
569 __ASM_EMIT(OP " v0.4s, " SEL("v0.4s", "v16.4s") ", " SEL("v16.4s", "v0.4s") ) \
570 __ASM_EMIT(OP " v1.4s, " SEL("v1.4s", "v17.4s") ", " SEL("v17.4s", "v1.4s") ) \
571 __ASM_EMIT(OP " v2.4s, " SEL("v2.4s", "v18.4s") ", " SEL("v18.4s", "v2.4s") ) \
572 __ASM_EMIT(OP " v3.4s, " SEL("v3.4s", "v19.4s") ", " SEL("v19.4s", "v3.4s") ) \
573 __ASM_EMIT(OP " v4.4s, " SEL("v4.4s", "v20.4s") ", " SEL("v20.4s", "v4.4s") ) \
574 __ASM_EMIT(OP " v5.4s, " SEL("v5.4s", "v21.4s") ", " SEL("v21.4s", "v5.4s") ) \
575 __ASM_EMIT(OP " v6.4s, " SEL("v6.4s", "v22.4s") ", " SEL("v22.4s", "v6.4s") ) \
576 __ASM_EMIT(OP " v7.4s, " SEL("v7.4s", "v23.4s") ", " SEL("v23.4s", "v7.4s") ) \
577 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
578 __ASM_EMIT("stp q2, q3, [%[" DST "], #0x20]") \
579 __ASM_EMIT("stp q4, q5, [%[" DST "], #0x40]") \
580 __ASM_EMIT("stp q6, q7, [%[" DST "], #0x60]") \
581 __ASM_EMIT("subs %[count], %[count], #32") \
582 __ASM_EMIT("add %[" DST "], %[" DST "], #0x80") \
583 __ASM_EMIT("add %[" SRC1 "], %[" SRC1 "], #0x80") \
584 __ASM_EMIT("add %[" SRC2 "], %[" SRC2 "], #0x80") \
585 __ASM_EMIT("bhs 1b") \
586 /* 16x block */ \
587 __ASM_EMIT("2:") \
588 __ASM_EMIT("adds %[count], %[count], #16") /* 32 - 16 */ \
589 __ASM_EMIT("blt 4f") \
590 __ASM_EMIT("ldp q16, q17, [%[" SRC2 "], #0x00]") \
591 __ASM_EMIT("ldp q18, q19, [%[" SRC2 "], #0x20]") \
592 __ASM_EMIT("ldp q0, q1, [%[" SRC1 "], #0x00]") \
593 __ASM_EMIT("ldp q2, q3, [%[" SRC1 "], #0x20]") \
594 __ASM_EMIT("fabs v16.4s, v16.4s") \
595 __ASM_EMIT("fabs v17.4s, v17.4s") \
596 __ASM_EMIT("fabs v18.4s, v18.4s") \
597 __ASM_EMIT("fabs v19.4s, v19.4s") \
598 __ASM_EMIT(OP " v0.4s, " SEL("v0.4s", "v16.4s") ", " SEL("v16.4s", "v0.4s") ) \
599 __ASM_EMIT(OP " v1.4s, " SEL("v1.4s", "v17.4s") ", " SEL("v17.4s", "v1.4s") ) \
600 __ASM_EMIT(OP " v2.4s, " SEL("v2.4s", "v18.4s") ", " SEL("v18.4s", "v2.4s") ) \
601 __ASM_EMIT(OP " v3.4s, " SEL("v3.4s", "v19.4s") ", " SEL("v19.4s", "v3.4s") ) \
602 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
603 __ASM_EMIT("stp q2, q3, [%[" DST "], #0x20]") \
604 __ASM_EMIT("sub %[count], %[count], #16") \
605 __ASM_EMIT("add %[" DST "], %[" DST "], #0x40") \
606 __ASM_EMIT("add %[" SRC1 "], %[" SRC1 "], #0x40") \
607 __ASM_EMIT("add %[" SRC2 "], %[" SRC2 "], #0x40") \
608 /* 8x block */ \
609 __ASM_EMIT("4:") \
610 __ASM_EMIT("adds %[count], %[count], #8") /* 16 - 8 */ \
611 __ASM_EMIT("blt 6f") \
612 __ASM_EMIT("ldp q16, q17, [%[" SRC2 "], #0x00]") \
613 __ASM_EMIT("ldp q0, q1, [%[" SRC1 "], #0x00]") \
614 __ASM_EMIT("fabs v16.4s, v16.4s") \
615 __ASM_EMIT("fabs v17.4s, v17.4s") \
616 __ASM_EMIT(OP " v0.4s, " SEL("v0.4s", "v16.4s") ", " SEL("v16.4s", "v0.4s") ) \
617 __ASM_EMIT(OP " v1.4s, " SEL("v1.4s", "v17.4s") ", " SEL("v17.4s", "v1.4s") ) \
618 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
619 __ASM_EMIT("sub %[count], %[count], #8") \
620 __ASM_EMIT("add %[" DST "], %[" DST "], #0x20") \
621 __ASM_EMIT("add %[" SRC1 "], %[" SRC1 "], #0x20") \
622 __ASM_EMIT("add %[" SRC2 "], %[" SRC2 "], #0x20") \
623 /* 4x block */ \
624 __ASM_EMIT("6:") \
625 __ASM_EMIT("adds %[count], %[count], #4") /* 8 - 4 */ \
626 __ASM_EMIT("blt 8f") \
627 __ASM_EMIT("ldr q16, [%[" SRC2 "], #0x00]") \
628 __ASM_EMIT("ldr q0, [%[" SRC1 "], #0x00]") \
629 __ASM_EMIT("fabs v16.4s, v16.4s") \
630 __ASM_EMIT(OP " v0.4s, " SEL("v0.4s", "v16.4s") ", " SEL("v16.4s", "v0.4s") ) \
631 __ASM_EMIT("str q0, [%[" DST "], #0x00]") \
632 __ASM_EMIT("sub %[count], %[count], #4") \
633 __ASM_EMIT("add %[" DST "], %[" DST "], #0x10") \
634 __ASM_EMIT("add %[" SRC1 "], %[" SRC1 "], #0x10") \
635 __ASM_EMIT("add %[" SRC2 "], %[" SRC2 "], #0x10") \
636 /* 1x block */ \
637 __ASM_EMIT("8:") \
638 __ASM_EMIT("adds %[count], %[count], #3") /* 4 - 3 */ \
639 __ASM_EMIT("blt 10f") \
640 __ASM_EMIT("9:") \
641 __ASM_EMIT("ld1r {v16.4s}, [%[" SRC2 "]]") \
642 __ASM_EMIT("ld1r {v0.4s}, [%[" SRC1 "]]") \
643 __ASM_EMIT("fabs v16.4s, v16.4s") \
644 __ASM_EMIT(OP " v0.4s, " SEL("v0.4s", "v16.4s") ", " SEL("v16.4s", "v0.4s") ) \
645 __ASM_EMIT("st1 {v0.s}[0], [%[" DST "]]") \
646 __ASM_EMIT("subs %[count], %[count], #1") \
647 __ASM_EMIT("add %[" DST "], %[" DST "], #0x04") \
648 __ASM_EMIT("add %[" SRC1 "], %[" SRC1 "], #0x04") \
649 __ASM_EMIT("add %[" SRC2 "], %[" SRC2 "], #0x04") \
650 __ASM_EMIT("bge 9b") \
651 __ASM_EMIT("10:")
652
653 void abs_add3(float *dst, const float *src1, const float *src2, size_t count)
654 {
655 ARCH_AARCH64_ASM
656 (
657 OP_ABS_VV3_CORE("dst", "src1", "src2", "fadd", OP_DSEL)
658 : [dst] "+r" (dst), [src1] "+r" (src1), [src2] "+r" (src2),
659 [count] "+r" (count)
660 :
661 : "cc", "memory",
662 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
663 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"
664 );
665 }
666
667 void abs_sub3(float *dst, const float *src1, const float *src2, size_t count)
668 {
669 ARCH_AARCH64_ASM
670 (
671 OP_ABS_VV3_CORE("dst", "src1", "src2", "fsub", OP_DSEL)
672 : [dst] "+r" (dst), [src1] "+r" (src1), [src2] "+r" (src2),
673 [count] "+r" (count)
674 :
675 : "cc", "memory",
676 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
677 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"
678 );
679 }
680
681 void abs_rsub3(float *dst, const float *src1, const float *src2, size_t count)
682 {
683 ARCH_AARCH64_ASM
684 (
685 OP_ABS_VV3_CORE("dst", "src1", "src2", "fsub", OP_RSEL)
686 : [dst] "+r" (dst), [src1] "+r" (src1), [src2] "+r" (src2),
687 [count] "+r" (count)
688 :
689 : "cc", "memory",
690 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
691 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"
692 );
693 }
694
695 void abs_mul3(float *dst, const float *src1, const float *src2, size_t count)
696 {
697 ARCH_AARCH64_ASM
698 (
699 OP_ABS_VV3_CORE("dst", "src1", "src2", "fmul", OP_DSEL)
700 : [dst] "+r" (dst), [src1] "+r" (src1), [src2] "+r" (src2),
701 [count] "+r" (count)
702 :
703 : "cc", "memory",
704 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
705 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"
706 );
707 }
708
709 void abs_max3(float *dst, const float *src1, const float *src2, size_t count)
710 {
711 ARCH_AARCH64_ASM
712 (
713 OP_ABS_VV3_CORE("dst", "src1", "src2", "fmax", OP_DSEL)
714 : [dst] "+r" (dst), [src1] "+r" (src1), [src2] "+r" (src2),
715 [count] "+r" (count)
716 :
717 : "cc", "memory",
718 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
719 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"
720 );
721 }
722
723 void abs_min3(float *dst, const float *src1, const float *src2, size_t count)
724 {
725 ARCH_AARCH64_ASM
726 (
727 OP_ABS_VV3_CORE("dst", "src1", "src2", "fmin", OP_DSEL)
728 : [dst] "+r" (dst), [src1] "+r" (src1), [src2] "+r" (src2),
729 [count] "+r" (count)
730 :
731 : "cc", "memory",
732 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
733 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"
734 );
735 }
736
737 #undef OP_ABS_VV3_CORE
738
739 #define OP_ABS_DIV3_CORE(DST, SRC1, SRC2, SEL) \
740 __ASM_EMIT("subs %[count], %[count], #16") \
741 __ASM_EMIT("blo 2f") \
742 /* 16x blocks */ \
743 __ASM_EMIT("1:") \
744 __ASM_EMIT("ldp q0, q1, [%[" SEL(SRC2, SRC1) "], #0x00]") \
745 __ASM_EMIT("ldp q2, q3, [%[" SEL(SRC2, SRC1) "], #0x20]") \
746 __ASM_EMIT("ldp q4, q5, [%[" SEL(SRC1, SRC2) "], #0x00]") \
747 __ASM_EMIT("ldp q6, q7, [%[" SEL(SRC1, SRC2) "], #0x20]") \
748 __ASM_EMIT("fabs " SEL("v0.4s, v0.4s", "v4.4s, v4.4s")) \
749 __ASM_EMIT("fabs " SEL("v1.4s, v1.4s", "v5.4s, v5.4s")) \
750 __ASM_EMIT("fabs " SEL("v2.4s, v2.4s", "v6.4s, v6.4s")) \
751 __ASM_EMIT("fabs " SEL("v3.4s, v3.4s", "v7.4s, v7.4s")) \
752 __ASM_EMIT("frecpe v16.4s, v0.4s") /* v16 = s2 */ \
753 __ASM_EMIT("frecpe v17.4s, v1.4s") \
754 __ASM_EMIT("frecpe v18.4s, v2.4s") \
755 __ASM_EMIT("frecpe v19.4s, v3.4s") \
756 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2) */ \
757 __ASM_EMIT("frecps v25.4s, v17.4s, v1.4s") \
758 __ASM_EMIT("frecps v26.4s, v18.4s, v2.4s") \
759 __ASM_EMIT("frecps v27.4s, v19.4s, v3.4s") \
760 __ASM_EMIT("fmul v16.4s, v24.4s, v16.4s") /* v16 = s2' = s2 * (2 - R*s2) */ \
761 __ASM_EMIT("fmul v17.4s, v25.4s, v17.4s") \
762 __ASM_EMIT("fmul v18.4s, v26.4s, v18.4s") \
763 __ASM_EMIT("fmul v19.4s, v27.4s, v19.4s") \
764 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2') */ \
765 __ASM_EMIT("frecps v25.4s, v17.4s, v1.4s") \
766 __ASM_EMIT("frecps v26.4s, v18.4s, v2.4s") \
767 __ASM_EMIT("frecps v27.4s, v19.4s, v3.4s") \
768 __ASM_EMIT("fmul v0.4s, v24.4s, v16.4s") /* v0 = s2" = s2' * (2 - R*s2) = 1/s2 */ \
769 __ASM_EMIT("fmul v1.4s, v25.4s, v17.4s") \
770 __ASM_EMIT("fmul v2.4s, v26.4s, v18.4s") \
771 __ASM_EMIT("fmul v3.4s, v27.4s, v19.4s") \
772 __ASM_EMIT("fmul v0.4s, v0.4s, v4.4s") /* s1 / s2 */ \
773 __ASM_EMIT("fmul v1.4s, v1.4s, v5.4s") \
774 __ASM_EMIT("fmul v2.4s, v2.4s, v6.4s") \
775 __ASM_EMIT("fmul v3.4s, v3.4s, v7.4s") \
776 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
777 __ASM_EMIT("stp q2, q3, [%[" DST "], #0x20]") \
778 __ASM_EMIT("subs %[count], %[count], #16") \
779 __ASM_EMIT("add %[" DST "], %[" DST "], #0x40") \
780 __ASM_EMIT("add %[" SRC1 "], %[" SRC1 "], #0x40") \
781 __ASM_EMIT("add %[" SRC2 "], %[" SRC2 "], #0x40") \
782 __ASM_EMIT("bhs 1b") \
783 /* 8x block */ \
784 __ASM_EMIT("2:") \
785 __ASM_EMIT("adds %[count], %[count], #8") /* 16 - 8 */ \
786 __ASM_EMIT("blt 4f") \
787 __ASM_EMIT("ldp q0, q1, [%[" SEL(SRC2, SRC1) "], #0x00]") \
788 __ASM_EMIT("ldp q4, q5, [%[" SEL(SRC1, SRC2) "], #0x00]") \
789 __ASM_EMIT("fabs " SEL("v0.4s, v0.4s", "v4.4s, v4.4s")) \
790 __ASM_EMIT("fabs " SEL("v1.4s, v1.4s", "v5.4s, v5.4s")) \
791 __ASM_EMIT("frecpe v16.4s, v0.4s") /* v16 = s2 */ \
792 __ASM_EMIT("frecpe v17.4s, v1.4s") \
793 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2) */ \
794 __ASM_EMIT("frecps v25.4s, v17.4s, v1.4s") \
795 __ASM_EMIT("fmul v16.4s, v24.4s, v16.4s") /* v16 = s2' = s2 * (2 - R*s2) */ \
796 __ASM_EMIT("fmul v17.4s, v25.4s, v17.4s") \
797 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2') */ \
798 __ASM_EMIT("frecps v25.4s, v17.4s, v1.4s") \
799 __ASM_EMIT("fmul v0.4s, v24.4s, v16.4s") /* v0 = s2" = s2' * (2 - R*s2) = 1/s2 */ \
800 __ASM_EMIT("fmul v1.4s, v25.4s, v17.4s") \
801 __ASM_EMIT("fmul v0.4s, v0.4s, v4.4s") /* s1 / s2 */ \
802 __ASM_EMIT("fmul v1.4s, v1.4s, v5.4s") \
803 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
804 __ASM_EMIT("sub %[count], %[count], #8") \
805 __ASM_EMIT("add %[" DST "], %[" DST "], #0x20") \
806 __ASM_EMIT("add %[" SRC1 "], %[" SRC1 "], #0x20") \
807 __ASM_EMIT("add %[" SRC2 "], %[" SRC2 "], #0x20") \
808 /* 4x block */ \
809 __ASM_EMIT("4:") \
810 __ASM_EMIT("adds %[count], %[count], #4") /* 8 - 4 */ \
811 __ASM_EMIT("blt 6f") \
812 __ASM_EMIT("ldr q0, [%[" SEL(SRC2, SRC1) "], #0x00]") \
813 __ASM_EMIT("ldr q4, [%[" SEL(SRC1, SRC2) "], #0x00]") \
814 __ASM_EMIT("fabs " SEL("v0.4s, v0.4s", "v4.4s, v4.4s")) \
815 __ASM_EMIT("frecpe v16.4s, v0.4s") /* v16 = s2 */ \
816 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2) */ \
817 __ASM_EMIT("fmul v16.4s, v24.4s, v16.4s") /* v16 = s2' = s2 * (2 - R*s2) */ \
818 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2') */ \
819 __ASM_EMIT("fmul v0.4s, v24.4s, v16.4s") /* v0 = s2" = s2' * (2 - R*s2) = 1/s2 */ \
820 __ASM_EMIT("fmul v0.4s, v0.4s, v4.4s") /* s1 / s2 */ \
821 __ASM_EMIT("str q0, [%[" DST "], #0x00]") \
822 __ASM_EMIT("sub %[count], %[count], #4") \
823 __ASM_EMIT("add %[" DST "], %[" DST "], #0x10") \
824 __ASM_EMIT("add %[" SRC1 "], %[" SRC1 "], #0x10") \
825 __ASM_EMIT("add %[" SRC2 "], %[" SRC2 "], #0x10") \
826 /* 1x block */ \
827 __ASM_EMIT("6:") \
828 __ASM_EMIT("adds %[count], %[count], #3") /* 4 - 3 */ \
829 __ASM_EMIT("blt 8f") \
830 __ASM_EMIT("7:") \
831 __ASM_EMIT("ld1r {v0.4s}, [%[" SEL(SRC2, SRC1) "]]") \
832 __ASM_EMIT("ld1r {v4.4s}, [%[" SEL(SRC1, SRC2) "]]") \
833 __ASM_EMIT("fabs " SEL("v0.4s, v0.4s", "v4.4s, v4.4s")) \
834 __ASM_EMIT("frecpe v16.4s, v0.4s") /* v16 = s2 */ \
835 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2) */ \
836 __ASM_EMIT("fmul v16.4s, v24.4s, v16.4s") /* v16 = s2' = s2 * (2 - R*s2) */ \
837 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2') */ \
838 __ASM_EMIT("fmul v0.4s, v24.4s, v16.4s") /* v0 = s2" = s2' * (2 - R*s2) = 1/s2 */ \
839 __ASM_EMIT("fmul v0.4s, v0.4s, v4.4s") /* s1 / s2 */ \
840 __ASM_EMIT("st1 {v0.s}[0], [%[" DST "]]") \
841 __ASM_EMIT("subs %[count], %[count], #1") \
842 __ASM_EMIT("add %[" DST "], %[" DST "], #0x04") \
843 __ASM_EMIT("add %[" SRC1 "], %[" SRC1 "], #0x04") \
844 __ASM_EMIT("add %[" SRC2 "], %[" SRC2 "], #0x04") \
845 __ASM_EMIT("bge 7b") \
846 __ASM_EMIT("8:")
847
848 void abs_div3(float *dst, const float *src1, const float *src2, size_t count)
849 {
850 ARCH_AARCH64_ASM
851 (
852 OP_ABS_DIV3_CORE("dst", "src1", "src2", OP_DSEL)
853 : [dst] "+r" (dst), [src1] "+r" (src1), [src2] "+r" (src2),
854 [count] "+r" (count)
855 :
856 : "cc", "memory",
857 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
858 "v16", "v17", "v18", "v19",
859 "v24", "v25", "v26", "v27"
860 );
861 }
862
863 void abs_rdiv3(float *dst, const float *src1, const float *src2, size_t count)
864 {
865 ARCH_AARCH64_ASM
866 (
867 OP_ABS_DIV3_CORE("dst", "src1", "src2", OP_RSEL)
868 : [dst] "+r" (dst), [src1] "+r" (src1), [src2] "+r" (src2),
869 [count] "+r" (count)
870 :
871 : "cc", "memory",
872 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
873 "v16", "v17", "v18", "v19",
874 "v24", "v25", "v26", "v27"
875 );
876 }
877
878 #undef OP_ABS_DIV3_CORE
879
880 #undef OP_DSEL
881 #undef OP_RSEL
882
883 } /* namespace asimd */
884 } /* namespace lsp */
885
886 #endif /* PRIVATE_DSP_ARCH_AARCH64_ASIMD_PMATH_ABS_VV_H_ */