Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/aarch64/asimd/pmath/op_vv.h

raw
1 /*
2 * Copyright (C) 2020 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2020 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 31 мар. 2020 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_AARCH64_ASIMD_PMATH_OP_VV_H_
23 #define PRIVATE_DSP_ARCH_AARCH64_ASIMD_PMATH_OP_VV_H_
24
25 #ifndef PRIVATE_DSP_ARCH_AARCH64_ASIMD_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_AARCH64_ASIMD_IMPL */
28
29 namespace lsp
30 {
31 namespace asimd
32 {
33 #define OP_SDSEL(a, b) a
34 #define OP_SRSEL(a, b) b
35
36 #define OP_VV2_CORE(DST, SRC, OP, SEL) \
37 __ASM_EMIT("subs %[count], %[count], #32") \
38 __ASM_EMIT("b.lo 2f") \
39 /* 32x blocks */ \
40 __ASM_EMIT("1:") \
41 __ASM_EMIT("ldp q0, q1, [%[" SEL(DST, SRC) "], #0x00]") \
42 __ASM_EMIT("ldp q2, q3, [%[" SEL(DST, SRC) "], #0x20]") \
43 __ASM_EMIT("ldp q4, q5, [%[" SEL(DST, SRC) "], #0x40]") \
44 __ASM_EMIT("ldp q6, q7, [%[" SEL(DST, SRC) "], #0x60]") \
45 __ASM_EMIT("ldp q16, q17, [%[" SEL(SRC, DST) "], #0x00]") \
46 __ASM_EMIT("ldp q18, q19, [%[" SEL(SRC, DST) "], #0x20]") \
47 __ASM_EMIT("ldp q20, q21, [%[" SEL(SRC, DST) "], #0x40]") \
48 __ASM_EMIT("ldp q22, q23, [%[" SEL(SRC, DST) "], #0x60]") \
49 __ASM_EMIT(OP " v0.4s, v0.4s, v16.4s") \
50 __ASM_EMIT(OP " v1.4s, v1.4s, v17.4s") \
51 __ASM_EMIT(OP " v2.4s, v2.4s, v18.4s") \
52 __ASM_EMIT(OP " v3.4s, v3.4s, v19.4s") \
53 __ASM_EMIT(OP " v4.4s, v4.4s, v20.4s") \
54 __ASM_EMIT(OP " v5.4s, v5.4s, v21.4s") \
55 __ASM_EMIT(OP " v6.4s, v6.4s, v22.4s") \
56 __ASM_EMIT(OP " v7.4s, v7.4s, v23.4s") \
57 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
58 __ASM_EMIT("stp q2, q3, [%[" DST "], #0x20]") \
59 __ASM_EMIT("stp q4, q5, [%[" DST "], #0x40]") \
60 __ASM_EMIT("stp q6, q7, [%[" DST "], #0x60]") \
61 __ASM_EMIT("subs %[count], %[count], #32") \
62 __ASM_EMIT("add %[" DST "], %[" DST "], #0x80") \
63 __ASM_EMIT("add %[" SRC "], %[" SRC "], #0x80") \
64 __ASM_EMIT("b.hs 1b") \
65 /* 16x block */ \
66 __ASM_EMIT("2:") \
67 __ASM_EMIT("adds %[count], %[count], #16") /* 32 - 16 */ \
68 __ASM_EMIT("b.lt 4f") \
69 __ASM_EMIT("ldp q0, q1, [%[" SEL(DST, SRC) "], #0x00]") \
70 __ASM_EMIT("ldp q2, q3, [%[" SEL(DST, SRC) "], #0x20]") \
71 __ASM_EMIT("ldp q16, q17, [%[" SEL(SRC, DST) "], #0x00]") \
72 __ASM_EMIT("ldp q18, q19, [%[" SEL(SRC, DST) "], #0x20]") \
73 __ASM_EMIT(OP " v0.4s, v0.4s, v16.4s") \
74 __ASM_EMIT(OP " v1.4s, v1.4s, v17.4s") \
75 __ASM_EMIT(OP " v2.4s, v2.4s, v18.4s") \
76 __ASM_EMIT(OP " v3.4s, v3.4s, v19.4s") \
77 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
78 __ASM_EMIT("stp q2, q3, [%[" DST "], #0x20]") \
79 __ASM_EMIT("sub %[count], %[count], #16") \
80 __ASM_EMIT("add %[" DST "], %[" DST "], #0x40") \
81 __ASM_EMIT("add %[" SRC "], %[" SRC "], #0x40") \
82 /* 8x block */ \
83 __ASM_EMIT("4:") \
84 __ASM_EMIT("adds %[count], %[count], #8") /* 16 - 8 */ \
85 __ASM_EMIT("b.lt 6f") \
86 __ASM_EMIT("ldp q0, q1, [%[" SEL(DST, SRC) "], #0x00]") \
87 __ASM_EMIT("ldp q2, q3, [%[" SEL(DST, SRC) "], #0x20]") \
88 __ASM_EMIT("ldp q16, q17, [%[" SEL(SRC, DST) "], #0x00]") \
89 __ASM_EMIT("ldp q18, q19, [%[" SEL(SRC, DST) "], #0x20]") \
90 __ASM_EMIT(OP " v0.4s, v0.4s, v16.4s") \
91 __ASM_EMIT(OP " v1.4s, v1.4s, v17.4s") \
92 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
93 __ASM_EMIT("sub %[count], %[count], #8") \
94 __ASM_EMIT("add %[" DST "], %[" DST "], #0x20") \
95 __ASM_EMIT("add %[" SRC "], %[" SRC "], #0x20") \
96 /* 4x block */ \
97 __ASM_EMIT("6:") \
98 __ASM_EMIT("adds %[count], %[count], #4") /* 8 - 4 */ \
99 __ASM_EMIT("b.lt 8f") \
100 __ASM_EMIT("ldr q0, [%[" SEL(DST, SRC) "], #0x00]") \
101 __ASM_EMIT("ldr q16, [%[" SEL(SRC, DST) "], #0x00]") \
102 __ASM_EMIT(OP " v0.4s, v0.4s, v16.4s") \
103 __ASM_EMIT("str q0, [%[" DST "], #0x00]") \
104 __ASM_EMIT("sub %[count], %[count], #4") \
105 __ASM_EMIT("add %[" DST "], %[" DST "], #0x10") \
106 __ASM_EMIT("add %[" SRC "], %[" SRC "], #0x10") \
107 /* 1x block */ \
108 __ASM_EMIT("8:") \
109 __ASM_EMIT("adds %[count], %[count], #3") /* 4 - 3 */ \
110 __ASM_EMIT("b.lt 10f") \
111 __ASM_EMIT("9:") \
112 __ASM_EMIT("ld1r {v0.4s}, [%[" SEL(DST, SRC) "]]") \
113 __ASM_EMIT("ld1r {v16.4s}, [%[" SEL(SRC, DST) "]]") \
114 __ASM_EMIT(OP " v0.4s, v0.4s, v16.4s") \
115 __ASM_EMIT("st1 {v0.s}[0], [%[" DST "]]") \
116 __ASM_EMIT("subs %[count], %[count], #1") \
117 __ASM_EMIT("add %[" DST "], %[" DST "], #0x04") \
118 __ASM_EMIT("add %[" SRC "], %[" SRC "], #0x04") \
119 __ASM_EMIT("bge 9b") \
120 __ASM_EMIT("10:")
121
122 void add2(float *dst, const float *src, size_t count)
123 {
124 ARCH_AARCH64_ASM
125 (
126 OP_VV2_CORE("dst", "src", "fadd", OP_SDSEL)
127 : [dst] "+r" (dst), [src] "+r" (src),
128 [count] "+r" (count)
129 :
130 : "cc", "memory",
131 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
132 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"
133 );
134 }
135
136 void sub2(float *dst, const float *src, size_t count)
137 {
138 ARCH_AARCH64_ASM
139 (
140 OP_VV2_CORE("dst", "src", "fsub", OP_SDSEL)
141 : [dst] "+r" (dst), [src] "+r" (src),
142 [count] "+r" (count)
143 :
144 : "cc", "memory",
145 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
146 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"
147 );
148 }
149
150 void rsub2(float *dst, const float *src, size_t count)
151 {
152 ARCH_AARCH64_ASM
153 (
154 OP_VV2_CORE("dst", "src", "fsub", OP_SRSEL)
155 : [dst] "+r" (dst), [src] "+r" (src),
156 [count] "+r" (count)
157 :
158 : "cc", "memory",
159 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
160 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"
161 );
162 }
163
164 void mul2(float *dst, const float *src, size_t count)
165 {
166 ARCH_AARCH64_ASM
167 (
168 OP_VV2_CORE("dst", "src", "fmul", OP_SDSEL)
169 : [dst] "+r" (dst), [src] "+r" (src),
170 [count] "+r" (count)
171 :
172 : "cc", "memory",
173 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
174 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"
175 );
176 }
177
178 #undef OP_VV2_CORE
179
180 #define OP_DIV2_CORE(DST, SRC, SEL) \
181 __ASM_EMIT("subs %[count], %[count], #16") \
182 __ASM_EMIT("b.lo 2f") \
183 /* 16x blocks */ \
184 __ASM_EMIT("1:") \
185 __ASM_EMIT("ldp q0, q1, [%[" SEL(SRC, DST) "], #0x00]") \
186 __ASM_EMIT("ldp q2, q3, [%[" SEL(SRC, DST) "], #0x20]") \
187 __ASM_EMIT("ldp q4, q5, [%[" SEL(DST, SRC) "], #0x00]") \
188 __ASM_EMIT("ldp q6, q7, [%[" SEL(DST, SRC) "], #0x20]") \
189 __ASM_EMIT("frecpe v16.4s, v0.4s") /* v16 = s2 */ \
190 __ASM_EMIT("frecpe v17.4s, v1.4s") \
191 __ASM_EMIT("frecpe v18.4s, v2.4s") \
192 __ASM_EMIT("frecpe v19.4s, v3.4s") \
193 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2) */ \
194 __ASM_EMIT("frecps v25.4s, v17.4s, v1.4s") \
195 __ASM_EMIT("frecps v26.4s, v18.4s, v2.4s") \
196 __ASM_EMIT("frecps v27.4s, v19.4s, v3.4s") \
197 __ASM_EMIT("fmul v16.4s, v24.4s, v16.4s") /* v16 = s2' = s2 * (2 - R*s2) */ \
198 __ASM_EMIT("fmul v17.4s, v25.4s, v17.4s") \
199 __ASM_EMIT("fmul v18.4s, v26.4s, v18.4s") \
200 __ASM_EMIT("fmul v19.4s, v27.4s, v19.4s") \
201 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2') */ \
202 __ASM_EMIT("frecps v25.4s, v17.4s, v1.4s") \
203 __ASM_EMIT("frecps v26.4s, v18.4s, v2.4s") \
204 __ASM_EMIT("frecps v27.4s, v19.4s, v3.4s") \
205 __ASM_EMIT("fmul v0.4s, v24.4s, v16.4s") /* v0 = s2" = s2' * (2 - R*s2) = 1/s2 */ \
206 __ASM_EMIT("fmul v1.4s, v25.4s, v17.4s") \
207 __ASM_EMIT("fmul v2.4s, v26.4s, v18.4s") \
208 __ASM_EMIT("fmul v3.4s, v27.4s, v19.4s") \
209 __ASM_EMIT("fmul v0.4s, v0.4s, v4.4s") /* s1 / s2 */ \
210 __ASM_EMIT("fmul v1.4s, v1.4s, v5.4s") \
211 __ASM_EMIT("fmul v2.4s, v2.4s, v6.4s") \
212 __ASM_EMIT("fmul v3.4s, v3.4s, v7.4s") \
213 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
214 __ASM_EMIT("stp q2, q3, [%[" DST "], #0x20]") \
215 __ASM_EMIT("subs %[count], %[count], #16") \
216 __ASM_EMIT("add %[" DST "], %[" DST "], #0x40") \
217 __ASM_EMIT("add %[" SRC "], %[" SRC "], #0x40") \
218 __ASM_EMIT("b.hs 1b") \
219 /* 8x block */ \
220 __ASM_EMIT("2:") \
221 __ASM_EMIT("adds %[count], %[count], #8") /* 16 - 8 */ \
222 __ASM_EMIT("b.lt 4f") \
223 __ASM_EMIT("ldp q0, q1, [%[" SEL(SRC, DST) "], #0x00]") \
224 __ASM_EMIT("ldp q4, q5, [%[" SEL(DST, SRC) "], #0x00]") \
225 __ASM_EMIT("frecpe v16.4s, v0.4s") /* v16 = s2 */ \
226 __ASM_EMIT("frecpe v17.4s, v1.4s") \
227 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2) */ \
228 __ASM_EMIT("frecps v25.4s, v17.4s, v1.4s") \
229 __ASM_EMIT("fmul v16.4s, v24.4s, v16.4s") /* v16 = s2' = s2 * (2 - R*s2) */ \
230 __ASM_EMIT("fmul v17.4s, v25.4s, v17.4s") \
231 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2') */ \
232 __ASM_EMIT("frecps v25.4s, v17.4s, v1.4s") \
233 __ASM_EMIT("fmul v0.4s, v24.4s, v16.4s") /* v0 = s2" = s2' * (2 - R*s2) = 1/s2 */ \
234 __ASM_EMIT("fmul v1.4s, v25.4s, v17.4s") \
235 __ASM_EMIT("fmul v0.4s, v0.4s, v4.4s") /* s1 / s2 */ \
236 __ASM_EMIT("fmul v1.4s, v1.4s, v5.4s") \
237 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
238 __ASM_EMIT("sub %[count], %[count], #8") \
239 __ASM_EMIT("add %[" DST "], %[" DST "], #0x20") \
240 __ASM_EMIT("add %[" SRC "], %[" SRC "], #0x20") \
241 /* 4x block */ \
242 __ASM_EMIT("4:") \
243 __ASM_EMIT("adds %[count], %[count], #4") /* 8 - 4 */ \
244 __ASM_EMIT("b.lt 6f") \
245 __ASM_EMIT("ldr q0, [%[" SEL(SRC, DST) "], #0x00]") \
246 __ASM_EMIT("ldr q4, [%[" SEL(DST, SRC) "], #0x00]") \
247 __ASM_EMIT("frecpe v16.4s, v0.4s") /* v16 = s2 */ \
248 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2) */ \
249 __ASM_EMIT("fmul v16.4s, v24.4s, v16.4s") /* v16 = s2' = s2 * (2 - R*s2) */ \
250 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2') */ \
251 __ASM_EMIT("fmul v0.4s, v24.4s, v16.4s") /* v0 = s2" = s2' * (2 - R*s2) = 1/s2 */ \
252 __ASM_EMIT("fmul v0.4s, v0.4s, v4.4s") /* s1 / s2 */ \
253 __ASM_EMIT("str q0, [%[" DST "], #0x00]") \
254 __ASM_EMIT("sub %[count], %[count], #4") \
255 __ASM_EMIT("add %[" DST "], %[" DST "], #0x10") \
256 __ASM_EMIT("add %[" SRC "], %[" SRC "], #0x10") \
257 /* 1x block */ \
258 __ASM_EMIT("6:") \
259 __ASM_EMIT("adds %[count], %[count], #3") /* 4 - 3 */ \
260 __ASM_EMIT("b.lt 8f") \
261 __ASM_EMIT("7:") \
262 __ASM_EMIT("ld1r {v0.4s}, [%[" SEL(SRC, DST) "]]") \
263 __ASM_EMIT("ld1r {v4.4s}, [%[" SEL(DST, SRC) "]]") \
264 __ASM_EMIT("frecpe v16.4s, v0.4s") /* v16 = s2 */ \
265 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2) */ \
266 __ASM_EMIT("fmul v16.4s, v24.4s, v16.4s") /* v16 = s2' = s2 * (2 - R*s2) */ \
267 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2') */ \
268 __ASM_EMIT("fmul v0.4s, v24.4s, v16.4s") /* v0 = s2" = s2' * (2 - R*s2) = 1/s2 */ \
269 __ASM_EMIT("fmul v0.4s, v0.4s, v4.4s") /* s1 / s2 */ \
270 __ASM_EMIT("st1 {v0.s}[0], [%[" DST "]]") \
271 __ASM_EMIT("subs %[count], %[count], #1") \
272 __ASM_EMIT("add %[" DST "], %[" DST "], #0x04") \
273 __ASM_EMIT("add %[" SRC "], %[" SRC "], #0x04") \
274 __ASM_EMIT("bge 7b") \
275 __ASM_EMIT("8:")
276
277 void div2(float *dst, const float *src, size_t count)
278 {
279 ARCH_AARCH64_ASM
280 (
281 OP_DIV2_CORE("dst", "src", OP_SDSEL)
282 : [dst] "+r" (dst), [src] "+r" (src),
283 [count] "+r" (count)
284 :
285 : "cc", "memory",
286 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
287 "v16", "v17", "v18", "v19",
288 "v24", "v25", "v26", "v27"
289 );
290 }
291
292 void rdiv2(float *dst, const float *src, size_t count)
293 {
294 ARCH_AARCH64_ASM
295 (
296 OP_DIV2_CORE("dst", "src", OP_SRSEL)
297 : [dst] "+r" (dst), [src] "+r" (src),
298 [count] "+r" (count)
299 :
300 : "cc", "memory",
301 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
302 "v16", "v17", "v18", "v19",
303 "v24", "v25", "v26", "v27"
304 );
305 }
306
307 #undef OP_DIV2_CORE
308
309 #define OP_VV3_CORE(DST, SRC1, SRC2, OP) \
310 __ASM_EMIT("subs %[count], %[count], #32") \
311 __ASM_EMIT("b.lo 2f") \
312 /* 32x blocks */ \
313 __ASM_EMIT("1:") \
314 __ASM_EMIT("ldp q0, q1, [%[" SRC1 "], #0x00]") \
315 __ASM_EMIT("ldp q2, q3, [%[" SRC1 "], #0x20]") \
316 __ASM_EMIT("ldp q4, q5, [%[" SRC1 "], #0x40]") \
317 __ASM_EMIT("ldp q6, q7, [%[" SRC1 "], #0x60]") \
318 __ASM_EMIT("ldp q16, q17, [%[" SRC2 "], #0x00]") \
319 __ASM_EMIT("ldp q18, q19, [%[" SRC2 "], #0x20]") \
320 __ASM_EMIT("ldp q20, q21, [%[" SRC2 "], #0x40]") \
321 __ASM_EMIT("ldp q22, q23, [%[" SRC2 "], #0x60]") \
322 __ASM_EMIT(OP " v0.4s, v0.4s, v16.4s") \
323 __ASM_EMIT(OP " v1.4s, v1.4s, v17.4s") \
324 __ASM_EMIT(OP " v2.4s, v2.4s, v18.4s") \
325 __ASM_EMIT(OP " v3.4s, v3.4s, v19.4s") \
326 __ASM_EMIT(OP " v4.4s, v4.4s, v20.4s") \
327 __ASM_EMIT(OP " v5.4s, v5.4s, v21.4s") \
328 __ASM_EMIT(OP " v6.4s, v6.4s, v22.4s") \
329 __ASM_EMIT(OP " v7.4s, v7.4s, v23.4s") \
330 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
331 __ASM_EMIT("stp q2, q3, [%[" DST "], #0x20]") \
332 __ASM_EMIT("stp q4, q5, [%[" DST "], #0x40]") \
333 __ASM_EMIT("stp q6, q7, [%[" DST "], #0x60]") \
334 __ASM_EMIT("subs %[count], %[count], #32") \
335 __ASM_EMIT("add %[" DST "], %[" DST "], #0x80") \
336 __ASM_EMIT("add %[" SRC1 "], %[" SRC1 "], #0x80") \
337 __ASM_EMIT("add %[" SRC2 "], %[" SRC2 "], #0x80") \
338 __ASM_EMIT("b.hs 1b") \
339 /* 16x block */ \
340 __ASM_EMIT("2:") \
341 __ASM_EMIT("adds %[count], %[count], #16") /* 32 - 16 */ \
342 __ASM_EMIT("b.lt 4f") \
343 __ASM_EMIT("ldp q0, q1, [%[" SRC1 "], #0x00]") \
344 __ASM_EMIT("ldp q2, q3, [%[" SRC1 "], #0x20]") \
345 __ASM_EMIT("ldp q16, q17, [%[" SRC2 "], #0x00]") \
346 __ASM_EMIT("ldp q18, q19, [%[" SRC2 "], #0x20]") \
347 __ASM_EMIT(OP " v0.4s, v0.4s, v16.4s") \
348 __ASM_EMIT(OP " v1.4s, v1.4s, v17.4s") \
349 __ASM_EMIT(OP " v2.4s, v2.4s, v18.4s") \
350 __ASM_EMIT(OP " v3.4s, v3.4s, v19.4s") \
351 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
352 __ASM_EMIT("stp q2, q3, [%[" DST "], #0x20]") \
353 __ASM_EMIT("sub %[count], %[count], #16") \
354 __ASM_EMIT("add %[" DST "], %[" DST "], #0x40") \
355 __ASM_EMIT("add %[" SRC1 "], %[" SRC1 "], #0x40") \
356 __ASM_EMIT("add %[" SRC2 "], %[" SRC2 "], #0x40") \
357 /* 8x block */ \
358 __ASM_EMIT("4:") \
359 __ASM_EMIT("adds %[count], %[count], #8") /* 16 - 8 */ \
360 __ASM_EMIT("b.lt 6f") \
361 __ASM_EMIT("ldp q0, q1, [%[" SRC1 "], #0x00]") \
362 __ASM_EMIT("ldp q16, q17, [%[" SRC2 "], #0x00]") \
363 __ASM_EMIT(OP " v0.4s, v0.4s, v16.4s") \
364 __ASM_EMIT(OP " v1.4s, v1.4s, v17.4s") \
365 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
366 __ASM_EMIT("sub %[count], %[count], #8") \
367 __ASM_EMIT("add %[" DST "], %[" DST "], #0x20") \
368 __ASM_EMIT("add %[" SRC1 "], %[" SRC1 "], #0x20") \
369 __ASM_EMIT("add %[" SRC2 "], %[" SRC2 "], #0x20") \
370 /* 4x block */ \
371 __ASM_EMIT("6:") \
372 __ASM_EMIT("adds %[count], %[count], #4") /* 8 - 4 */ \
373 __ASM_EMIT("b.lt 8f") \
374 __ASM_EMIT("ldr q0, [%[" SRC1 "], #0x00]") \
375 __ASM_EMIT("ldr q16, [%[" SRC2 "], #0x00]") \
376 __ASM_EMIT(OP " v0.4s, v0.4s, v16.4s") \
377 __ASM_EMIT("str q0, [%[" DST "], #0x00]") \
378 __ASM_EMIT("sub %[count], %[count], #4") \
379 __ASM_EMIT("add %[" DST "], %[" DST "], #0x10") \
380 __ASM_EMIT("add %[" SRC1 "], %[" SRC1 "], #0x10") \
381 __ASM_EMIT("add %[" SRC2 "], %[" SRC2 "], #0x10") \
382 /* 1x block */ \
383 __ASM_EMIT("8:") \
384 __ASM_EMIT("adds %[count], %[count], #3") /* 4 - 3 */ \
385 __ASM_EMIT("b.lt 10f") \
386 __ASM_EMIT("9:") \
387 __ASM_EMIT("ld1r {v0.4s}, [%[" SRC1 "]]") \
388 __ASM_EMIT("ld1r {v16.4s}, [%[" SRC2 "]]") \
389 __ASM_EMIT(OP " v0.4s, v0.4s, v16.4s") \
390 __ASM_EMIT("st1 {v0.s}[0], [%[" DST "]]") \
391 __ASM_EMIT("subs %[count], %[count], #1") \
392 __ASM_EMIT("add %[" DST "], %[" DST "], #0x04") \
393 __ASM_EMIT("add %[" SRC1 "], %[" SRC1 "], #0x04") \
394 __ASM_EMIT("add %[" SRC2 "], %[" SRC2 "], #0x04") \
395 __ASM_EMIT("bge 9b") \
396 __ASM_EMIT("10:")
397
398 void add3(float *dst, const float *src1, const float *src2, size_t count)
399 {
400 ARCH_AARCH64_ASM
401 (
402 OP_VV3_CORE("dst", "src1", "src2", "fadd")
403 : [dst] "+r" (dst), [src1] "+r" (src1), [src2] "+r" (src2),
404 [count] "+r" (count)
405 :
406 : "cc", "memory",
407 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
408 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"
409 );
410 }
411
412 void sub3(float *dst, const float *src1, const float *src2, size_t count)
413 {
414 ARCH_AARCH64_ASM
415 (
416 OP_VV3_CORE("dst", "src1", "src2", "fsub")
417 : [dst] "+r" (dst), [src1] "+r" (src1), [src2] "+r" (src2),
418 [count] "+r" (count)
419 :
420 : "cc", "memory",
421 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
422 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"
423 );
424 }
425
426 void mul3(float *dst, const float *src1, const float *src2, size_t count)
427 {
428 ARCH_AARCH64_ASM
429 (
430 OP_VV3_CORE("dst", "src1", "src2", "fmul")
431 : [dst] "+r" (dst), [src1] "+r" (src1), [src2] "+r" (src2),
432 [count] "+r" (count)
433 :
434 : "cc", "memory",
435 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
436 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"
437 );
438 }
439
440 #undef OP_VV3_CORE
441
442 #define OP_DIV3_CORE(DST, SRC1, SRC2) \
443 __ASM_EMIT("subs %[count], %[count], #16") \
444 __ASM_EMIT("b.lo 2f") \
445 /* 16x blocks */ \
446 __ASM_EMIT("1:") \
447 __ASM_EMIT("ldp q0, q1, [%[" SRC2 "], #0x00]") \
448 __ASM_EMIT("ldp q2, q3, [%[" SRC2 "], #0x20]") \
449 __ASM_EMIT("ldp q4, q5, [%[" SRC1 "], #0x00]") \
450 __ASM_EMIT("ldp q6, q7, [%[" SRC1 "], #0x20]") \
451 __ASM_EMIT("frecpe v16.4s, v0.4s") /* v16 = s2 */ \
452 __ASM_EMIT("frecpe v17.4s, v1.4s") \
453 __ASM_EMIT("frecpe v18.4s, v2.4s") \
454 __ASM_EMIT("frecpe v19.4s, v3.4s") \
455 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2) */ \
456 __ASM_EMIT("frecps v25.4s, v17.4s, v1.4s") \
457 __ASM_EMIT("frecps v26.4s, v18.4s, v2.4s") \
458 __ASM_EMIT("frecps v27.4s, v19.4s, v3.4s") \
459 __ASM_EMIT("fmul v16.4s, v24.4s, v16.4s") /* v16 = s2' = s2 * (2 - R*s2) */ \
460 __ASM_EMIT("fmul v17.4s, v25.4s, v17.4s") \
461 __ASM_EMIT("fmul v18.4s, v26.4s, v18.4s") \
462 __ASM_EMIT("fmul v19.4s, v27.4s, v19.4s") \
463 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2') */ \
464 __ASM_EMIT("frecps v25.4s, v17.4s, v1.4s") \
465 __ASM_EMIT("frecps v26.4s, v18.4s, v2.4s") \
466 __ASM_EMIT("frecps v27.4s, v19.4s, v3.4s") \
467 __ASM_EMIT("fmul v0.4s, v24.4s, v16.4s") /* v0 = s2" = s2' * (2 - R*s2) = 1/s2 */ \
468 __ASM_EMIT("fmul v1.4s, v25.4s, v17.4s") \
469 __ASM_EMIT("fmul v2.4s, v26.4s, v18.4s") \
470 __ASM_EMIT("fmul v3.4s, v27.4s, v19.4s") \
471 __ASM_EMIT("fmul v0.4s, v0.4s, v4.4s") /* s1 / s2 */ \
472 __ASM_EMIT("fmul v1.4s, v1.4s, v5.4s") \
473 __ASM_EMIT("fmul v2.4s, v2.4s, v6.4s") \
474 __ASM_EMIT("fmul v3.4s, v3.4s, v7.4s") \
475 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
476 __ASM_EMIT("stp q2, q3, [%[" DST "], #0x20]") \
477 __ASM_EMIT("subs %[count], %[count], #16") \
478 __ASM_EMIT("add %[" DST "], %[" DST "], #0x40") \
479 __ASM_EMIT("add %[" SRC1 "], %[" SRC1 "], #0x40") \
480 __ASM_EMIT("add %[" SRC2 "], %[" SRC2 "], #0x40") \
481 __ASM_EMIT("b.hs 1b") \
482 /* 8x block */ \
483 __ASM_EMIT("2:") \
484 __ASM_EMIT("adds %[count], %[count], #8") /* 16 - 8 */ \
485 __ASM_EMIT("b.lt 4f") \
486 __ASM_EMIT("ldp q0, q1, [%[" SRC2 "], #0x00]") \
487 __ASM_EMIT("ldp q4, q5, [%[" SRC1 "], #0x00]") \
488 __ASM_EMIT("frecpe v16.4s, v0.4s") /* v16 = s2 */ \
489 __ASM_EMIT("frecpe v17.4s, v1.4s") \
490 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2) */ \
491 __ASM_EMIT("frecps v25.4s, v17.4s, v1.4s") \
492 __ASM_EMIT("fmul v16.4s, v24.4s, v16.4s") /* v16 = s2' = s2 * (2 - R*s2) */ \
493 __ASM_EMIT("fmul v17.4s, v25.4s, v17.4s") \
494 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2') */ \
495 __ASM_EMIT("frecps v25.4s, v17.4s, v1.4s") \
496 __ASM_EMIT("fmul v0.4s, v24.4s, v16.4s") /* v0 = s2" = s2' * (2 - R*s2) = 1/s2 */ \
497 __ASM_EMIT("fmul v1.4s, v25.4s, v17.4s") \
498 __ASM_EMIT("fmul v0.4s, v0.4s, v4.4s") /* s1 / s2 */ \
499 __ASM_EMIT("fmul v1.4s, v1.4s, v5.4s") \
500 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
501 __ASM_EMIT("sub %[count], %[count], #8") \
502 __ASM_EMIT("add %[" DST "], %[" DST "], #0x20") \
503 __ASM_EMIT("add %[" SRC1 "], %[" SRC1 "], #0x20") \
504 __ASM_EMIT("add %[" SRC2 "], %[" SRC2 "], #0x20") \
505 /* 4x block */ \
506 __ASM_EMIT("4:") \
507 __ASM_EMIT("adds %[count], %[count], #4") /* 8 - 4 */ \
508 __ASM_EMIT("b.lt 6f") \
509 __ASM_EMIT("ldr q0, [%[" SRC2 "], #0x00]") \
510 __ASM_EMIT("ldr q4, [%[" SRC1 "], #0x00]") \
511 __ASM_EMIT("frecpe v16.4s, v0.4s") /* v16 = s2 */ \
512 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2) */ \
513 __ASM_EMIT("fmul v16.4s, v24.4s, v16.4s") /* v16 = s2' = s2 * (2 - R*s2) */ \
514 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2') */ \
515 __ASM_EMIT("fmul v0.4s, v24.4s, v16.4s") /* v0 = s2" = s2' * (2 - R*s2) = 1/s2 */ \
516 __ASM_EMIT("fmul v0.4s, v0.4s, v4.4s") /* s1 / s2 */ \
517 __ASM_EMIT("str q0, [%[" DST "], #0x00]") \
518 __ASM_EMIT("sub %[count], %[count], #4") \
519 __ASM_EMIT("add %[" DST "], %[" DST "], #0x10") \
520 __ASM_EMIT("add %[" SRC1 "], %[" SRC1 "], #0x10") \
521 __ASM_EMIT("add %[" SRC2 "], %[" SRC2 "], #0x10") \
522 /* 1x block */ \
523 __ASM_EMIT("6:") \
524 __ASM_EMIT("adds %[count], %[count], #3") /* 4 - 3 */ \
525 __ASM_EMIT("b.lt 8f") \
526 __ASM_EMIT("7:") \
527 __ASM_EMIT("ld1r {v0.4s}, [%[" SRC2 "]]") \
528 __ASM_EMIT("ld1r {v4.4s}, [%[" SRC1 "]]") \
529 __ASM_EMIT("frecpe v16.4s, v0.4s") /* v16 = s2 */ \
530 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2) */ \
531 __ASM_EMIT("fmul v16.4s, v24.4s, v16.4s") /* v16 = s2' = s2 * (2 - R*s2) */ \
532 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2') */ \
533 __ASM_EMIT("fmul v0.4s, v24.4s, v16.4s") /* v0 = s2" = s2' * (2 - R*s2) = 1/s2 */ \
534 __ASM_EMIT("fmul v0.4s, v0.4s, v4.4s") /* s1 / s2 */ \
535 __ASM_EMIT("st1 {v0.s}[0], [%[" DST "]]") \
536 __ASM_EMIT("subs %[count], %[count], #1") \
537 __ASM_EMIT("add %[" DST "], %[" DST "], #0x04") \
538 __ASM_EMIT("add %[" SRC1 "], %[" SRC1 "], #0x04") \
539 __ASM_EMIT("add %[" SRC2 "], %[" SRC2 "], #0x04") \
540 __ASM_EMIT("bge 7b") \
541 __ASM_EMIT("8:")
542
543 void div3(float *dst, const float *src1, const float *src2, size_t count)
544 {
545 ARCH_AARCH64_ASM
546 (
547 OP_DIV3_CORE("dst", "src1", "src2")
548 : [dst] "+r" (dst), [src1] "+r" (src1), [src2] "+r" (src2),
549 [count] "+r" (count)
550 :
551 : "cc", "memory",
552 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
553 "v16", "v17", "v18", "v19",
554 "v24", "v25", "v26", "v27"
555 );
556 }
557
558 #undef OP_DIV3_CORE
559
560 #define INC_ON(x) x
561 #define INC_OFF(x)
562
563 #define FMOD_VV_CORE(DST, SRC1, INC1, SRC2, INC2, SEL) \
564 __ASM_EMIT("subs %[count], %[count], #16") \
565 __ASM_EMIT("b.lo 2f") \
566 /* 16x blocks */ \
567 __ASM_EMIT("1:") \
568 __ASM_EMIT("ldp q0, q1, [%[" SRC1 "], #0x00]") \
569 __ASM_EMIT("ldp q2, q3, [%[" SRC1 "], #0x20]") \
570 __ASM_EMIT("ldp q4, q5, [%[" SRC2 "], #0x00]") \
571 __ASM_EMIT("ldp q6, q7, [%[" SRC2 "], #0x20]") \
572 __ASM_EMIT("frecpe v16.4s, " SEL("v4.4s", "v0.4s")) /* v16 = d */ \
573 __ASM_EMIT("frecpe v17.4s, " SEL("v5.4s", "v1.4s")) \
574 __ASM_EMIT("frecpe v18.4s, " SEL("v6.4s", "v2.4s")) \
575 __ASM_EMIT("frecpe v19.4s, " SEL("v7.4s", "v3.4s")) \
576 __ASM_EMIT("frecps v20.4s, v16.4s, " SEL("v4.4s", "v0.4s")) /* v20 = (2 - R*s2) */ \
577 __ASM_EMIT("frecps v21.4s, v17.4s, " SEL("v5.4s", "v1.4s")) \
578 __ASM_EMIT("frecps v22.4s, v18.4s, " SEL("v6.4s", "v2.4s")) \
579 __ASM_EMIT("frecps v23.4s, v19.4s, " SEL("v7.4s", "v3.4s")) \
580 __ASM_EMIT("fmul v16.4s, v20.4s, v16.4s") /* v16 = s2' = s2 * (2 - R*s2) */ \
581 __ASM_EMIT("fmul v17.4s, v21.4s, v17.4s") \
582 __ASM_EMIT("fmul v18.4s, v22.4s, v18.4s") \
583 __ASM_EMIT("fmul v19.4s, v23.4s, v19.4s") \
584 __ASM_EMIT("frecps v20.4s, v16.4s, " SEL("v4.4s", "v0.4s")) /* v20 = (2 - R*s2') */ \
585 __ASM_EMIT("frecps v21.4s, v17.4s, " SEL("v5.4s", "v1.4s")) \
586 __ASM_EMIT("frecps v22.4s, v18.4s, " SEL("v6.4s", "v2.4s")) \
587 __ASM_EMIT("frecps v23.4s, v19.4s, " SEL("v7.4s", "v3.4s")) \
588 __ASM_EMIT("fmul v16.4s, v20.4s, v16.4s") /* v16 = s2" = s2' * (2 - R*s2) = 1/d */ \
589 __ASM_EMIT("fmul v17.4s, v21.4s, v17.4s") \
590 __ASM_EMIT("fmul v18.4s, v22.4s, v18.4s") \
591 __ASM_EMIT("fmul v19.4s, v23.4s, v19.4s") \
592 __ASM_EMIT("fmul v16.4s, v16.4s, " SEL("v0.4s", "v4.4s")) /* v16 = s/d */ \
593 __ASM_EMIT("fmul v17.4s, v17.4s, " SEL("v1.4s", "v5.4s")) \
594 __ASM_EMIT("fmul v18.4s, v18.4s, " SEL("v2.4s", "v6.4s")) \
595 __ASM_EMIT("fmul v19.4s, v19.4s, " SEL("v3.4s", "v7.4s")) \
596 __ASM_EMIT("frintz v16.4s, v16.4s") \
597 __ASM_EMIT("frintz v17.4s, v17.4s") \
598 __ASM_EMIT("frintz v18.4s, v18.4s") \
599 __ASM_EMIT("frintz v19.4s, v19.4s") \
600 __ASM_EMIT("fmls " SEL("v0.4s", "v4.4s") ", v16.4s, " SEL("v4.4s", "v0.4s")) \
601 __ASM_EMIT("fmls " SEL("v1.4s", "v5.4s") ", v17.4s, " SEL("v5.4s", "v1.4s")) \
602 __ASM_EMIT("fmls " SEL("v2.4s", "v6.4s") ", v18.4s, " SEL("v6.4s", "v2.4s")) \
603 __ASM_EMIT("fmls " SEL("v3.4s", "v7.4s") ", v19.4s, " SEL("v7.4s", "v3.4s")) \
604 __ASM_EMIT("stp " SEL("q0, q1", "q4, q5") ", [%[" DST "], #0x00]") \
605 __ASM_EMIT("stp " SEL("q2, q3", "q6, q7") ", [%[" DST "], #0x20]") \
606 __ASM_EMIT("subs %[count], %[count], #16") \
607 __ASM_EMIT(INC1("add %[" SRC1 "], %[" SRC1 "], #0x40")) \
608 __ASM_EMIT(INC2("add %[" SRC2 "], %[" SRC2 "], #0x40")) \
609 __ASM_EMIT("add %[" DST "], %[" DST "], #0x40") \
610 __ASM_EMIT("b.hs 1b") \
611 /* 8x block */ \
612 __ASM_EMIT("2:") \
613 __ASM_EMIT("adds %[count], %[count], #8") \
614 __ASM_EMIT("b.lt 4f") \
615 __ASM_EMIT("ldp q0, q1, [%[" SRC1 "], #0x00]") \
616 __ASM_EMIT("ldp q4, q5, [%[" SRC2 "], #0x00]") \
617 __ASM_EMIT("frecpe v16.4s, " SEL("v4.4s", "v0.4s")) /* v16 = d */ \
618 __ASM_EMIT("frecpe v17.4s, " SEL("v5.4s", "v1.4s")) \
619 __ASM_EMIT("frecps v20.4s, v16.4s, " SEL("v4.4s", "v0.4s")) /* v20 = (2 - R*s2) */ \
620 __ASM_EMIT("frecps v21.4s, v17.4s, " SEL("v5.4s", "v1.4s")) \
621 __ASM_EMIT("fmul v16.4s, v20.4s, v16.4s") /* v16 = s2' = s2 * (2 - R*s2) */ \
622 __ASM_EMIT("fmul v17.4s, v21.4s, v17.4s") \
623 __ASM_EMIT("frecps v20.4s, v16.4s, " SEL("v4.4s", "v0.4s")) /* v20 = (2 - R*s2') */ \
624 __ASM_EMIT("frecps v21.4s, v17.4s, " SEL("v5.4s", "v1.4s")) \
625 __ASM_EMIT("fmul v16.4s, v20.4s, v16.4s") /* v16 = s2" = s2' * (2 - R*s2) = 1/d */ \
626 __ASM_EMIT("fmul v17.4s, v21.4s, v17.4s") \
627 __ASM_EMIT("fmul v16.4s, v16.4s, " SEL("v0.4s", "v4.4s")) /* v16 = s/d */ \
628 __ASM_EMIT("fmul v17.4s, v17.4s, " SEL("v1.4s", "v5.4s")) \
629 __ASM_EMIT("frintz v16.4s, v16.4s") \
630 __ASM_EMIT("frintz v17.4s, v17.4s") \
631 __ASM_EMIT("fmls " SEL("v0.4s", "v4.4s") ", v16.4s, " SEL("v4.4s", "v0.4s")) \
632 __ASM_EMIT("fmls " SEL("v1.4s", "v5.4s") ", v17.4s, " SEL("v5.4s", "v1.4s")) \
633 __ASM_EMIT("stp " SEL("q0, q1", "q4, q5") ", [%[" DST "], #0x00]") \
634 __ASM_EMIT("sub %[count], %[count], #8") \
635 __ASM_EMIT(INC1("add %[" SRC1 "], %[" SRC1 "], #0x20")) \
636 __ASM_EMIT(INC2("add %[" SRC2 "], %[" SRC2 "], #0x20")) \
637 __ASM_EMIT("add %[" DST "], %[" DST "], #0x20") \
638 /* 4x blocks */ \
639 __ASM_EMIT("4:") \
640 __ASM_EMIT("adds %[count], %[count], #4") \
641 __ASM_EMIT("b.lt 6f") \
642 __ASM_EMIT("ldr q0, [%[" SRC1 "]]") \
643 __ASM_EMIT("ldr q4, [%[" SRC2 "]]") \
644 __ASM_EMIT("frecpe v16.4s, " SEL("v4.4s", "v0.4s")) /* v16 = d */ \
645 __ASM_EMIT("frecps v20.4s, v16.4s, " SEL("v4.4s", "v0.4s")) /* v20 = (2 - R*s2) */ \
646 __ASM_EMIT("fmul v16.4s, v20.4s, v16.4s") /* v16 = s2' = s2 * (2 - R*s2) */ \
647 __ASM_EMIT("frecps v20.4s, v16.4s, " SEL("v4.4s", "v0.4s")) /* v20 = (2 - R*s2') */ \
648 __ASM_EMIT("fmul v16.4s, v20.4s, v16.4s") /* v16 = s2" = s2' * (2 - R*s2) = 1/d */ \
649 __ASM_EMIT("fmul v16.4s, v16.4s, " SEL("v0.4s", "v4.4s")) /* v16 = s/d */ \
650 __ASM_EMIT("frintz v16.4s, v16.4s") \
651 __ASM_EMIT("fmls " SEL("v0.4s", "v4.4s") ", v16.4s, " SEL("v4.4s", "v0.4s")) \
652 __ASM_EMIT("str " SEL("q0", "q4") ", [%[" DST "]]") \
653 __ASM_EMIT("sub %[count], %[count], #4") \
654 __ASM_EMIT(INC1("add %[" SRC1 "], %[" SRC1 "], #0x10")) \
655 __ASM_EMIT(INC2("add %[" SRC2 "], %[" SRC2 "], #0x10")) \
656 __ASM_EMIT("add %[" DST "], %[" DST "], #0x10") \
657 /* 1x blocks */ \
658 __ASM_EMIT("6:") \
659 __ASM_EMIT("adds %[count], %[count], #3") \
660 __ASM_EMIT("b.lt 8f") \
661 __ASM_EMIT("7:") \
662 __ASM_EMIT("ld1r {v0.4s}, [%[" SRC1 "]]") \
663 __ASM_EMIT("ld1r {v4.4s}, [%[" SRC2 "]]") \
664 __ASM_EMIT("frecpe v16.4s, " SEL("v4.4s", "v0.4s")) /* v16 = d */ \
665 __ASM_EMIT("frecps v20.4s, v16.4s, " SEL("v4.4s", "v0.4s")) /* v20 = (2 - R*s2) */ \
666 __ASM_EMIT("fmul v16.4s, v20.4s, v16.4s") /* v16 = s2' = s2 * (2 - R*s2) */ \
667 __ASM_EMIT("frecps v20.4s, v16.4s, " SEL("v4.4s", "v0.4s")) /* v20 = (2 - R*s2') */ \
668 __ASM_EMIT("fmul v16.4s, v20.4s, v16.4s") /* v16 = s2" = s2' * (2 - R*s2) = 1/d */ \
669 __ASM_EMIT("fmul v16.4s, v16.4s, " SEL("v0.4s", "v4.4s")) /* v16 = s/d */ \
670 __ASM_EMIT("frintz v16.4s, v16.4s") \
671 __ASM_EMIT("fmls " SEL("v0.4s", "v4.4s") ", v16.4s, " SEL("v4.4s", "v0.4s")) \
672 __ASM_EMIT("st1 " SEL("{v0.s}[0]", "{v4.s}[0]") ", [%[" DST "]]") \
673 __ASM_EMIT("subs %[count], %[count], #1") \
674 __ASM_EMIT(INC1("add %[" SRC1 "], %[" SRC1 "], #0x04")) \
675 __ASM_EMIT(INC2("add %[" SRC2 "], %[" SRC2 "], #0x04")) \
676 __ASM_EMIT("add %[" DST "], %[" DST "], #0x04") \
677 __ASM_EMIT("bge 7b") \
678 __ASM_EMIT("8:")
679
680 void mod2(float *dst, const float *src, size_t count)
681 {
682 ARCH_AARCH64_ASM
683 (
684 FMOD_VV_CORE("dst", "dst", INC_OFF, "src", INC_ON, OP_SDSEL)
685 : [dst] "+r" (dst), [src] "+r" (src),
686 [count] "+r" (count)
687 :
688 : "cc", "memory",
689 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
690 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"
691 );
692 }
693
694 void rmod2(float *dst, const float *src, size_t count)
695 {
696 ARCH_AARCH64_ASM
697 (
698 FMOD_VV_CORE("dst", "dst", INC_OFF, "src", INC_ON, OP_SRSEL)
699 : [dst] "+r" (dst), [src] "+r" (src),
700 [count] "+r" (count)
701 :
702 : "cc", "memory",
703 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
704 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"
705 );
706 }
707
708 void mod3(float *dst, const float *src1, const float *src2, size_t count)
709 {
710 ARCH_AARCH64_ASM
711 (
712 FMOD_VV_CORE("dst", "src1", INC_ON, "src2", INC_ON, OP_SDSEL)
713 : [dst] "+r" (dst), [src1] "+r" (src1), [src2] "+r" (src2),
714 [count] "+r" (count)
715 :
716 : "cc", "memory",
717 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
718 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"
719 );
720 }
721
722 #undef FMOD_VV_CORE
723 #undef INC_ON
724 #undef INC_OFF
725
726 #undef OP_SDSEL
727 #undef OP_SRSEL
728 }
729 }
730
731 #endif /* PRIVATE_DSP_ARCH_AARCH64_ASIMD_PMATH_OP_VV_H_ */