Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/aarch64/asimd/pmath/fmop_kx.h

raw
1 /*
2 * Copyright (C) 2020 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2020 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 31 мар. 2020 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_AARCH64_ASIMD_PMATH_FMOP_KX_H_
23 #define PRIVATE_DSP_ARCH_AARCH64_ASIMD_PMATH_FMOP_KX_H_
24
25 #ifndef PRIVATE_DSP_ARCH_AARCH64_ASIMD_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_AARCH64_ASIMD_IMPL */
28
29 namespace lsp
30 {
31 namespace asimd
32 {
33 #define OP_DSEL(a, b) a
34 #define OP_RSEL(a, b) b
35
36 #define FMADDSUB_K3_CORE(DST, SRC, OP) \
37 __ASM_EMIT("subs %[count], %[count], #32") \
38 __ASM_EMIT("mov v25.16b, v24.16b") \
39 __ASM_EMIT("blo 2f") \
40 /* 32x blocks */ \
41 __ASM_EMIT("1:") \
42 __ASM_EMIT("ldp q0, q1, [%[" DST "], #0x00]") \
43 __ASM_EMIT("ldp q2, q3, [%[" DST "], #0x20]") \
44 __ASM_EMIT("ldp q4, q5, [%[" DST "], #0x40]") \
45 __ASM_EMIT("ldp q6, q7, [%[" DST "], #0x60]") \
46 __ASM_EMIT("ldp q16, q17, [%[" SRC "], #0x00]") \
47 __ASM_EMIT("ldp q18, q19, [%[" SRC "], #0x20]") \
48 __ASM_EMIT("ldp q20, q21, [%[" SRC "], #0x40]") \
49 __ASM_EMIT("ldp q22, q23, [%[" SRC "], #0x60]") \
50 __ASM_EMIT(OP " v0.4s, v16.4s, v24.4s") \
51 __ASM_EMIT(OP " v1.4s, v17.4s, v25.4s") \
52 __ASM_EMIT(OP " v2.4s, v18.4s, v24.4s") \
53 __ASM_EMIT(OP " v3.4s, v19.4s, v25.4s") \
54 __ASM_EMIT(OP " v4.4s, v20.4s, v24.4s") \
55 __ASM_EMIT(OP " v5.4s, v21.4s, v25.4s") \
56 __ASM_EMIT(OP " v6.4s, v22.4s, v24.4s") \
57 __ASM_EMIT(OP " v7.4s, v23.4s, v25.4s") \
58 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
59 __ASM_EMIT("stp q2, q3, [%[" DST "], #0x20]") \
60 __ASM_EMIT("stp q4, q5, [%[" DST "], #0x40]") \
61 __ASM_EMIT("stp q6, q7, [%[" DST "], #0x60]") \
62 __ASM_EMIT("subs %[count], %[count], #32") \
63 __ASM_EMIT("add %[" DST "], %[" DST "], #0x80") \
64 __ASM_EMIT("add %[" SRC "], %[" SRC "], #0x80") \
65 __ASM_EMIT("bhs 1b") \
66 /* 16x block */ \
67 __ASM_EMIT("2:") \
68 __ASM_EMIT("adds %[count], %[count], #16") /* 32-16 */ \
69 __ASM_EMIT("blt 4f") \
70 __ASM_EMIT("ldp q0, q1, [%[" DST "], #0x00]") \
71 __ASM_EMIT("ldp q2, q3, [%[" DST "], #0x20]") \
72 __ASM_EMIT("ldp q16, q17, [%[" SRC "], #0x00]") \
73 __ASM_EMIT("ldp q18, q19, [%[" SRC "], #0x20]") \
74 __ASM_EMIT(OP " v0.4s, v16.4s, v24.4s") \
75 __ASM_EMIT(OP " v1.4s, v17.4s, v25.4s") \
76 __ASM_EMIT(OP " v2.4s, v18.4s, v24.4s") \
77 __ASM_EMIT(OP " v3.4s, v19.4s, v25.4s") \
78 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
79 __ASM_EMIT("stp q2, q3, [%[" DST "], #0x20]") \
80 __ASM_EMIT("sub %[count], %[count], #16") \
81 __ASM_EMIT("add %[" DST "], %[" DST "], #0x40") \
82 __ASM_EMIT("add %[" SRC "], %[" SRC "], #0x40") \
83 /* 8x block */ \
84 __ASM_EMIT("4:") \
85 __ASM_EMIT("adds %[count], %[count], #8") /* 16-8 */ \
86 __ASM_EMIT("blt 6f") \
87 __ASM_EMIT("ldp q0, q1, [%[" DST "], #0x00]") \
88 __ASM_EMIT("ldp q16, q17, [%[" SRC "], #0x00]") \
89 __ASM_EMIT(OP " v0.4s, v16.4s, v24.4s") \
90 __ASM_EMIT(OP " v1.4s, v17.4s, v25.4s") \
91 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
92 __ASM_EMIT("sub %[count], %[count], #8") \
93 __ASM_EMIT("add %[" DST "], %[" DST "], #0x20") \
94 __ASM_EMIT("add %[" SRC "], %[" SRC "], #0x20") \
95 /* 4x block */ \
96 __ASM_EMIT("6:") \
97 __ASM_EMIT("adds %[count], %[count], #4") /* 8 - 4 */ \
98 __ASM_EMIT("blt 8f") \
99 __ASM_EMIT("ldr q0, [%[" DST "], #0x00]") \
100 __ASM_EMIT("ldr q16, [%[" SRC "], #0x00]") \
101 __ASM_EMIT(OP " v0.4s, v16.4s, v24.4s") \
102 __ASM_EMIT("str q0, [%[" DST "], #0x00]") \
103 __ASM_EMIT("sub %[count], %[count], #4") \
104 __ASM_EMIT("add %[" DST "], %[" DST "], #0x10") \
105 __ASM_EMIT("add %[" SRC "], %[" SRC "], #0x10") \
106 /* 1x block */ \
107 __ASM_EMIT("8:") \
108 __ASM_EMIT("adds %[count], %[count], #3") /* 4 - 3 */ \
109 __ASM_EMIT("blt 10f") \
110 __ASM_EMIT("9:") \
111 __ASM_EMIT("ld1r {v0.4s}, [%[" DST "]]") \
112 __ASM_EMIT("ld1r {v16.4s}, [%[" SRC "]]") \
113 __ASM_EMIT(OP " v0.4s, v16.4s, v24.4s") \
114 __ASM_EMIT("st1 {v0.s}[0], [%[" DST "]]") \
115 __ASM_EMIT("subs %[count], %[count], #1") \
116 __ASM_EMIT("add %[" DST "], %[" DST "], #0x04") \
117 __ASM_EMIT("add %[" SRC "], %[" SRC "], #0x04") \
118 __ASM_EMIT("bge 9b") \
119 __ASM_EMIT("10:")
120
121 void fmadd_k3(float *dst, const float *src, float k, size_t count)
122 {
123 IF_ARCH_AARCH64(float *pk = &k);
124 ARCH_AARCH64_ASM
125 (
126 __ASM_EMIT("ld1r {v24.4s}, [%[k]]")
127 FMADDSUB_K3_CORE("dst", "src", "fmla")
128 : [dst] "+r" (dst), [src] "+r" (src),
129 [count] "+r" (count)
130 : [k] "r" (pk)
131 : "cc", "memory",
132 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
133 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23",
134 "v24", "v25"
135 );
136 }
137
138 void fmsub_k3(float *dst, const float *src, float k, size_t count)
139 {
140 IF_ARCH_AARCH64(float *pk = &k);
141 ARCH_AARCH64_ASM
142 (
143 __ASM_EMIT("ld1r {v24.4s}, [%[k]]")
144 FMADDSUB_K3_CORE("dst", "src", "fmls")
145 : [dst] "+r" (dst), [src] "+r" (src),
146 [count] "+r" (count)
147 : [k] "r" (pk)
148 : "cc", "memory",
149 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
150 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23",
151 "v24", "v25"
152 );
153 }
154
155 #undef FMADDSUB_K3_CORE
156
157 #define FMOP_K3_CORE(DST, SRC, OP, SEL) \
158 __ASM_EMIT("subs %[count], %[count], #32") \
159 __ASM_EMIT("mov v25.16b, v24.16b") \
160 __ASM_EMIT("blo 2f") \
161 /* 32x blocks */ \
162 __ASM_EMIT("1:") \
163 __ASM_EMIT("ldp q16, q17, [%[" SRC "], #0x00]") \
164 __ASM_EMIT("ldp q18, q19, [%[" SRC "], #0x20]") \
165 __ASM_EMIT("ldp q20, q21, [%[" SRC "], #0x40]") \
166 __ASM_EMIT("ldp q22, q23, [%[" SRC "], #0x60]") \
167 __ASM_EMIT("ldp q0, q1, [%[" DST "], #0x00]") \
168 __ASM_EMIT("ldp q2, q3, [%[" DST "], #0x20]") \
169 __ASM_EMIT("ldp q4, q5, [%[" DST "], #0x40]") \
170 __ASM_EMIT("ldp q6, q7, [%[" DST "], #0x60]") \
171 __ASM_EMIT("fmul v16.4s, v16.4s, v24.4s") \
172 __ASM_EMIT("fmul v17.4s, v17.4s, v25.4s") \
173 __ASM_EMIT("fmul v18.4s, v18.4s, v24.4s") \
174 __ASM_EMIT("fmul v19.4s, v19.4s, v25.4s") \
175 __ASM_EMIT("fmul v20.4s, v20.4s, v24.4s") \
176 __ASM_EMIT("fmul v21.4s, v21.4s, v25.4s") \
177 __ASM_EMIT("fmul v22.4s, v22.4s, v24.4s") \
178 __ASM_EMIT("fmul v23.4s, v23.4s, v25.4s") \
179 __ASM_EMIT(OP " v0.4s, " SEL("v0.4s", "v16.4s") ", " SEL("v16.4s", "v0.4s")) \
180 __ASM_EMIT(OP " v1.4s, " SEL("v1.4s", "v17.4s") ", " SEL("v17.4s", "v1.4s")) \
181 __ASM_EMIT(OP " v2.4s, " SEL("v2.4s", "v18.4s") ", " SEL("v18.4s", "v2.4s")) \
182 __ASM_EMIT(OP " v3.4s, " SEL("v3.4s", "v19.4s") ", " SEL("v19.4s", "v3.4s")) \
183 __ASM_EMIT(OP " v4.4s, " SEL("v4.4s", "v20.4s") ", " SEL("v20.4s", "v4.4s")) \
184 __ASM_EMIT(OP " v5.4s, " SEL("v5.4s", "v21.4s") ", " SEL("v21.4s", "v5.4s")) \
185 __ASM_EMIT(OP " v6.4s, " SEL("v6.4s", "v22.4s") ", " SEL("v22.4s", "v6.4s")) \
186 __ASM_EMIT(OP " v7.4s, " SEL("v7.4s", "v23.4s") ", " SEL("v23.4s", "v7.4s")) \
187 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
188 __ASM_EMIT("stp q2, q3, [%[" DST "], #0x20]") \
189 __ASM_EMIT("stp q4, q5, [%[" DST "], #0x40]") \
190 __ASM_EMIT("stp q6, q7, [%[" DST "], #0x60]") \
191 __ASM_EMIT("subs %[count], %[count], #32") \
192 __ASM_EMIT("add %[" DST "], %[" DST "], #0x80") \
193 __ASM_EMIT("add %[" SRC "], %[" SRC "], #0x80") \
194 __ASM_EMIT("bhs 1b") \
195 /* 16x block */ \
196 __ASM_EMIT("2:") \
197 __ASM_EMIT("adds %[count], %[count], #16") /* 32-16 */ \
198 __ASM_EMIT("blt 4f") \
199 __ASM_EMIT("ldp q16, q17, [%[" SRC "], #0x00]") \
200 __ASM_EMIT("ldp q18, q19, [%[" SRC "], #0x20]") \
201 __ASM_EMIT("ldp q0, q1, [%[" DST "], #0x00]") \
202 __ASM_EMIT("ldp q2, q3, [%[" DST "], #0x20]") \
203 __ASM_EMIT("fmul v16.4s, v16.4s, v24.4s") \
204 __ASM_EMIT("fmul v17.4s, v17.4s, v25.4s") \
205 __ASM_EMIT("fmul v18.4s, v18.4s, v24.4s") \
206 __ASM_EMIT("fmul v19.4s, v19.4s, v25.4s") \
207 __ASM_EMIT(OP " v0.4s, " SEL("v0.4s", "v16.4s") ", " SEL("v16.4s", "v0.4s")) \
208 __ASM_EMIT(OP " v1.4s, " SEL("v1.4s", "v17.4s") ", " SEL("v17.4s", "v1.4s")) \
209 __ASM_EMIT(OP " v2.4s, " SEL("v2.4s", "v18.4s") ", " SEL("v18.4s", "v2.4s")) \
210 __ASM_EMIT(OP " v3.4s, " SEL("v3.4s", "v19.4s") ", " SEL("v19.4s", "v3.4s")) \
211 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
212 __ASM_EMIT("stp q2, q3, [%[" DST "], #0x20]") \
213 __ASM_EMIT("sub %[count], %[count], #16") \
214 __ASM_EMIT("add %[" DST "], %[" DST "], #0x40") \
215 __ASM_EMIT("add %[" SRC "], %[" SRC "], #0x40") \
216 /* 8x block */ \
217 __ASM_EMIT("4:") \
218 __ASM_EMIT("adds %[count], %[count], #8") /* 16-8 */ \
219 __ASM_EMIT("blt 6f") \
220 __ASM_EMIT("ldp q16, q17, [%[" SRC "], #0x00]") \
221 __ASM_EMIT("ldp q0, q1, [%[" DST "], #0x00]") \
222 __ASM_EMIT("fmul v16.4s, v16.4s, v24.4s") \
223 __ASM_EMIT("fmul v17.4s, v17.4s, v25.4s") \
224 __ASM_EMIT(OP " v0.4s, " SEL("v0.4s", "v16.4s") ", " SEL("v16.4s", "v0.4s")) \
225 __ASM_EMIT(OP " v1.4s, " SEL("v1.4s", "v17.4s") ", " SEL("v17.4s", "v1.4s")) \
226 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
227 __ASM_EMIT("sub %[count], %[count], #8") \
228 __ASM_EMIT("add %[" DST "], %[" DST "], #0x20") \
229 __ASM_EMIT("add %[" SRC "], %[" SRC "], #0x20") \
230 /* 4x block */ \
231 __ASM_EMIT("6:") \
232 __ASM_EMIT("adds %[count], %[count], #4") /* 8 - 4 */ \
233 __ASM_EMIT("blt 8f") \
234 __ASM_EMIT("ldr q16, [%[" SRC "], #0x00]") \
235 __ASM_EMIT("ldr q0, [%[" DST "], #0x00]") \
236 __ASM_EMIT("fmul v16.4s, v16.4s, v24.4s") \
237 __ASM_EMIT(OP " v0.4s, " SEL("v0.4s", "v16.4s") ", " SEL("v16.4s", "v0.4s")) \
238 __ASM_EMIT("str q0, [%[" DST "], #0x00]") \
239 __ASM_EMIT("sub %[count], %[count], #4") \
240 __ASM_EMIT("add %[" DST "], %[" DST "], #0x10") \
241 __ASM_EMIT("add %[" SRC "], %[" SRC "], #0x10") \
242 /* 1x block */ \
243 __ASM_EMIT("8:") \
244 __ASM_EMIT("adds %[count], %[count], #3") /* 4 - 3 */ \
245 __ASM_EMIT("blt 10f") \
246 __ASM_EMIT("9:") \
247 __ASM_EMIT("ld1r {v16.4s}, [%[" SRC "]]") \
248 __ASM_EMIT("ld1r {v0.4s}, [%[" DST "]]") \
249 __ASM_EMIT("fmul v16.4s, v16.4s, v24.4s") \
250 __ASM_EMIT(OP " v0.4s, " SEL("v0.4s", "v16.4s") ", " SEL("v16.4s", "v0.4s")) \
251 __ASM_EMIT("st1 {v0.s}[0], [%[" DST "]]") \
252 __ASM_EMIT("subs %[count], %[count], #1") \
253 __ASM_EMIT("add %[" DST "], %[" DST "], #0x04") \
254 __ASM_EMIT("add %[" SRC "], %[" SRC "], #0x04") \
255 __ASM_EMIT("bge 9b") \
256 __ASM_EMIT("10:")
257
258 void fmmul_k3(float *dst, const float *src, float k, size_t count)
259 {
260 IF_ARCH_AARCH64(float *pk = &k);
261 ARCH_AARCH64_ASM
262 (
263 __ASM_EMIT("ld1r {v24.4s}, [%[k]]")
264 FMOP_K3_CORE("dst", "src", "fmul", OP_DSEL)
265 : [dst] "+r" (dst), [src] "+r" (src),
266 [count] "+r" (count)
267 : [k] "r" (pk)
268 : "cc", "memory",
269 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
270 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23",
271 "v24", "v25"
272 );
273 }
274
275 void fmrsub_k3(float *dst, const float *src, float k, size_t count)
276 {
277 IF_ARCH_AARCH64(float *pk = &k);
278 ARCH_AARCH64_ASM
279 (
280 __ASM_EMIT("ld1r {v24.4s}, [%[k]]")
281 FMOP_K3_CORE("dst", "src", "fsub", OP_RSEL)
282 : [dst] "+r" (dst), [src] "+r" (src),
283 [count] "+r" (count)
284 : [k] "r" (pk)
285 : "cc", "memory",
286 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
287 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23",
288 "v24", "v25"
289 );
290 }
291
292 #undef FMOP_K3_CORE
293
294 #define FMDIV_K3_CORE(DST, SRC, SEL) \
295 __ASM_EMIT("subs %[count], %[count], #16") \
296 __ASM_EMIT("mov v21.16b, v20.16b") \
297 __ASM_EMIT("blo 2f") \
298 /* 16x blocks */ \
299 __ASM_EMIT("1:") \
300 __ASM_EMIT("ldp q0, q1, [%[" SEL(SRC, DST) "], #0x00]") \
301 __ASM_EMIT("ldp q2, q3, [%[" SEL(SRC, DST) "], #0x20]") \
302 __ASM_EMIT("ldp q4, q5, [%[" SEL(DST, SRC) "], #0x00]") \
303 __ASM_EMIT("ldp q6, q7, [%[" SEL(DST, SRC) "], #0x20]") \
304 __ASM_EMIT("frecpe v16.4s, v0.4s") /* v16 = s2 */ \
305 __ASM_EMIT("frecpe v17.4s, v1.4s") \
306 __ASM_EMIT("frecpe v18.4s, v2.4s") \
307 __ASM_EMIT("frecpe v19.4s, v3.4s") \
308 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2) */ \
309 __ASM_EMIT("frecps v25.4s, v17.4s, v1.4s") \
310 __ASM_EMIT("frecps v26.4s, v18.4s, v2.4s") \
311 __ASM_EMIT("frecps v27.4s, v19.4s, v3.4s") \
312 __ASM_EMIT("fmul v16.4s, v24.4s, v16.4s") /* v16 = s2' = s2 * (2 - R*s2) */ \
313 __ASM_EMIT("fmul v17.4s, v25.4s, v17.4s") \
314 __ASM_EMIT("fmul v18.4s, v26.4s, v18.4s") \
315 __ASM_EMIT("fmul v19.4s, v27.4s, v19.4s") \
316 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2') */ \
317 __ASM_EMIT("frecps v25.4s, v17.4s, v1.4s") \
318 __ASM_EMIT("frecps v26.4s, v18.4s, v2.4s") \
319 __ASM_EMIT("frecps v27.4s, v19.4s, v3.4s") \
320 __ASM_EMIT("fmul v0.4s, v24.4s, v16.4s") /* v0 = s2" = s2' * (2 - R*s2) = 1/s2 */ \
321 __ASM_EMIT("fmul v1.4s, v25.4s, v17.4s") \
322 __ASM_EMIT("fmul v2.4s, v26.4s, v18.4s") \
323 __ASM_EMIT("fmul v3.4s, v27.4s, v19.4s") \
324 __ASM_EMIT("fmul v0.4s, v0.4s, v4.4s") /* s1 / s2 */ \
325 __ASM_EMIT("fmul v1.4s, v1.4s, v5.4s") \
326 __ASM_EMIT("fmul v2.4s, v2.4s, v6.4s") \
327 __ASM_EMIT("fmul v3.4s, v3.4s, v7.4s") \
328 __ASM_EMIT("fmul v0.4s, v0.4s, v20.4s") /* k * s1 / s2 */ \
329 __ASM_EMIT("fmul v1.4s, v1.4s, v21.4s") \
330 __ASM_EMIT("fmul v2.4s, v2.4s, v20.4s") \
331 __ASM_EMIT("fmul v3.4s, v3.4s, v21.4s") \
332 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
333 __ASM_EMIT("stp q2, q3, [%[" DST "], #0x20]") \
334 __ASM_EMIT("subs %[count], %[count], #16") \
335 __ASM_EMIT("add %[" DST "], %[" DST "], #0x40") \
336 __ASM_EMIT("add %[" SRC "], %[" SRC "], #0x40") \
337 __ASM_EMIT("bhs 1b") \
338 /* 8x block */ \
339 __ASM_EMIT("2:") \
340 __ASM_EMIT("adds %[count], %[count], #8") /* 16 - 8 */ \
341 __ASM_EMIT("blt 4f") \
342 __ASM_EMIT("ldp q0, q1, [%[" SEL(SRC, DST) "], #0x00]") \
343 __ASM_EMIT("ldp q4, q5, [%[" SEL(DST, SRC) "], #0x00]") \
344 __ASM_EMIT("frecpe v16.4s, v0.4s") /* v16 = s2 */ \
345 __ASM_EMIT("frecpe v17.4s, v1.4s") \
346 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2) */ \
347 __ASM_EMIT("frecps v25.4s, v17.4s, v1.4s") \
348 __ASM_EMIT("fmul v16.4s, v24.4s, v16.4s") /* v16 = s2' = s2 * (2 - R*s2) */ \
349 __ASM_EMIT("fmul v17.4s, v25.4s, v17.4s") \
350 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2') */ \
351 __ASM_EMIT("frecps v25.4s, v17.4s, v1.4s") \
352 __ASM_EMIT("fmul v0.4s, v24.4s, v16.4s") /* v0 = s2" = s2' * (2 - R*s2) = 1/s2 */ \
353 __ASM_EMIT("fmul v1.4s, v25.4s, v17.4s") \
354 __ASM_EMIT("fmul v0.4s, v0.4s, v4.4s") /* s1 / s2 */ \
355 __ASM_EMIT("fmul v1.4s, v1.4s, v5.4s") \
356 __ASM_EMIT("fmul v0.4s, v0.4s, v20.4s") /* k * s1 / s2 */ \
357 __ASM_EMIT("fmul v1.4s, v1.4s, v21.4s") \
358 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
359 __ASM_EMIT("sub %[count], %[count], #8") \
360 __ASM_EMIT("add %[" DST "], %[" DST "], #0x20") \
361 __ASM_EMIT("add %[" SRC "], %[" SRC "], #0x20") \
362 /* 4x block */ \
363 __ASM_EMIT("4:") \
364 __ASM_EMIT("adds %[count], %[count], #4") /* 8 - 4 */ \
365 __ASM_EMIT("blt 6f") \
366 __ASM_EMIT("ldr q0, [%[" SEL(SRC, DST) "], #0x00]") \
367 __ASM_EMIT("ldr q4, [%[" SEL(DST, SRC) "], #0x00]") \
368 __ASM_EMIT("frecpe v16.4s, v0.4s") /* v16 = s2 */ \
369 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2) */ \
370 __ASM_EMIT("fmul v16.4s, v24.4s, v16.4s") /* v16 = s2' = s2 * (2 - R*s2) */ \
371 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2') */ \
372 __ASM_EMIT("fmul v0.4s, v24.4s, v16.4s") /* v0 = s2" = s2' * (2 - R*s2) = 1/s2 */ \
373 __ASM_EMIT("fmul v0.4s, v0.4s, v4.4s") /* s1 / s2 */ \
374 __ASM_EMIT("fmul v0.4s, v0.4s, v20.4s") /* k * s1 / s2 */ \
375 __ASM_EMIT("str q0, [%[" DST "], #0x00]") \
376 __ASM_EMIT("sub %[count], %[count], #4") \
377 __ASM_EMIT("add %[" DST "], %[" DST "], #0x10") \
378 __ASM_EMIT("add %[" SRC "], %[" SRC "], #0x10") \
379 /* 1x block */ \
380 __ASM_EMIT("6:") \
381 __ASM_EMIT("adds %[count], %[count], #3") /* 4 - 3 */ \
382 __ASM_EMIT("blt 8f") \
383 __ASM_EMIT("7:") \
384 __ASM_EMIT("ld1r {v0.4s}, [%[" SEL(SRC, DST) "]]") \
385 __ASM_EMIT("ld1r {v4.4s}, [%[" SEL(DST, SRC) "]]") \
386 __ASM_EMIT("frecpe v16.4s, v0.4s") /* v16 = s2 */ \
387 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2) */ \
388 __ASM_EMIT("fmul v16.4s, v24.4s, v16.4s") /* v16 = s2' = s2 * (2 - R*s2) */ \
389 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2') */ \
390 __ASM_EMIT("fmul v0.4s, v24.4s, v16.4s") /* v0 = s2" = s2' * (2 - R*s2) = 1/s2 */ \
391 __ASM_EMIT("fmul v0.4s, v0.4s, v4.4s") /* s1 / s2 */ \
392 __ASM_EMIT("fmul v0.4s, v0.4s, v20.4s") /* k * s1 / s2 */ \
393 __ASM_EMIT("st1 {v0.s}[0], [%[" DST "]]") \
394 __ASM_EMIT("subs %[count], %[count], #1") \
395 __ASM_EMIT("add %[" DST "], %[" DST "], #0x04") \
396 __ASM_EMIT("add %[" SRC "], %[" SRC "], #0x04") \
397 __ASM_EMIT("bge 7b") \
398 __ASM_EMIT("8:")
399
400
401 void fmdiv_k3(float *dst, const float *src, float k, size_t count)
402 {
403 IF_ARCH_AARCH64(float *pk = &k);
404 ARCH_AARCH64_ASM
405 (
406 __ASM_EMIT("ld1r {v0.4s}, [%[k]]")
407 __ASM_EMIT("frecpe v16.4s, v0.4s") /* v16 = k */
408 __ASM_EMIT("frecps v17.4s, v16.4s, v0.4s") /* v17 = (2 - R*k) */
409 __ASM_EMIT("fmul v16.4s, v17.4s, v16.4s") /* v16 = k' = k * (2 - R*k) */
410 __ASM_EMIT("frecps v17.4s, v16.4s, v0.4s") /* v17 = (2 - R*k') */
411 __ASM_EMIT("fmul v20.4s, v17.4s, v16.4s") /* v20 = k" = k' * (2 - R*k) = 1/k */
412 FMDIV_K3_CORE("dst", "src", OP_DSEL)
413 : [dst] "+r" (dst), [src] "+r" (src),
414 [count] "+r" (count)
415 : [k] "r" (pk)
416 : "cc", "memory",
417 "v0", "v1", "v2", "v3" , "v4", "v5", "v6", "v7",
418 "v16", "v17", "v18", "v19", "v20", "v21",
419 "v24", "v25", "v26", "v27"
420 );
421 }
422
423 void fmrdiv_k3(float *dst, const float *src, float k, size_t count)
424 {
425 IF_ARCH_AARCH64(float *pk = &k);
426 ARCH_AARCH64_ASM
427 (
428 __ASM_EMIT("ld1r {v20.4s}, [%[k]]")
429 FMDIV_K3_CORE("dst", "src", OP_RSEL)
430 : [dst] "+r" (dst), [src] "+r" (src),
431 [count] "+r" (count)
432 : [k] "r" (pk)
433 : "cc", "memory",
434 "v0", "v1", "v2", "v3" , "v4", "v5", "v6", "v7",
435 "v16", "v17", "v18", "v19", "v20", "v21",
436 "v24", "v25", "v26", "v27"
437 );
438 }
439
440 #undef FMDIV_K3_CORE
441
442 #define FMADDSUB_K4_CORE(DST, SRC1, SRC2, OP) \
443 __ASM_EMIT("subs %[count], %[count], #32") \
444 __ASM_EMIT("mov v25.16b, v24.16b") \
445 __ASM_EMIT("blo 2f") \
446 /* 32x blocks */ \
447 __ASM_EMIT("1:") \
448 __ASM_EMIT("ldp q0, q1, [%[" SRC1 "], #0x00]") \
449 __ASM_EMIT("ldp q2, q3, [%[" SRC1 "], #0x20]") \
450 __ASM_EMIT("ldp q4, q5, [%[" SRC1 "], #0x40]") \
451 __ASM_EMIT("ldp q6, q7, [%[" SRC1 "], #0x60]") \
452 __ASM_EMIT("ldp q16, q17, [%[" SRC2 "], #0x00]") \
453 __ASM_EMIT("ldp q18, q19, [%[" SRC2 "], #0x20]") \
454 __ASM_EMIT("ldp q20, q21, [%[" SRC2 "], #0x40]") \
455 __ASM_EMIT("ldp q22, q23, [%[" SRC2 "], #0x60]") \
456 __ASM_EMIT(OP " v0.4s, v16.4s, v24.4s") \
457 __ASM_EMIT(OP " v1.4s, v17.4s, v25.4s") \
458 __ASM_EMIT(OP " v2.4s, v18.4s, v24.4s") \
459 __ASM_EMIT(OP " v3.4s, v19.4s, v25.4s") \
460 __ASM_EMIT(OP " v4.4s, v20.4s, v24.4s") \
461 __ASM_EMIT(OP " v5.4s, v21.4s, v25.4s") \
462 __ASM_EMIT(OP " v6.4s, v22.4s, v24.4s") \
463 __ASM_EMIT(OP " v7.4s, v23.4s, v25.4s") \
464 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
465 __ASM_EMIT("stp q2, q3, [%[" DST "], #0x20]") \
466 __ASM_EMIT("stp q4, q5, [%[" DST "], #0x40]") \
467 __ASM_EMIT("stp q6, q7, [%[" DST "], #0x60]") \
468 __ASM_EMIT("subs %[count], %[count], #32") \
469 __ASM_EMIT("add %[" SRC1 "], %[" SRC1 "], #0x80") \
470 __ASM_EMIT("add %[" SRC2 "], %[" SRC2 "], #0x80") \
471 __ASM_EMIT("add %[" DST "], %[" DST "], #0x80") \
472 __ASM_EMIT("bhs 1b") \
473 /* 16x block */ \
474 __ASM_EMIT("2:") \
475 __ASM_EMIT("adds %[count], %[count], #16") /* 32-16 */ \
476 __ASM_EMIT("blt 4f") \
477 __ASM_EMIT("ldp q0, q1, [%[" SRC1 "], #0x00]") \
478 __ASM_EMIT("ldp q2, q3, [%[" SRC1 "], #0x20]") \
479 __ASM_EMIT("ldp q16, q17, [%[" SRC2 "], #0x00]") \
480 __ASM_EMIT("ldp q18, q19, [%[" SRC2 "], #0x20]") \
481 __ASM_EMIT(OP " v0.4s, v16.4s, v24.4s") \
482 __ASM_EMIT(OP " v1.4s, v17.4s, v25.4s") \
483 __ASM_EMIT(OP " v2.4s, v18.4s, v24.4s") \
484 __ASM_EMIT(OP " v3.4s, v19.4s, v25.4s") \
485 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
486 __ASM_EMIT("stp q2, q3, [%[" DST "], #0x20]") \
487 __ASM_EMIT("sub %[count], %[count], #16") \
488 __ASM_EMIT("add %[" SRC1 "], %[" SRC1 "], #0x40") \
489 __ASM_EMIT("add %[" SRC2 "], %[" SRC2 "], #0x40") \
490 __ASM_EMIT("add %[" DST "], %[" DST "], #0x40") \
491 /* 8x block */ \
492 __ASM_EMIT("4:") \
493 __ASM_EMIT("adds %[count], %[count], #8") /* 16-8 */ \
494 __ASM_EMIT("blt 6f") \
495 __ASM_EMIT("ldp q0, q1, [%[" SRC1 "], #0x00]") \
496 __ASM_EMIT("ldp q16, q17, [%[" SRC2 "], #0x00]") \
497 __ASM_EMIT(OP " v0.4s, v16.4s, v24.4s") \
498 __ASM_EMIT(OP " v1.4s, v17.4s, v25.4s") \
499 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
500 __ASM_EMIT("sub %[count], %[count], #8") \
501 __ASM_EMIT("add %[" SRC1 "], %[" SRC1 "], #0x20") \
502 __ASM_EMIT("add %[" SRC2 "], %[" SRC2 "], #0x20") \
503 __ASM_EMIT("add %[" DST "], %[" DST "], #0x20") \
504 /* 4x block */ \
505 __ASM_EMIT("6:") \
506 __ASM_EMIT("adds %[count], %[count], #4") /* 8 - 4 */ \
507 __ASM_EMIT("blt 8f") \
508 __ASM_EMIT("ldr q0, [%[" SRC1 "], #0x00]") \
509 __ASM_EMIT("ldr q16, [%[" SRC2 "], #0x00]") \
510 __ASM_EMIT(OP " v0.4s, v16.4s, v24.4s") \
511 __ASM_EMIT("str q0, [%[" DST "], #0x00]") \
512 __ASM_EMIT("sub %[count], %[count], #4") \
513 __ASM_EMIT("add %[" SRC1 "], %[" SRC1 "], #0x10") \
514 __ASM_EMIT("add %[" SRC2 "], %[" SRC2 "], #0x10") \
515 __ASM_EMIT("add %[" DST "], %[" DST "], #0x10") \
516 /* 1x block */ \
517 __ASM_EMIT("8:") \
518 __ASM_EMIT("adds %[count], %[count], #3") /* 4 - 3 */ \
519 __ASM_EMIT("blt 10f") \
520 __ASM_EMIT("9:") \
521 __ASM_EMIT("ld1r {v0.4s}, [%[" SRC1 "]]") \
522 __ASM_EMIT("ld1r {v16.4s}, [%[" SRC2 "]]") \
523 __ASM_EMIT(OP " v0.4s, v16.4s, v24.4s") \
524 __ASM_EMIT("st1 {v0.s}[0], [%[" DST "]]") \
525 __ASM_EMIT("subs %[count], %[count], #1") \
526 __ASM_EMIT("add %[" SRC1 "], %[" SRC1 "], #0x04") \
527 __ASM_EMIT("add %[" SRC2 "], %[" SRC2 "], #0x04") \
528 __ASM_EMIT("add %[" DST "], %[" DST "], #0x04") \
529 __ASM_EMIT("bge 9b") \
530 __ASM_EMIT("10:")
531
532 void fmadd_k4(float *dst, const float *src1, const float *src2, float k, size_t count)
533 {
534 IF_ARCH_AARCH64(float *pk = &k);
535 ARCH_AARCH64_ASM
536 (
537 __ASM_EMIT("ld1r {v24.4s}, [%[k]]")
538 FMADDSUB_K4_CORE("dst", "src1", "src2", "fmla")
539 : [dst] "+r" (dst), [src1] "+r" (src1), [src2] "+r" (src2),
540 [count] "+r" (count)
541 : [k] "r" (pk)
542 : "cc", "memory",
543 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
544 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23",
545 "v24", "v25"
546 );
547 }
548
549 void fmsub_k4(float *dst, const float *src1, const float *src2, float k, size_t count)
550 {
551 IF_ARCH_AARCH64(float *pk = &k);
552 ARCH_AARCH64_ASM
553 (
554 __ASM_EMIT("ld1r {v24.4s}, [%[k]]")
555 FMADDSUB_K4_CORE("dst", "src1", "src2", "fmls")
556 : [dst] "+r" (dst), [src1] "+r" (src1), [src2] "+r" (src2),
557 [count] "+r" (count)
558 : [k] "r" (pk)
559 : "cc", "memory",
560 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
561 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23",
562 "v24", "v25"
563 );
564 }
565
566 #undef FMADDSUB_K4_CORE
567
568 #define FMOP_K4_CORE(DST, SRC1, SRC2, OP, SEL) \
569 __ASM_EMIT("subs %[count], %[count], #32") \
570 __ASM_EMIT("mov v25.16b, v24.16b") \
571 __ASM_EMIT("blo 2f") \
572 /* 32x blocks */ \
573 __ASM_EMIT("1:") \
574 __ASM_EMIT("ldp q16, q17, [%[" SRC2 "], #0x00]") \
575 __ASM_EMIT("ldp q18, q19, [%[" SRC2 "], #0x20]") \
576 __ASM_EMIT("ldp q20, q21, [%[" SRC2 "], #0x40]") \
577 __ASM_EMIT("ldp q22, q23, [%[" SRC2 "], #0x60]") \
578 __ASM_EMIT("ldp q0, q1, [%[" SRC1 "], #0x00]") \
579 __ASM_EMIT("ldp q2, q3, [%[" SRC1 "], #0x20]") \
580 __ASM_EMIT("ldp q4, q5, [%[" SRC1 "], #0x40]") \
581 __ASM_EMIT("ldp q6, q7, [%[" SRC1 "], #0x60]") \
582 __ASM_EMIT("fmul v16.4s, v16.4s, v24.4s") \
583 __ASM_EMIT("fmul v17.4s, v17.4s, v25.4s") \
584 __ASM_EMIT("fmul v18.4s, v18.4s, v24.4s") \
585 __ASM_EMIT("fmul v19.4s, v19.4s, v25.4s") \
586 __ASM_EMIT("fmul v20.4s, v20.4s, v24.4s") \
587 __ASM_EMIT("fmul v21.4s, v21.4s, v25.4s") \
588 __ASM_EMIT("fmul v22.4s, v22.4s, v24.4s") \
589 __ASM_EMIT("fmul v23.4s, v23.4s, v25.4s") \
590 __ASM_EMIT(OP " v0.4s, " SEL("v0.4s", "v16.4s") ", " SEL("v16.4s", "v0.4s")) \
591 __ASM_EMIT(OP " v1.4s, " SEL("v1.4s", "v17.4s") ", " SEL("v17.4s", "v1.4s")) \
592 __ASM_EMIT(OP " v2.4s, " SEL("v2.4s", "v18.4s") ", " SEL("v18.4s", "v2.4s")) \
593 __ASM_EMIT(OP " v3.4s, " SEL("v3.4s", "v19.4s") ", " SEL("v19.4s", "v3.4s")) \
594 __ASM_EMIT(OP " v4.4s, " SEL("v4.4s", "v20.4s") ", " SEL("v20.4s", "v4.4s")) \
595 __ASM_EMIT(OP " v5.4s, " SEL("v5.4s", "v21.4s") ", " SEL("v21.4s", "v5.4s")) \
596 __ASM_EMIT(OP " v6.4s, " SEL("v6.4s", "v22.4s") ", " SEL("v22.4s", "v6.4s")) \
597 __ASM_EMIT(OP " v7.4s, " SEL("v7.4s", "v23.4s") ", " SEL("v23.4s", "v7.4s")) \
598 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
599 __ASM_EMIT("stp q2, q3, [%[" DST "], #0x20]") \
600 __ASM_EMIT("stp q4, q5, [%[" DST "], #0x40]") \
601 __ASM_EMIT("stp q6, q7, [%[" DST "], #0x60]") \
602 __ASM_EMIT("subs %[count], %[count], #32") \
603 __ASM_EMIT("add %[" SRC1 "], %[" SRC1 "], #0x80") \
604 __ASM_EMIT("add %[" SRC2 "], %[" SRC2 "], #0x80") \
605 __ASM_EMIT("add %[" DST "], %[" DST "], #0x80") \
606 __ASM_EMIT("bhs 1b") \
607 /* 16x block */ \
608 __ASM_EMIT("2:") \
609 __ASM_EMIT("adds %[count], %[count], #16") /* 32-16 */ \
610 __ASM_EMIT("blt 4f") \
611 __ASM_EMIT("ldp q16, q17, [%[" SRC2 "], #0x00]") \
612 __ASM_EMIT("ldp q18, q19, [%[" SRC2 "], #0x20]") \
613 __ASM_EMIT("ldp q0, q1, [%[" SRC1 "], #0x00]") \
614 __ASM_EMIT("ldp q2, q3, [%[" SRC1 "], #0x20]") \
615 __ASM_EMIT("fmul v16.4s, v16.4s, v24.4s") \
616 __ASM_EMIT("fmul v17.4s, v17.4s, v25.4s") \
617 __ASM_EMIT("fmul v18.4s, v18.4s, v24.4s") \
618 __ASM_EMIT("fmul v19.4s, v19.4s, v25.4s") \
619 __ASM_EMIT(OP " v0.4s, " SEL("v0.4s", "v16.4s") ", " SEL("v16.4s", "v0.4s")) \
620 __ASM_EMIT(OP " v1.4s, " SEL("v1.4s", "v17.4s") ", " SEL("v17.4s", "v1.4s")) \
621 __ASM_EMIT(OP " v2.4s, " SEL("v2.4s", "v18.4s") ", " SEL("v18.4s", "v2.4s")) \
622 __ASM_EMIT(OP " v3.4s, " SEL("v3.4s", "v19.4s") ", " SEL("v19.4s", "v3.4s")) \
623 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
624 __ASM_EMIT("stp q2, q3, [%[" DST "], #0x20]") \
625 __ASM_EMIT("sub %[count], %[count], #16") \
626 __ASM_EMIT("add %[" SRC1 "], %[" SRC1 "], #0x40") \
627 __ASM_EMIT("add %[" SRC2 "], %[" SRC2 "], #0x40") \
628 __ASM_EMIT("add %[" DST "], %[" DST "], #0x40") \
629 /* 8x block */ \
630 __ASM_EMIT("4:") \
631 __ASM_EMIT("adds %[count], %[count], #8") /* 16-8 */ \
632 __ASM_EMIT("blt 6f") \
633 __ASM_EMIT("ldp q16, q17, [%[" SRC2 "], #0x00]") \
634 __ASM_EMIT("ldp q0, q1, [%[" SRC1 "], #0x00]") \
635 __ASM_EMIT("fmul v16.4s, v16.4s, v24.4s") \
636 __ASM_EMIT("fmul v17.4s, v17.4s, v25.4s") \
637 __ASM_EMIT(OP " v0.4s, " SEL("v0.4s", "v16.4s") ", " SEL("v16.4s", "v0.4s")) \
638 __ASM_EMIT(OP " v1.4s, " SEL("v1.4s", "v17.4s") ", " SEL("v17.4s", "v1.4s")) \
639 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
640 __ASM_EMIT("sub %[count], %[count], #8") \
641 __ASM_EMIT("add %[" SRC1 "], %[" SRC1 "], #0x20") \
642 __ASM_EMIT("add %[" SRC2 "], %[" SRC2 "], #0x20") \
643 __ASM_EMIT("add %[" DST "], %[" DST "], #0x20") \
644 /* 4x block */ \
645 __ASM_EMIT("6:") \
646 __ASM_EMIT("adds %[count], %[count], #4") /* 8 - 4 */ \
647 __ASM_EMIT("blt 8f") \
648 __ASM_EMIT("ldr q16, [%[" SRC2 "], #0x00]") \
649 __ASM_EMIT("ldr q0, [%[" SRC1 "], #0x00]") \
650 __ASM_EMIT("fmul v16.4s, v16.4s, v24.4s") \
651 __ASM_EMIT(OP " v0.4s, " SEL("v0.4s", "v16.4s") ", " SEL("v16.4s", "v0.4s")) \
652 __ASM_EMIT("str q0, [%[" DST "], #0x00]") \
653 __ASM_EMIT("sub %[count], %[count], #4") \
654 __ASM_EMIT("add %[" SRC1 "], %[" SRC1 "], #0x10") \
655 __ASM_EMIT("add %[" SRC2 "], %[" SRC2 "], #0x10") \
656 __ASM_EMIT("add %[" DST "], %[" DST "], #0x10") \
657 /* 1x block */ \
658 __ASM_EMIT("8:") \
659 __ASM_EMIT("adds %[count], %[count], #3") /* 4 - 3 */ \
660 __ASM_EMIT("blt 10f") \
661 __ASM_EMIT("9:") \
662 __ASM_EMIT("ld1r {v16.4s}, [%[" SRC2 "]]") \
663 __ASM_EMIT("ld1r {v0.4s}, [%[" SRC1 "]]") \
664 __ASM_EMIT("fmul v16.4s, v16.4s, v24.4s") \
665 __ASM_EMIT(OP " v0.4s, " SEL("v0.4s", "v16.4s") ", " SEL("v16.4s", "v0.4s")) \
666 __ASM_EMIT("st1 {v0.s}[0], [%[" DST "]]") \
667 __ASM_EMIT("subs %[count], %[count], #1") \
668 __ASM_EMIT("add %[" SRC1 "], %[" SRC1 "], #0x04") \
669 __ASM_EMIT("add %[" SRC2 "], %[" SRC2 "], #0x04") \
670 __ASM_EMIT("add %[" DST "], %[" DST "], #0x04") \
671 __ASM_EMIT("bge 9b") \
672 __ASM_EMIT("10:")
673
674 void fmmul_k4(float *dst, const float *src1, const float *src2, float k, size_t count)
675 {
676 IF_ARCH_AARCH64(float *pk = &k);
677 ARCH_AARCH64_ASM
678 (
679 __ASM_EMIT("ld1r {v24.4s}, [%[k]]")
680 FMOP_K4_CORE("dst", "src1", "src2", "fmul", OP_DSEL)
681 : [dst] "+r" (dst), [src1] "+r" (src1), [src2] "+r" (src2),
682 [count] "+r" (count)
683 : [k] "r" (pk)
684 : "cc", "memory",
685 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
686 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23",
687 "v24", "v25"
688 );
689 }
690
691 void fmrsub_k4(float *dst, const float *src1, const float *src2, float k, size_t count)
692 {
693 IF_ARCH_AARCH64(float *pk = &k);
694 ARCH_AARCH64_ASM
695 (
696 __ASM_EMIT("ld1r {v24.4s}, [%[k]]")
697 FMOP_K4_CORE("dst", "src1", "src2", "fsub", OP_RSEL)
698 : [dst] "+r" (dst), [src1] "+r" (src1), [src2] "+r" (src2),
699 [count] "+r" (count)
700 : [k] "r" (pk)
701 : "cc", "memory",
702 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
703 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23",
704 "v24", "v25"
705 );
706 }
707
708 #undef FMOP_K4_CORE
709
710 #define FMDIV_K4_CORE(DST, SRC1, SRC2, SEL) \
711 __ASM_EMIT("subs %[count], %[count], #16") \
712 __ASM_EMIT("mov v21.16b, v20.16b") \
713 __ASM_EMIT("blo 2f") \
714 /* 16x blocks */ \
715 __ASM_EMIT("1:") \
716 __ASM_EMIT("ldp q0, q1, [%[" SEL(SRC2, SRC1) "], #0x00]") \
717 __ASM_EMIT("ldp q2, q3, [%[" SEL(SRC2, SRC1) "], #0x20]") \
718 __ASM_EMIT("ldp q4, q5, [%[" SEL(SRC1, SRC2) "], #0x00]") \
719 __ASM_EMIT("ldp q6, q7, [%[" SEL(SRC1, SRC2) "], #0x20]") \
720 __ASM_EMIT("frecpe v16.4s, v0.4s") /* v16 = s2 */ \
721 __ASM_EMIT("frecpe v17.4s, v1.4s") \
722 __ASM_EMIT("frecpe v18.4s, v2.4s") \
723 __ASM_EMIT("frecpe v19.4s, v3.4s") \
724 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2) */ \
725 __ASM_EMIT("frecps v25.4s, v17.4s, v1.4s") \
726 __ASM_EMIT("frecps v26.4s, v18.4s, v2.4s") \
727 __ASM_EMIT("frecps v27.4s, v19.4s, v3.4s") \
728 __ASM_EMIT("fmul v16.4s, v24.4s, v16.4s") /* v16 = s2' = s2 * (2 - R*s2) */ \
729 __ASM_EMIT("fmul v17.4s, v25.4s, v17.4s") \
730 __ASM_EMIT("fmul v18.4s, v26.4s, v18.4s") \
731 __ASM_EMIT("fmul v19.4s, v27.4s, v19.4s") \
732 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2') */ \
733 __ASM_EMIT("frecps v25.4s, v17.4s, v1.4s") \
734 __ASM_EMIT("frecps v26.4s, v18.4s, v2.4s") \
735 __ASM_EMIT("frecps v27.4s, v19.4s, v3.4s") \
736 __ASM_EMIT("fmul v0.4s, v24.4s, v16.4s") /* v0 = s2" = s2' * (2 - R*s2) = 1/s2 */ \
737 __ASM_EMIT("fmul v1.4s, v25.4s, v17.4s") \
738 __ASM_EMIT("fmul v2.4s, v26.4s, v18.4s") \
739 __ASM_EMIT("fmul v3.4s, v27.4s, v19.4s") \
740 __ASM_EMIT("fmul v0.4s, v0.4s, v4.4s") /* s1 / s2 */ \
741 __ASM_EMIT("fmul v1.4s, v1.4s, v5.4s") \
742 __ASM_EMIT("fmul v2.4s, v2.4s, v6.4s") \
743 __ASM_EMIT("fmul v3.4s, v3.4s, v7.4s") \
744 __ASM_EMIT("fmul v0.4s, v0.4s, v20.4s") /* k * s1 / s2 */ \
745 __ASM_EMIT("fmul v1.4s, v1.4s, v21.4s") \
746 __ASM_EMIT("fmul v2.4s, v2.4s, v20.4s") \
747 __ASM_EMIT("fmul v3.4s, v3.4s, v21.4s") \
748 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
749 __ASM_EMIT("stp q2, q3, [%[" DST "], #0x20]") \
750 __ASM_EMIT("subs %[count], %[count], #16") \
751 __ASM_EMIT("add %[" SRC1 "], %[" SRC1 "], #0x40") \
752 __ASM_EMIT("add %[" SRC2 "], %[" SRC2 "], #0x40") \
753 __ASM_EMIT("add %[" DST "], %[" DST "], #0x40") \
754 __ASM_EMIT("bhs 1b") \
755 /* 8x block */ \
756 __ASM_EMIT("2:") \
757 __ASM_EMIT("adds %[count], %[count], #8") /* 16 - 8 */ \
758 __ASM_EMIT("blt 4f") \
759 __ASM_EMIT("ldp q0, q1, [%[" SEL(SRC2, SRC1) "], #0x00]") \
760 __ASM_EMIT("ldp q4, q5, [%[" SEL(SRC1, SRC2) "], #0x00]") \
761 __ASM_EMIT("frecpe v16.4s, v0.4s") /* v16 = s2 */ \
762 __ASM_EMIT("frecpe v17.4s, v1.4s") \
763 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2) */ \
764 __ASM_EMIT("frecps v25.4s, v17.4s, v1.4s") \
765 __ASM_EMIT("fmul v16.4s, v24.4s, v16.4s") /* v16 = s2' = s2 * (2 - R*s2) */ \
766 __ASM_EMIT("fmul v17.4s, v25.4s, v17.4s") \
767 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2') */ \
768 __ASM_EMIT("frecps v25.4s, v17.4s, v1.4s") \
769 __ASM_EMIT("fmul v0.4s, v24.4s, v16.4s") /* v0 = s2" = s2' * (2 - R*s2) = 1/s2 */ \
770 __ASM_EMIT("fmul v1.4s, v25.4s, v17.4s") \
771 __ASM_EMIT("fmul v0.4s, v0.4s, v4.4s") /* s1 / s2 */ \
772 __ASM_EMIT("fmul v1.4s, v1.4s, v5.4s") \
773 __ASM_EMIT("fmul v0.4s, v0.4s, v20.4s") /* k * s1 / s2 */ \
774 __ASM_EMIT("fmul v1.4s, v1.4s, v21.4s") \
775 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
776 __ASM_EMIT("sub %[count], %[count], #8") \
777 __ASM_EMIT("add %[" SRC1 "], %[" SRC1 "], #0x20") \
778 __ASM_EMIT("add %[" SRC2 "], %[" SRC2 "], #0x20") \
779 __ASM_EMIT("add %[" DST "], %[" DST "], #0x20") \
780 /* 4x block */ \
781 __ASM_EMIT("4:") \
782 __ASM_EMIT("adds %[count], %[count], #4") /* 8 - 4 */ \
783 __ASM_EMIT("blt 6f") \
784 __ASM_EMIT("ldr q0, [%[" SEL(SRC2, SRC1) "], #0x00]") \
785 __ASM_EMIT("ldr q4, [%[" SEL(SRC1, SRC2) "], #0x00]") \
786 __ASM_EMIT("frecpe v16.4s, v0.4s") /* v16 = s2 */ \
787 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2) */ \
788 __ASM_EMIT("fmul v16.4s, v24.4s, v16.4s") /* v16 = s2' = s2 * (2 - R*s2) */ \
789 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2') */ \
790 __ASM_EMIT("fmul v0.4s, v24.4s, v16.4s") /* v0 = s2" = s2' * (2 - R*s2) = 1/s2 */ \
791 __ASM_EMIT("fmul v0.4s, v0.4s, v4.4s") /* s1 / s2 */ \
792 __ASM_EMIT("fmul v0.4s, v0.4s, v20.4s") /* k * s1 / s2 */ \
793 __ASM_EMIT("str q0, [%[" DST "], #0x00]") \
794 __ASM_EMIT("sub %[count], %[count], #4") \
795 __ASM_EMIT("add %[" SRC1 "], %[" SRC1 "], #0x10") \
796 __ASM_EMIT("add %[" SRC2 "], %[" SRC2 "], #0x10") \
797 __ASM_EMIT("add %[" DST "], %[" DST "], #0x10") \
798 /* 1x block */ \
799 __ASM_EMIT("6:") \
800 __ASM_EMIT("adds %[count], %[count], #3") /* 4 - 3 */ \
801 __ASM_EMIT("blt 8f") \
802 __ASM_EMIT("7:") \
803 __ASM_EMIT("ld1r {v0.4s}, [%[" SEL(SRC2, SRC1) "]]") \
804 __ASM_EMIT("ld1r {v4.4s}, [%[" SEL(SRC1, SRC2) "]]") \
805 __ASM_EMIT("frecpe v16.4s, v0.4s") /* v16 = s2 */ \
806 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2) */ \
807 __ASM_EMIT("fmul v16.4s, v24.4s, v16.4s") /* v16 = s2' = s2 * (2 - R*s2) */ \
808 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2') */ \
809 __ASM_EMIT("fmul v0.4s, v24.4s, v16.4s") /* v0 = s2" = s2' * (2 - R*s2) = 1/s2 */ \
810 __ASM_EMIT("fmul v0.4s, v0.4s, v4.4s") /* s1 / s2 */ \
811 __ASM_EMIT("fmul v0.4s, v0.4s, v20.4s") /* k * s1 / s2 */ \
812 __ASM_EMIT("st1 {v0.s}[0], [%[" DST "]]") \
813 __ASM_EMIT("subs %[count], %[count], #1") \
814 __ASM_EMIT("add %[" SRC1 "], %[" SRC1 "], #0x04") \
815 __ASM_EMIT("add %[" SRC2 "], %[" SRC2 "], #0x04") \
816 __ASM_EMIT("add %[" DST "], %[" DST "], #0x04") \
817 __ASM_EMIT("bge 7b") \
818 __ASM_EMIT("8:")
819
820
821 void fmdiv_k4(float *dst, const float *src1, const float *src2, float k, size_t count)
822 {
823 IF_ARCH_AARCH64(float *pk = &k);
824 ARCH_AARCH64_ASM
825 (
826 __ASM_EMIT("ld1r {v0.4s}, [%[k]]")
827 __ASM_EMIT("frecpe v16.4s, v0.4s") /* v16 = k */
828 __ASM_EMIT("frecps v17.4s, v16.4s, v0.4s") /* v17 = (2 - R*k) */
829 __ASM_EMIT("fmul v16.4s, v17.4s, v16.4s") /* v16 = k' = k * (2 - R*k) */
830 __ASM_EMIT("frecps v17.4s, v16.4s, v0.4s") /* v17 = (2 - R*k') */
831 __ASM_EMIT("fmul v20.4s, v17.4s, v16.4s") /* v20 = k" = k' * (2 - R*k) = 1/k */
832 FMDIV_K4_CORE("dst", "src1", "src2", OP_DSEL)
833 : [dst] "+r" (dst), [src1] "+r" (src1), [src2] "+r" (src2),
834 [count] "+r" (count)
835 : [k] "r" (pk)
836 : "cc", "memory",
837 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
838 "v16", "v17", "v18", "v19", "v20", "v21",
839 "v24", "v25", "v26", "v27"
840 );
841 }
842
843 void fmrdiv_k4(float *dst, const float *src1, const float *src2, float k, size_t count)
844 {
845 IF_ARCH_AARCH64(float *pk = &k);
846 ARCH_AARCH64_ASM
847 (
848 __ASM_EMIT("ld1r {v20.4s}, [%[k]]")
849 FMDIV_K4_CORE("dst", "src1", "src2", OP_RSEL)
850 : [dst] "+r" (dst), [src1] "+r" (src1), [src2] "+r" (src2),
851 [count] "+r" (count)
852 : [k] "r" (pk)
853 : "cc", "memory",
854 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
855 "v16", "v17", "v18", "v19", "v20", "v21",
856 "v24", "v25", "v26", "v27"
857 );
858 }
859
860 #define INC_ON(x) x
861 #define INC_OFF(x)
862
863 #undef FMDIV_K4_CORE
864
865 #define FMMOD_VV_CORE(DST, INCD, SRC1, INC1, SRC2, INC2, SEL) \
866 __ASM_EMIT("subs %[count], %[count], #16") \
867 __ASM_EMIT("mov v21.16b, v20.16b") \
868 __ASM_EMIT("b.lo 2f") \
869 /* 16x blocks */ \
870 __ASM_EMIT("1:") \
871 __ASM_EMIT("ldp q4, q5, [%[" SEL(SRC2, SRC1) "], #0x00]") \
872 __ASM_EMIT("ldp q6, q7, [%[" SEL(SRC2, SRC1) "], #0x20]") \
873 __ASM_EMIT(SEL("fmul v4.4s, v4.4s, v20.4s", "")) /* v4 = d */ \
874 __ASM_EMIT(SEL("fmul v5.4s, v5.4s, v21.4s", "")) \
875 __ASM_EMIT(SEL("fmul v6.4s, v6.4s, v20.4s", "")) \
876 __ASM_EMIT(SEL("fmul v7.4s, v7.4s, v21.4s", "")) \
877 __ASM_EMIT("frecpe v16.4s, v4.4s") /* v16 = s2 */ \
878 __ASM_EMIT("frecpe v17.4s, v5.4s") \
879 __ASM_EMIT("frecpe v18.4s, v6.4s") \
880 __ASM_EMIT("frecpe v19.4s, v7.4s") \
881 __ASM_EMIT("frecps v0.4s, v16.4s, v4.4s") /* v0 = (2 - R*s2) */ \
882 __ASM_EMIT("frecps v1.4s, v17.4s, v5.4s") \
883 __ASM_EMIT("frecps v2.4s, v18.4s, v6.4s") \
884 __ASM_EMIT("frecps v3.4s, v19.4s, v7.4s") \
885 __ASM_EMIT("fmul v16.4s, v0.4s, v16.4s") /* v16 = s2' = s2 * (2 - R*s2) */ \
886 __ASM_EMIT("fmul v17.4s, v1.4s, v17.4s") \
887 __ASM_EMIT("fmul v18.4s, v2.4s, v18.4s") \
888 __ASM_EMIT("fmul v19.4s, v3.4s, v19.4s") \
889 __ASM_EMIT("frecps v0.4s, v16.4s, v4.4s") /* v0 = (2 - R*s2') */ \
890 __ASM_EMIT("frecps v1.4s, v17.4s, v5.4s") \
891 __ASM_EMIT("frecps v2.4s, v18.4s, v6.4s") \
892 __ASM_EMIT("frecps v3.4s, v19.4s, v7.4s") \
893 __ASM_EMIT("fmul v16.4s, v0.4s, v16.4s") /* v16 = s2" = s2' * (2 - R*s2) = 1/d */ \
894 __ASM_EMIT("fmul v17.4s, v1.4s, v17.4s") \
895 __ASM_EMIT("fmul v18.4s, v2.4s, v18.4s") \
896 __ASM_EMIT("fmul v19.4s, v3.4s, v19.4s") \
897 __ASM_EMIT("ldp q0, q1, [%[" SEL(SRC1, SRC2) "], #0x00]") \
898 __ASM_EMIT("ldp q2, q3, [%[" SEL(SRC1, SRC2) "], #0x20]") \
899 __ASM_EMIT(SEL("", "fmul v0.4s, v0.4s, v20.4s")) /* v0 = s */ \
900 __ASM_EMIT(SEL("", "fmul v1.4s, v1.4s, v21.4s"))\
901 __ASM_EMIT(SEL("", "fmul v2.4s, v2.4s, v20.4s")) \
902 __ASM_EMIT(SEL("", "fmul v3.4s, v3.4s, v21.4s")) \
903 __ASM_EMIT("fmul v16.4s, v16.4s, v0.4s") /* v16 = s/d */ \
904 __ASM_EMIT("fmul v17.4s, v17.4s, v1.4s") \
905 __ASM_EMIT("fmul v18.4s, v18.4s, v2.4s") \
906 __ASM_EMIT("fmul v19.4s, v19.4s, v3.4s") \
907 __ASM_EMIT("frintz v16.4s, v16.4s") \
908 __ASM_EMIT("frintz v17.4s, v17.4s") \
909 __ASM_EMIT("frintz v18.4s, v18.4s") \
910 __ASM_EMIT("frintz v19.4s, v19.4s") \
911 __ASM_EMIT("fmls v0.4s, v4.4s, v16.4s") /* v0 = s - d*int(s/d) */ \
912 __ASM_EMIT("fmls v1.4s, v5.4s, v17.4s") \
913 __ASM_EMIT("fmls v2.4s, v6.4s, v18.4s") \
914 __ASM_EMIT("fmls v3.4s, v7.4s, v19.4s") \
915 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
916 __ASM_EMIT("stp q2, q3, [%[" DST "], #0x20]") \
917 __ASM_EMIT("subs %[count], %[count], #16") \
918 __ASM_EMIT(INC1("add %[" SRC1 "], %[" SRC1 "], #0x40")) \
919 __ASM_EMIT(INC2("add %[" SRC2 "], %[" SRC2 "], #0x40")) \
920 __ASM_EMIT(INCD("add %[" DST "], %[" DST "], #0x40")) \
921 __ASM_EMIT("b.hs 1b") \
922 /* 8x block */ \
923 __ASM_EMIT("2:") \
924 __ASM_EMIT("adds %[count], %[count], #8") \
925 __ASM_EMIT("b.lt 4f") \
926 __ASM_EMIT("ldp q4, q5, [%[" SEL(SRC2, SRC1) "], #0x00]") \
927 __ASM_EMIT(SEL("fmul v4.4s, v4.4s, v20.4s", "")) /* v4 = d */ \
928 __ASM_EMIT(SEL("fmul v5.4s, v5.4s, v21.4s", "")) \
929 __ASM_EMIT("frecpe v16.4s, v4.4s") /* v16 = s2 */ \
930 __ASM_EMIT("frecpe v17.4s, v5.4s") \
931 __ASM_EMIT("frecps v0.4s, v16.4s, v4.4s") /* v0 = (2 - R*s2) */ \
932 __ASM_EMIT("frecps v1.4s, v17.4s, v5.4s") \
933 __ASM_EMIT("fmul v16.4s, v0.4s, v16.4s") /* v16 = s2' = s2 * (2 - R*s2) */ \
934 __ASM_EMIT("fmul v17.4s, v1.4s, v17.4s") \
935 __ASM_EMIT("frecps v0.4s, v16.4s, v4.4s") /* v0 = (2 - R*s2') */ \
936 __ASM_EMIT("frecps v1.4s, v17.4s, v5.4s") \
937 __ASM_EMIT("fmul v16.4s, v0.4s, v16.4s") /* v16 = s2" = s2' * (2 - R*s2) = 1/d */ \
938 __ASM_EMIT("fmul v17.4s, v1.4s, v17.4s") \
939 __ASM_EMIT("ldp q0, q1, [%[" SEL(SRC1, SRC2) "], #0x00]") \
940 __ASM_EMIT(SEL("", "fmul v0.4s, v0.4s, v20.4s")) /* v0 = s */ \
941 __ASM_EMIT(SEL("", "fmul v1.4s, v1.4s, v21.4s"))\
942 __ASM_EMIT("fmul v16.4s, v16.4s, v0.4s") /* v16 = s/d */ \
943 __ASM_EMIT("fmul v17.4s, v17.4s, v1.4s") \
944 __ASM_EMIT("frintz v16.4s, v16.4s") \
945 __ASM_EMIT("frintz v17.4s, v17.4s") \
946 __ASM_EMIT("fmls v0.4s, v4.4s, v16.4s") /* v0 = s - d*int(s/d) */ \
947 __ASM_EMIT("fmls v1.4s, v5.4s, v17.4s") \
948 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
949 __ASM_EMIT("sub %[count], %[count], #8") \
950 __ASM_EMIT(INC1("add %[" SRC1 "], %[" SRC1 "], #0x20")) \
951 __ASM_EMIT(INC2("add %[" SRC2 "], %[" SRC2 "], #0x20")) \
952 __ASM_EMIT(INCD("add %[" DST "], %[" DST "], #0x20")) \
953 /* 4x blocks */ \
954 __ASM_EMIT("4:") \
955 __ASM_EMIT("adds %[count], %[count], #4") \
956 __ASM_EMIT("b.lt 6f") \
957 __ASM_EMIT("ldr q4, [%[" SEL(SRC2, SRC1) "]]") \
958 __ASM_EMIT(SEL("fmul v4.4s, v4.4s, v20.4s", "")) /* v4 = d */ \
959 __ASM_EMIT("frecpe v16.4s, v4.4s") /* v16 = s2 */ \
960 __ASM_EMIT("frecps v0.4s, v16.4s, v4.4s") /* v0 = (2 - R*s2) */ \
961 __ASM_EMIT("fmul v16.4s, v0.4s, v16.4s") /* v16 = s2' = s2 * (2 - R*s2) */ \
962 __ASM_EMIT("frecps v0.4s, v16.4s, v4.4s") /* v0 = (2 - R*s2') */ \
963 __ASM_EMIT("fmul v16.4s, v0.4s, v16.4s") /* v16 = s2" = s2' * (2 - R*s2) = 1/d */ \
964 __ASM_EMIT("ldr q0, [%[" SEL(SRC1, SRC2) "]]") \
965 __ASM_EMIT(SEL("", "fmul v0.4s, v0.4s, v20.4s")) /* v0 = s */ \
966 __ASM_EMIT("fmul v16.4s, v16.4s, v0.4s") /* v16 = s/d */ \
967 __ASM_EMIT("frintz v16.4s, v16.4s") \
968 __ASM_EMIT("fmls v0.4s, v4.4s, v16.4s") /* v0 = s - d*int(s/d) */ \
969 __ASM_EMIT("str q0, [%[" DST "]]") \
970 __ASM_EMIT("sub %[count], %[count], #4") \
971 __ASM_EMIT(INC1("add %[" SRC1 "], %[" SRC1 "], #0x10")) \
972 __ASM_EMIT(INC2("add %[" SRC2 "], %[" SRC2 "], #0x10")) \
973 __ASM_EMIT(INCD("add %[" DST "], %[" DST "], #0x10")) \
974 /* 1x blocks */ \
975 __ASM_EMIT("6:") \
976 __ASM_EMIT("adds %[count], %[count], #3") \
977 __ASM_EMIT("b.lt 8f") \
978 __ASM_EMIT("7:") \
979 __ASM_EMIT("ld1r {v4.4s}, [%[" SEL(SRC2, SRC1) "]]") \
980 __ASM_EMIT(SEL("fmul v4.4s, v4.4s, v20.4s", "")) /* v4 = d */ \
981 __ASM_EMIT("frecpe v16.4s, v4.4s") /* v16 = s2 */ \
982 __ASM_EMIT("frecps v0.4s, v16.4s, v4.4s") /* v0 = (2 - R*s2) */ \
983 __ASM_EMIT("fmul v16.4s, v0.4s, v16.4s") /* v16 = s2' = s2 * (2 - R*s2) */ \
984 __ASM_EMIT("frecps v0.4s, v16.4s, v4.4s") /* v0 = (2 - R*s2') */ \
985 __ASM_EMIT("fmul v16.4s, v0.4s, v16.4s") /* v16 = s2" = s2' * (2 - R*s2) = 1/d */ \
986 __ASM_EMIT("ld1r {v0.4s}, [%[" SEL(SRC1, SRC2) "]]") \
987 __ASM_EMIT(SEL("", "fmul v0.4s, v0.4s, v20.4s")) /* v0 = s */ \
988 __ASM_EMIT("fmul v16.4s, v16.4s, v0.4s") /* v16 = s/d */ \
989 __ASM_EMIT("frintz v16.4s, v16.4s") \
990 __ASM_EMIT("fmls v0.4s, v4.4s, v16.4s") /* v0 = s - d*int(s/d) */ \
991 __ASM_EMIT("st1 {v0.s}[0], [%[" DST "]]") \
992 __ASM_EMIT("subs %[count], %[count], #1") \
993 __ASM_EMIT(INC1("add %[" SRC1 "], %[" SRC1 "], #0x04")) \
994 __ASM_EMIT(INC2("add %[" SRC2 "], %[" SRC2 "], #0x04")) \
995 __ASM_EMIT(INCD("add %[" DST "], %[" DST "], #0x04")) \
996 __ASM_EMIT("bge 7b") \
997 __ASM_EMIT("8:")
998
999 void fmmod_k3(float *dst, const float *src, float k, size_t count)
1000 {
1001 IF_ARCH_AARCH64(float *pk = &k);
1002 ARCH_AARCH64_ASM
1003 (
1004 __ASM_EMIT("ld1r {v20.4s}, [%[k]]")
1005 FMMOD_VV_CORE("dst", INC_ON, "dst", INC_OFF, "src", INC_ON, OP_DSEL)
1006 : [dst] "+r" (dst), [src] "+r" (src),
1007 [count] "+r" (count)
1008 : [k] "r" (pk)
1009 : "cc", "memory",
1010 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
1011 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"
1012 );
1013 }
1014
1015 void fmrmod_k3(float *dst, const float *src, float k, size_t count)
1016 {
1017 IF_ARCH_AARCH64(float *pk = &k);
1018 ARCH_AARCH64_ASM
1019 (
1020 __ASM_EMIT("ld1r {v20.4s}, [%[k]]")
1021 FMMOD_VV_CORE("dst", INC_ON, "dst", INC_OFF, "src", INC_ON, OP_RSEL)
1022 : [dst] "+r" (dst), [src] "+r" (src),
1023 [count] "+r" (count)
1024 : [k] "r" (pk)
1025 : "cc", "memory",
1026 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
1027 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"
1028 );
1029 }
1030
1031 void fmmod_k4(float *dst, const float *src1, const float *src2, float k, size_t count)
1032 {
1033 IF_ARCH_AARCH64(float *pk = &k);
1034 ARCH_AARCH64_ASM
1035 (
1036 __ASM_EMIT("ld1r {v20.4s}, [%[k]]")
1037 FMMOD_VV_CORE("dst", INC_ON, "src1", INC_ON, "src2", INC_ON, OP_DSEL)
1038 : [dst] "+r" (dst), [src1] "+r" (src1), [src2] "+r" (src2),
1039 [count] "+r" (count)
1040 : [k] "r" (pk)
1041 : "cc", "memory",
1042 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
1043 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"
1044 );
1045 }
1046
1047 void fmrmod_k4(float *dst, const float *src1, const float *src2, float k, size_t count)
1048 {
1049 IF_ARCH_AARCH64(float *pk = &k);
1050 ARCH_AARCH64_ASM
1051 (
1052 __ASM_EMIT("ld1r {v20.4s}, [%[k]]")
1053 FMMOD_VV_CORE("dst", INC_ON, "src1", INC_ON, "src2", INC_ON, OP_RSEL)
1054 : [dst] "+r" (dst), [src1] "+r" (src1), [src2] "+r" (src2),
1055 [count] "+r" (count)
1056 : [k] "r" (pk)
1057 : "cc", "memory",
1058 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
1059 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"
1060 );
1061 }
1062
1063 #undef FMMOD_VV_CORE
1064
1065 #undef INC_ON
1066 #undef INC_OFF
1067
1068 #undef OP_DSEL
1069 #undef OP_RSEL
1070 }
1071 }
1072
1073 #endif /* PRIVATE_DSP_ARCH_AARCH64_ASIMD_PMATH_FMOP_KX_H_ */