Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/aarch64/asimd/pmath/fmop_vv.h

raw
1 /*
2 * Copyright (C) 2020 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2020 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 31 мар. 2020 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_AARCH64_ASIMD_PMATH_FMOP_VV_H_
23 #define PRIVATE_DSP_ARCH_AARCH64_ASIMD_PMATH_FMOP_VV_H_
24
25 #ifndef PRIVATE_DSP_ARCH_AARCH64_ASIMD_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_AARCH64_ASIMD_IMPL */
28
29 namespace lsp
30 {
31 namespace asimd
32 {
33 #define OP_DSEL(a, b) a
34 #define OP_RSEL(a, b) b
35
36 #define FMADDSUB_V3_CORE(DST, A, B, OP) \
37 __ASM_EMIT("subs %[count], %[count], #32") \
38 __ASM_EMIT("blo 2f") \
39 /* 32x blocks */ \
40 __ASM_EMIT("1:") \
41 __ASM_EMIT("ldp q0, q1, [%[" DST "], #0x00]") \
42 __ASM_EMIT("ldp q2, q3, [%[" DST "], #0x20]") \
43 __ASM_EMIT("ldp q4, q5, [%[" DST "], #0x40]") \
44 __ASM_EMIT("ldp q6, q7, [%[" DST "], #0x60]") \
45 __ASM_EMIT("ldp q16, q17, [%[" A "], #0x00]") \
46 __ASM_EMIT("ldp q18, q19, [%[" A "], #0x20]") \
47 __ASM_EMIT("ldp q20, q21, [%[" A "], #0x40]") \
48 __ASM_EMIT("ldp q22, q23, [%[" A "], #0x60]") \
49 __ASM_EMIT("ldp q24, q25, [%[" B "], #0x00]") \
50 __ASM_EMIT("ldp q26, q27, [%[" B "], #0x20]") \
51 __ASM_EMIT("ldp q28, q29, [%[" B "], #0x40]") \
52 __ASM_EMIT("ldp q30, q31, [%[" B "], #0x60]") \
53 __ASM_EMIT(OP " v0.4s, v16.4s, v24.4s") \
54 __ASM_EMIT(OP " v1.4s, v17.4s, v25.4s") \
55 __ASM_EMIT(OP " v2.4s, v18.4s, v26.4s") \
56 __ASM_EMIT(OP " v3.4s, v19.4s, v27.4s") \
57 __ASM_EMIT(OP " v4.4s, v20.4s, v28.4s") \
58 __ASM_EMIT(OP " v5.4s, v21.4s, v29.4s") \
59 __ASM_EMIT(OP " v6.4s, v22.4s, v30.4s") \
60 __ASM_EMIT(OP " v7.4s, v23.4s, v31.4s") \
61 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
62 __ASM_EMIT("stp q2, q3, [%[" DST "], #0x20]") \
63 __ASM_EMIT("stp q4, q5, [%[" DST "], #0x40]") \
64 __ASM_EMIT("stp q6, q7, [%[" DST "], #0x60]") \
65 __ASM_EMIT("subs %[count], %[count], #32") \
66 __ASM_EMIT("add %[" DST "], %[" DST "], #0x80") \
67 __ASM_EMIT("add %[" A "], %[" A "], #0x80") \
68 __ASM_EMIT("add %[" B "], %[" B "], #0x80") \
69 __ASM_EMIT("bhs 1b") \
70 /* 16x block */ \
71 __ASM_EMIT("2:") \
72 __ASM_EMIT("adds %[count], %[count], #16") /* 32-16 */ \
73 __ASM_EMIT("blt 4f") \
74 __ASM_EMIT("ldp q0, q1, [%[" DST "], #0x00]") \
75 __ASM_EMIT("ldp q2, q3, [%[" DST "], #0x20]") \
76 __ASM_EMIT("ldp q16, q17, [%[" A "], #0x00]") \
77 __ASM_EMIT("ldp q18, q19, [%[" A "], #0x20]") \
78 __ASM_EMIT("ldp q24, q25, [%[" B "], #0x00]") \
79 __ASM_EMIT("ldp q26, q27, [%[" B "], #0x20]") \
80 __ASM_EMIT(OP " v0.4s, v16.4s, v24.4s") \
81 __ASM_EMIT(OP " v1.4s, v17.4s, v25.4s") \
82 __ASM_EMIT(OP " v2.4s, v18.4s, v26.4s") \
83 __ASM_EMIT(OP " v3.4s, v19.4s, v27.4s") \
84 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
85 __ASM_EMIT("stp q2, q3, [%[" DST "], #0x20]") \
86 __ASM_EMIT("sub %[count], %[count], #16") \
87 __ASM_EMIT("add %[" DST "], %[" DST "], #0x40") \
88 __ASM_EMIT("add %[" A "], %[" A "], #0x40") \
89 __ASM_EMIT("add %[" B "], %[" B "], #0x40") \
90 /* 8x block */ \
91 __ASM_EMIT("4:") \
92 __ASM_EMIT("adds %[count], %[count], #8") /* 16-8 */ \
93 __ASM_EMIT("blt 6f") \
94 __ASM_EMIT("ldp q0, q1, [%[" DST "], #0x00]") \
95 __ASM_EMIT("ldp q16, q17, [%[" A "], #0x00]") \
96 __ASM_EMIT("ldp q24, q25, [%[" B "], #0x00]") \
97 __ASM_EMIT(OP " v0.4s, v16.4s, v24.4s") \
98 __ASM_EMIT(OP " v1.4s, v17.4s, v25.4s") \
99 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
100 __ASM_EMIT("sub %[count], %[count], #8") \
101 __ASM_EMIT("add %[" DST "], %[" DST "], #0x20") \
102 __ASM_EMIT("add %[" A "], %[" A "], #0x20") \
103 __ASM_EMIT("add %[" B "], %[" B "], #0x20") \
104 /* 4x block */ \
105 __ASM_EMIT("6:") \
106 __ASM_EMIT("adds %[count], %[count], #4") /* 8 - 4 */ \
107 __ASM_EMIT("blt 8f") \
108 __ASM_EMIT("ldr q0, [%[" DST "], #0x00]") \
109 __ASM_EMIT("ldr q16, [%[" A "], #0x00]") \
110 __ASM_EMIT("ldr q24, [%[" B "], #0x00]") \
111 __ASM_EMIT(OP " v0.4s, v16.4s, v24.4s") \
112 __ASM_EMIT("str q0, [%[" DST "], #0x00]") \
113 __ASM_EMIT("sub %[count], %[count], #4") \
114 __ASM_EMIT("add %[" DST "], %[" DST "], #0x10") \
115 __ASM_EMIT("add %[" A "], %[" A "], #0x10") \
116 __ASM_EMIT("add %[" B "], %[" B "], #0x10") \
117 /* 1x block */ \
118 __ASM_EMIT("8:") \
119 __ASM_EMIT("adds %[count], %[count], #3") /* 4 - 3 */ \
120 __ASM_EMIT("blt 10f") \
121 __ASM_EMIT("9:") \
122 __ASM_EMIT("ld1r {v0.4s}, [%[" DST "]]") \
123 __ASM_EMIT("ld1r {v16.4s}, [%[" A "]]") \
124 __ASM_EMIT("ld1r {v24.4s}, [%[" B "]]") \
125 __ASM_EMIT(OP " v0.4s, v16.4s, v24.4s") \
126 __ASM_EMIT("st1 {v0.s}[0], [%[" DST "]]") \
127 __ASM_EMIT("subs %[count], %[count], #1") \
128 __ASM_EMIT("add %[" DST "], %[" DST "], #0x04") \
129 __ASM_EMIT("add %[" A "], %[" A "], #0x04") \
130 __ASM_EMIT("add %[" B "], %[" B "], #0x04") \
131 __ASM_EMIT("bge 9b") \
132 __ASM_EMIT("10:")
133
134 void fmadd3(float *dst, const float *a, const float *b, size_t count)
135 {
136 ARCH_AARCH64_ASM
137 (
138 FMADDSUB_V3_CORE("dst", "a", "b", "fmla")
139 : [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b),
140 [count] "+r" (count)
141 :
142 : "cc", "memory",
143 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
144 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23",
145 "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31"
146 );
147 }
148
149 void fmsub3(float *dst, const float *a, const float *b, size_t count)
150 {
151 ARCH_AARCH64_ASM
152 (
153 FMADDSUB_V3_CORE("dst", "a", "b", "fmls")
154 : [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b),
155 [count] "+r" (count)
156 :
157 : "cc", "memory",
158 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
159 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23",
160 "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31"
161 );
162 }
163
164 #undef FMADDSUB_V3_CORE
165
166 #define FMOP_V3_CORE(DST, A, B, OP, SEL) \
167 __ASM_EMIT("subs %[count], %[count], #32") \
168 __ASM_EMIT("blo 2f") \
169 /* 32x blocks */ \
170 __ASM_EMIT("1:") \
171 __ASM_EMIT("ldp q16, q17, [%[" A "], #0x00]") \
172 __ASM_EMIT("ldp q18, q19, [%[" A "], #0x20]") \
173 __ASM_EMIT("ldp q20, q21, [%[" A "], #0x40]") \
174 __ASM_EMIT("ldp q22, q23, [%[" A "], #0x60]") \
175 __ASM_EMIT("ldp q24, q25, [%[" B "], #0x00]") \
176 __ASM_EMIT("ldp q26, q27, [%[" B "], #0x20]") \
177 __ASM_EMIT("ldp q28, q29, [%[" B "], #0x40]") \
178 __ASM_EMIT("ldp q30, q31, [%[" B "], #0x60]") \
179 __ASM_EMIT("ldp q0, q1, [%[" DST "], #0x00]") \
180 __ASM_EMIT("ldp q2, q3, [%[" DST "], #0x20]") \
181 __ASM_EMIT("ldp q4, q5, [%[" DST "], #0x40]") \
182 __ASM_EMIT("ldp q6, q7, [%[" DST "], #0x60]") \
183 __ASM_EMIT("fmul v16.4s, v16.4s, v24.4s") \
184 __ASM_EMIT("fmul v17.4s, v17.4s, v25.4s") \
185 __ASM_EMIT("fmul v18.4s, v18.4s, v26.4s") \
186 __ASM_EMIT("fmul v19.4s, v19.4s, v27.4s") \
187 __ASM_EMIT("fmul v20.4s, v20.4s, v28.4s") \
188 __ASM_EMIT("fmul v21.4s, v21.4s, v29.4s") \
189 __ASM_EMIT("fmul v22.4s, v22.4s, v30.4s") \
190 __ASM_EMIT("fmul v23.4s, v23.4s, v31.4s") \
191 __ASM_EMIT(OP " v0.4s, " SEL("v0.4s", "v16.4s") ", " SEL("v16.4s", "v0.4s")) \
192 __ASM_EMIT(OP " v1.4s, " SEL("v1.4s", "v17.4s") ", " SEL("v17.4s", "v1.4s")) \
193 __ASM_EMIT(OP " v2.4s, " SEL("v2.4s", "v18.4s") ", " SEL("v18.4s", "v2.4s")) \
194 __ASM_EMIT(OP " v3.4s, " SEL("v3.4s", "v19.4s") ", " SEL("v19.4s", "v3.4s")) \
195 __ASM_EMIT(OP " v4.4s, " SEL("v4.4s", "v20.4s") ", " SEL("v20.4s", "v4.4s")) \
196 __ASM_EMIT(OP " v5.4s, " SEL("v5.4s", "v21.4s") ", " SEL("v21.4s", "v5.4s")) \
197 __ASM_EMIT(OP " v6.4s, " SEL("v6.4s", "v22.4s") ", " SEL("v22.4s", "v6.4s")) \
198 __ASM_EMIT(OP " v7.4s, " SEL("v7.4s", "v23.4s") ", " SEL("v23.4s", "v7.4s")) \
199 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
200 __ASM_EMIT("stp q2, q3, [%[" DST "], #0x20]") \
201 __ASM_EMIT("stp q4, q5, [%[" DST "], #0x40]") \
202 __ASM_EMIT("stp q6, q7, [%[" DST "], #0x60]") \
203 __ASM_EMIT("subs %[count], %[count], #32") \
204 __ASM_EMIT("add %[" DST "], %[" DST "], #0x80") \
205 __ASM_EMIT("add %[" A "], %[" A "], #0x80") \
206 __ASM_EMIT("add %[" B "], %[" B "], #0x80") \
207 __ASM_EMIT("bhs 1b") \
208 /* 16x block */ \
209 __ASM_EMIT("2:") \
210 __ASM_EMIT("adds %[count], %[count], #16") /* 32-16 */ \
211 __ASM_EMIT("blt 4f") \
212 __ASM_EMIT("ldp q16, q17, [%[" A "], #0x00]") \
213 __ASM_EMIT("ldp q18, q19, [%[" A "], #0x20]") \
214 __ASM_EMIT("ldp q24, q25, [%[" B "], #0x00]") \
215 __ASM_EMIT("ldp q26, q27, [%[" B "], #0x20]") \
216 __ASM_EMIT("ldp q0, q1, [%[" DST "], #0x00]") \
217 __ASM_EMIT("ldp q2, q3, [%[" DST "], #0x20]") \
218 __ASM_EMIT("fmul v16.4s, v16.4s, v24.4s") \
219 __ASM_EMIT("fmul v17.4s, v17.4s, v25.4s") \
220 __ASM_EMIT("fmul v18.4s, v18.4s, v26.4s") \
221 __ASM_EMIT("fmul v19.4s, v19.4s, v27.4s") \
222 __ASM_EMIT(OP " v0.4s, " SEL("v0.4s", "v16.4s") ", " SEL("v16.4s", "v0.4s")) \
223 __ASM_EMIT(OP " v1.4s, " SEL("v1.4s", "v17.4s") ", " SEL("v17.4s", "v1.4s")) \
224 __ASM_EMIT(OP " v2.4s, " SEL("v2.4s", "v18.4s") ", " SEL("v18.4s", "v2.4s")) \
225 __ASM_EMIT(OP " v3.4s, " SEL("v3.4s", "v19.4s") ", " SEL("v19.4s", "v3.4s")) \
226 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
227 __ASM_EMIT("stp q2, q3, [%[" DST "], #0x20]") \
228 __ASM_EMIT("sub %[count], %[count], #16") \
229 __ASM_EMIT("add %[" DST "], %[" DST "], #0x40") \
230 __ASM_EMIT("add %[" A "], %[" A "], #0x40") \
231 __ASM_EMIT("add %[" B "], %[" B "], #0x40") \
232 /* 8x block */ \
233 __ASM_EMIT("4:") \
234 __ASM_EMIT("adds %[count], %[count], #8") /* 16-8 */ \
235 __ASM_EMIT("blt 6f") \
236 __ASM_EMIT("ldp q16, q17, [%[" A "], #0x00]") \
237 __ASM_EMIT("ldp q24, q25, [%[" B "], #0x00]") \
238 __ASM_EMIT("ldp q0, q1, [%[" DST "], #0x00]") \
239 __ASM_EMIT("fmul v16.4s, v16.4s, v24.4s") \
240 __ASM_EMIT("fmul v17.4s, v17.4s, v25.4s") \
241 __ASM_EMIT(OP " v0.4s, " SEL("v0.4s", "v16.4s") ", " SEL("v16.4s", "v0.4s")) \
242 __ASM_EMIT(OP " v1.4s, " SEL("v1.4s", "v17.4s") ", " SEL("v17.4s", "v1.4s")) \
243 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
244 __ASM_EMIT("sub %[count], %[count], #8") \
245 __ASM_EMIT("add %[" DST "], %[" DST "], #0x20") \
246 __ASM_EMIT("add %[" A "], %[" A "], #0x20") \
247 __ASM_EMIT("add %[" B "], %[" B "], #0x20") \
248 /* 4x block */ \
249 __ASM_EMIT("6:") \
250 __ASM_EMIT("adds %[count], %[count], #4") /* 8 - 4 */ \
251 __ASM_EMIT("blt 8f") \
252 __ASM_EMIT("ldr q16, [%[" A "], #0x00]") \
253 __ASM_EMIT("ldr q24, [%[" B "], #0x00]") \
254 __ASM_EMIT("ldr q0, [%[" DST "], #0x00]") \
255 __ASM_EMIT("fmul v16.4s, v16.4s, v24.4s") \
256 __ASM_EMIT(OP " v0.4s, " SEL("v0.4s", "v16.4s") ", " SEL("v16.4s", "v0.4s")) \
257 __ASM_EMIT("str q0, [%[" DST "], #0x00]") \
258 __ASM_EMIT("sub %[count], %[count], #4") \
259 __ASM_EMIT("add %[" DST "], %[" DST "], #0x10") \
260 __ASM_EMIT("add %[" A "], %[" A "], #0x10") \
261 __ASM_EMIT("add %[" B "], %[" B "], #0x10") \
262 /* 1x block */ \
263 __ASM_EMIT("8:") \
264 __ASM_EMIT("adds %[count], %[count], #3") /* 4 - 3 */ \
265 __ASM_EMIT("blt 10f") \
266 __ASM_EMIT("9:") \
267 __ASM_EMIT("ld1r {v16.4s}, [%[" A "]]") \
268 __ASM_EMIT("ld1r {v24.4s}, [%[" B "]]") \
269 __ASM_EMIT("ld1r {v0.4s}, [%[" DST "]]") \
270 __ASM_EMIT("fmul v16.4s, v16.4s, v24.4s") \
271 __ASM_EMIT(OP " v0.4s, " SEL("v0.4s", "v16.4s") ", " SEL("v16.4s", "v0.4s")) \
272 __ASM_EMIT("st1 {v0.s}[0], [%[" DST "]]") \
273 __ASM_EMIT("subs %[count], %[count], #1") \
274 __ASM_EMIT("add %[" DST "], %[" DST "], #0x04") \
275 __ASM_EMIT("add %[" A "], %[" A "], #0x04") \
276 __ASM_EMIT("add %[" B "], %[" B "], #0x04") \
277 __ASM_EMIT("bge 9b") \
278 __ASM_EMIT("10:")
279
280 void fmmul3(float *dst, const float *a, const float *b, size_t count)
281 {
282 ARCH_AARCH64_ASM
283 (
284 FMOP_V3_CORE("dst", "a", "b", "fmul", OP_DSEL)
285 : [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b),
286 [count] "+r" (count)
287 :
288 : "cc", "memory",
289 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
290 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23",
291 "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31"
292 );
293 }
294
295 void fmrsub3(float *dst, const float *a, const float *b, size_t count)
296 {
297 ARCH_AARCH64_ASM
298 (
299 FMOP_V3_CORE("dst", "a", "b", "fsub", OP_RSEL)
300 : [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b),
301 [count] "+r" (count)
302 :
303 : "cc", "memory",
304 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
305 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23",
306 "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31"
307 );
308 }
309
310 #undef FMOP_V3_CORE
311
312 #define FMDIV_V3_CORE(DST, A, B, SEL) \
313 __ASM_EMIT("subs %[count], %[count], #16") \
314 __ASM_EMIT("mov v21.16b, v20.16b") \
315 __ASM_EMIT("blo 2f") \
316 /* 16x blocks */ \
317 __ASM_EMIT("1:") \
318 __ASM_EMIT("ldp q0, q1, [%[" SEL(A, DST) "], #0x00]") \
319 __ASM_EMIT("ldp q2, q3, [%[" SEL(A, DST) "], #0x20]") \
320 __ASM_EMIT("ldp q4, q5, [%[" SEL(DST, A) "], #0x00]") \
321 __ASM_EMIT("ldp q6, q7, [%[" SEL(DST, A) "], #0x20]") \
322 __ASM_EMIT("ldp q16, q17, [%[" B "], #0x00]") \
323 __ASM_EMIT("ldp q18, q19, [%[" B "], #0x20]") \
324 __ASM_EMIT("fmul " SEL("v0.4s, v0.4s", "v4.4s, v4.4s") ", v16.4s") /* {v0, v4} = {A*B, DST} | {DST, A*B} */ \
325 __ASM_EMIT("fmul " SEL("v1.4s, v1.4s", "v5.4s, v5.4s") ", v17.4s") \
326 __ASM_EMIT("fmul " SEL("v2.4s, v2.4s", "v6.4s, v6.4s") ", v18.4s") \
327 __ASM_EMIT("fmul " SEL("v3.4s, v3.4s", "v7.4s, v7.4s") ", v19.4s") \
328 __ASM_EMIT("frecpe v16.4s, v0.4s") /* v16 = s2 */ \
329 __ASM_EMIT("frecpe v17.4s, v1.4s") \
330 __ASM_EMIT("frecpe v18.4s, v2.4s") \
331 __ASM_EMIT("frecpe v19.4s, v3.4s") \
332 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2) */ \
333 __ASM_EMIT("frecps v25.4s, v17.4s, v1.4s") \
334 __ASM_EMIT("frecps v26.4s, v18.4s, v2.4s") \
335 __ASM_EMIT("frecps v27.4s, v19.4s, v3.4s") \
336 __ASM_EMIT("fmul v16.4s, v24.4s, v16.4s") /* v16 = s2' = s2 * (2 - R*s2) */ \
337 __ASM_EMIT("fmul v17.4s, v25.4s, v17.4s") \
338 __ASM_EMIT("fmul v18.4s, v26.4s, v18.4s") \
339 __ASM_EMIT("fmul v19.4s, v27.4s, v19.4s") \
340 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2') */ \
341 __ASM_EMIT("frecps v25.4s, v17.4s, v1.4s") \
342 __ASM_EMIT("frecps v26.4s, v18.4s, v2.4s") \
343 __ASM_EMIT("frecps v27.4s, v19.4s, v3.4s") \
344 __ASM_EMIT("fmul v0.4s, v24.4s, v16.4s") /* v0 = s2" = s2' * (2 - R*s2) = 1/s2 = 1/(A*B) | 1/DST */ \
345 __ASM_EMIT("fmul v1.4s, v25.4s, v17.4s") \
346 __ASM_EMIT("fmul v2.4s, v26.4s, v18.4s") \
347 __ASM_EMIT("fmul v3.4s, v27.4s, v19.4s") \
348 __ASM_EMIT("fmul v0.4s, v0.4s, v4.4s") /* DST/(A*B) | (A*B)/DST */ \
349 __ASM_EMIT("fmul v1.4s, v1.4s, v5.4s") \
350 __ASM_EMIT("fmul v2.4s, v2.4s, v6.4s") \
351 __ASM_EMIT("fmul v3.4s, v3.4s, v7.4s") \
352 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
353 __ASM_EMIT("stp q2, q3, [%[" DST "], #0x20]") \
354 __ASM_EMIT("subs %[count], %[count], #16") \
355 __ASM_EMIT("add %[" DST "], %[" DST "], #0x40") \
356 __ASM_EMIT("add %[" A "], %[" A "], #0x40") \
357 __ASM_EMIT("add %[" B "], %[" B "], #0x40") \
358 __ASM_EMIT("bhs 1b") \
359 /* 8x block */ \
360 __ASM_EMIT("2:") \
361 __ASM_EMIT("adds %[count], %[count], #8") /* 16 - 8 */ \
362 __ASM_EMIT("blt 4f") \
363 __ASM_EMIT("ldp q0, q1, [%[" SEL(A, DST) "], #0x00]") \
364 __ASM_EMIT("ldp q4, q5, [%[" SEL(DST, A) "], #0x00]") \
365 __ASM_EMIT("ldp q16, q17, [%[" B "], #0x00]") \
366 __ASM_EMIT("fmul " SEL("v0.4s, v0.4s", "v4.4s, v4.4s") ", v16.4s") /* {v0, v4} = {A*B, DST} | {DST, A*B} */ \
367 __ASM_EMIT("fmul " SEL("v1.4s, v1.4s", "v5.4s, v5.4s") ", v17.4s") \
368 __ASM_EMIT("frecpe v16.4s, v0.4s") /* v16 = s2 */ \
369 __ASM_EMIT("frecpe v17.4s, v1.4s") \
370 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2) */ \
371 __ASM_EMIT("frecps v25.4s, v17.4s, v1.4s") \
372 __ASM_EMIT("fmul v16.4s, v24.4s, v16.4s") /* v16 = s2' = s2 * (2 - R*s2) */ \
373 __ASM_EMIT("fmul v17.4s, v25.4s, v17.4s") \
374 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2') */ \
375 __ASM_EMIT("frecps v25.4s, v17.4s, v1.4s") \
376 __ASM_EMIT("fmul v0.4s, v24.4s, v16.4s") /* v0 = s2" = s2' * (2 - R*s2) = 1/s2 = 1/(A*B) | 1/DST */ \
377 __ASM_EMIT("fmul v1.4s, v25.4s, v17.4s") \
378 __ASM_EMIT("fmul v0.4s, v0.4s, v4.4s") /* DST/(A*B) | (A*B)/DST */ \
379 __ASM_EMIT("fmul v1.4s, v1.4s, v5.4s") \
380 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
381 __ASM_EMIT("sub %[count], %[count], #8") \
382 __ASM_EMIT("add %[" DST "], %[" DST "], #0x20") \
383 __ASM_EMIT("add %[" A "], %[" A "], #0x20") \
384 __ASM_EMIT("add %[" B "], %[" B "], #0x20") \
385 /* 4x block */ \
386 __ASM_EMIT("4:") \
387 __ASM_EMIT("adds %[count], %[count], #4") /* 8 - 4 */ \
388 __ASM_EMIT("blt 6f") \
389 __ASM_EMIT("ldr q0, [%[" SEL(A, DST) "], #0x00]") \
390 __ASM_EMIT("ldr q4, [%[" SEL(DST, A) "], #0x00]") \
391 __ASM_EMIT("ldr q16, [%[" B "], #0x00]") \
392 __ASM_EMIT("fmul " SEL("v0.4s, v0.4s", "v4.4s, v4.4s") ", v16.4s") /* {v0, v4} = {A*B, DST} | {DST, A*B} */ \
393 __ASM_EMIT("frecpe v16.4s, v0.4s") /* v16 = s2 */ \
394 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2) */ \
395 __ASM_EMIT("fmul v16.4s, v24.4s, v16.4s") /* v16 = s2' = s2 * (2 - R*s2) */ \
396 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2') */ \
397 __ASM_EMIT("fmul v0.4s, v24.4s, v16.4s") /* v0 = s2" = s2' * (2 - R*s2) = 1/s2 = 1/(A*B) | 1/DST */ \
398 __ASM_EMIT("fmul v0.4s, v0.4s, v4.4s") /* DST/(A*B) | (A*B)/DST */ \
399 __ASM_EMIT("str q0, [%[" DST "], #0x00]") \
400 __ASM_EMIT("sub %[count], %[count], #4") \
401 __ASM_EMIT("add %[" DST "], %[" DST "], #0x10") \
402 __ASM_EMIT("add %[" A "], %[" A "], #0x10") \
403 __ASM_EMIT("add %[" B "], %[" B "], #0x10") \
404 /* 1x block */ \
405 __ASM_EMIT("6:") \
406 __ASM_EMIT("adds %[count], %[count], #3") /* 4 - 3 */ \
407 __ASM_EMIT("blt 8f") \
408 __ASM_EMIT("7:") \
409 __ASM_EMIT("ld1r {v0.4s}, [%[" SEL(A, DST) "]]") \
410 __ASM_EMIT("ld1r {v4.4s}, [%[" SEL(DST, A) "]]") \
411 __ASM_EMIT("ld1r {v16.4s}, [%[" B "]]") \
412 __ASM_EMIT("fmul " SEL("v0.4s, v0.4s", "v4.4s, v4.4s") ", v16.4s") /* {v0, v4} = {A*B, DST} | {DST, A*B} */ \
413 __ASM_EMIT("frecpe v16.4s, v0.4s") /* v16 = s2 */ \
414 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2) */ \
415 __ASM_EMIT("fmul v16.4s, v24.4s, v16.4s") /* v16 = s2' = s2 * (2 - R*s2) */ \
416 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2') */ \
417 __ASM_EMIT("fmul v0.4s, v24.4s, v16.4s") /* v0 = s2" = s2' * (2 - R*s2) = 1/s2 = 1/(A*B) | 1/DST */ \
418 __ASM_EMIT("fmul v0.4s, v0.4s, v4.4s") /* DST/(A*B) | (A*B)/DST */ \
419 __ASM_EMIT("st1 {v0.s}[0], [%[" DST "]]") \
420 __ASM_EMIT("subs %[count], %[count], #1") \
421 __ASM_EMIT("add %[" DST "], %[" DST "], #0x04") \
422 __ASM_EMIT("add %[" A "], %[" A "], #0x04") \
423 __ASM_EMIT("add %[" B "], %[" B "], #0x04") \
424 __ASM_EMIT("bge 7b") \
425 __ASM_EMIT("8:")
426
427 void fmdiv3(float *dst, const float *a, const float *b, size_t count)
428 {
429 ARCH_AARCH64_ASM
430 (
431 FMDIV_V3_CORE("dst", "a", "b", OP_DSEL)
432 : [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b),
433 [count] "+r" (count)
434 :
435 : "cc", "memory",
436 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
437 "v16", "v17", "v18", "v19",
438 "v24", "v25", "v26", "v27"
439 );
440 }
441
442 void fmrdiv3(float *dst, const float *a, const float *b, size_t count)
443 {
444 ARCH_AARCH64_ASM
445 (
446 FMDIV_V3_CORE("dst", "a", "b", OP_RSEL)
447 : [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b),
448 [count] "+r" (count)
449 :
450 : "cc", "memory",
451 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
452 "v16", "v17", "v18", "v19",
453 "v24", "v25", "v26", "v27"
454 );
455 }
456
457 #undef FMDIV_V3_CORE
458
459 #define FMADDSUB_V4_CORE(DST, A, B, C, OP) \
460 __ASM_EMIT("subs %[count], %[count], #32") \
461 __ASM_EMIT("blo 2f") \
462 /* 32x blocks */ \
463 __ASM_EMIT("1:") \
464 __ASM_EMIT("ldp q0, q1, [%[" A "], #0x00]") \
465 __ASM_EMIT("ldp q2, q3, [%[" A "], #0x20]") \
466 __ASM_EMIT("ldp q4, q5, [%[" A "], #0x40]") \
467 __ASM_EMIT("ldp q6, q7, [%[" A "], #0x60]") \
468 __ASM_EMIT("ldp q16, q17, [%[" B "], #0x00]") \
469 __ASM_EMIT("ldp q18, q19, [%[" B "], #0x20]") \
470 __ASM_EMIT("ldp q20, q21, [%[" B "], #0x40]") \
471 __ASM_EMIT("ldp q22, q23, [%[" B "], #0x60]") \
472 __ASM_EMIT("ldp q24, q25, [%[" C "], #0x00]") \
473 __ASM_EMIT("ldp q26, q27, [%[" C "], #0x20]") \
474 __ASM_EMIT("ldp q28, q29, [%[" C "], #0x40]") \
475 __ASM_EMIT("ldp q30, q31, [%[" C "], #0x60]") \
476 __ASM_EMIT(OP " v0.4s, v16.4s, v24.4s") \
477 __ASM_EMIT(OP " v1.4s, v17.4s, v25.4s") \
478 __ASM_EMIT(OP " v2.4s, v18.4s, v26.4s") \
479 __ASM_EMIT(OP " v3.4s, v19.4s, v27.4s") \
480 __ASM_EMIT(OP " v4.4s, v20.4s, v28.4s") \
481 __ASM_EMIT(OP " v5.4s, v21.4s, v29.4s") \
482 __ASM_EMIT(OP " v6.4s, v22.4s, v30.4s") \
483 __ASM_EMIT(OP " v7.4s, v23.4s, v31.4s") \
484 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
485 __ASM_EMIT("stp q2, q3, [%[" DST "], #0x20]") \
486 __ASM_EMIT("stp q4, q5, [%[" DST "], #0x40]") \
487 __ASM_EMIT("stp q6, q7, [%[" DST "], #0x60]") \
488 __ASM_EMIT("subs %[count], %[count], #32") \
489 __ASM_EMIT("add %[" DST "], %[" DST "], #0x80") \
490 __ASM_EMIT("add %[" A "], %[" A "], #0x80") \
491 __ASM_EMIT("add %[" B "], %[" B "], #0x80") \
492 __ASM_EMIT("add %[" C "], %[" C "], #0x80") \
493 __ASM_EMIT("bhs 1b") \
494 /* 16x block */ \
495 __ASM_EMIT("2:") \
496 __ASM_EMIT("adds %[count], %[count], #16") /* 32-16 */ \
497 __ASM_EMIT("blt 4f") \
498 __ASM_EMIT("ldp q0, q1, [%[" A "], #0x00]") \
499 __ASM_EMIT("ldp q2, q3, [%[" A "], #0x20]") \
500 __ASM_EMIT("ldp q16, q17, [%[" B "], #0x00]") \
501 __ASM_EMIT("ldp q18, q19, [%[" B "], #0x20]") \
502 __ASM_EMIT("ldp q24, q25, [%[" C "], #0x00]") \
503 __ASM_EMIT("ldp q26, q27, [%[" C "], #0x20]") \
504 __ASM_EMIT(OP " v0.4s, v16.4s, v24.4s") \
505 __ASM_EMIT(OP " v1.4s, v17.4s, v25.4s") \
506 __ASM_EMIT(OP " v2.4s, v18.4s, v26.4s") \
507 __ASM_EMIT(OP " v3.4s, v19.4s, v27.4s") \
508 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
509 __ASM_EMIT("stp q2, q3, [%[" DST "], #0x20]") \
510 __ASM_EMIT("sub %[count], %[count], #16") \
511 __ASM_EMIT("add %[" DST "], %[" DST "], #0x40") \
512 __ASM_EMIT("add %[" A "], %[" A "], #0x40") \
513 __ASM_EMIT("add %[" B "], %[" B "], #0x40") \
514 __ASM_EMIT("add %[" C "], %[" C "], #0x40") \
515 /* 8x block */ \
516 __ASM_EMIT("4:") \
517 __ASM_EMIT("adds %[count], %[count], #8") /* 16-8 */ \
518 __ASM_EMIT("blt 6f") \
519 __ASM_EMIT("ldp q0, q1, [%[" A "], #0x00]") \
520 __ASM_EMIT("ldp q16, q17, [%[" B "], #0x00]") \
521 __ASM_EMIT("ldp q24, q25, [%[" C "], #0x00]") \
522 __ASM_EMIT(OP " v0.4s, v16.4s, v24.4s") \
523 __ASM_EMIT(OP " v1.4s, v17.4s, v25.4s") \
524 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
525 __ASM_EMIT("sub %[count], %[count], #8") \
526 __ASM_EMIT("add %[" DST "], %[" DST "], #0x20") \
527 __ASM_EMIT("add %[" A "], %[" A "], #0x20") \
528 __ASM_EMIT("add %[" B "], %[" B "], #0x20") \
529 __ASM_EMIT("add %[" C "], %[" C "], #0x20") \
530 /* 4x block */ \
531 __ASM_EMIT("6:") \
532 __ASM_EMIT("adds %[count], %[count], #4") /* 8 - 4 */ \
533 __ASM_EMIT("blt 8f") \
534 __ASM_EMIT("ldr q0, [%[" A "], #0x00]") \
535 __ASM_EMIT("ldr q16, [%[" B "], #0x00]") \
536 __ASM_EMIT("ldr q24, [%[" C "], #0x00]") \
537 __ASM_EMIT(OP " v0.4s, v16.4s, v24.4s") \
538 __ASM_EMIT("str q0, [%[" DST "], #0x00]") \
539 __ASM_EMIT("sub %[count], %[count], #4") \
540 __ASM_EMIT("add %[" DST "], %[" DST "], #0x10") \
541 __ASM_EMIT("add %[" A "], %[" A "], #0x10") \
542 __ASM_EMIT("add %[" B "], %[" B "], #0x10") \
543 __ASM_EMIT("add %[" C "], %[" C "], #0x10") \
544 /* 1x block */ \
545 __ASM_EMIT("8:") \
546 __ASM_EMIT("adds %[count], %[count], #3") /* 4 - 3 */ \
547 __ASM_EMIT("blt 10f") \
548 __ASM_EMIT("9:") \
549 __ASM_EMIT("ld1r {v0.4s}, [%[" A "]]") \
550 __ASM_EMIT("ld1r {v16.4s}, [%[" B "]]") \
551 __ASM_EMIT("ld1r {v24.4s}, [%[" C "]]") \
552 __ASM_EMIT(OP " v0.4s, v16.4s, v24.4s") \
553 __ASM_EMIT("st1 {v0.s}[0], [%[" DST "]]") \
554 __ASM_EMIT("subs %[count], %[count], #1") \
555 __ASM_EMIT("add %[" DST "], %[" DST "], #0x04") \
556 __ASM_EMIT("add %[" A "], %[" A "], #0x04") \
557 __ASM_EMIT("add %[" B "], %[" B "], #0x04") \
558 __ASM_EMIT("add %[" C "], %[" C "], #0x04") \
559 __ASM_EMIT("bge 9b") \
560 __ASM_EMIT("10:")
561
562 void fmadd4(float *dst, const float *a, const float *b, const float *c, size_t count)
563 {
564 ARCH_AARCH64_ASM
565 (
566 FMADDSUB_V4_CORE("dst", "a", "b", "c", "fmla")
567 : [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b), [c] "+r" (c),
568 [count] "+r" (count)
569 :
570 : "cc", "memory",
571 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
572 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23",
573 "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31"
574 );
575 }
576
577 void fmsub4(float *dst, const float *a, const float *b, const float *c, size_t count)
578 {
579 ARCH_AARCH64_ASM
580 (
581 FMADDSUB_V4_CORE("dst", "a", "b", "c", "fmls")
582 : [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b), [c] "+r" (c),
583 [count] "+r" (count)
584 :
585 : "cc", "memory",
586 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
587 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23",
588 "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31"
589 );
590 }
591
592 #undef FMADDSUB_V4_CORE
593
594 #define FMOP_V4_CORE(DST, A, B, C, OP, SEL) \
595 __ASM_EMIT("subs %[count], %[count], #32") \
596 __ASM_EMIT("blo 2f") \
597 /* 32x blocks */ \
598 __ASM_EMIT("1:") \
599 __ASM_EMIT("ldp q16, q17, [%[" B "], #0x00]") \
600 __ASM_EMIT("ldp q18, q19, [%[" B "], #0x20]") \
601 __ASM_EMIT("ldp q20, q21, [%[" B "], #0x40]") \
602 __ASM_EMIT("ldp q22, q23, [%[" B "], #0x60]") \
603 __ASM_EMIT("ldp q24, q25, [%[" C "], #0x00]") \
604 __ASM_EMIT("ldp q26, q27, [%[" C "], #0x20]") \
605 __ASM_EMIT("ldp q28, q29, [%[" C "], #0x40]") \
606 __ASM_EMIT("ldp q30, q31, [%[" C "], #0x60]") \
607 __ASM_EMIT("ldp q0, q1, [%[" A "], #0x00]") \
608 __ASM_EMIT("ldp q2, q3, [%[" A "], #0x20]") \
609 __ASM_EMIT("ldp q4, q5, [%[" A "], #0x40]") \
610 __ASM_EMIT("ldp q6, q7, [%[" A "], #0x60]") \
611 __ASM_EMIT("fmul v16.4s, v16.4s, v24.4s") \
612 __ASM_EMIT("fmul v17.4s, v17.4s, v25.4s") \
613 __ASM_EMIT("fmul v18.4s, v18.4s, v26.4s") \
614 __ASM_EMIT("fmul v19.4s, v19.4s, v27.4s") \
615 __ASM_EMIT("fmul v20.4s, v20.4s, v28.4s") \
616 __ASM_EMIT("fmul v21.4s, v21.4s, v29.4s") \
617 __ASM_EMIT("fmul v22.4s, v22.4s, v30.4s") \
618 __ASM_EMIT("fmul v23.4s, v23.4s, v31.4s") \
619 __ASM_EMIT(OP " v0.4s, " SEL("v0.4s", "v16.4s") ", " SEL("v16.4s", "v0.4s")) \
620 __ASM_EMIT(OP " v1.4s, " SEL("v1.4s", "v17.4s") ", " SEL("v17.4s", "v1.4s")) \
621 __ASM_EMIT(OP " v2.4s, " SEL("v2.4s", "v18.4s") ", " SEL("v18.4s", "v2.4s")) \
622 __ASM_EMIT(OP " v3.4s, " SEL("v3.4s", "v19.4s") ", " SEL("v19.4s", "v3.4s")) \
623 __ASM_EMIT(OP " v4.4s, " SEL("v4.4s", "v20.4s") ", " SEL("v20.4s", "v4.4s")) \
624 __ASM_EMIT(OP " v5.4s, " SEL("v5.4s", "v21.4s") ", " SEL("v21.4s", "v5.4s")) \
625 __ASM_EMIT(OP " v6.4s, " SEL("v6.4s", "v22.4s") ", " SEL("v22.4s", "v6.4s")) \
626 __ASM_EMIT(OP " v7.4s, " SEL("v7.4s", "v23.4s") ", " SEL("v23.4s", "v7.4s")) \
627 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
628 __ASM_EMIT("stp q2, q3, [%[" DST "], #0x20]") \
629 __ASM_EMIT("stp q4, q5, [%[" DST "], #0x40]") \
630 __ASM_EMIT("stp q6, q7, [%[" DST "], #0x60]") \
631 __ASM_EMIT("subs %[count], %[count], #32") \
632 __ASM_EMIT("add %[" DST "], %[" DST "], #0x80") \
633 __ASM_EMIT("add %[" A "], %[" A "], #0x80") \
634 __ASM_EMIT("add %[" B "], %[" B "], #0x80") \
635 __ASM_EMIT("add %[" C "], %[" C "], #0x80") \
636 __ASM_EMIT("bhs 1b") \
637 /* 16x block */ \
638 __ASM_EMIT("2:") \
639 __ASM_EMIT("adds %[count], %[count], #16") /* 32-16 */ \
640 __ASM_EMIT("blt 4f") \
641 __ASM_EMIT("ldp q16, q17, [%[" B "], #0x00]") \
642 __ASM_EMIT("ldp q18, q19, [%[" B "], #0x20]") \
643 __ASM_EMIT("ldp q24, q25, [%[" C "], #0x00]") \
644 __ASM_EMIT("ldp q26, q27, [%[" C "], #0x20]") \
645 __ASM_EMIT("ldp q0, q1, [%[" A "], #0x00]") \
646 __ASM_EMIT("ldp q2, q3, [%[" A "], #0x20]") \
647 __ASM_EMIT("fmul v16.4s, v16.4s, v24.4s") \
648 __ASM_EMIT("fmul v17.4s, v17.4s, v25.4s") \
649 __ASM_EMIT("fmul v18.4s, v18.4s, v26.4s") \
650 __ASM_EMIT("fmul v19.4s, v19.4s, v27.4s") \
651 __ASM_EMIT(OP " v0.4s, " SEL("v0.4s", "v16.4s") ", " SEL("v16.4s", "v0.4s")) \
652 __ASM_EMIT(OP " v1.4s, " SEL("v1.4s", "v17.4s") ", " SEL("v17.4s", "v1.4s")) \
653 __ASM_EMIT(OP " v2.4s, " SEL("v2.4s", "v18.4s") ", " SEL("v18.4s", "v2.4s")) \
654 __ASM_EMIT(OP " v3.4s, " SEL("v3.4s", "v19.4s") ", " SEL("v19.4s", "v3.4s")) \
655 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
656 __ASM_EMIT("stp q2, q3, [%[" DST "], #0x20]") \
657 __ASM_EMIT("sub %[count], %[count], #16") \
658 __ASM_EMIT("add %[" DST "], %[" DST "], #0x40") \
659 __ASM_EMIT("add %[" A "], %[" A "], #0x40") \
660 __ASM_EMIT("add %[" B "], %[" B "], #0x40") \
661 __ASM_EMIT("add %[" C "], %[" C "], #0x40") \
662 /* 8x block */ \
663 __ASM_EMIT("4:") \
664 __ASM_EMIT("adds %[count], %[count], #8") /* 16-8 */ \
665 __ASM_EMIT("blt 6f") \
666 __ASM_EMIT("ldp q16, q17, [%[" B "], #0x00]") \
667 __ASM_EMIT("ldp q24, q25, [%[" C "], #0x00]") \
668 __ASM_EMIT("ldp q0, q1, [%[" A "], #0x00]") \
669 __ASM_EMIT("fmul v16.4s, v16.4s, v24.4s") \
670 __ASM_EMIT("fmul v17.4s, v17.4s, v25.4s") \
671 __ASM_EMIT(OP " v0.4s, " SEL("v0.4s", "v16.4s") ", " SEL("v16.4s", "v0.4s")) \
672 __ASM_EMIT(OP " v1.4s, " SEL("v1.4s", "v17.4s") ", " SEL("v17.4s", "v1.4s")) \
673 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
674 __ASM_EMIT("sub %[count], %[count], #8") \
675 __ASM_EMIT("add %[" DST "], %[" DST "], #0x20") \
676 __ASM_EMIT("add %[" A "], %[" A "], #0x20") \
677 __ASM_EMIT("add %[" B "], %[" B "], #0x20") \
678 __ASM_EMIT("add %[" C "], %[" C "], #0x20") \
679 /* 4x block */ \
680 __ASM_EMIT("6:") \
681 __ASM_EMIT("adds %[count], %[count], #4") /* 8 - 4 */ \
682 __ASM_EMIT("blt 8f") \
683 __ASM_EMIT("ldr q16, [%[" B "], #0x00]") \
684 __ASM_EMIT("ldr q24, [%[" C "], #0x00]") \
685 __ASM_EMIT("ldr q0, [%[" A "], #0x00]") \
686 __ASM_EMIT("fmul v16.4s, v16.4s, v24.4s") \
687 __ASM_EMIT(OP " v0.4s, " SEL("v0.4s", "v16.4s") ", " SEL("v16.4s", "v0.4s")) \
688 __ASM_EMIT("str q0, [%[" DST "], #0x00]") \
689 __ASM_EMIT("sub %[count], %[count], #4") \
690 __ASM_EMIT("add %[" DST "], %[" DST "], #0x10") \
691 __ASM_EMIT("add %[" A "], %[" A "], #0x10") \
692 __ASM_EMIT("add %[" B "], %[" B "], #0x10") \
693 __ASM_EMIT("add %[" C "], %[" C "], #0x10") \
694 /* 1x block */ \
695 __ASM_EMIT("8:") \
696 __ASM_EMIT("adds %[count], %[count], #3") /* 4 - 3 */ \
697 __ASM_EMIT("blt 10f") \
698 __ASM_EMIT("9:") \
699 __ASM_EMIT("ld1r {v16.4s}, [%[" B "]]") \
700 __ASM_EMIT("ld1r {v24.4s}, [%[" C "]]") \
701 __ASM_EMIT("ld1r {v0.4s}, [%[" A "]]") \
702 __ASM_EMIT("fmul v16.4s, v16.4s, v24.4s") \
703 __ASM_EMIT(OP " v0.4s, " SEL("v0.4s", "v16.4s") ", " SEL("v16.4s", "v0.4s")) \
704 __ASM_EMIT("st1 {v0.s}[0], [%[" DST "]]") \
705 __ASM_EMIT("subs %[count], %[count], #1") \
706 __ASM_EMIT("add %[" DST "], %[" DST "], #0x04") \
707 __ASM_EMIT("add %[" A "], %[" A "], #0x04") \
708 __ASM_EMIT("add %[" B "], %[" B "], #0x04") \
709 __ASM_EMIT("add %[" C "], %[" C "], #0x04") \
710 __ASM_EMIT("bge 9b") \
711 __ASM_EMIT("10:")
712
713 void fmmul4(float *dst, const float *a, const float *b, const float *c, size_t count)
714 {
715 ARCH_AARCH64_ASM
716 (
717 FMOP_V4_CORE("dst", "a", "b", "c", "fmul", OP_DSEL)
718 : [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b), [c] "+r" (c),
719 [count] "+r" (count)
720 :
721 : "cc", "memory",
722 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
723 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23",
724 "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31"
725 );
726 }
727
728 void fmrsub4(float *dst, const float *a, const float *b, const float *c, size_t count)
729 {
730 ARCH_AARCH64_ASM
731 (
732 FMOP_V4_CORE("dst", "a", "b", "c", "fsub", OP_RSEL)
733 : [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b), [c] "+r" (c),
734 [count] "+r" (count)
735 :
736 : "cc", "memory",
737 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
738 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23",
739 "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31"
740 );
741 }
742
743 #undef FMOP_V4_CORE
744
745 #define FMDIV_V4_CORE(DST, A, B, C, SEL) \
746 __ASM_EMIT("subs %[count], %[count], #16") \
747 __ASM_EMIT("mov v21.16b, v20.16b") \
748 __ASM_EMIT("blo 2f") \
749 /* 16x blocks */ \
750 __ASM_EMIT("1:") \
751 __ASM_EMIT("ldp q0, q1, [%[" SEL(B, A) "], #0x00]") \
752 __ASM_EMIT("ldp q2, q3, [%[" SEL(B, A) "], #0x20]") \
753 __ASM_EMIT("ldp q4, q5, [%[" SEL(A, B) "], #0x00]") \
754 __ASM_EMIT("ldp q6, q7, [%[" SEL(A, B) "], #0x20]") \
755 __ASM_EMIT("ldp q16, q17, [%[" C "], #0x00]") \
756 __ASM_EMIT("ldp q18, q19, [%[" C "], #0x20]") \
757 __ASM_EMIT("fmul " SEL("v0.4s, v0.4s", "v4.4s, v4.4s") ", v16.4s") /* {v0, v4} = {B*C, DST} | {DST, B*C} */ \
758 __ASM_EMIT("fmul " SEL("v1.4s, v1.4s", "v5.4s, v5.4s") ", v17.4s") \
759 __ASM_EMIT("fmul " SEL("v2.4s, v2.4s", "v6.4s, v6.4s") ", v18.4s") \
760 __ASM_EMIT("fmul " SEL("v3.4s, v3.4s", "v7.4s, v7.4s") ", v19.4s") \
761 __ASM_EMIT("frecpe v16.4s, v0.4s") /* v16 = s2 */ \
762 __ASM_EMIT("frecpe v17.4s, v1.4s") \
763 __ASM_EMIT("frecpe v18.4s, v2.4s") \
764 __ASM_EMIT("frecpe v19.4s, v3.4s") \
765 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2) */ \
766 __ASM_EMIT("frecps v25.4s, v17.4s, v1.4s") \
767 __ASM_EMIT("frecps v26.4s, v18.4s, v2.4s") \
768 __ASM_EMIT("frecps v27.4s, v19.4s, v3.4s") \
769 __ASM_EMIT("fmul v16.4s, v24.4s, v16.4s") /* v16 = s2' = s2 * (2 - R*s2) */ \
770 __ASM_EMIT("fmul v17.4s, v25.4s, v17.4s") \
771 __ASM_EMIT("fmul v18.4s, v26.4s, v18.4s") \
772 __ASM_EMIT("fmul v19.4s, v27.4s, v19.4s") \
773 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2') */ \
774 __ASM_EMIT("frecps v25.4s, v17.4s, v1.4s") \
775 __ASM_EMIT("frecps v26.4s, v18.4s, v2.4s") \
776 __ASM_EMIT("frecps v27.4s, v19.4s, v3.4s") \
777 __ASM_EMIT("fmul v0.4s, v24.4s, v16.4s") /* v0 = s2" = s2' * (2 - R*s2) = 1/s2 = 1/(B*C) | 1/DST */ \
778 __ASM_EMIT("fmul v1.4s, v25.4s, v17.4s") \
779 __ASM_EMIT("fmul v2.4s, v26.4s, v18.4s") \
780 __ASM_EMIT("fmul v3.4s, v27.4s, v19.4s") \
781 __ASM_EMIT("fmul v0.4s, v0.4s, v4.4s") /* DST/(B*C) | (B*C)/DST */ \
782 __ASM_EMIT("fmul v1.4s, v1.4s, v5.4s") \
783 __ASM_EMIT("fmul v2.4s, v2.4s, v6.4s") \
784 __ASM_EMIT("fmul v3.4s, v3.4s, v7.4s") \
785 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
786 __ASM_EMIT("stp q2, q3, [%[" DST "], #0x20]") \
787 __ASM_EMIT("subs %[count], %[count], #16") \
788 __ASM_EMIT("add %[" DST "], %[" DST "], #0x40") \
789 __ASM_EMIT("add %[" A "], %[" A "], #0x40") \
790 __ASM_EMIT("add %[" B "], %[" B "], #0x40") \
791 __ASM_EMIT("add %[" C "], %[" C "], #0x40") \
792 __ASM_EMIT("bhs 1b") \
793 /* 8x block */ \
794 __ASM_EMIT("2:") \
795 __ASM_EMIT("adds %[count], %[count], #8") /* 16 - 8 */ \
796 __ASM_EMIT("blt 4f") \
797 __ASM_EMIT("ldp q0, q1, [%[" SEL(B, A) "], #0x00]") \
798 __ASM_EMIT("ldp q4, q5, [%[" SEL(A, B) "], #0x00]") \
799 __ASM_EMIT("ldp q16, q17, [%[" C "], #0x00]") \
800 __ASM_EMIT("fmul " SEL("v0.4s, v0.4s", "v4.4s, v4.4s") ", v16.4s") /* {v0, v4} = {B*C, DST} | {DST, B*C} */ \
801 __ASM_EMIT("fmul " SEL("v1.4s, v1.4s", "v5.4s, v5.4s") ", v17.4s") \
802 __ASM_EMIT("frecpe v16.4s, v0.4s") /* v16 = s2 */ \
803 __ASM_EMIT("frecpe v17.4s, v1.4s") \
804 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2) */ \
805 __ASM_EMIT("frecps v25.4s, v17.4s, v1.4s") \
806 __ASM_EMIT("fmul v16.4s, v24.4s, v16.4s") /* v16 = s2' = s2 * (2 - R*s2) */ \
807 __ASM_EMIT("fmul v17.4s, v25.4s, v17.4s") \
808 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2') */ \
809 __ASM_EMIT("frecps v25.4s, v17.4s, v1.4s") \
810 __ASM_EMIT("fmul v0.4s, v24.4s, v16.4s") /* v0 = s2" = s2' * (2 - R*s2) = 1/s2 = 1/(B*C) | 1/DST */ \
811 __ASM_EMIT("fmul v1.4s, v25.4s, v17.4s") \
812 __ASM_EMIT("fmul v0.4s, v0.4s, v4.4s") /* DST/(B*C) | (B*C)/DST */ \
813 __ASM_EMIT("fmul v1.4s, v1.4s, v5.4s") \
814 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
815 __ASM_EMIT("sub %[count], %[count], #8") \
816 __ASM_EMIT("add %[" DST "], %[" DST "], #0x20") \
817 __ASM_EMIT("add %[" A "], %[" A "], #0x20") \
818 __ASM_EMIT("add %[" B "], %[" B "], #0x20") \
819 __ASM_EMIT("add %[" C "], %[" C "], #0x20") \
820 /* 4x block */ \
821 __ASM_EMIT("4:") \
822 __ASM_EMIT("adds %[count], %[count], #4") /* 8 - 4 */ \
823 __ASM_EMIT("blt 6f") \
824 __ASM_EMIT("ldr q0, [%[" SEL(B, A) "], #0x00]") \
825 __ASM_EMIT("ldr q4, [%[" SEL(A, B) "], #0x00]") \
826 __ASM_EMIT("ldr q16, [%[" C "], #0x00]") \
827 __ASM_EMIT("fmul " SEL("v0.4s, v0.4s", "v4.4s, v4.4s") ", v16.4s") /* {v0, v4} = {B*C, DST} | {DST, B*C} */ \
828 __ASM_EMIT("frecpe v16.4s, v0.4s") /* v16 = s2 */ \
829 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2) */ \
830 __ASM_EMIT("fmul v16.4s, v24.4s, v16.4s") /* v16 = s2' = s2 * (2 - R*s2) */ \
831 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2') */ \
832 __ASM_EMIT("fmul v0.4s, v24.4s, v16.4s") /* v0 = s2" = s2' * (2 - R*s2) = 1/s2 = 1/(B*C) | 1/DST */ \
833 __ASM_EMIT("fmul v0.4s, v0.4s, v4.4s") /* DST/(B*C) | (B*C)/DST */ \
834 __ASM_EMIT("str q0, [%[" DST "], #0x00]") \
835 __ASM_EMIT("sub %[count], %[count], #4") \
836 __ASM_EMIT("add %[" DST "], %[" DST "], #0x10") \
837 __ASM_EMIT("add %[" A "], %[" A "], #0x10") \
838 __ASM_EMIT("add %[" B "], %[" B "], #0x10") \
839 __ASM_EMIT("add %[" C "], %[" C "], #0x10") \
840 /* 1x block */ \
841 __ASM_EMIT("6:") \
842 __ASM_EMIT("adds %[count], %[count], #3") /* 4 - 3 */ \
843 __ASM_EMIT("blt 8f") \
844 __ASM_EMIT("7:") \
845 __ASM_EMIT("ld1r {v0.4s}, [%[" SEL(B, A) "]]") \
846 __ASM_EMIT("ld1r {v4.4s}, [%[" SEL(A, B) "]]") \
847 __ASM_EMIT("ld1r {v16.4s}, [%[" C "]]") \
848 __ASM_EMIT("fmul " SEL("v0.4s, v0.4s", "v4.4s, v4.4s") ", v16.4s") /* {v0, v4} = {B*C, DST} | {DST, B*C} */ \
849 __ASM_EMIT("frecpe v16.4s, v0.4s") /* v16 = s2 */ \
850 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2) */ \
851 __ASM_EMIT("fmul v16.4s, v24.4s, v16.4s") /* v16 = s2' = s2 * (2 - R*s2) */ \
852 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2') */ \
853 __ASM_EMIT("fmul v0.4s, v24.4s, v16.4s") /* v0 = s2" = s2' * (2 - R*s2) = 1/s2 = 1/(B*C) | 1/DST */ \
854 __ASM_EMIT("fmul v0.4s, v0.4s, v4.4s") /* DST/(B*C) | (B*C)/DST */ \
855 __ASM_EMIT("st1 {v0.s}[0], [%[" DST "]]") \
856 __ASM_EMIT("subs %[count], %[count], #1") \
857 __ASM_EMIT("add %[" DST "], %[" DST "], #0x04") \
858 __ASM_EMIT("add %[" A "], %[" A "], #0x04") \
859 __ASM_EMIT("add %[" B "], %[" B "], #0x04") \
860 __ASM_EMIT("add %[" C "], %[" C "], #0x04") \
861 __ASM_EMIT("bge 7b") \
862 __ASM_EMIT("8:")
863
864 void fmdiv4(float *dst, const float *a, const float *b, const float *c, size_t count)
865 {
866 ARCH_AARCH64_ASM
867 (
868 FMDIV_V4_CORE("dst", "a", "b", "c", OP_DSEL)
869 : [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b), [c] "+r" (c),
870 [count] "+r" (count)
871 :
872 : "cc", "memory",
873 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
874 "v16", "v17", "v18", "v19",
875 "v24", "v25", "v26", "v27"
876 );
877 }
878
879 void fmrdiv4(float *dst, const float *a, const float *b, const float *c, size_t count)
880 {
881 ARCH_AARCH64_ASM
882 (
883 FMDIV_V4_CORE("dst", "a", "b", "c", OP_RSEL)
884 : [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b), [c] "+r" (c),
885 [count] "+r" (count)
886 :
887 : "cc", "memory",
888 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
889 "v16", "v17", "v18", "v19",
890 "v24", "v25", "v26", "v27"
891 );
892 }
893
894 #undef FMDIV_V3_CORE
895
896 #define INC_ON(x) x
897 #define INC_OFF(x)
898
899 #define FMMOD_VV_CORE(DST, INCD, A, INCA, B, INCB, C, INCC, SEL) \
900 __ASM_EMIT("subs %[count], %[count], #16") \
901 __ASM_EMIT("b.lo 2f") \
902 /* 16x blocks */ \
903 __ASM_EMIT("1:") \
904 __ASM_EMIT("ldp q20, q21, [%[" C "], #0x00]") \
905 __ASM_EMIT("ldp q22, q23, [%[" C "], #0x20]") \
906 __ASM_EMIT("ldp q4, q5, [%[" SEL(B, A) "], #0x00]") \
907 __ASM_EMIT("ldp q6, q7, [%[" SEL(B, A) "], #0x20]") \
908 __ASM_EMIT(SEL("fmul v4.4s, v4.4s, v20.4s", "")) /* v4 = d */ \
909 __ASM_EMIT(SEL("fmul v5.4s, v5.4s, v21.4s", "")) \
910 __ASM_EMIT(SEL("fmul v6.4s, v6.4s, v22.4s", "")) \
911 __ASM_EMIT(SEL("fmul v7.4s, v7.4s, v23.4s", "")) \
912 __ASM_EMIT("frecpe v16.4s, v4.4s") /* v16 = s2 */ \
913 __ASM_EMIT("frecpe v17.4s, v5.4s") \
914 __ASM_EMIT("frecpe v18.4s, v6.4s") \
915 __ASM_EMIT("frecpe v19.4s, v7.4s") \
916 __ASM_EMIT("frecps v0.4s, v16.4s, v4.4s") /* v0 = (2 - R*s2) */ \
917 __ASM_EMIT("frecps v1.4s, v17.4s, v5.4s") \
918 __ASM_EMIT("frecps v2.4s, v18.4s, v6.4s") \
919 __ASM_EMIT("frecps v3.4s, v19.4s, v7.4s") \
920 __ASM_EMIT("fmul v16.4s, v0.4s, v16.4s") /* v16 = s2' = s2 * (2 - R*s2) */ \
921 __ASM_EMIT("fmul v17.4s, v1.4s, v17.4s") \
922 __ASM_EMIT("fmul v18.4s, v2.4s, v18.4s") \
923 __ASM_EMIT("fmul v19.4s, v3.4s, v19.4s") \
924 __ASM_EMIT("frecps v0.4s, v16.4s, v4.4s") /* v0 = (2 - R*s2') */ \
925 __ASM_EMIT("frecps v1.4s, v17.4s, v5.4s") \
926 __ASM_EMIT("frecps v2.4s, v18.4s, v6.4s") \
927 __ASM_EMIT("frecps v3.4s, v19.4s, v7.4s") \
928 __ASM_EMIT("fmul v16.4s, v0.4s, v16.4s") /* v16 = s2" = s2' * (2 - R*s2) = 1/d */ \
929 __ASM_EMIT("fmul v17.4s, v1.4s, v17.4s") \
930 __ASM_EMIT("fmul v18.4s, v2.4s, v18.4s") \
931 __ASM_EMIT("fmul v19.4s, v3.4s, v19.4s") \
932 __ASM_EMIT("ldp q0, q1, [%[" SEL(A, B) "], #0x00]") \
933 __ASM_EMIT("ldp q2, q3, [%[" SEL(A, B) "], #0x20]") \
934 __ASM_EMIT(SEL("", "fmul v0.4s, v0.4s, v20.4s")) /* v0 = s */ \
935 __ASM_EMIT(SEL("", "fmul v1.4s, v1.4s, v21.4s"))\
936 __ASM_EMIT(SEL("", "fmul v2.4s, v2.4s, v22.4s")) \
937 __ASM_EMIT(SEL("", "fmul v3.4s, v3.4s, v23.4s")) \
938 __ASM_EMIT("fmul v16.4s, v16.4s, v0.4s") /* v16 = s/d */ \
939 __ASM_EMIT("fmul v17.4s, v17.4s, v1.4s") \
940 __ASM_EMIT("fmul v18.4s, v18.4s, v2.4s") \
941 __ASM_EMIT("fmul v19.4s, v19.4s, v3.4s") \
942 __ASM_EMIT("frintz v16.4s, v16.4s") \
943 __ASM_EMIT("frintz v17.4s, v17.4s") \
944 __ASM_EMIT("frintz v18.4s, v18.4s") \
945 __ASM_EMIT("frintz v19.4s, v19.4s") \
946 __ASM_EMIT("fmls v0.4s, v4.4s, v16.4s") /* v0 = s - d*int(s/d) */ \
947 __ASM_EMIT("fmls v1.4s, v5.4s, v17.4s") \
948 __ASM_EMIT("fmls v2.4s, v6.4s, v18.4s") \
949 __ASM_EMIT("fmls v3.4s, v7.4s, v19.4s") \
950 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
951 __ASM_EMIT("stp q2, q3, [%[" DST "], #0x20]") \
952 __ASM_EMIT("subs %[count], %[count], #16") \
953 __ASM_EMIT(INCA("add %[" A "], %[" A "], #0x40")) \
954 __ASM_EMIT(INCB("add %[" B "], %[" B "], #0x40")) \
955 __ASM_EMIT(INCC("add %[" C "], %[" C "], #0x40")) \
956 __ASM_EMIT(INCD("add %[" DST "], %[" DST "], #0x40")) \
957 __ASM_EMIT("b.hs 1b") \
958 /* 8x block */ \
959 __ASM_EMIT("2:") \
960 __ASM_EMIT("adds %[count], %[count], #8") \
961 __ASM_EMIT("b.lt 4f") \
962 __ASM_EMIT("ldp q20, q21, [%[" C "], #0x00]") \
963 __ASM_EMIT("ldp q4, q5, [%[" SEL(B, A) "], #0x00]") \
964 __ASM_EMIT(SEL("fmul v4.4s, v4.4s, v20.4s", "")) /* v4 = d */ \
965 __ASM_EMIT(SEL("fmul v5.4s, v5.4s, v21.4s", "")) \
966 __ASM_EMIT("frecpe v16.4s, v4.4s") /* v16 = s2 */ \
967 __ASM_EMIT("frecpe v17.4s, v5.4s") \
968 __ASM_EMIT("frecps v0.4s, v16.4s, v4.4s") /* v0 = (2 - R*s2) */ \
969 __ASM_EMIT("frecps v1.4s, v17.4s, v5.4s") \
970 __ASM_EMIT("fmul v16.4s, v0.4s, v16.4s") /* v16 = s2' = s2 * (2 - R*s2) */ \
971 __ASM_EMIT("fmul v17.4s, v1.4s, v17.4s") \
972 __ASM_EMIT("frecps v0.4s, v16.4s, v4.4s") /* v0 = (2 - R*s2') */ \
973 __ASM_EMIT("frecps v1.4s, v17.4s, v5.4s") \
974 __ASM_EMIT("fmul v16.4s, v0.4s, v16.4s") /* v16 = s2" = s2' * (2 - R*s2) = 1/d */ \
975 __ASM_EMIT("fmul v17.4s, v1.4s, v17.4s") \
976 __ASM_EMIT("ldp q0, q1, [%[" SEL(A, B) "], #0x00]") \
977 __ASM_EMIT(SEL("", "fmul v0.4s, v0.4s, v20.4s")) /* v0 = s */ \
978 __ASM_EMIT(SEL("", "fmul v1.4s, v1.4s, v21.4s"))\
979 __ASM_EMIT("fmul v16.4s, v16.4s, v0.4s") /* v16 = s/d */ \
980 __ASM_EMIT("fmul v17.4s, v17.4s, v1.4s") \
981 __ASM_EMIT("frintz v16.4s, v16.4s") \
982 __ASM_EMIT("frintz v17.4s, v17.4s") \
983 __ASM_EMIT("fmls v0.4s, v4.4s, v16.4s") /* v0 = s - d*int(s/d) */ \
984 __ASM_EMIT("fmls v1.4s, v5.4s, v17.4s") \
985 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
986 __ASM_EMIT("sub %[count], %[count], #8") \
987 __ASM_EMIT(INCA("add %[" A "], %[" A "], #0x20")) \
988 __ASM_EMIT(INCB("add %[" B "], %[" B "], #0x20")) \
989 __ASM_EMIT(INCC("add %[" C "], %[" C "], #0x20")) \
990 __ASM_EMIT(INCD("add %[" DST "], %[" DST "], #0x20")) \
991 /* 4x blocks */ \
992 __ASM_EMIT("4:") \
993 __ASM_EMIT("adds %[count], %[count], #4") \
994 __ASM_EMIT("b.lt 6f") \
995 __ASM_EMIT("ldr q20, [%[" C "]]") \
996 __ASM_EMIT("ldr q4, [%[" SEL(B, A) "]]") \
997 __ASM_EMIT(SEL("fmul v4.4s, v4.4s, v20.4s", "")) /* v4 = d */ \
998 __ASM_EMIT("frecpe v16.4s, v4.4s") /* v16 = s2 */ \
999 __ASM_EMIT("frecps v0.4s, v16.4s, v4.4s") /* v0 = (2 - R*s2) */ \
1000 __ASM_EMIT("fmul v16.4s, v0.4s, v16.4s") /* v16 = s2' = s2 * (2 - R*s2) */ \
1001 __ASM_EMIT("frecps v0.4s, v16.4s, v4.4s") /* v0 = (2 - R*s2') */ \
1002 __ASM_EMIT("fmul v16.4s, v0.4s, v16.4s") /* v16 = s2" = s2' * (2 - R*s2) = 1/d */ \
1003 __ASM_EMIT("ldr q0,[%[" SEL(A, B) "]]") \
1004 __ASM_EMIT(SEL("", "fmul v0.4s, v0.4s, v20.4s")) /* v0 = s */ \
1005 __ASM_EMIT("fmul v16.4s, v16.4s, v0.4s") /* v16 = s/d */ \
1006 __ASM_EMIT("frintz v16.4s, v16.4s") \
1007 __ASM_EMIT("fmls v0.4s, v4.4s, v16.4s") /* v0 = s - d*int(s/d) */ \
1008 __ASM_EMIT("str q0, [%[" DST "]]") \
1009 __ASM_EMIT("sub %[count], %[count], #4") \
1010 __ASM_EMIT(INCA("add %[" A "], %[" A "], #0x10")) \
1011 __ASM_EMIT(INCB("add %[" B "], %[" B "], #0x10")) \
1012 __ASM_EMIT(INCC("add %[" C "], %[" C "], #0x10")) \
1013 __ASM_EMIT(INCD("add %[" DST "], %[" DST "], #0x10")) \
1014 /* 1x blocks */ \
1015 __ASM_EMIT("6:") \
1016 __ASM_EMIT("adds %[count], %[count], #3") \
1017 __ASM_EMIT("b.lt 8f") \
1018 __ASM_EMIT("7:") \
1019 __ASM_EMIT("ld1r {v20.4s}, [%[" C "]]") \
1020 __ASM_EMIT("ld1r {v4.4s}, [%[" SEL(B, A) "]]") \
1021 __ASM_EMIT(SEL("fmul v4.4s, v4.4s, v20.4s", "")) /* v4 = d */ \
1022 __ASM_EMIT("frecpe v16.4s, v4.4s") /* v16 = s2 */ \
1023 __ASM_EMIT("frecps v0.4s, v16.4s, v4.4s") /* v0 = (2 - R*s2) */ \
1024 __ASM_EMIT("fmul v16.4s, v0.4s, v16.4s") /* v16 = s2' = s2 * (2 - R*s2) */ \
1025 __ASM_EMIT("frecps v0.4s, v16.4s, v4.4s") /* v0 = (2 - R*s2') */ \
1026 __ASM_EMIT("fmul v16.4s, v0.4s, v16.4s") /* v16 = s2" = s2' * (2 - R*s2) = 1/d */ \
1027 __ASM_EMIT("ld1r {v0.4s}, [%[" SEL(A, B) "]]") \
1028 __ASM_EMIT(SEL("", "fmul v0.4s, v0.4s, v20.4s")) /* v0 = s */ \
1029 __ASM_EMIT("fmul v16.4s, v16.4s, v0.4s") /* v16 = s/d */ \
1030 __ASM_EMIT("frintz v16.4s, v16.4s") \
1031 __ASM_EMIT("fmls v0.4s, v4.4s, v16.4s") /* v0 = s - d*int(s/d) */ \
1032 __ASM_EMIT("st1 {v0.s}[0], [%[" DST "]]") \
1033 __ASM_EMIT("subs %[count], %[count], #1") \
1034 __ASM_EMIT(INCA("add %[" A "], %[" A "], #0x04")) \
1035 __ASM_EMIT(INCB("add %[" B "], %[" B "], #0x04")) \
1036 __ASM_EMIT(INCC("add %[" C "], %[" C "], #0x04")) \
1037 __ASM_EMIT(INCD("add %[" DST "], %[" DST "], #0x04")) \
1038 __ASM_EMIT("b.ge 7b") \
1039 __ASM_EMIT("8:")
1040
1041 void fmmod3(float *dst, const float *a, const float *b, size_t count)
1042 {
1043 ARCH_AARCH64_ASM
1044 (
1045 FMMOD_VV_CORE("dst", INC_ON, "dst", INC_OFF, "a", INC_ON, "b", INC_ON, OP_DSEL)
1046 : [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b),
1047 [count] "+r" (count)
1048 :
1049 : "cc", "memory",
1050 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
1051 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"
1052 );
1053 }
1054
1055 void fmrmod3(float *dst, const float *a, const float *b, size_t count)
1056 {
1057 ARCH_AARCH64_ASM
1058 (
1059 FMMOD_VV_CORE("dst", INC_ON, "dst", INC_OFF, "a", INC_ON, "b", INC_ON, OP_RSEL)
1060 : [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b),
1061 [count] "+r" (count)
1062 :
1063 : "cc", "memory",
1064 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
1065 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"
1066 );
1067 }
1068
1069 void fmmod4(float *dst, const float *a, const float *b, const float *c, size_t count)
1070 {
1071 ARCH_AARCH64_ASM
1072 (
1073 FMMOD_VV_CORE("dst", INC_ON, "a", INC_ON, "b", INC_ON, "c", INC_ON, OP_DSEL)
1074 : [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b), [c] "+r" (c),
1075 [count] "+r" (count)
1076 :
1077 : "cc", "memory",
1078 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
1079 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"
1080 );
1081 }
1082
1083 void fmrmod4(float *dst, const float *a, const float *b, const float *c, size_t count)
1084 {
1085 ARCH_AARCH64_ASM
1086 (
1087 FMMOD_VV_CORE("dst", INC_ON, "a", INC_ON, "b", INC_ON, "c", INC_ON, OP_RSEL)
1088 : [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b), [c] "+r" (c),
1089 [count] "+r" (count)
1090 :
1091 : "cc", "memory",
1092 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
1093 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"
1094 );
1095 }
1096
1097 #undef FMMOD_VV_CORE
1098
1099 #undef INC_ON
1100 #undef INC_OFF
1101
1102 #undef OP_DSEL
1103 #undef OP_RSEL
1104 }
1105 }
1106
1107 #endif /* PRIVATE_DSP_ARCH_AARCH64_ASIMD_PMATH_FMOP_VV_H_ */