Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/aarch64/asimd/pmath/op_kx.h

raw
1 /*
2 * Copyright (C) 2020 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2020 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 31 мар. 2020 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_AARCH64_ASIMD_PMATH_OP_KX_H_
23 #define PRIVATE_DSP_ARCH_AARCH64_ASIMD_PMATH_OP_KX_H_
24
25 #ifndef PRIVATE_DSP_ARCH_AARCH64_ASIMD_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_AARCH64_ASIMD_IMPL */
28
29 namespace lsp
30 {
31 namespace asimd
32 {
33 #define OP_DORDER(a, b) a "," b
34 #define OP_RORDER(a, b) b "," a
35
36 #define OP_K2_CORE(DST, OP, ORDER) \
37 __ASM_EMIT("subs %[count], %[count], #32") \
38 __ASM_EMIT("mov v17.16b, v16.16b") \
39 __ASM_EMIT("b.lo 2f") \
40 /* 32x blocks */ \
41 __ASM_EMIT("1:") \
42 __ASM_EMIT("ldp q0, q1, [%[" DST "], #0x00]") \
43 __ASM_EMIT("ldp q2, q3, [%[" DST "], #0x20]") \
44 __ASM_EMIT("ldp q4, q5, [%[" DST "], #0x40]") \
45 __ASM_EMIT("ldp q6, q7, [%[" DST "], #0x60]") \
46 __ASM_EMIT(OP " v0.4s, " ORDER("v0.4s", "v16.4s")) \
47 __ASM_EMIT(OP " v1.4s, " ORDER("v1.4s", "v17.4s")) \
48 __ASM_EMIT(OP " v2.4s, " ORDER("v2.4s", "v16.4s")) \
49 __ASM_EMIT(OP " v3.4s, " ORDER("v3.4s", "v17.4s")) \
50 __ASM_EMIT(OP " v4.4s, " ORDER("v4.4s", "v16.4s")) \
51 __ASM_EMIT(OP " v5.4s, " ORDER("v5.4s", "v17.4s")) \
52 __ASM_EMIT(OP " v6.4s, " ORDER("v6.4s", "v16.4s")) \
53 __ASM_EMIT(OP " v7.4s, " ORDER("v7.4s", "v17.4s")) \
54 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
55 __ASM_EMIT("stp q2, q3, [%[" DST "], #0x20]") \
56 __ASM_EMIT("stp q4, q5, [%[" DST "], #0x40]") \
57 __ASM_EMIT("stp q6, q7, [%[" DST "], #0x60]") \
58 __ASM_EMIT("subs %[count], %[count], #32") \
59 __ASM_EMIT("add %[dst], %[dst], #0x80") \
60 __ASM_EMIT("b.hs 1b") \
61 /* 16x block */ \
62 __ASM_EMIT("2:") \
63 __ASM_EMIT("adds %[count], %[count], #16") /* 32 - 16 */ \
64 __ASM_EMIT("b.lt 4f") \
65 __ASM_EMIT("ldp q0, q1, [%[" DST "], #0x00]") \
66 __ASM_EMIT("ldp q2, q3, [%[" DST "], #0x20]") \
67 __ASM_EMIT(OP " v0.4s, " ORDER("v0.4s", "v16.4s")) \
68 __ASM_EMIT(OP " v1.4s, " ORDER("v1.4s", "v17.4s")) \
69 __ASM_EMIT(OP " v2.4s, " ORDER("v2.4s", "v16.4s")) \
70 __ASM_EMIT(OP " v3.4s, " ORDER("v3.4s", "v17.4s")) \
71 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
72 __ASM_EMIT("stp q2, q3, [%[" DST "], #0x20]") \
73 __ASM_EMIT("sub %[count], %[count], #16") \
74 __ASM_EMIT("add %[dst], %[dst], #0x40") \
75 /* 8x block */ \
76 __ASM_EMIT("4:") \
77 __ASM_EMIT("adds %[count], %[count], #8") /* 16 - 8 */ \
78 __ASM_EMIT("b.lt 6f") \
79 __ASM_EMIT("ldp q0, q1, [%[" DST "], #0x00]") \
80 __ASM_EMIT(OP " v0.4s, " ORDER("v0.4s", "v16.4s")) \
81 __ASM_EMIT(OP " v1.4s, " ORDER("v1.4s", "v17.4s")) \
82 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
83 __ASM_EMIT("sub %[count], %[count], #8") \
84 __ASM_EMIT("add %[dst], %[dst], #0x20") \
85 /* 4x block */ \
86 __ASM_EMIT("6:") \
87 __ASM_EMIT("adds %[count], %[count], #4") /* 8 - 4 */ \
88 __ASM_EMIT("b.lt 8f") \
89 __ASM_EMIT("ldr q0, [%[" DST "], #0x00]") \
90 __ASM_EMIT(OP " v0.4s, " ORDER("v0.4s", "v16.4s")) \
91 __ASM_EMIT("str q0, [%[" DST "], #0x00]") \
92 __ASM_EMIT("sub %[count], %[count], #4") \
93 __ASM_EMIT("add %[dst], %[dst], #0x10") \
94 /* 1x block */ \
95 __ASM_EMIT("8:") \
96 __ASM_EMIT("adds %[count], %[count], #3") /* 4 - 3 */ \
97 __ASM_EMIT("b.lt 10f") \
98 __ASM_EMIT("9:") \
99 __ASM_EMIT("ld1r {v0.4s}, [%[" DST "]]") \
100 __ASM_EMIT(OP " v0.4s, " ORDER("v0.4s", "v16.4s")) \
101 __ASM_EMIT("st1 {v0.s}[0], [%[" DST "]]") \
102 __ASM_EMIT("subs %[count], %[count], #1") \
103 __ASM_EMIT("add %[dst], %[dst], #0x04") \
104 __ASM_EMIT("bge 9b") \
105 __ASM_EMIT("10:")
106
107 #define OP_K3_CORE(DST, SRC, OP, ORDER) \
108 __ASM_EMIT("subs %[count], %[count], #32") \
109 __ASM_EMIT("mov v17.16b, v16.16b") \
110 __ASM_EMIT("b.lo 2f") \
111 /* 32x blocks */ \
112 __ASM_EMIT("1:") \
113 __ASM_EMIT("ldp q0, q1, [%[" SRC "], #0x00]") \
114 __ASM_EMIT("ldp q2, q3, [%[" SRC "], #0x20]") \
115 __ASM_EMIT("ldp q4, q5, [%[" SRC "], #0x40]") \
116 __ASM_EMIT("ldp q6, q7, [%[" SRC "], #0x60]") \
117 __ASM_EMIT(OP " v0.4s, " ORDER("v0.4s", "v16.4s")) \
118 __ASM_EMIT(OP " v1.4s, " ORDER("v1.4s", "v17.4s")) \
119 __ASM_EMIT(OP " v2.4s, " ORDER("v2.4s", "v16.4s")) \
120 __ASM_EMIT(OP " v3.4s, " ORDER("v3.4s", "v17.4s")) \
121 __ASM_EMIT(OP " v4.4s, " ORDER("v4.4s", "v16.4s")) \
122 __ASM_EMIT(OP " v5.4s, " ORDER("v5.4s", "v17.4s")) \
123 __ASM_EMIT(OP " v6.4s, " ORDER("v6.4s", "v16.4s")) \
124 __ASM_EMIT(OP " v7.4s, " ORDER("v7.4s", "v17.4s")) \
125 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
126 __ASM_EMIT("stp q2, q3, [%[" DST "], #0x20]") \
127 __ASM_EMIT("stp q4, q5, [%[" DST "], #0x40]") \
128 __ASM_EMIT("stp q6, q7, [%[" DST "], #0x60]") \
129 __ASM_EMIT("subs %[count], %[count], #32") \
130 __ASM_EMIT("add %[src], %[src], #0x80") \
131 __ASM_EMIT("add %[dst], %[dst], #0x80") \
132 __ASM_EMIT("b.hs 1b") \
133 /* 16x block */ \
134 __ASM_EMIT("2:") \
135 __ASM_EMIT("adds %[count], %[count], #16") /* 32 - 16 */ \
136 __ASM_EMIT("b.lt 4f") \
137 __ASM_EMIT("ldp q0, q1, [%[" SRC "], #0x00]") \
138 __ASM_EMIT("ldp q2, q3, [%[" SRC "], #0x20]") \
139 __ASM_EMIT(OP " v0.4s, " ORDER("v0.4s", "v16.4s")) \
140 __ASM_EMIT(OP " v1.4s, " ORDER("v1.4s", "v17.4s")) \
141 __ASM_EMIT(OP " v2.4s, " ORDER("v2.4s", "v16.4s")) \
142 __ASM_EMIT(OP " v3.4s, " ORDER("v3.4s", "v17.4s")) \
143 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
144 __ASM_EMIT("stp q2, q3, [%[" DST "], #0x20]") \
145 __ASM_EMIT("sub %[count], %[count], #16") \
146 __ASM_EMIT("add %[src], %[src], #0x40") \
147 __ASM_EMIT("add %[dst], %[dst], #0x40") \
148 /* 8x block */ \
149 __ASM_EMIT("4:") \
150 __ASM_EMIT("adds %[count], %[count], #8") /* 16 - 8 */ \
151 __ASM_EMIT("b.lt 6f") \
152 __ASM_EMIT("ldp q0, q1, [%[" SRC "], #0x00]") \
153 __ASM_EMIT(OP " v0.4s, " ORDER("v0.4s", "v16.4s")) \
154 __ASM_EMIT(OP " v1.4s, " ORDER("v1.4s", "v17.4s")) \
155 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
156 __ASM_EMIT("sub %[count], %[count], #8") \
157 __ASM_EMIT("add %[src], %[src], #0x20") \
158 __ASM_EMIT("add %[dst], %[dst], #0x20") \
159 /* 4x block */ \
160 __ASM_EMIT("6:") \
161 __ASM_EMIT("adds %[count], %[count], #4") /* 8 - 4 */ \
162 __ASM_EMIT("b.lt 8f") \
163 __ASM_EMIT("ldr q0, [%[" SRC "], #0x00]") \
164 __ASM_EMIT(OP " v0.4s, " ORDER("v0.4s", "v16.4s")) \
165 __ASM_EMIT("str q0, [%[" DST "], #0x00]") \
166 __ASM_EMIT("sub %[count], %[count], #4") \
167 __ASM_EMIT("add %[src], %[src], #0x10") \
168 __ASM_EMIT("add %[dst], %[dst], #0x10") \
169 /* 1x block */ \
170 __ASM_EMIT("8:") \
171 __ASM_EMIT("adds %[count], %[count], #3") /* 4 - 3 */ \
172 __ASM_EMIT("b.lt 10f") \
173 __ASM_EMIT("9:") \
174 __ASM_EMIT("ld1r {v0.4s}, [%[" SRC "]]") \
175 __ASM_EMIT(OP " v0.4s, " ORDER("v0.4s", "v16.4s")) \
176 __ASM_EMIT("st1 {v0.s}[0], [%[" DST "]]") \
177 __ASM_EMIT("subs %[count], %[count], #1") \
178 __ASM_EMIT("add %[src], %[src], #0x04") \
179 __ASM_EMIT("add %[dst], %[dst], #0x04") \
180 __ASM_EMIT("bge 9b") \
181 __ASM_EMIT("10:")
182
183 #define OP_DIV2_CORE(DST) \
184 __ASM_EMIT("subs %[count], %[count], #32") \
185 __ASM_EMIT("mov v9.16b, v8.16b") \
186 __ASM_EMIT("b.lo 2f") \
187 /* 32x blocks */ \
188 __ASM_EMIT("1:") \
189 __ASM_EMIT("ldp q0, q1, [%[" DST "], #0x00]") \
190 __ASM_EMIT("ldp q2, q3, [%[" DST "], #0x20]") \
191 __ASM_EMIT("ldp q4, q5, [%[" DST "], #0x40]") \
192 __ASM_EMIT("ldp q6, q7, [%[" DST "], #0x60]") \
193 __ASM_EMIT("frecpe v16.4s, v0.4s") /* v16 = s2 */ \
194 __ASM_EMIT("frecpe v17.4s, v1.4s") \
195 __ASM_EMIT("frecpe v18.4s, v2.4s") \
196 __ASM_EMIT("frecpe v19.4s, v3.4s") \
197 __ASM_EMIT("frecpe v20.4s, v4.4s") \
198 __ASM_EMIT("frecpe v21.4s, v5.4s") \
199 __ASM_EMIT("frecpe v22.4s, v6.4s") \
200 __ASM_EMIT("frecpe v23.4s, v7.4s") \
201 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2) */ \
202 __ASM_EMIT("frecps v25.4s, v17.4s, v1.4s") \
203 __ASM_EMIT("frecps v26.4s, v18.4s, v2.4s") \
204 __ASM_EMIT("frecps v27.4s, v19.4s, v3.4s") \
205 __ASM_EMIT("frecps v28.4s, v20.4s, v4.4s") \
206 __ASM_EMIT("frecps v29.4s, v21.4s, v5.4s") \
207 __ASM_EMIT("frecps v30.4s, v22.4s, v6.4s") \
208 __ASM_EMIT("frecps v31.4s, v23.4s, v7.4s") \
209 __ASM_EMIT("fmul v16.4s, v24.4s, v16.4s") /* v16 = s2' = s2 * (2 - R*s2) */ \
210 __ASM_EMIT("fmul v17.4s, v25.4s, v17.4s") \
211 __ASM_EMIT("fmul v18.4s, v26.4s, v18.4s") \
212 __ASM_EMIT("fmul v19.4s, v27.4s, v19.4s") \
213 __ASM_EMIT("fmul v20.4s, v28.4s, v20.4s") \
214 __ASM_EMIT("fmul v21.4s, v29.4s, v21.4s") \
215 __ASM_EMIT("fmul v22.4s, v30.4s, v22.4s") \
216 __ASM_EMIT("fmul v23.4s, v31.4s, v23.4s") \
217 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2') */ \
218 __ASM_EMIT("frecps v25.4s, v17.4s, v1.4s") \
219 __ASM_EMIT("frecps v26.4s, v18.4s, v2.4s") \
220 __ASM_EMIT("frecps v27.4s, v19.4s, v3.4s") \
221 __ASM_EMIT("frecps v28.4s, v20.4s, v4.4s") \
222 __ASM_EMIT("frecps v29.4s, v21.4s, v5.4s") \
223 __ASM_EMIT("frecps v30.4s, v22.4s, v6.4s") \
224 __ASM_EMIT("frecps v31.4s, v23.4s, v7.4s") \
225 __ASM_EMIT("fmul v0.4s, v24.4s, v16.4s") /* v0 = s2" = s2' * (2 - R*s2) = 1/s2 */ \
226 __ASM_EMIT("fmul v1.4s, v25.4s, v17.4s") \
227 __ASM_EMIT("fmul v2.4s, v26.4s, v18.4s") \
228 __ASM_EMIT("fmul v3.4s, v27.4s, v19.4s") \
229 __ASM_EMIT("fmul v4.4s, v28.4s, v20.4s") \
230 __ASM_EMIT("fmul v5.4s, v29.4s, v21.4s") \
231 __ASM_EMIT("fmul v6.4s, v30.4s, v22.4s") \
232 __ASM_EMIT("fmul v7.4s, v31.4s, v23.4s") \
233 __ASM_EMIT("fmul v0.4s, v0.4s, v8.4s") /* s1 / s2 */ \
234 __ASM_EMIT("fmul v1.4s, v1.4s, v9.4s") \
235 __ASM_EMIT("fmul v2.4s, v2.4s, v8.4s") \
236 __ASM_EMIT("fmul v3.4s, v3.4s, v9.4s") \
237 __ASM_EMIT("fmul v4.4s, v4.4s, v8.4s") \
238 __ASM_EMIT("fmul v5.4s, v5.4s, v9.4s") \
239 __ASM_EMIT("fmul v6.4s, v6.4s, v8.4s") \
240 __ASM_EMIT("fmul v7.4s, v7.4s, v9.4s") \
241 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
242 __ASM_EMIT("stp q2, q3, [%[" DST "], #0x20]") \
243 __ASM_EMIT("stp q4, q5, [%[" DST "], #0x40]") \
244 __ASM_EMIT("stp q6, q7, [%[" DST "], #0x60]") \
245 __ASM_EMIT("subs %[count], %[count], #32") \
246 __ASM_EMIT("add %[dst], %[dst], #0x80") \
247 __ASM_EMIT("b.hs 1b") \
248 /* 16x block */ \
249 __ASM_EMIT("2:") \
250 __ASM_EMIT("adds %[count], %[count], #16") /* 32 - 16 */ \
251 __ASM_EMIT("b.lt 4f") \
252 __ASM_EMIT("ldp q0, q1, [%[" DST "], #0x00]") \
253 __ASM_EMIT("ldp q2, q3, [%[" DST "], #0x20]") \
254 __ASM_EMIT("frecpe v16.4s, v0.4s") /* v16 = s2 */ \
255 __ASM_EMIT("frecpe v17.4s, v1.4s") \
256 __ASM_EMIT("frecpe v18.4s, v2.4s") \
257 __ASM_EMIT("frecpe v19.4s, v3.4s") \
258 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2) */ \
259 __ASM_EMIT("frecps v25.4s, v17.4s, v1.4s") \
260 __ASM_EMIT("frecps v26.4s, v18.4s, v2.4s") \
261 __ASM_EMIT("frecps v27.4s, v19.4s, v3.4s") \
262 __ASM_EMIT("fmul v16.4s, v24.4s, v16.4s") /* v16 = s2' = s2 * (2 - R*s2) */ \
263 __ASM_EMIT("fmul v17.4s, v25.4s, v17.4s") \
264 __ASM_EMIT("fmul v18.4s, v26.4s, v18.4s") \
265 __ASM_EMIT("fmul v19.4s, v27.4s, v19.4s") \
266 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2') */ \
267 __ASM_EMIT("frecps v25.4s, v17.4s, v1.4s") \
268 __ASM_EMIT("frecps v26.4s, v18.4s, v2.4s") \
269 __ASM_EMIT("frecps v27.4s, v19.4s, v3.4s") \
270 __ASM_EMIT("fmul v0.4s, v24.4s, v16.4s") /* v0 = s2" = s2' * (2 - R*s2) = 1/s2 */ \
271 __ASM_EMIT("fmul v1.4s, v25.4s, v17.4s") \
272 __ASM_EMIT("fmul v2.4s, v26.4s, v18.4s") \
273 __ASM_EMIT("fmul v3.4s, v27.4s, v19.4s") \
274 __ASM_EMIT("fmul v0.4s, v0.4s, v8.4s") /* s1 / s2 */ \
275 __ASM_EMIT("fmul v1.4s, v1.4s, v9.4s") \
276 __ASM_EMIT("fmul v2.4s, v2.4s, v8.4s") \
277 __ASM_EMIT("fmul v3.4s, v3.4s, v9.4s") \
278 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
279 __ASM_EMIT("stp q2, q3, [%[" DST "], #0x20]") \
280 __ASM_EMIT("sub %[count], %[count], #16") \
281 __ASM_EMIT("add %[dst], %[dst], #0x40") \
282 /* 8x block */ \
283 __ASM_EMIT("4:") \
284 __ASM_EMIT("adds %[count], %[count], #8") /* 16 - 8 */ \
285 __ASM_EMIT("b.lt 6f") \
286 __ASM_EMIT("ldp q0, q1, [%[" DST "], #0x00]") \
287 __ASM_EMIT("frecpe v16.4s, v0.4s") /* v16 = s2 */ \
288 __ASM_EMIT("frecpe v17.4s, v1.4s") \
289 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2) */ \
290 __ASM_EMIT("frecps v25.4s, v17.4s, v1.4s") \
291 __ASM_EMIT("fmul v16.4s, v24.4s, v16.4s") /* v16 = s2' = s2 * (2 - R*s2) */ \
292 __ASM_EMIT("fmul v17.4s, v25.4s, v17.4s") \
293 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2') */ \
294 __ASM_EMIT("frecps v25.4s, v17.4s, v1.4s") \
295 __ASM_EMIT("fmul v0.4s, v24.4s, v16.4s") /* v0 = s2" = s2' * (2 - R*s2) = 1/s2 */ \
296 __ASM_EMIT("fmul v1.4s, v25.4s, v17.4s") \
297 __ASM_EMIT("fmul v0.4s, v0.4s, v8.4s") /* s1 / s2 */ \
298 __ASM_EMIT("fmul v1.4s, v1.4s, v9.4s") \
299 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
300 __ASM_EMIT("sub %[count], %[count], #8") \
301 __ASM_EMIT("add %[dst], %[dst], #0x20") \
302 /* 4x block */ \
303 __ASM_EMIT("6:") \
304 __ASM_EMIT("adds %[count], %[count], #4") /* 8 - 4 */ \
305 __ASM_EMIT("b.lt 8f") \
306 __ASM_EMIT("ldr q0, [%[" DST "], #0x00]") \
307 __ASM_EMIT("frecpe v16.4s, v0.4s") /* v16 = s2 */ \
308 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2) */ \
309 __ASM_EMIT("fmul v16.4s, v24.4s, v16.4s") /* v16 = s2' = s2 * (2 - R*s2) */ \
310 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2') */ \
311 __ASM_EMIT("fmul v0.4s, v24.4s, v16.4s") /* v0 = s2" = s2' * (2 - R*s2) = 1/s2 */ \
312 __ASM_EMIT("fmul v0.4s, v0.4s, v8.4s") /* s1 / s2 */ \
313 __ASM_EMIT("str q0, [%[" DST "], #0x00]") \
314 __ASM_EMIT("sub %[count], %[count], #4") \
315 __ASM_EMIT("add %[dst], %[dst], #0x10") \
316 /* 1x block */ \
317 __ASM_EMIT("8:") \
318 __ASM_EMIT("adds %[count], %[count], #3") /* 4 - 3 */ \
319 __ASM_EMIT("b.lt 10f") \
320 __ASM_EMIT("9:") \
321 __ASM_EMIT("ld1r {v0.4s}, [%[" DST "]]") \
322 __ASM_EMIT("frecpe v16.4s, v0.4s") /* v16 = s2 */ \
323 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2) */ \
324 __ASM_EMIT("fmul v16.4s, v24.4s, v16.4s") /* v16 = s2' = s2 * (2 - R*s2) */ \
325 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2') */ \
326 __ASM_EMIT("fmul v0.4s, v24.4s, v16.4s") /* v0 = s2" = s2' * (2 - R*s2) = 1/s2 */ \
327 __ASM_EMIT("fmul v0.4s, v0.4s, v8.4s") /* s1 / s2 */ \
328 __ASM_EMIT("st1 {v0.s}[0], [%[" DST "]]") \
329 __ASM_EMIT("subs %[count], %[count], #1") \
330 __ASM_EMIT("add %[dst], %[dst], #0x04") \
331 __ASM_EMIT("bge 9b") \
332 __ASM_EMIT("10:")
333
334 #define OP_DIV3_CORE(DST, SRC) \
335 __ASM_EMIT("subs %[count], %[count], #32") \
336 __ASM_EMIT("mov v9.16b, v8.16b") \
337 __ASM_EMIT("b.lo 2f") \
338 /* 32x blocks */ \
339 __ASM_EMIT("1:") \
340 __ASM_EMIT("ldp q0, q1, [%[" SRC "], #0x00]") \
341 __ASM_EMIT("ldp q2, q3, [%[" SRC "], #0x20]") \
342 __ASM_EMIT("ldp q4, q5, [%[" SRC "], #0x40]") \
343 __ASM_EMIT("ldp q6, q7, [%[" SRC "], #0x60]") \
344 __ASM_EMIT("frecpe v16.4s, v0.4s") /* v16 = s2 */ \
345 __ASM_EMIT("frecpe v17.4s, v1.4s") \
346 __ASM_EMIT("frecpe v18.4s, v2.4s") \
347 __ASM_EMIT("frecpe v19.4s, v3.4s") \
348 __ASM_EMIT("frecpe v20.4s, v4.4s") \
349 __ASM_EMIT("frecpe v21.4s, v5.4s") \
350 __ASM_EMIT("frecpe v22.4s, v6.4s") \
351 __ASM_EMIT("frecpe v23.4s, v7.4s") \
352 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2) */ \
353 __ASM_EMIT("frecps v25.4s, v17.4s, v1.4s") \
354 __ASM_EMIT("frecps v26.4s, v18.4s, v2.4s") \
355 __ASM_EMIT("frecps v27.4s, v19.4s, v3.4s") \
356 __ASM_EMIT("frecps v28.4s, v20.4s, v4.4s") \
357 __ASM_EMIT("frecps v29.4s, v21.4s, v5.4s") \
358 __ASM_EMIT("frecps v30.4s, v22.4s, v6.4s") \
359 __ASM_EMIT("frecps v31.4s, v23.4s, v7.4s") \
360 __ASM_EMIT("fmul v16.4s, v24.4s, v16.4s") /* v16 = s2' = s2 * (2 - R*s2) */ \
361 __ASM_EMIT("fmul v17.4s, v25.4s, v17.4s") \
362 __ASM_EMIT("fmul v18.4s, v26.4s, v18.4s") \
363 __ASM_EMIT("fmul v19.4s, v27.4s, v19.4s") \
364 __ASM_EMIT("fmul v20.4s, v28.4s, v20.4s") \
365 __ASM_EMIT("fmul v21.4s, v29.4s, v21.4s") \
366 __ASM_EMIT("fmul v22.4s, v30.4s, v22.4s") \
367 __ASM_EMIT("fmul v23.4s, v31.4s, v23.4s") \
368 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2') */ \
369 __ASM_EMIT("frecps v25.4s, v17.4s, v1.4s") \
370 __ASM_EMIT("frecps v26.4s, v18.4s, v2.4s") \
371 __ASM_EMIT("frecps v27.4s, v19.4s, v3.4s") \
372 __ASM_EMIT("frecps v28.4s, v20.4s, v4.4s") \
373 __ASM_EMIT("frecps v29.4s, v21.4s, v5.4s") \
374 __ASM_EMIT("frecps v30.4s, v22.4s, v6.4s") \
375 __ASM_EMIT("frecps v31.4s, v23.4s, v7.4s") \
376 __ASM_EMIT("fmul v0.4s, v24.4s, v16.4s") /* v0 = s2" = s2' * (2 - R*s2) = 1/s2 */ \
377 __ASM_EMIT("fmul v1.4s, v25.4s, v17.4s") \
378 __ASM_EMIT("fmul v2.4s, v26.4s, v18.4s") \
379 __ASM_EMIT("fmul v3.4s, v27.4s, v19.4s") \
380 __ASM_EMIT("fmul v4.4s, v28.4s, v20.4s") \
381 __ASM_EMIT("fmul v5.4s, v29.4s, v21.4s") \
382 __ASM_EMIT("fmul v6.4s, v30.4s, v22.4s") \
383 __ASM_EMIT("fmul v7.4s, v31.4s, v23.4s") \
384 __ASM_EMIT("fmul v0.4s, v0.4s, v8.4s") /* s1 / s2 */ \
385 __ASM_EMIT("fmul v1.4s, v1.4s, v9.4s") \
386 __ASM_EMIT("fmul v2.4s, v2.4s, v8.4s") \
387 __ASM_EMIT("fmul v3.4s, v3.4s, v9.4s") \
388 __ASM_EMIT("fmul v4.4s, v4.4s, v8.4s") \
389 __ASM_EMIT("fmul v5.4s, v5.4s, v9.4s") \
390 __ASM_EMIT("fmul v6.4s, v6.4s, v8.4s") \
391 __ASM_EMIT("fmul v7.4s, v7.4s, v9.4s") \
392 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
393 __ASM_EMIT("stp q2, q3, [%[" DST "], #0x20]") \
394 __ASM_EMIT("stp q4, q5, [%[" DST "], #0x40]") \
395 __ASM_EMIT("stp q6, q7, [%[" DST "], #0x60]") \
396 __ASM_EMIT("subs %[count], %[count], #32") \
397 __ASM_EMIT("add %[src], %[src], #0x80") \
398 __ASM_EMIT("add %[dst], %[dst], #0x80") \
399 __ASM_EMIT("b.hs 1b") \
400 /* 16x block */ \
401 __ASM_EMIT("2:") \
402 __ASM_EMIT("adds %[count], %[count], #16") /* 32 - 16 */ \
403 __ASM_EMIT("b.lt 4f") \
404 __ASM_EMIT("ldp q0, q1, [%[" SRC "], #0x00]") \
405 __ASM_EMIT("ldp q2, q3, [%[" SRC "], #0x20]") \
406 __ASM_EMIT("frecpe v16.4s, v0.4s") /* v16 = s2 */ \
407 __ASM_EMIT("frecpe v17.4s, v1.4s") \
408 __ASM_EMIT("frecpe v18.4s, v2.4s") \
409 __ASM_EMIT("frecpe v19.4s, v3.4s") \
410 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2) */ \
411 __ASM_EMIT("frecps v25.4s, v17.4s, v1.4s") \
412 __ASM_EMIT("frecps v26.4s, v18.4s, v2.4s") \
413 __ASM_EMIT("frecps v27.4s, v19.4s, v3.4s") \
414 __ASM_EMIT("fmul v16.4s, v24.4s, v16.4s") /* v16 = s2' = s2 * (2 - R*s2) */ \
415 __ASM_EMIT("fmul v17.4s, v25.4s, v17.4s") \
416 __ASM_EMIT("fmul v18.4s, v26.4s, v18.4s") \
417 __ASM_EMIT("fmul v19.4s, v27.4s, v19.4s") \
418 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2') */ \
419 __ASM_EMIT("frecps v25.4s, v17.4s, v1.4s") \
420 __ASM_EMIT("frecps v26.4s, v18.4s, v2.4s") \
421 __ASM_EMIT("frecps v27.4s, v19.4s, v3.4s") \
422 __ASM_EMIT("fmul v0.4s, v24.4s, v16.4s") /* v0 = s2" = s2' * (2 - R*s2) = 1/s2 */ \
423 __ASM_EMIT("fmul v1.4s, v25.4s, v17.4s") \
424 __ASM_EMIT("fmul v2.4s, v26.4s, v18.4s") \
425 __ASM_EMIT("fmul v3.4s, v27.4s, v19.4s") \
426 __ASM_EMIT("fmul v0.4s, v0.4s, v8.4s") /* s1 / s2 */ \
427 __ASM_EMIT("fmul v1.4s, v1.4s, v9.4s") \
428 __ASM_EMIT("fmul v2.4s, v2.4s, v8.4s") \
429 __ASM_EMIT("fmul v3.4s, v3.4s, v9.4s") \
430 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
431 __ASM_EMIT("stp q2, q3, [%[" DST "], #0x20]") \
432 __ASM_EMIT("sub %[count], %[count], #16") \
433 __ASM_EMIT("add %[src], %[src], #0x40") \
434 __ASM_EMIT("add %[dst], %[dst], #0x40") \
435 /* 8x block */ \
436 __ASM_EMIT("4:") \
437 __ASM_EMIT("adds %[count], %[count], #8") /* 16 - 8 */ \
438 __ASM_EMIT("b.lt 6f") \
439 __ASM_EMIT("ldp q0, q1, [%[" SRC "], #0x00]") \
440 __ASM_EMIT("frecpe v16.4s, v0.4s") /* v16 = s2 */ \
441 __ASM_EMIT("frecpe v17.4s, v1.4s") \
442 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2) */ \
443 __ASM_EMIT("frecps v25.4s, v17.4s, v1.4s") \
444 __ASM_EMIT("fmul v16.4s, v24.4s, v16.4s") /* v16 = s2' = s2 * (2 - R*s2) */ \
445 __ASM_EMIT("fmul v17.4s, v25.4s, v17.4s") \
446 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2') */ \
447 __ASM_EMIT("frecps v25.4s, v17.4s, v1.4s") \
448 __ASM_EMIT("fmul v0.4s, v24.4s, v16.4s") /* v0 = s2" = s2' * (2 - R*s2) = 1/s2 */ \
449 __ASM_EMIT("fmul v1.4s, v25.4s, v17.4s") \
450 __ASM_EMIT("fmul v0.4s, v0.4s, v8.4s") /* s1 / s2 */ \
451 __ASM_EMIT("fmul v1.4s, v1.4s, v9.4s") \
452 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
453 __ASM_EMIT("sub %[count], %[count], #8") \
454 __ASM_EMIT("add %[src], %[src], #0x20") \
455 __ASM_EMIT("add %[dst], %[dst], #0x20") \
456 /* 4x block */ \
457 __ASM_EMIT("6:") \
458 __ASM_EMIT("adds %[count], %[count], #4") /* 8 - 4 */ \
459 __ASM_EMIT("b.lt 8f") \
460 __ASM_EMIT("ldr q0, [%[" SRC "], #0x00]") \
461 __ASM_EMIT("frecpe v16.4s, v0.4s") /* v16 = s2 */ \
462 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2) */ \
463 __ASM_EMIT("fmul v16.4s, v24.4s, v16.4s") /* v16 = s2' = s2 * (2 - R*s2) */ \
464 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2') */ \
465 __ASM_EMIT("fmul v0.4s, v24.4s, v16.4s") /* v0 = s2" = s2' * (2 - R*s2) = 1/s2 */ \
466 __ASM_EMIT("fmul v0.4s, v0.4s, v8.4s") /* s1 / s2 */ \
467 __ASM_EMIT("str q0, [%[" DST "], #0x00]") \
468 __ASM_EMIT("sub %[count], %[count], #4") \
469 __ASM_EMIT("add %[src], %[src], #0x10") \
470 __ASM_EMIT("add %[dst], %[dst], #0x10") \
471 /* 1x block */ \
472 __ASM_EMIT("8:") \
473 __ASM_EMIT("adds %[count], %[count], #3") /* 4 - 3 */ \
474 __ASM_EMIT("b.lt 10f") \
475 __ASM_EMIT("9:") \
476 __ASM_EMIT("ld1r {v0.4s}, [%[" SRC "]]") \
477 __ASM_EMIT("frecpe v16.4s, v0.4s") /* v16 = s2 */ \
478 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2) */ \
479 __ASM_EMIT("fmul v16.4s, v24.4s, v16.4s") /* v16 = s2' = s2 * (2 - R*s2) */ \
480 __ASM_EMIT("frecps v24.4s, v16.4s, v0.4s") /* v24 = (2 - R*s2') */ \
481 __ASM_EMIT("fmul v0.4s, v24.4s, v16.4s") /* v0 = s2" = s2' * (2 - R*s2) = 1/s2 */ \
482 __ASM_EMIT("fmul v0.4s, v0.4s, v8.4s") /* s1 / s2 */ \
483 __ASM_EMIT("st1 {v0.s}[0], [%[" DST "]]") \
484 __ASM_EMIT("subs %[count], %[count], #1") \
485 __ASM_EMIT("add %[src], %[src], #0x04") \
486 __ASM_EMIT("add %[dst], %[dst], #0x04") \
487 __ASM_EMIT("bge 9b") \
488 __ASM_EMIT("10:")
489
490 void add_k2(float *dst, float k, size_t count)
491 {
492 IF_ARCH_AARCH64(float *pk = &k);
493 ARCH_AARCH64_ASM
494 (
495 __ASM_EMIT("ld1r {v16.4s}, [%[k]]")
496 OP_K2_CORE("dst", "fadd", OP_DORDER)
497 : [dst] "+r" (dst),
498 [count] "+r" (count)
499 : [k] "r" (pk)
500 : "cc", "memory",
501 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
502 "v16", "v17"
503 );
504 }
505
506 void sub_k2(float *dst, float k, size_t count)
507 {
508 IF_ARCH_AARCH64(float *pk = &k);
509 ARCH_AARCH64_ASM
510 (
511 __ASM_EMIT("ld1r {v16.4s}, [%[k]]")
512 OP_K2_CORE("dst", "fsub", OP_DORDER)
513 : [dst] "+r" (dst),
514 [count] "+r" (count)
515 : [k] "r" (pk)
516 : "cc", "memory",
517 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
518 "v16", "v17"
519 );
520 }
521
522 void rsub_k2(float *dst, float k, size_t count)
523 {
524 IF_ARCH_AARCH64(float *pk = &k);
525 ARCH_AARCH64_ASM
526 (
527 __ASM_EMIT("ld1r {v16.4s}, [%[k]]")
528 OP_K2_CORE("dst", "fsub", OP_RORDER)
529 : [dst] "+r" (dst),
530 [count] "+r" (count)
531 : [k] "r" (pk)
532 : "cc", "memory",
533 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
534 "v16", "v17"
535 );
536 }
537
538 void mul_k2(float *dst, float k, size_t count)
539 {
540 IF_ARCH_AARCH64(float *pk = &k);
541 ARCH_AARCH64_ASM
542 (
543 __ASM_EMIT("ld1r {v16.4s}, [%[k]]")
544 OP_K2_CORE("dst", "fmul", OP_DORDER)
545 : [dst] "+r" (dst),
546 [count] "+r" (count)
547 : [k] "r" (pk)
548 : "cc", "memory",
549 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
550 "v16", "v17"
551 );
552 }
553
554 void div_k2(float *dst, float k, size_t count)
555 {
556 IF_ARCH_AARCH64(float *pk = &k);
557 ARCH_AARCH64_ASM
558 (
559 __ASM_EMIT("ld1r {v0.4s}, [%[k]]")
560 __ASM_EMIT("frecpe v16.4s, v0.4s") /* v16 = k */
561 __ASM_EMIT("frecps v17.4s, v16.4s, v0.4s") /* v17 = (2 - R*k) */
562 __ASM_EMIT("fmul v16.4s, v17.4s, v16.4s") /* v16 = k' = k * (2 - R*k) */
563 __ASM_EMIT("frecps v17.4s, v16.4s, v0.4s") /* v17 = (2 - R*k') */
564 __ASM_EMIT("fmul v16.4s, v17.4s, v16.4s") /* v16 = k" = k' * (2 - R*k) = 1/k */
565 OP_K2_CORE("dst", "fmul", OP_DORDER)
566 : [dst] "+r" (dst),
567 [count] "+r" (count)
568 : [k] "r" (pk)
569 : "cc", "memory",
570 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
571 "v16", "v17"
572 );
573 }
574
575 void rdiv_k2(float *dst, float k, size_t count)
576 {
577 IF_ARCH_AARCH64(float *pk = &k);
578 ARCH_AARCH64_ASM
579 (
580 __ASM_EMIT("ld1r {v8.4s}, [%[k]]")
581 OP_DIV2_CORE("dst")
582 : [dst] "+r" (dst),
583 [count] "+r" (count)
584 : [k] "r" (pk)
585 : "cc", "memory",
586 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
587 "v8", "v9",
588 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23",
589 "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31"
590 );
591 }
592
593 void add_k3(float *dst, const float *src, float k, size_t count)
594 {
595 IF_ARCH_AARCH64(float *pk = &k);
596 ARCH_AARCH64_ASM
597 (
598 __ASM_EMIT("ld1r {v16.4s}, [%[k]]")
599 OP_K3_CORE("dst", "src", "fadd", OP_DORDER)
600 : [dst] "+r" (dst), [src] "+r" (src),
601 [count] "+r" (count)
602 : [k] "r" (pk)
603 : "cc", "memory",
604 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
605 "v16", "v17"
606 );
607 }
608
609 void sub_k3(float *dst, const float *src, float k, size_t count)
610 {
611 IF_ARCH_AARCH64(float *pk = &k);
612 ARCH_AARCH64_ASM
613 (
614 __ASM_EMIT("ld1r {v16.4s}, [%[k]]")
615 OP_K3_CORE("dst", "src", "fsub", OP_DORDER)
616 : [dst] "+r" (dst), [src] "+r" (src),
617 [count] "+r" (count)
618 : [k] "r" (pk)
619 : "cc", "memory",
620 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
621 "v16", "v17"
622 );
623 }
624
625 void rsub_k3(float *dst, const float *src, float k, size_t count)
626 {
627 IF_ARCH_AARCH64(float *pk = &k);
628 ARCH_AARCH64_ASM
629 (
630 __ASM_EMIT("ld1r {v16.4s}, [%[k]]")
631 OP_K3_CORE("dst", "src", "fsub", OP_RORDER)
632 : [dst] "+r" (dst), [src] "+r" (src),
633 [count] "+r" (count)
634 : [k] "r" (pk)
635 : "cc", "memory",
636 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
637 "v16", "v17"
638 );
639 }
640
641 void mul_k3(float *dst, const float *src, float k, size_t count)
642 {
643 IF_ARCH_AARCH64(float *pk = &k);
644 ARCH_AARCH64_ASM
645 (
646 __ASM_EMIT("ld1r {v16.4s}, [%[k]]")
647 OP_K3_CORE("dst", "src", "fmul", OP_DORDER)
648 : [dst] "+r" (dst), [src] "+r" (src),
649 [count] "+r" (count)
650 : [k] "r" (pk)
651 : "cc", "memory",
652 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
653 "v16", "v17"
654 );
655 }
656
657 void div_k3(float *dst, const float *src, float k, size_t count)
658 {
659 IF_ARCH_AARCH64(float *pk = &k);
660 ARCH_AARCH64_ASM
661 (
662 __ASM_EMIT("ld1r {v0.4s}, [%[k]]")
663 __ASM_EMIT("frecpe v16.4s, v0.4s") /* v16 = k */
664 __ASM_EMIT("frecps v17.4s, v16.4s, v0.4s") /* v17 = (2 - R*k) */
665 __ASM_EMIT("fmul v16.4s, v17.4s, v16.4s") /* v16 = k' = k * (2 - R*k) */
666 __ASM_EMIT("frecps v17.4s, v16.4s, v0.4s") /* v17 = (2 - R*k') */
667 __ASM_EMIT("fmul v16.4s, v17.4s, v16.4s") /* v16 = k" = k' * (2 - R*k) = 1/k */
668 OP_K3_CORE("dst", "src", "fmul", OP_DORDER)
669 : [dst] "+r" (dst), [src] "+r" (src),
670 [count] "+r" (count)
671 : [k] "r" (pk)
672 : "cc", "memory",
673 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
674 "v16", "v17"
675 );
676 }
677
678 void rdiv_k3(float *dst, const float *src, float k, size_t count)
679 {
680 IF_ARCH_AARCH64(float *pk = &k);
681 ARCH_AARCH64_ASM
682 (
683 __ASM_EMIT("ld1r {v8.4s}, [%[k]]")
684 OP_DIV3_CORE("dst", "src")
685 : [dst] "+r" (dst), [src] "+r" (src),
686 [count] "+r" (count)
687 : [k] "r" (pk)
688 : "cc", "memory",
689 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
690 "v8", "v9",
691 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23",
692 "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31"
693 );
694 }
695
696 #define INC_ON(x) x
697 #define INC_OFF(x)
698
699 #define MOD_KX_CORE(DST, SRC, INC) \
700 __ASM_EMIT("subs %[count], %[count], #16") \
701 __ASM_EMIT("frecpe v16.4s, v4.4s") \
702 __ASM_EMIT("mov v5.16b, v4.16b") /* v5 = k */ \
703 __ASM_EMIT("frecps v20.4s, v16.4s, v4.4s") \
704 __ASM_EMIT("fmul v16.4s, v20.4s, v16.4s") \
705 __ASM_EMIT("frecps v20.4s, v16.4s, v4.4s") \
706 __ASM_EMIT("fmul v6.4s, v20.4s, v16.4s") /* v6 = 1/k */ \
707 __ASM_EMIT("mov v7.16b, v6.16b") /* v7 = 1/k */ \
708 __ASM_EMIT("b.lo 2f") \
709 /* 16x blocks */ \
710 __ASM_EMIT("1:") \
711 __ASM_EMIT("ldp q0, q1, [%[" SRC "], #0x00]") \
712 __ASM_EMIT("ldp q2, q3, [%[" SRC "], #0x20]") \
713 __ASM_EMIT("fmul v16.4s, v0.4s, v6.4s") /* v0 = x/k */ \
714 __ASM_EMIT("fmul v17.4s, v1.4s, v7.4s") \
715 __ASM_EMIT("fmul v18.4s, v2.4s, v6.4s") \
716 __ASM_EMIT("fmul v19.4s, v3.4s, v7.4s") \
717 __ASM_EMIT("frintz v16.4s, v16.4s") \
718 __ASM_EMIT("frintz v17.4s, v17.4s") \
719 __ASM_EMIT("frintz v18.4s, v18.4s") \
720 __ASM_EMIT("frintz v19.4s, v19.4s") \
721 __ASM_EMIT("fmls v0.4s, v16.4s, v4.4s") \
722 __ASM_EMIT("fmls v1.4s, v17.4s, v5.4s") \
723 __ASM_EMIT("fmls v2.4s, v18.4s, v4.4s") \
724 __ASM_EMIT("fmls v3.4s, v19.4s, v5.4s") \
725 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
726 __ASM_EMIT("stp q2, q3, [%[" DST "], #0x20]") \
727 __ASM_EMIT("subs %[count], %[count], #16") \
728 __ASM_EMIT(INC("add %[" SRC "], %[" SRC "], #0x40")) \
729 __ASM_EMIT("add %[" DST "], %[" DST "], #0x40") \
730 __ASM_EMIT("b.hs 1b") \
731 /* 8x block */ \
732 __ASM_EMIT("2:") \
733 __ASM_EMIT("adds %[count], %[count], #8") \
734 __ASM_EMIT("b.lt 4f") \
735 __ASM_EMIT("ldp q0, q1, [%[" SRC "], #0x00]") \
736 __ASM_EMIT("fmul v16.4s, v0.4s, v6.4s") /* v0 = x/k */ \
737 __ASM_EMIT("fmul v17.4s, v1.4s, v7.4s") \
738 __ASM_EMIT("frintz v16.4s, v16.4s") \
739 __ASM_EMIT("frintz v17.4s, v17.4s") \
740 __ASM_EMIT("fmls v0.4s, v16.4s, v4.4s") \
741 __ASM_EMIT("fmls v1.4s, v17.4s, v5.4s") \
742 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
743 __ASM_EMIT("sub %[count], %[count], #8") \
744 __ASM_EMIT(INC("add %[" SRC "], %[" SRC "], #0x20")) \
745 __ASM_EMIT("add %[" DST "], %[" DST "], #0x20") \
746 /* 4x blocks */ \
747 __ASM_EMIT("4:") \
748 __ASM_EMIT("adds %[count], %[count], #4") \
749 __ASM_EMIT("b.lt 6f") \
750 __ASM_EMIT("ldr q0, [%[" SRC "]]") \
751 __ASM_EMIT("fmul v16.4s, v0.4s, v6.4s") /* v0 = x/k */ \
752 __ASM_EMIT("frintz v16.4s, v16.4s") \
753 __ASM_EMIT("fmls v0.4s, v16.4s, v4.4s") \
754 __ASM_EMIT("str q0, [%[" DST "]]") \
755 __ASM_EMIT("sub %[count], %[count], #4") \
756 __ASM_EMIT(INC("add %[" SRC "], %[" SRC "], #0x10")) \
757 __ASM_EMIT("add %[" DST "], %[" DST "], #0x10") \
758 /* 1x blocks */ \
759 __ASM_EMIT("6:") \
760 __ASM_EMIT("adds %[count], %[count], #3") \
761 __ASM_EMIT("b.lt 8f") \
762 __ASM_EMIT("7:") \
763 __ASM_EMIT("ld1r {v0.4s}, [%[" SRC "]]") \
764 __ASM_EMIT("fmul v16.4s, v0.4s, v6.4s") /* v0 = x/k */ \
765 __ASM_EMIT("frintz v16.4s, v16.4s") \
766 __ASM_EMIT("fmls v0.4s, v16.4s, v4.4s") \
767 __ASM_EMIT("st1 {v0.s}[0], [%[" DST "]]") \
768 __ASM_EMIT("subs %[count], %[count], #1") \
769 __ASM_EMIT(INC("add %[" SRC "], %[" SRC "], #0x04")) \
770 __ASM_EMIT("add %[" DST "], %[" DST "], #0x04") \
771 __ASM_EMIT("bge 7b") \
772 __ASM_EMIT("8:")
773
774 #define RMOD_KX_CORE(DST, SRC, INC) \
775 __ASM_EMIT("subs %[count], %[count], #16") \
776 __ASM_EMIT("mov v5.16b, v4.16b") /* v5 = k */ \
777 __ASM_EMIT("b.lo 2f") \
778 /* 16x blocks */ \
779 __ASM_EMIT("1:") \
780 __ASM_EMIT("ldp q0, q1, [%[" SRC "], #0x00]") \
781 __ASM_EMIT("ldp q2, q3, [%[" SRC "], #0x20]") \
782 __ASM_EMIT("frecpe v16.4s, v0.4s") /* v16 = s2 */ \
783 __ASM_EMIT("frecpe v17.4s, v1.4s") \
784 __ASM_EMIT("frecpe v18.4s, v2.4s") \
785 __ASM_EMIT("frecpe v19.4s, v3.4s") \
786 __ASM_EMIT("frecps v20.4s, v16.4s, v0.4s") /* v20 = (2 - R*s2) */ \
787 __ASM_EMIT("frecps v21.4s, v17.4s, v1.4s") \
788 __ASM_EMIT("frecps v22.4s, v18.4s, v2.4s") \
789 __ASM_EMIT("frecps v23.4s, v19.4s, v3.4s") \
790 __ASM_EMIT("fmul v16.4s, v20.4s, v16.4s") /* v16 = s2' = s2 * (2 - R*s2) */ \
791 __ASM_EMIT("fmul v17.4s, v21.4s, v17.4s") \
792 __ASM_EMIT("fmul v18.4s, v22.4s, v18.4s") \
793 __ASM_EMIT("fmul v19.4s, v23.4s, v19.4s") \
794 __ASM_EMIT("frecps v20.4s, v16.4s, v0.4s") /* v20 = (2 - R*s2') */ \
795 __ASM_EMIT("frecps v21.4s, v17.4s, v1.4s") \
796 __ASM_EMIT("frecps v22.4s, v18.4s, v2.4s") \
797 __ASM_EMIT("frecps v23.4s, v19.4s, v3.4s") \
798 __ASM_EMIT("fmul v16.4s, v20.4s, v16.4s") /* v16 = s2" = s2' * (2 - R*s2) = 1/x */ \
799 __ASM_EMIT("fmul v17.4s, v21.4s, v17.4s") \
800 __ASM_EMIT("fmul v18.4s, v22.4s, v18.4s") \
801 __ASM_EMIT("fmul v19.4s, v23.4s, v19.4s") \
802 __ASM_EMIT("fmul v16.4s, v16.4s, v4.4s") /* v16 = k/x */ \
803 __ASM_EMIT("fmul v17.4s, v17.4s, v5.4s") \
804 __ASM_EMIT("fmul v18.4s, v18.4s, v4.4s") \
805 __ASM_EMIT("fmul v19.4s, v19.4s, v5.4s") \
806 __ASM_EMIT("frintz v16.4s, v16.4s") \
807 __ASM_EMIT("frintz v17.4s, v17.4s") \
808 __ASM_EMIT("frintz v18.4s, v18.4s") \
809 __ASM_EMIT("frintz v19.4s, v19.4s") \
810 __ASM_EMIT("fmul v0.4s, v0.4s, v16.4s") /* v0 = x * int(k/x) */ \
811 __ASM_EMIT("fmul v1.4s, v1.4s, v17.4s") \
812 __ASM_EMIT("fmul v2.4s, v2.4s, v18.4s") \
813 __ASM_EMIT("fmul v3.4s, v3.4s, v19.4s") \
814 __ASM_EMIT("fsub v0.4s, v4.4s, v0.4s") /* v0 = k - x * int(k/x) */ \
815 __ASM_EMIT("fsub v1.4s, v5.4s, v1.4s") \
816 __ASM_EMIT("fsub v2.4s, v4.4s, v2.4s") \
817 __ASM_EMIT("fsub v3.4s, v5.4s, v3.4s") \
818 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
819 __ASM_EMIT("stp q2, q3, [%[" DST "], #0x20]") \
820 __ASM_EMIT("subs %[count], %[count], #16") \
821 __ASM_EMIT(INC("add %[" SRC "], %[" SRC "], #0x40")) \
822 __ASM_EMIT("add %[" DST "], %[" DST "], #0x40") \
823 __ASM_EMIT("b.hs 1b") \
824 /* 8x block */ \
825 __ASM_EMIT("2:") \
826 __ASM_EMIT("adds %[count], %[count], #8") \
827 __ASM_EMIT("b.lt 4f") \
828 __ASM_EMIT("ldp q0, q1, [%[" SRC "], #0x00]") \
829 __ASM_EMIT("frecpe v16.4s, v0.4s") /* v16 = s2 */ \
830 __ASM_EMIT("frecpe v17.4s, v1.4s") \
831 __ASM_EMIT("frecps v20.4s, v16.4s, v0.4s") /* v20 = (2 - R*s2) */ \
832 __ASM_EMIT("frecps v21.4s, v17.4s, v1.4s") \
833 __ASM_EMIT("fmul v16.4s, v20.4s, v16.4s") /* v16 = s2' = s2 * (2 - R*s2) */ \
834 __ASM_EMIT("fmul v17.4s, v21.4s, v17.4s") \
835 __ASM_EMIT("frecps v20.4s, v16.4s, v0.4s") /* v20 = (2 - R*s2') */ \
836 __ASM_EMIT("frecps v21.4s, v17.4s, v1.4s") \
837 __ASM_EMIT("fmul v16.4s, v20.4s, v16.4s") /* v16 = s2" = s2' * (2 - R*s2) = 1/x */ \
838 __ASM_EMIT("fmul v17.4s, v21.4s, v17.4s") \
839 __ASM_EMIT("fmul v16.4s, v16.4s, v4.4s") /* v16 = k/x */ \
840 __ASM_EMIT("fmul v17.4s, v17.4s, v5.4s") \
841 __ASM_EMIT("frintz v16.4s, v16.4s") \
842 __ASM_EMIT("frintz v17.4s, v17.4s") \
843 __ASM_EMIT("fmul v0.4s, v0.4s, v16.4s") /* v0 = x * int(k/x) */ \
844 __ASM_EMIT("fmul v1.4s, v1.4s, v17.4s") \
845 __ASM_EMIT("fsub v0.4s, v4.4s, v0.4s") /* v0 = k - x * int(k/x) */ \
846 __ASM_EMIT("fsub v1.4s, v5.4s, v1.4s") \
847 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
848 __ASM_EMIT("sub %[count], %[count], #8") \
849 __ASM_EMIT(INC("add %[" SRC "], %[" SRC "], #0x20")) \
850 __ASM_EMIT("add %[" DST "], %[" DST "], #0x20") \
851 /* 4x blocks */ \
852 __ASM_EMIT("4:") \
853 __ASM_EMIT("adds %[count], %[count], #4") \
854 __ASM_EMIT("b.lt 6f") \
855 __ASM_EMIT("ldr q0, [%[" SRC "]]") \
856 __ASM_EMIT("frecpe v16.4s, v0.4s") /* v16 = s2 */ \
857 __ASM_EMIT("frecps v20.4s, v16.4s, v0.4s") /* v20 = (2 - R*s2) */ \
858 __ASM_EMIT("fmul v16.4s, v20.4s, v16.4s") /* v16 = s2' = s2 * (2 - R*s2) */ \
859 __ASM_EMIT("frecps v20.4s, v16.4s, v0.4s") /* v20 = (2 - R*s2') */ \
860 __ASM_EMIT("fmul v16.4s, v20.4s, v16.4s") /* v16 = s2" = s2' * (2 - R*s2) = 1/x */ \
861 __ASM_EMIT("fmul v16.4s, v16.4s, v4.4s") /* v16 = k/x */ \
862 __ASM_EMIT("frintz v16.4s, v16.4s") \
863 __ASM_EMIT("fmul v0.4s, v0.4s, v16.4s") /* v0 = x * int(k/x) */ \
864 __ASM_EMIT("fsub v0.4s, v4.4s, v0.4s") /* v0 = k - x * int(k/x) */ \
865 __ASM_EMIT("str q0, [%[" DST "]]") \
866 __ASM_EMIT("sub %[count], %[count], #4") \
867 __ASM_EMIT(INC("add %[" SRC "], %[" SRC "], #0x10")) \
868 __ASM_EMIT("add %[" DST "], %[" DST "], #0x10") \
869 /* 1x blocks */ \
870 __ASM_EMIT("6:") \
871 __ASM_EMIT("adds %[count], %[count], #3") \
872 __ASM_EMIT("b.lt 8f") \
873 __ASM_EMIT("7:") \
874 __ASM_EMIT("ld1r {v0.4s}, [%[" SRC "]]") \
875 __ASM_EMIT("frecpe v16.4s, v0.4s") /* v16 = s2 */ \
876 __ASM_EMIT("frecps v20.4s, v16.4s, v0.4s") /* v20 = (2 - R*s2) */ \
877 __ASM_EMIT("fmul v16.4s, v20.4s, v16.4s") /* v16 = s2' = s2 * (2 - R*s2) */ \
878 __ASM_EMIT("frecps v20.4s, v16.4s, v0.4s") /* v20 = (2 - R*s2') */ \
879 __ASM_EMIT("fmul v16.4s, v20.4s, v16.4s") /* v16 = s2" = s2' * (2 - R*s2) = 1/x */ \
880 __ASM_EMIT("fmul v16.4s, v16.4s, v4.4s") /* v16 = k/x */ \
881 __ASM_EMIT("frintz v16.4s, v16.4s") \
882 __ASM_EMIT("fmul v0.4s, v0.4s, v16.4s") /* v0 = x * int(k/x) */ \
883 __ASM_EMIT("fsub v0.4s, v4.4s, v0.4s") /* v0 = k - x * int(k/x) */ \
884 __ASM_EMIT("st1 {v0.s}[0], [%[" DST "]]") \
885 __ASM_EMIT("subs %[count], %[count], #1") \
886 __ASM_EMIT(INC("add %[" SRC "], %[" SRC "], #0x04")) \
887 __ASM_EMIT("add %[" DST "], %[" DST "], #0x04") \
888 __ASM_EMIT("bge 7b") \
889 __ASM_EMIT("8:")
890
891 void mod_k2(float *dst, float k, size_t count)
892 {
893 IF_ARCH_AARCH64(float *pk = &k);
894 ARCH_AARCH64_ASM
895 (
896 __ASM_EMIT("ld1r {v4.4s}, [%[k]]")
897 MOD_KX_CORE("dst", "dst", INC_OFF)
898 : [dst] "+r" (dst),
899 [count] "+r" (count)
900 : [k] "r" (pk)
901 : "cc", "memory",
902 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
903 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"
904 );
905 }
906
907 void rmod_k2(float *dst, float k, size_t count)
908 {
909 IF_ARCH_AARCH64(float *pk = &k);
910 ARCH_AARCH64_ASM
911 (
912 __ASM_EMIT("ld1r {v4.4s}, [%[k]]")
913 RMOD_KX_CORE("dst", "dst", INC_OFF)
914 : [dst] "+r" (dst),
915 [count] "+r" (count)
916 : [k] "r" (pk)
917 : "cc", "memory",
918 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
919 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"
920 );
921 }
922
923 void mod_k3(float *dst, const float *src, float k, size_t count)
924 {
925 IF_ARCH_AARCH64(float *pk = &k);
926 ARCH_AARCH64_ASM
927 (
928 __ASM_EMIT("ld1r {v4.4s}, [%[k]]")
929 MOD_KX_CORE("dst", "src", INC_ON)
930 : [dst] "+r" (dst), [src] "+r" (src),
931 [count] "+r" (count)
932 : [k] "r" (pk)
933 : "cc", "memory",
934 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
935 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"
936 );
937 }
938
939 void rmod_k3(float *dst, const float *src, float k, size_t count)
940 {
941 IF_ARCH_AARCH64(float *pk = &k);
942 ARCH_AARCH64_ASM
943 (
944 __ASM_EMIT("ld1r {v4.4s}, [%[k]]")
945 RMOD_KX_CORE("dst", "src", INC_ON)
946 : [dst] "+r" (dst), [src] "+r" (src),
947 [count] "+r" (count)
948 : [k] "r" (pk)
949 : "cc", "memory",
950 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
951 "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"
952 );
953 }
954
955 #undef INC_ON
956 #undef INC_OFF
957
958 #undef OP_DORDER
959 #undef OP_RORDER
960 #undef OP_K2_CORE
961 #undef OP_K3_CORE
962 #undef OP_DIV2_CORE
963 #undef OP_DIV3_CORE
964 #undef MOD_KX_CORE
965 #undef RMOD_KX_CORE
966 }
967 }
968
969 #endif /* PRIVATE_DSP_ARCH_AARCH64_ASIMD_PMATH_OP_KX_H_ */