Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/aarch64/asimd/pmath/lramp.h

raw
1 /*
2 * Copyright (C) 2023 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2023 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 3 февр. 2023 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_AARCH64_ASIMD_PMATH_LRAMP_H_
23 #define PRIVATE_DSP_ARCH_AARCH64_ASIMD_PMATH_LRAMP_H_
24
25 #ifndef PRIVATE_DSP_ARCH_AARCH64_ASIMD_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_AARCH64_ASIMD_IMPL */
28
29 namespace lsp
30 {
31 namespace asimd
32 {
33 IF_ARCH_AARCH64(
34 static const float lramp_const[] __lsp_aligned16 =
35 {
36 0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, /* Initial values 0..7 */
37 8.0f, 9.0f, 10.0f, 11.0f, 12.0f, 13.0f, 14.0f, 15.0f, /* Initial values 8..15 */
38 16.0f, 16.0f, 16.0f, 16.0f, 16.0f, 16.0f, 16.0f, 16.0f /* Step */
39 };
40 )
41
42 void lramp_set1(float *dst, float v1, float v2, size_t count)
43 {
44 float delta = v2 - v1;
45 if (delta == 0.0f)
46 {
47 dsp::fill(dst, v1, count);
48 return;
49 }
50 if (count == 0)
51 return;
52
53 delta /= count;
54 ARCH_AARCH64_ASM(
55 __ASM_EMIT("ld1r {v14.4s}, [%[v1]]")
56 __ASM_EMIT("ld1r {v15.4s}, [%[delta]]")
57 __ASM_EMIT("ldp q8, q9, [%[CC], #0x00]")
58 __ASM_EMIT("ldp q10, q11, [%[CC], #0x20]")
59 __ASM_EMIT("ldp q12, q13, [%[CC], #0x40]")
60 /* v8.4s = i[0], v9.4s = i[1], v10.4s = i[2], v11.4s = i[3] */
61 /* v12.4s = step, v13.4s = step */
62 /* v14.4s = v1, v15.4s = delta */
63 /* 16x blocks */
64 __ASM_EMIT("subs %[count], %[count], #16")
65 __ASM_EMIT("b.lo 2f")
66 __ASM_EMIT("1:")
67 __ASM_EMIT("fmul v0.4s, v8.4s, v15.4s") /* v0.4s = i[0] * delta */
68 __ASM_EMIT("fmul v1.4s, v9.4s, v15.4s") /* v1.4s = i[1] * delta */
69 __ASM_EMIT("fmul v2.4s, v10.4s, v15.4s") /* v2.4s = i[2] * delta */
70 __ASM_EMIT("fmul v3.4s, v11.4s, v15.4s") /* v3.4s = i[3] * delta */
71 __ASM_EMIT("fadd v0.4s, v0.4s, v14.4s") /* v0.4s = v1 + i[0] * delta */
72 __ASM_EMIT("fadd v1.4s, v1.4s, v14.4s") /* v1.4s = v1 + i[0] * delta */
73 __ASM_EMIT("fadd v2.4s, v2.4s, v14.4s") /* v2.4s = v1 + i[0] * delta */
74 __ASM_EMIT("fadd v3.4s, v3.4s, v14.4s") /* v3.4s = v1 + i[0] * delta */
75 __ASM_EMIT("fadd v8.4s, v8.4s, v12.4s") /* v8.4s = i[0]' = i[0] + step */
76 __ASM_EMIT("fadd v9.4s, v9.4s, v13.4s") /* v9.4s = i[1]' = i[1] + step */
77 __ASM_EMIT("fadd v10.4s, v10.4s, v12.4s") /* v10.4s = i[2]' = i[2] + step */
78 __ASM_EMIT("fadd v11.4s, v11.4s, v13.4s") /* v11.4s = i[3]' = i[3] + step */
79 __ASM_EMIT("stp q0, q1, [%[dst], 0x00]")
80 __ASM_EMIT("stp q2, q3, [%[dst], 0x20]")
81 __ASM_EMIT("subs %[count], %[count], #16")
82 __ASM_EMIT("add %[dst], %[dst], #0x40")
83 __ASM_EMIT("b.hs 1b")
84 __ASM_EMIT("2:")
85 /* 8x blocks */
86 __ASM_EMIT("adds %[count], %[count], #8")
87 __ASM_EMIT("b.lt 4f")
88 __ASM_EMIT("fmul v0.4s, v8.4s, v15.4s") /* v0.4s = i[0] * delta */
89 __ASM_EMIT("fmul v1.4s, v9.4s, v15.4s") /* v1.4s = i[1] * delta */
90 __ASM_EMIT("fadd v0.4s, v0.4s, v14.4s") /* v0.4s = v1 + i[0] * delta */
91 __ASM_EMIT("fadd v1.4s, v1.4s, v14.4s") /* v1.4s = v1 + i[0] * delta */
92 __ASM_EMIT("mov v8.16b, v10.16b") /* v8.4s = i[0]' = i[0] + step */
93 __ASM_EMIT("mov v9.16b, v11.16b") /* v9.4s = i[1]' = i[1] + step */
94 __ASM_EMIT("stp q0, q1, [%[dst], 0x00]")
95 __ASM_EMIT("sub %[count], %[count], #8")
96 __ASM_EMIT("add %[dst], %[dst], #0x20")
97 __ASM_EMIT("4:")
98 /* 4x block */
99 __ASM_EMIT("adds %[count], %[count], #4")
100 __ASM_EMIT("b.lt 6f")
101 __ASM_EMIT("fmul v0.4s, v8.4s, v15.4s") /* v0.4s = i[0] * delta */
102 __ASM_EMIT("mov v8.16b, v9.16b") /* v8.4s = i[0]' = i[0] + step */
103 __ASM_EMIT("fadd v0.4s, v0.4s, v14.4s") /* v0.4s = v1 + i[0] * delta */
104 __ASM_EMIT("str q0, [%[dst], 0x00]")
105 __ASM_EMIT("sub %[count], %[count], #4")
106 __ASM_EMIT("add %[dst], %[dst], #0x10")
107 __ASM_EMIT("6:")
108 /* 1x blocks */
109 __ASM_EMIT("adds %[count], %[count], #3")
110 __ASM_EMIT("b.lt 8f")
111 __ASM_EMIT("7:")
112 __ASM_EMIT("fmul v0.4s, v8.4s, v15.4s") /* v0.4s = i[0] * delta */
113 __ASM_EMIT("ext v8.16b, v8.16b, v8.16b, #4") /* v8.4s = i[0]' = i[0] + step */
114 __ASM_EMIT("fadd v0.4s, v0.4s, v14.4s") /* v0.4s = v1 + i[0] * delta */
115 __ASM_EMIT("st1 {v0.s}[0], [%[dst]]")
116 __ASM_EMIT("subs %[count], %[count], #1")
117 __ASM_EMIT("add %[dst], %[dst], #0x04")
118 __ASM_EMIT("b.ge 7b")
119 __ASM_EMIT("8:")
120
121 : [count] "+r" (count), [dst] "+r" (dst)
122 : [CC] "r" (&lramp_const[0]),
123 [delta] "r" (&delta),
124 [v1] "r" (&v1)
125 : "cc", "memory",
126 "v0", "v1", "v2", "v3",
127 "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15"
128 );
129 }
130
131 #define OP_SINC(a) a
132 #define OP_SNOINC(a)
133
134 #define LRAMP_CORE(DST, SRC, SINC) \
135 __ASM_EMIT("ld1r {v14.4s}, [%[v1]]") \
136 __ASM_EMIT("ld1r {v15.4s}, [%[delta]]") \
137 __ASM_EMIT("ldp q8, q9, [%[CC], #0x00]") \
138 __ASM_EMIT("ldp q10, q11, [%[CC], #0x20]") \
139 __ASM_EMIT("ldp q12, q13, [%[CC], #0x40]") \
140 /* v8.4s = i[0], v9.4s = i[1], v10.4s = i[2], v11.4s = i[3] */ \
141 /* v12.4s = step, v13.4s = step */ \
142 /* v14.4s = v1, v15.4s = delta */ \
143 /* 16x blocks */ \
144 __ASM_EMIT("subs %[count], %[count], #16") \
145 __ASM_EMIT("b.lo 2f") \
146 __ASM_EMIT("1:") \
147 __ASM_EMIT("fmul v0.4s, v8.4s, v15.4s") /* v0.4s = i[0] * delta */ \
148 __ASM_EMIT("fmul v1.4s, v9.4s, v15.4s") /* v1.4s = i[1] * delta */ \
149 __ASM_EMIT("fmul v2.4s, v10.4s, v15.4s") /* v2.4s = i[2] * delta */ \
150 __ASM_EMIT("fmul v3.4s, v11.4s, v15.4s") /* v3.4s = i[3] * delta */ \
151 __ASM_EMIT("ldp q4, q5, [%[" SRC "], 0x00]") \
152 __ASM_EMIT("ldp q6, q7, [%[" SRC "], 0x20]") \
153 __ASM_EMIT("fadd v0.4s, v0.4s, v14.4s") /* v0.4s = v1 + i[0] * delta */ \
154 __ASM_EMIT("fadd v1.4s, v1.4s, v14.4s") /* v1.4s = v1 + i[0] * delta */ \
155 __ASM_EMIT("fadd v2.4s, v2.4s, v14.4s") /* v2.4s = v1 + i[0] * delta */ \
156 __ASM_EMIT("fadd v3.4s, v3.4s, v14.4s") /* v3.4s = v1 + i[0] * delta */ \
157 __ASM_EMIT("fadd v8.4s, v8.4s, v12.4s") /* v8.4s = i[0]' = i[0] + step */ \
158 __ASM_EMIT("fadd v9.4s, v9.4s, v13.4s") /* v9.4s = i[1]' = i[1] + step */ \
159 __ASM_EMIT("fadd v10.4s, v10.4s, v12.4s") /* v10.4s = i[2]' = i[2] + step */ \
160 __ASM_EMIT("fadd v11.4s, v11.4s, v13.4s") /* v11.4s = i[3]' = i[3] + step */ \
161 __ASM_EMIT("fmul v0.4s, v0.4s, v4.4s") /* v0.4s = src[0] * (v1 + i[0] * delta) */ \
162 __ASM_EMIT("fmul v1.4s, v1.4s, v5.4s") /* v1.4s = src[1] * (v1 + i[1] * delta) */ \
163 __ASM_EMIT("fmul v2.4s, v2.4s, v6.4s") /* v2.4s = src[2] * (v1 + i[2] * delta) */ \
164 __ASM_EMIT("fmul v3.4s, v3.4s, v7.4s") /* v3.4s = src[3] * (v1 + i[3] * delta) */ \
165 __ASM_EMIT("stp q0, q1, [%[" DST "], 0x00]") \
166 __ASM_EMIT("stp q2, q3, [%[" DST "], 0x20]") \
167 __ASM_EMIT("subs %[count], %[count], #16") \
168 __ASM_EMIT(SINC("add %[" SRC "], %[" SRC "], #0x40")) \
169 __ASM_EMIT("add %[" DST "], %[" DST "], #0x40") \
170 __ASM_EMIT("b.hs 1b") \
171 __ASM_EMIT("2:") \
172 /* 8x blocks */ \
173 __ASM_EMIT("adds %[count], %[count], #8") \
174 __ASM_EMIT("b.lt 4f") \
175 __ASM_EMIT("fmul v0.4s, v8.4s, v15.4s") /* v0.4s = i[0] * delta */ \
176 __ASM_EMIT("fmul v1.4s, v9.4s, v15.4s") /* v1.4s = i[1] * delta */ \
177 __ASM_EMIT("ldp q4, q5, [%[" SRC "], 0x00]") \
178 __ASM_EMIT("fadd v0.4s, v0.4s, v14.4s") /* v0.4s = v1 + i[0] * delta */ \
179 __ASM_EMIT("fadd v1.4s, v1.4s, v14.4s") /* v1.4s = v1 + i[0] * delta */ \
180 __ASM_EMIT("mov v8.16b, v10.16b") /* v8.4s = i[0]' = i[0] + step */ \
181 __ASM_EMIT("mov v9.16b, v11.16b") /* v9.4s = i[1]' = i[1] + step */ \
182 __ASM_EMIT("fmul v0.4s, v0.4s, v4.4s") /* v0.4s = src[0] * (v1 + i[0] * delta) */ \
183 __ASM_EMIT("fmul v1.4s, v1.4s, v5.4s") /* v1.4s = src[1] * (v1 + i[1] * delta) */ \
184 __ASM_EMIT("stp q0, q1, [%[" DST "], 0x00]") \
185 __ASM_EMIT("sub %[count], %[count], #8") \
186 __ASM_EMIT(SINC("add %[" SRC "], %[" SRC "], #0x20")) \
187 __ASM_EMIT("add %[" DST "], %[" DST "], #0x20") \
188 __ASM_EMIT("4:") \
189 /* 4x block */ \
190 __ASM_EMIT("adds %[count], %[count], #4") \
191 __ASM_EMIT("b.lt 6f") \
192 __ASM_EMIT("fmul v0.4s, v8.4s, v15.4s") /* v0.4s = i[0] * delta */ \
193 __ASM_EMIT("ldr q4, [%[" SRC "], 0x00]") \
194 __ASM_EMIT("mov v8.16b, v9.16b") /* v8.4s = i[0]' = i[0] + step */ \
195 __ASM_EMIT("fadd v0.4s, v0.4s, v14.4s") /* v0.4s = v1 + i[0] * delta */ \
196 __ASM_EMIT("fmul v0.4s, v0.4s, v4.4s") /* v0.4s = src[0] * (v1 + i[0] * delta) */ \
197 __ASM_EMIT("str q0, [%[" DST "], 0x00]") \
198 __ASM_EMIT("sub %[count], %[count], #4") \
199 __ASM_EMIT(SINC("add %[" SRC "], %[" SRC "], #0x10")) \
200 __ASM_EMIT("add %[" DST "], %[" DST "], #0x10") \
201 __ASM_EMIT("6:") \
202 /* 1x blocks */ \
203 __ASM_EMIT("adds %[count], %[count], #3") \
204 __ASM_EMIT("b.lt 8f") \
205 __ASM_EMIT("7:") \
206 __ASM_EMIT("fmul v0.4s, v8.4s, v15.4s") /* v0.4s = i[0] * delta */ \
207 __ASM_EMIT("ld1r {v4.4s}, [%[" SRC "]]") \
208 __ASM_EMIT("ext v8.16b, v8.16b, v8.16b, #4") /* v8.4s = i[0]' = i[0] + step */ \
209 __ASM_EMIT("fadd v0.4s, v0.4s, v14.4s") /* v0.4s = v1 + i[0] * delta */ \
210 __ASM_EMIT("fmul v0.4s, v0.4s, v4.4s") /* v0.4s = src[0] * (v1 + i[0] * delta) */ \
211 __ASM_EMIT("st1 {v0.s}[0], [%[" DST "]]") \
212 __ASM_EMIT("subs %[count], %[count], #1") \
213 __ASM_EMIT(SINC("add %[" SRC "], %[" SRC "], #0x04")) \
214 __ASM_EMIT("add %[" DST "], %[" DST "], #0x04") \
215 __ASM_EMIT("b.ge 7b") \
216 __ASM_EMIT("8:")
217
218 void lramp1(float *dst, float v1, float v2, size_t count)
219 {
220 float delta = v2 - v1;
221 if (delta == 0.0f)
222 {
223 dsp::mul_k2(dst, v1, count);
224 return;
225 }
226 if (count == 0)
227 return;
228
229 delta /= count;
230 ARCH_AARCH64_ASM(
231 LRAMP_CORE("dst", "dst", OP_SNOINC)
232 : [count] "+r" (count), [dst] "+r" (dst)
233 : [CC] "r" (&lramp_const[0]),
234 [delta] "r" (&delta),
235 [v1] "r" (&v1)
236 : "cc", "memory",
237 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
238 "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15"
239 );
240 }
241
242 void lramp2(float *dst, const float *src, float v1, float v2, size_t count)
243 {
244 float delta = v2 - v1;
245 if (delta == 0.0f)
246 {
247 dsp::mul_k3(dst, src, v1, count);
248 return;
249 }
250 if (count == 0)
251 return;
252
253 delta /= count;
254 ARCH_AARCH64_ASM(
255 LRAMP_CORE("dst", "src", OP_SINC)
256 : [count] "+r" (count), [dst] "+r" (dst), [src] "+r" (src)
257 : [CC] "r" (&lramp_const[0]),
258 [delta] "r" (&delta),
259 [v1] "r" (&v1)
260 : "cc", "memory",
261 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
262 "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15"
263 );
264 }
265
266 #undef LRAMP_CORE
267
268 #define OP_DSEL(a, b) a
269 #define OP_RSEL(a, b) b
270
271 #define LRAMP_OP3_CORE(DST, A, B, OP, SEL, AINC) \
272 __ASM_EMIT("ld1r {v14.4s}, [%[v1]]") \
273 __ASM_EMIT("ld1r {v15.4s}, [%[delta]]") \
274 __ASM_EMIT("ldp q8, q9, [%[CC], #0x00]") \
275 __ASM_EMIT("ldp q10, q11, [%[CC], #0x20]") \
276 __ASM_EMIT("ldp q12, q13, [%[CC], #0x40]") \
277 /* v8.4s = i[0], v9.4s = i[1], v10.4s = i[2], v11.4s = i[3] */ \
278 /* v12.4s = step, v13.4s = step */ \
279 /* v14.4s = v1, v15.4s = delta */ \
280 /* 16x blocks */ \
281 __ASM_EMIT("subs %[count], %[count], #16") \
282 __ASM_EMIT("b.lo 2f") \
283 __ASM_EMIT("1:") \
284 __ASM_EMIT("fmul v0.4s, v8.4s, v15.4s") /* v0.4s = i[0] * delta */ \
285 __ASM_EMIT("fmul v1.4s, v9.4s, v15.4s") /* v1.4s = i[1] * delta */ \
286 __ASM_EMIT("fmul v2.4s, v10.4s, v15.4s") /* v2.4s = i[2] * delta */ \
287 __ASM_EMIT("fmul v3.4s, v11.4s, v15.4s") /* v3.4s = i[3] * delta */ \
288 __ASM_EMIT("ldp q4, q5, [%[" B "], 0x00]") \
289 __ASM_EMIT("ldp q6, q7, [%[" B "], 0x20]") \
290 __ASM_EMIT("fadd v0.4s, v0.4s, v14.4s") /* v0.4s = v1 + i[0] * delta */ \
291 __ASM_EMIT("fadd v1.4s, v1.4s, v14.4s") /* v1.4s = v1 + i[0] * delta */ \
292 __ASM_EMIT("fadd v2.4s, v2.4s, v14.4s") /* v2.4s = v1 + i[0] * delta */ \
293 __ASM_EMIT("fadd v3.4s, v3.4s, v14.4s") /* v3.4s = v1 + i[0] * delta */ \
294 __ASM_EMIT("fmul v0.4s, v0.4s, v4.4s") /* v0.4s = b[0] * (v1 + i[0] * delta) */ \
295 __ASM_EMIT("fmul v1.4s, v1.4s, v5.4s") /* v1.4s = b[1] * (v1 + i[1] * delta) */ \
296 __ASM_EMIT("fmul v2.4s, v2.4s, v6.4s") /* v2.4s = b[2] * (v1 + i[2] * delta) */ \
297 __ASM_EMIT("fmul v3.4s, v3.4s, v7.4s") /* v3.4s = b[3] * (v1 + i[3] * delta) */ \
298 __ASM_EMIT("ldp q4, q5, [%[" A "], 0x00]") \
299 __ASM_EMIT("ldp q6, q7, [%[" A "], 0x20]") \
300 __ASM_EMIT("fadd v8.4s, v8.4s, v12.4s") /* v8.4s = i[0]' = i[0] + step */ \
301 __ASM_EMIT("fadd v9.4s, v9.4s, v13.4s") /* v9.4s = i[1]' = i[1] + step */ \
302 __ASM_EMIT("fadd v10.4s, v10.4s, v12.4s") /* v10.4s = i[2]' = i[2] + step */ \
303 __ASM_EMIT("fadd v11.4s, v11.4s, v13.4s") /* v11.4s = i[3]' = i[3] + step */ \
304 __ASM_EMIT(OP " v0.4s, " SEL("v4.4s, v0.4s", "v0.4s, v4.4s")) /* v0.4s = a[0] OP b[0] * (v1 + i[0] * delta) */ \
305 __ASM_EMIT(OP " v1.4s, " SEL("v5.4s, v1.4s", "v1.4s, v5.4s")) /* v1.4s = a[1] OP b[1] * (v1 + i[1] * delta) */ \
306 __ASM_EMIT(OP " v2.4s, " SEL("v6.4s, v2.4s", "v2.4s, v6.4s")) /* v2.4s = a[2] OP b[2] * (v1 + i[2] * delta) */ \
307 __ASM_EMIT(OP " v3.4s, " SEL("v7.4s, v3.4s", "v3.4s, v7.4s")) /* v3.4s = a[3] OP b[3] * (v1 + i[3] * delta) */ \
308 __ASM_EMIT("subs %[count], %[count], #16") \
309 __ASM_EMIT("stp q0, q1, [%[" DST "], 0x00]") \
310 __ASM_EMIT("stp q2, q3, [%[" DST "], 0x20]") \
311 __ASM_EMIT("add %[" B "], %[" B "], #0x40") \
312 __ASM_EMIT(AINC("add %[" A "], %[" A "], #0x40")) \
313 __ASM_EMIT("add %[" DST "], %[" DST "], #0x40") \
314 __ASM_EMIT("b.hs 1b") \
315 __ASM_EMIT("2:") \
316 /* 8x blocks */ \
317 __ASM_EMIT("adds %[count], %[count], #8") \
318 __ASM_EMIT("b.lt 4f") \
319 __ASM_EMIT("fmul v0.4s, v8.4s, v15.4s") /* v0.4s = i[0] * delta */ \
320 __ASM_EMIT("fmul v1.4s, v9.4s, v15.4s") /* v1.4s = i[1] * delta */ \
321 __ASM_EMIT("ldp q4, q5, [%[" B "], 0x00]") \
322 __ASM_EMIT("fadd v0.4s, v0.4s, v14.4s") /* v0.4s = v1 + i[0] * delta */ \
323 __ASM_EMIT("fadd v1.4s, v1.4s, v14.4s") /* v1.4s = v1 + i[0] * delta */ \
324 __ASM_EMIT("fmul v0.4s, v0.4s, v4.4s") /* v0.4s = b[0] * (v1 + i[0] * delta) */ \
325 __ASM_EMIT("fmul v1.4s, v1.4s, v5.4s") /* v1.4s = b[1] * (v1 + i[1] * delta) */ \
326 __ASM_EMIT("ldp q4, q5, [%[" A "], 0x00]") \
327 __ASM_EMIT("mov v8.16b, v10.16b") /* v8.4s = i[0]' = i[0] + step */ \
328 __ASM_EMIT("mov v9.16b, v11.16b") /* v9.4s = i[1]' = i[1] + step */ \
329 __ASM_EMIT(OP " v0.4s, " SEL("v4.4s, v0.4s", "v0.4s, v4.4s")) /* v0.4s = a[0] OP b[0] * (v1 + i[0] * delta) */ \
330 __ASM_EMIT(OP " v1.4s, " SEL("v5.4s, v1.4s", "v1.4s, v5.4s")) /* v1.4s = a[1] OP b[1] * (v1 + i[1] * delta) */ \
331 __ASM_EMIT("sub %[count], %[count], #8") \
332 __ASM_EMIT("stp q0, q1, [%[" DST "], 0x00]") \
333 __ASM_EMIT("add %[" B "], %[" B "], #0x20") \
334 __ASM_EMIT(AINC("add %[" A "], %[" A "], #0x20")) \
335 __ASM_EMIT("add %[" DST "], %[" DST "], #0x20") \
336 __ASM_EMIT("4:") \
337 /* 4x block */ \
338 __ASM_EMIT("adds %[count], %[count], #4") \
339 __ASM_EMIT("b.lt 6f") \
340 __ASM_EMIT("fmul v0.4s, v8.4s, v15.4s") /* v0.4s = i[0] * delta */ \
341 __ASM_EMIT("ldr q4, [%[" B "], 0x00]") \
342 __ASM_EMIT("fadd v0.4s, v0.4s, v14.4s") /* v0.4s = v1 + i[0] * delta */ \
343 __ASM_EMIT("fmul v0.4s, v0.4s, v4.4s") /* v0.4s = b[0] * (v1 + i[0] * delta) */ \
344 __ASM_EMIT("ldr q4, [%[" A "], 0x00]") \
345 __ASM_EMIT("mov v8.16b, v9.16b") /* v8.4s = i[0]' = i[0] + step */ \
346 __ASM_EMIT(OP " v0.4s, " SEL("v4.4s, v0.4s", "v0.4s, v4.4s")) /* v0.4s = a[0] OP b[0] * (v1 + i[0] * delta) */ \
347 __ASM_EMIT("sub %[count], %[count], #4") \
348 __ASM_EMIT("str q0, [%[" DST "], 0x00]") \
349 __ASM_EMIT("add %[" B "], %[" B "], #0x10") \
350 __ASM_EMIT(AINC("add %[" A "], %[" A "], #0x10")) \
351 __ASM_EMIT("add %[" DST "], %[" DST "], #0x10") \
352 __ASM_EMIT("6:") \
353 /* 1x blocks */ \
354 __ASM_EMIT("adds %[count], %[count], #3") \
355 __ASM_EMIT("b.lt 8f") \
356 __ASM_EMIT("7:") \
357 __ASM_EMIT("fmul v0.4s, v8.4s, v15.4s") /* v0.4s = i[0] * delta */ \
358 __ASM_EMIT("ld1r {v4.4s}, [%[" B "]]") \
359 __ASM_EMIT("fadd v0.4s, v0.4s, v14.4s") /* v0.4s = v1 + i[0] * delta */ \
360 __ASM_EMIT("fmul v0.4s, v0.4s, v4.4s") /* v0.4s = b[0] * (v1 + i[0] * delta) */ \
361 __ASM_EMIT("ld1r {v4.4s}, [%[" A "]]") \
362 __ASM_EMIT("ext v8.16b, v8.16b, v8.16b, #4") /* v8.4s = i[0]' = i[0] + step */ \
363 __ASM_EMIT(OP " v0.4s, " SEL("v4.4s, v0.4s", "v0.4s, v4.4s")) /* v0.4s = a[0] OP b[0] * (v1 + i[0] * delta) */ \
364 __ASM_EMIT("subs %[count], %[count], #1") \
365 __ASM_EMIT("st1 {v0.s}[0], [%[" DST "]]") \
366 __ASM_EMIT("add %[" B "], %[" B "], #0x04") \
367 __ASM_EMIT(AINC("add %[" A "], %[" A "], #0x04")) \
368 __ASM_EMIT("add %[" DST "], %[" DST "], #0x04") \
369 __ASM_EMIT("b.ge 7b") \
370 __ASM_EMIT("8:")
371
372 void lramp_add2(float *dst, const float *src, float v1, float v2, size_t count)
373 {
374 float delta = v2 - v1;
375 if (delta == 0.0f)
376 {
377 dsp::fmadd_k3(dst, src, v1, count);
378 return;
379 }
380 if (count == 0)
381 return;
382
383 delta /= count;
384 ARCH_AARCH64_ASM(
385 LRAMP_OP3_CORE("dst", "dst", "src", "fadd", OP_DSEL, OP_SNOINC)
386 : [count] "+r" (count), [dst] "+r" (dst), [src] "+r" (src)
387 : [CC] "r" (&lramp_const[0]),
388 [delta] "r" (&delta),
389 [v1] "r" (&v1)
390 : "cc", "memory",
391 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
392 "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15"
393 );
394 }
395
396 void lramp_sub2(float *dst, const float *src, float v1, float v2, size_t count)
397 {
398 float delta = v2 - v1;
399 if (delta == 0.0f)
400 {
401 dsp::fmsub_k3(dst, src, v1, count);
402 return;
403 }
404 if (count == 0)
405 return;
406
407 delta /= count;
408 ARCH_AARCH64_ASM(
409 LRAMP_OP3_CORE("dst", "dst", "src", "fsub", OP_DSEL, OP_SNOINC)
410 : [count] "+r" (count), [dst] "+r" (dst), [src] "+r" (src)
411 : [CC] "r" (&lramp_const[0]),
412 [delta] "r" (&delta),
413 [v1] "r" (&v1)
414 : "cc", "memory",
415 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
416 "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15"
417 );
418 }
419
420 void lramp_rsub2(float *dst, const float *src, float v1, float v2, size_t count)
421 {
422 float delta = v2 - v1;
423 if (delta == 0.0f)
424 {
425 dsp::fmrsub_k3(dst, src, v1, count);
426 return;
427 }
428 if (count == 0)
429 return;
430
431 delta /= count;
432 ARCH_AARCH64_ASM(
433 LRAMP_OP3_CORE("dst", "dst", "src", "fsub", OP_RSEL, OP_SNOINC)
434 : [count] "+r" (count), [dst] "+r" (dst), [src] "+r" (src)
435 : [CC] "r" (&lramp_const[0]),
436 [delta] "r" (&delta),
437 [v1] "r" (&v1)
438 : "cc", "memory",
439 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
440 "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15"
441 );
442 }
443
444 void lramp_mul2(float *dst, const float *src, float v1, float v2, size_t count)
445 {
446 float delta = v2 - v1;
447 if (delta == 0.0f)
448 {
449 dsp::fmmul_k3(dst, src, v1, count);
450 return;
451 }
452 if (count == 0)
453 return;
454
455 delta /= count;
456 ARCH_AARCH64_ASM(
457 LRAMP_OP3_CORE("dst", "dst", "src", "fmul", OP_DSEL, OP_SNOINC)
458 : [count] "+r" (count), [dst] "+r" (dst), [src] "+r" (src)
459 : [CC] "r" (&lramp_const[0]),
460 [delta] "r" (&delta),
461 [v1] "r" (&v1)
462 : "cc", "memory",
463 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
464 "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15"
465 );
466 }
467
468 void lramp_add3(float *dst, const float *a, const float *b, float v1, float v2, size_t count)
469 {
470 float delta = v2 - v1;
471 if (delta == 0.0f)
472 {
473 dsp::fmadd_k4(dst, a, b, v1, count);
474 return;
475 }
476 if (count == 0)
477 return;
478
479 delta /= count;
480 ARCH_AARCH64_ASM(
481 LRAMP_OP3_CORE("dst", "a", "b", "fadd", OP_DSEL, OP_SINC)
482 : [count] "+r" (count), [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b)
483 : [CC] "r" (&lramp_const[0]),
484 [delta] "r" (&delta),
485 [v1] "r" (&v1)
486 : "cc", "memory",
487 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
488 "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15"
489 );
490 }
491
492 void lramp_sub3(float *dst, const float *a, const float *b, float v1, float v2, size_t count)
493 {
494 float delta = v2 - v1;
495 if (delta == 0.0f)
496 {
497 dsp::fmsub_k4(dst, a, b, v1, count);
498 return;
499 }
500 if (count == 0)
501 return;
502
503 delta /= count;
504 ARCH_AARCH64_ASM(
505 LRAMP_OP3_CORE("dst", "a", "b", "fsub", OP_DSEL, OP_SINC)
506 : [count] "+r" (count), [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b)
507 : [CC] "r" (&lramp_const[0]),
508 [delta] "r" (&delta),
509 [v1] "r" (&v1)
510 : "cc", "memory",
511 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
512 "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15"
513 );
514 }
515
516 void lramp_rsub3(float *dst, const float *a, const float *b, float v1, float v2, size_t count)
517 {
518 float delta = v2 - v1;
519 if (delta == 0.0f)
520 {
521 dsp::fmrsub_k4(dst, a, b, v1, count);
522 return;
523 }
524 if (count == 0)
525 return;
526
527 delta /= count;
528 ARCH_AARCH64_ASM(
529 LRAMP_OP3_CORE("dst", "a", "b", "fsub", OP_RSEL, OP_SINC)
530 : [count] "+r" (count), [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b)
531 : [CC] "r" (&lramp_const[0]),
532 [delta] "r" (&delta),
533 [v1] "r" (&v1)
534 : "cc", "memory",
535 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
536 "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15"
537 );
538 }
539
540 void lramp_mul3(float *dst, const float *a, const float *b, float v1, float v2, size_t count)
541 {
542 float delta = v2 - v1;
543 if (delta == 0.0f)
544 {
545 dsp::fmmul_k4(dst, a, b, v1, count);
546 return;
547 }
548 if (count == 0)
549 return;
550
551 delta /= count;
552 ARCH_AARCH64_ASM(
553 LRAMP_OP3_CORE("dst", "a", "b", "fmul", OP_DSEL, OP_SINC)
554 : [count] "+r" (count), [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b)
555 : [CC] "r" (&lramp_const[0]),
556 [delta] "r" (&delta),
557 [v1] "r" (&v1)
558 : "cc", "memory",
559 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
560 "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15"
561 );
562 }
563
564 #undef LRAMP_OP3_CORE
565
566 IF_ARCH_AARCH64(
567 static const float lramp_div_const[] __lsp_aligned16 =
568 {
569 0.0f, 1.0f, 2.0f, 3.0f,
570 4.0f, 5.0f, 6.0f, 7.0f, /* Initial values 0..7 */
571 8.0f, 8.0f, 8.0f, 8.0f /* Step */
572 };
573 )
574
575 #define LRAMP_DIV3_CORE(DST, A, B, SEL, AINC) \
576 __ASM_EMIT("ld1r {v12.4s}, [%[v1]]") \
577 __ASM_EMIT("ld1r {v13.4s}, [%[delta]]") \
578 __ASM_EMIT("ldp q8, q9, [%[CC], #0x00]") \
579 __ASM_EMIT("ldr q10, [%[CC], #0x20]") \
580 /* v8.4s = i[0], v9.4s = i[1] */ \
581 /* v10.4s = step */ \
582 /* v12.4s = v1, v13.4s = delta */ \
583 /* 8x blocks */ \
584 __ASM_EMIT("subs %[count], %[count], #8") \
585 __ASM_EMIT("b.lo 2f") \
586 __ASM_EMIT("1:") \
587 __ASM_EMIT("fmul v4.4s, v8.4s, v13.4s") /* v4.4s = i[0] * delta */ \
588 __ASM_EMIT("fmul v5.4s, v9.4s, v13.4s") /* v5.4s = i[1] * delta */ \
589 __ASM_EMIT("ldp q0, q1, [%[" A "], #0x00]") /* v0.4s = a[0], v1.4s = a[1] */ \
590 __ASM_EMIT("ldp q2, q3, [%[" B "], #0x00]") /* v2.4s = b[0], v3.4s = b[1] */ \
591 __ASM_EMIT("fadd v4.4s, v4.4s, v12.4s") /* v4.4s = v1 + i[0] * delta */ \
592 __ASM_EMIT("fadd v5.4s, v5.4s, v12.4s") /* v5.4s = v1 + i[0] * delta */ \
593 __ASM_EMIT("fmul v2.4s, v2.4s, v4.4s") /* v2.4s = b[0] * (v1 + i[0] * delta) */ \
594 __ASM_EMIT("fmul v3.4s, v3.4s, v5.4s") /* v3.4s = b[0] * (v1 + i[0] * delta) */ \
595 __ASM_EMIT("fadd v8.4s, v8.4s, v10.4s") /* v8.4s = i[0]' = i[0] + step */ \
596 __ASM_EMIT("fadd v9.4s, v9.4s, v10.4s") /* v9.4s = i[1]' = i[1] + step */ \
597 __ASM_EMIT("frecpe v4.4s, " SEL("v2.4s", "v0.4s")) \
598 __ASM_EMIT("frecpe v5.4s, " SEL("v3.4s", "v1.4s")) \
599 __ASM_EMIT("frecps v6.4s, v4.4s, " SEL("v2.4s", "v0.4s")) \
600 __ASM_EMIT("frecps v7.4s, v5.4s, " SEL("v3.4s", "v1.4s")) \
601 __ASM_EMIT("fmul v4.4s, v6.4s, v4.4s") \
602 __ASM_EMIT("fmul v5.4s, v7.4s, v5.4s") \
603 __ASM_EMIT("frecps v6.4s, v4.4s, " SEL("v2.4s", "v0.4s")) \
604 __ASM_EMIT("frecps v7.4s, v5.4s, " SEL("v3.4s", "v1.4s")) \
605 __ASM_EMIT("fmul " SEL("v2.4s", "v0.4s") ", v6.4s, v4.4s") /* v2.4s = 1/(b[0] * (v1 + i[0] * delta)) */ \
606 __ASM_EMIT("fmul " SEL("v3.4s", "v1.4s") ", v7.4s, v5.4s") /* v3.4s = 1/(b[1] * (v1 + i[1] * delta)) */ \
607 __ASM_EMIT("fmul v0.4s, v0.4s, v2.4s") /* v0.4s = a[0] / b[0] * (v1 + i[0] * delta) */ \
608 __ASM_EMIT("fmul v1.4s, v1.4s, v3.4s") /* v1.4s = a[1] / b[1] * (v1 + i[1] * delta) */ \
609 __ASM_EMIT("subs %[count], %[count], #8") \
610 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
611 __ASM_EMIT("add %[" B "], %[" B "], #0x20") \
612 __ASM_EMIT(AINC("add %[" A "], %[" A "], #0x20")) \
613 __ASM_EMIT("add %[" DST "], %[" DST "], #0x20") \
614 __ASM_EMIT("b.hs 1b") \
615 __ASM_EMIT("2:") \
616 /* 4x blocks */ \
617 __ASM_EMIT("adds %[count], %[count], #4") \
618 __ASM_EMIT("b.lt 4f") \
619 __ASM_EMIT("fmul v4.4s, v8.4s, v13.4s") /* v4.4s = i[0] * delta */ \
620 __ASM_EMIT("ldr q0, [%[" A "], #0x00]") /* v0.4s = a[0] */ \
621 __ASM_EMIT("ldr q2, [%[" B "], #0x00]") /* v2.4s = b[0] */ \
622 __ASM_EMIT("fadd v4.4s, v4.4s, v12.4s") /* v4.4s = v1 + i[0] * delta */ \
623 __ASM_EMIT("fmul v2.4s, v2.4s, v4.4s") /* v2.4s = b[0] * (v1 + i[0] * delta) */ \
624 __ASM_EMIT("mov v8.16b, v9.16b") /* v8.4s = i[0]' = i[0] + step */ \
625 __ASM_EMIT("frecpe v4.4s, " SEL("v2.4s", "v0.4s")) \
626 __ASM_EMIT("frecps v6.4s, v4.4s, " SEL("v2.4s", "v0.4s")) \
627 __ASM_EMIT("fmul v4.4s, v6.4s, v4.4s") \
628 __ASM_EMIT("frecps v6.4s, v4.4s, " SEL("v2.4s", "v0.4s")) \
629 __ASM_EMIT("fmul " SEL("v2.4s", "v0.4s") ", v6.4s, v4.4s") /* v2.4s = 1/(b[0] * (v1 + i[0] * delta)) */ \
630 __ASM_EMIT("fmul v0.4s, v0.4s, v2.4s") /* v0.4s = a[0] / b[0] * (v1 + i[0] * delta) */ \
631 __ASM_EMIT("sub %[count], %[count], #4") \
632 __ASM_EMIT("str q0, [%[" DST "], #0x00]") \
633 __ASM_EMIT("add %[" B "], %[" B "], #0x10") \
634 __ASM_EMIT(AINC("add %[" A "], %[" A "], #0x10")) \
635 __ASM_EMIT("add %[" DST "], %[" DST "], #0x10") \
636 __ASM_EMIT("4:") \
637 /* 1x blocks */ \
638 __ASM_EMIT("adds %[count], %[count], #3") \
639 __ASM_EMIT("b.lt 6f") \
640 __ASM_EMIT("5:") \
641 __ASM_EMIT("fmul v4.4s, v8.4s, v13.4s") /* v4.4s = i[0] * delta */ \
642 __ASM_EMIT("ld1r {v0.4s}, [%[" A "]]") \
643 __ASM_EMIT("ld1r {v2.4s}, [%[" B "]]") \
644 __ASM_EMIT("fadd v4.4s, v4.4s, v12.4s") /* v4.4s = v1 + i[0] * delta */ \
645 __ASM_EMIT("fmul v2.4s, v2.4s, v4.4s") /* v2.4s = b[0] * (v1 + i[0] * delta) */ \
646 __ASM_EMIT("ext v8.16b, v8.16b, v8.16b, #4") /* v8.4s = i[0]' = i[0] + step */ \
647 __ASM_EMIT("frecpe v4.4s, " SEL("v2.4s", "v0.4s")) \
648 __ASM_EMIT("frecps v6.4s, v4.4s, " SEL("v2.4s", "v0.4s")) \
649 __ASM_EMIT("fmul v4.4s, v6.4s, v4.4s") \
650 __ASM_EMIT("frecps v6.4s, v4.4s, " SEL("v2.4s", "v0.4s")) \
651 __ASM_EMIT("fmul " SEL("v2.4s", "v0.4s") ", v6.4s, v4.4s") /* v2.4s = 1/(b[0] * (v1 + i[0] * delta)) */ \
652 __ASM_EMIT("fmul v0.4s, v0.4s, v2.4s") /* v0.4s = a[0] / b[0] * (v1 + i[0] * delta) */ \
653 __ASM_EMIT("subs %[count], %[count], #1") \
654 __ASM_EMIT("st1 {v0.s}[0], [%[" DST "]]") \
655 __ASM_EMIT("add %[" B "], %[" B "], #0x04") \
656 __ASM_EMIT(AINC("add %[" A "], %[" A "], #0x04")) \
657 __ASM_EMIT("add %[" DST "], %[" DST "], #0x04") \
658 __ASM_EMIT("b.ge 5b") \
659 __ASM_EMIT("6:")
660
661 void lramp_div2(float *dst, const float *src, float v1, float v2, size_t count)
662 {
663 float delta = v2 - v1;
664 if (delta == 0.0f)
665 {
666 dsp::fmdiv_k3(dst, src, v1, count);
667 return;
668 }
669 if (count == 0)
670 return;
671
672 delta /= count;
673 ARCH_AARCH64_ASM(
674 LRAMP_DIV3_CORE("dst", "dst", "src", OP_DSEL, OP_SNOINC)
675 : [count] "+r" (count), [dst] "+r" (dst), [src] "+r" (src)
676 : [CC] "r" (&lramp_div_const[0]),
677 [delta] "r" (&delta),
678 [v1] "r" (&v1)
679 : "cc", "memory",
680 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
681 "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15"
682 );
683 }
684
685 void lramp_rdiv2(float *dst, const float *src, float v1, float v2, size_t count)
686 {
687 float delta = v2 - v1;
688 if (delta == 0.0f)
689 {
690 dsp::fmrdiv_k3(dst, src, v1, count);
691 return;
692 }
693 if (count == 0)
694 return;
695
696 delta /= count;
697 ARCH_AARCH64_ASM(
698 LRAMP_DIV3_CORE("dst", "dst", "src", OP_RSEL, OP_SNOINC)
699 : [count] "+r" (count), [dst] "+r" (dst), [src] "+r" (src)
700 : [CC] "r" (&lramp_div_const[0]),
701 [delta] "r" (&delta),
702 [v1] "r" (&v1)
703 : "cc", "memory",
704 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
705 "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15"
706 );
707 }
708
709 void lramp_div3(float *dst, const float *a, const float *b, float v1, float v2, size_t count)
710 {
711 float delta = v2 - v1;
712 if (delta == 0.0f)
713 {
714 dsp::fmdiv_k4(dst, a, b, v1, count);
715 return;
716 }
717 if (count == 0)
718 return;
719
720 delta /= count;
721 ARCH_AARCH64_ASM(
722 LRAMP_DIV3_CORE("dst", "a", "b", OP_DSEL, OP_SINC)
723 : [count] "+r" (count), [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b)
724 : [CC] "r" (&lramp_div_const[0]),
725 [delta] "r" (&delta),
726 [v1] "r" (&v1)
727 : "cc", "memory",
728 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
729 "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15"
730 );
731 }
732
733 void lramp_rdiv3(float *dst, const float *a, const float *b, float v1, float v2, size_t count)
734 {
735 float delta = v2 - v1;
736 if (delta == 0.0f)
737 {
738 dsp::fmrdiv_k4(dst, a, b, v1, count);
739 return;
740 }
741 if (count == 0)
742 return;
743
744 delta /= count;
745 ARCH_AARCH64_ASM(
746 LRAMP_DIV3_CORE("dst", "a", "b", OP_RSEL, OP_SINC)
747 : [count] "+r" (count), [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b)
748 : [CC] "r" (&lramp_div_const[0]),
749 [delta] "r" (&delta),
750 [v1] "r" (&v1)
751 : "cc", "memory",
752 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
753 "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15"
754 );
755 }
756
757 #undef LRAMP_DIV3_CORE
758
759 #undef OP_DSEL
760 #undef OP_RSEL
761
762 #undef OP_SINC
763 #undef OP_SNOINC
764
765 } /* namespace asimd */
766 } /* namespace lsp */
767
768
769
770 #endif /* PRIVATE_DSP_ARCH_AARCH64_ASIMD_PMATH_LRAMP_H_ */