Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/aarch64/asimd/pmath/exp.h

raw
1 /*
2 * Copyright (C) 2023 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2023 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 31 мар. 2020 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_AARCH64_ASIMD_PMATH_EXP_H_
23 #define PRIVATE_DSP_ARCH_AARCH64_ASIMD_PMATH_EXP_H_
24
25 #ifndef PRIVATE_DSP_ARCH_AARCH64_ASIMD_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_AARCH64_ASIMD_IMPL */
28
29 namespace lsp
30 {
31 namespace asimd
32 {
33 IF_ARCH_AARCH64(
34 static const uint32_t EXP2_CONST[] __lsp_aligned16 =
35 {
36 LSP_DSP_VEC4(0x7f), // ME = 0x7f
37 LSP_DSP_VEC4(0x3f317218), // L2 = ln(2)
38 LSP_DSP_VEC4(0x40e00000), // C5 = 7!/6! = 7.0000000000000000
39 LSP_DSP_VEC4(0x42280000), // C4 = 7!/5! = 42.0000000000000000
40 LSP_DSP_VEC4(0x43520000), // C3 = 7!/4! = 210.0000000000000000
41 LSP_DSP_VEC4(0x44520000), // C2 = 7!/3! = 840.0000000000000000
42 LSP_DSP_VEC4(0x451d8000), // C1 = 7!/2! = 2520.0000000000000000
43 LSP_DSP_VEC4(0x459d8000), // C0 = 7!/1! = 5040.0000000000000000
44 LSP_DSP_VEC4(0x39500d01), // C6 = 1/7! = 0.0001984127011383
45 LSP_DSP_VEC4(0x3f800000), // C7 = 1.0
46 };
47
48 static const float EXP_LOG2E[] __lsp_aligned16 =
49 {
50 LSP_DSP_VEC4(M_LOG2E),
51 LSP_DSP_VEC4(M_LOG2E)
52 };
53 )
54
55 #define POW2_CORE_X8(ME, L2, C5, C4, C3, C2, C1, C0, C6, C7) \
56 /* v0 = x */ \
57 __ASM_EMIT("sshr v2.4s, v0.4s, #31") /* v2 = [ x < 0 ] */ \
58 __ASM_EMIT("sshr v3.4s, v1.4s, #31") \
59 __ASM_EMIT("fabs v0.4s, v0.4s") /* v0 = XP = fabs(x) */ \
60 __ASM_EMIT("fabs v1.4s, v1.4s") \
61 __ASM_EMIT("fcvtzs v4.4s, v0.4s") /* v4 = R = int(x) */ \
62 __ASM_EMIT("fcvtzs v5.4s, v1.4s") \
63 __ASM_EMIT("add v6.4s, v4.4s, " ME ".4s") /* v6 = R + (ME = 127) */ \
64 __ASM_EMIT("add v7.4s, v5.4s, " ME ".4s") \
65 __ASM_EMIT("scvtf v4.4s, v4.4s") /* v4 = float(R) */ \
66 __ASM_EMIT("scvtf v5.4s, v5.4s") \
67 __ASM_EMIT("shl v6.4s, v6.4s, #23") /* v6 = 1 << (R + 127) */ \
68 __ASM_EMIT("shl v7.4s, v7.4s, #23") \
69 __ASM_EMIT("fsub v0.4s, v0.4s, v4.4s") /* v0 = XP - float(R) */ \
70 __ASM_EMIT("fsub v1.4s, v1.4s, v5.4s") \
71 __ASM_EMIT("fmul v0.4s, v0.4s, " L2 ".4s") /* v0 = X = ln(2) * (XP - float(R)) */ \
72 __ASM_EMIT("fmul v1.4s, v1.4s, " L2 ".4s") \
73 /* v0 = X, v2 = [ x < 0 ], v6 = 1 << R */ \
74 __ASM_EMIT("fadd v4.4s, v0.4s, " C5 ".4s") /* v4 = C5+X */ \
75 __ASM_EMIT("fadd v5.4s, v1.4s, " C5 ".4s") \
76 __ASM_EMIT("fmul v4.4s, v4.4s, v0.4s") /* v4 = X*(C5+X) */ \
77 __ASM_EMIT("fmul v5.4s, v5.4s, v1.4s") \
78 __ASM_EMIT("fadd v4.4s, v4.4s, " C4 ".4s") /* v4 = C4+X*(C5+X) */ \
79 __ASM_EMIT("fadd v5.4s, v5.4s, " C4 ".4s") \
80 __ASM_EMIT("fmul v4.4s, v4.4s, v0.4s") /* v4 = X*(C4+X*(C5+X)) */ \
81 __ASM_EMIT("fmul v5.4s, v5.4s, v1.4s") \
82 __ASM_EMIT("fadd v4.4s, v4.4s, " C3 ".4s") /* v4 = C3+X*(C4+X*(C5+X)) */ \
83 __ASM_EMIT("fadd v5.4s, v5.4s, " C3 ".4s") \
84 __ASM_EMIT("fmul v4.4s, v4.4s, v0.4s") /* v4 = X*(C3+X*(C4+X*(C5+X))) */ \
85 __ASM_EMIT("fmul v5.4s, v5.4s, v1.4s") \
86 __ASM_EMIT("fadd v4.4s, v4.4s, " C2 ".4s") /* v4 = C2+X*(C3+X*(C4+X*(C5+X))) */ \
87 __ASM_EMIT("fadd v5.4s, v5.4s, " C2 ".4s") \
88 __ASM_EMIT("fmul v4.4s, v4.4s, v0.4s") /* v4 = X*(C2+X*(C3+X*(C4+X*(C5+X)))) */ \
89 __ASM_EMIT("fmul v5.4s, v5.4s, v1.4s") \
90 __ASM_EMIT("fadd v4.4s, v4.4s, " C1 ".4s") /* v4 = C1+X*(C2+X*(C3+X*(C4+X*(C5+X)))) */ \
91 __ASM_EMIT("fadd v5.4s, v5.4s, " C1 ".4s") \
92 __ASM_EMIT("fmul v4.4s, v4.4s, v0.4s") /* v4 = X*(C1+X*(C2+X*(C3+X*(C4+X*(C5+X))))) */ \
93 __ASM_EMIT("fmul v5.4s, v5.4s, v1.4s") \
94 __ASM_EMIT("fadd v4.4s, v4.4s, " C0 ".4s") /* v4 = C0+X*(C1+X*(C2+X*(C3+X*(C4+X*(C5+X))))) */ \
95 __ASM_EMIT("fadd v5.4s, v5.4s, " C0 ".4s") \
96 __ASM_EMIT("fmul v4.4s, v4.4s, v0.4s") /* v4 = X*(C0+X*(C1+X*(C2+X*(C3+X*(C4+X*(C5+X)))))) */ \
97 __ASM_EMIT("fmul v5.4s, v5.4s, v1.4s") \
98 __ASM_EMIT("fmul v4.4s, v4.4s, " C6 ".4s") /* v4 = 1/7! * X*(C0+X*(C1+X*(C2+X*(C3+X*(C4+X*(C5+X)))))) */ \
99 __ASM_EMIT("fmul v5.4s, v5.4s, " C6 ".4s") \
100 __ASM_EMIT("fadd v4.4s, v4.4s, " C7 ".4s") /* v4 = 1 + 1/7! * X*(C0+X*(C1+X*(C2+X*(C3+X*(C4+X*(C5+X)))))) */ \
101 __ASM_EMIT("fadd v5.4s, v5.4s, " C7 ".4s") \
102 __ASM_EMIT("fmul v4.4s, v4.4s, v6.4s") /* v4 = E = (1 << R) * (1 + 1/7! * X*(C0+X*(C1+X*(C2+X*(C3+X*(C4+X*(C5+X))))))) */ \
103 __ASM_EMIT("fmul v5.4s, v5.4s, v7.4s") \
104 /* Calc reciprocals into v0 */ \
105 __ASM_EMIT("frecpe v8.4s, v4.4s") /* v8 = e2 */ \
106 __ASM_EMIT("frecpe v9.4s, v5.4s") \
107 __ASM_EMIT("frecps v10.4s, v8.4s, v4.4s") /* v10 = (2 - E*e2) */ \
108 __ASM_EMIT("frecps v11.4s, v9.4s, v5.4s") \
109 __ASM_EMIT("fmul v8.4s, v10.4s, v8.4s") /* v8 = e2' = e2 * (2 - E*e2) */ \
110 __ASM_EMIT("fmul v9.4s, v11.4s, v9.4s") \
111 __ASM_EMIT("frecps v10.4s, v8.4s, v4.4s") /* v10 = (2 - E*e2') */ \
112 __ASM_EMIT("frecps v11.4s, v9.4s, v5.4s") \
113 __ASM_EMIT("fmul v0.4s, v10.4s, v8.4s") /* v0 = 1/E = e2" = e2' * (2 - E*e2) */ \
114 __ASM_EMIT("fmul v1.4s, v11.4s, v9.4s") \
115 /* Perform conditional output */ \
116 __ASM_EMIT("bif v0.16b, v4.16b, v2.16b") /* v0 = ((1/E) & [ x < 0 ]) | (E & [x >= 0]) */ \
117 __ASM_EMIT("bif v1.16b, v5.16b, v3.16b")
118
119 #define POW2_CORE_X4(ME, L2, C5, C4, C3, C2, C1, C0, C6, C7) \
120 /* v0 = x */ \
121 __ASM_EMIT("sshr v2.4s, v0.4s, #31") /* v2 = [ x < 0 ] */ \
122 __ASM_EMIT("fabs v0.4s, v0.4s") /* v0 = XP = fabs(x) */ \
123 __ASM_EMIT("fcvtzs v4.4s, v0.4s") /* v4 = R = int(x) */ \
124 __ASM_EMIT("add v6.4s, v4.4s, " ME ".4s") /* v6 = R + (ME = 127) */ \
125 __ASM_EMIT("scvtf v4.4s, v4.4s") /* v4 = float(R) */ \
126 __ASM_EMIT("shl v6.4s, v6.4s, #23") /* v6 = 1 << (R + 127) */ \
127 __ASM_EMIT("fsub v0.4s, v0.4s, v4.4s") /* v0 = XP - float(R) */ \
128 __ASM_EMIT("fmul v0.4s, v0.4s, " L2 ".4s") /* v0 = X = ln(2) * (XP - float(R)) */ \
129 /* v0 = X, v2 = [ x < 0 ], v6 = 1 << R */ \
130 __ASM_EMIT("fadd v4.4s, v0.4s, " C5 ".4s") /* v4 = C5+X */ \
131 __ASM_EMIT("fmul v4.4s, v4.4s, v0.4s") /* v4 = X*(C5+X) */ \
132 __ASM_EMIT("fadd v4.4s, v4.4s, " C4 ".4s") /* v4 = C4+X*(C5+X) */ \
133 __ASM_EMIT("fmul v4.4s, v4.4s, v0.4s") /* v4 = X*(C4+X*(C5+X)) */ \
134 __ASM_EMIT("fadd v4.4s, v4.4s, " C3 ".4s") /* v4 = C3+X*(C4+X*(C5+X)) */ \
135 __ASM_EMIT("fmul v4.4s, v4.4s, v0.4s") /* v4 = X*(C3+X*(C4+X*(C5+X))) */ \
136 __ASM_EMIT("fadd v4.4s, v4.4s, " C2 ".4s") /* v4 = C2+X*(C3+X*(C4+X*(C5+X))) */ \
137 __ASM_EMIT("fmul v4.4s, v4.4s, v0.4s") /* v4 = X*(C2+X*(C3+X*(C4+X*(C5+X)))) */ \
138 __ASM_EMIT("fadd v4.4s, v4.4s, " C1 ".4s") /* v4 = C1+X*(C2+X*(C3+X*(C4+X*(C5+X)))) */ \
139 __ASM_EMIT("fmul v4.4s, v4.4s, v0.4s") /* v4 = X*(C1+X*(C2+X*(C3+X*(C4+X*(C5+X))))) */ \
140 __ASM_EMIT("fadd v4.4s, v4.4s, " C0 ".4s") /* v4 = C0+X*(C1+X*(C2+X*(C3+X*(C4+X*(C5+X))))) */ \
141 __ASM_EMIT("fmul v4.4s, v4.4s, v0.4s") /* v4 = X*(C0+X*(C1+X*(C2+X*(C3+X*(C4+X*(C5+X)))))) */ \
142 __ASM_EMIT("fmul v4.4s, v4.4s, " C6 ".4s") /* v4 = 1/7! * X*(C0+X*(C1+X*(C2+X*(C3+X*(C4+X*(C5+X)))))) */ \
143 __ASM_EMIT("fadd v4.4s, v4.4s, " C7 ".4s") /* v4 = 1 + 1/7! * X*(C0+X*(C1+X*(C2+X*(C3+X*(C4+X*(C5+X)))))) */ \
144 __ASM_EMIT("fmul v4.4s, v4.4s, v6.4s") /* v4 = E = (1 << R) * (1 + 1/7! * X*(C0+X*(C1+X*(C2+X*(C3+X*(C4+X*(C5+X))))))) */ \
145 /* Calc reciprocals into v0 */ \
146 __ASM_EMIT("frecpe v8.4s, v4.4s") /* v8 = e2 */ \
147 __ASM_EMIT("frecps v10.4s, v8.4s, v4.4s") /* v10 = (2 - E*e2) */ \
148 __ASM_EMIT("fmul v8.4s, v10.4s, v8.4s") /* v8 = e2' = e2 * (2 - E*e2) */ \
149 __ASM_EMIT("frecps v10.4s, v8.4s, v4.4s") /* v10 = (2 - E*e2') */ \
150 __ASM_EMIT("fmul v0.4s, v10.4s, v8.4s") /* v0 = 1/E = e2" = e2' * (2 - E*e2) */ \
151 /* Perform conditional output */ \
152 __ASM_EMIT("bif v0.16b, v4.16b, v2.16b") /* v0 = ((1/E) & [ x < 0 ]) | (E & [x >= 0]) */
153
154 #define EXP_CORE_LOAD \
155 __ASM_EMIT("ldp q26, q27, [%[LOG2E]]") \
156 __ASM_EMIT("ldp q16, q17, [%[E2C], #0x00]") /* v16 = ME, v17 = L2 */ \
157 __ASM_EMIT("ldp q18, q19, [%[E2C], #0x20]") /* v18 = C5, v19 = C4 */ \
158 __ASM_EMIT("ldp q20, q21, [%[E2C], #0x40]") /* v20 = C3, v21 = C2 */ \
159 __ASM_EMIT("ldp q22, q23, [%[E2C], #0x60]") /* v22 = C1, v23 = C0 */ \
160 __ASM_EMIT("ldp q24, q25, [%[E2C], #0x80]") /* v22 = C6, v23 = C7 */
161
162 #define EXP_CORE_X8_NOLOAD \
163 /* in: v0 = x0, v1 = x1 */ \
164 __ASM_EMIT("fmul v0.4s, v0.4s, v26.4s") \
165 __ASM_EMIT("fmul v1.4s, v1.4s, v27.4s") \
166 POW2_CORE_X8("v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25") \
167 /* out: v0 = exp(x0), v1 = exp(x1) */
168
169 #define EXP_CORE_X4_NOLOAD \
170 /* in: v0 = x0 */ \
171 __ASM_EMIT("fmul v0.4s, v0.4s, v26.4s") \
172 POW2_CORE_X4("v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25") \
173 /* out: v0 = exp(x0) */
174
175 #define EXP_CORE_X8 \
176 /* in: v0 = x0, v1 = x1 */ \
177 EXP_CORE_LOAD \
178 EXP_CORE_X8_NOLOAD \
179 /* out: v0 = exp(x0), v1 = exp(x1) */
180
181 #define EXP_CORE_X4 \
182 /* in: v0 = x0 */ \
183 EXP_CORE_LOAD \
184 EXP_CORE_X4_NOLOAD \
185 /* out: v0 = exp(x0) */
186
187 void exp2(float *dst, const float *src, size_t count)
188 {
189 ARCH_AARCH64_ASM(
190 EXP_CORE_LOAD
191 // x8 blocks
192 __ASM_EMIT("subs %[count], %[count], #8")
193 __ASM_EMIT("b.lo 2f")
194 __ASM_EMIT("1:")
195 __ASM_EMIT("ldp q0, q1, [%[src]]")
196 EXP_CORE_X8_NOLOAD
197 __ASM_EMIT("subs %[count], %[count], #8")
198 __ASM_EMIT("stp q0, q1, [%[dst]]")
199 __ASM_EMIT("add %[src], %[src], #0x20")
200 __ASM_EMIT("add %[dst], %[dst], #0x20")
201 __ASM_EMIT("bhs 1b")
202 __ASM_EMIT("2:")
203 // x4 block
204 __ASM_EMIT("adds %[count], %[count], #4")
205 __ASM_EMIT("blt 4f")
206 __ASM_EMIT("ldr q0, [%[src]]")
207 EXP_CORE_X4_NOLOAD
208 __ASM_EMIT("sub %[count], %[count], #4")
209 __ASM_EMIT("str q0, [%[dst]]")
210 __ASM_EMIT("add %[src], %[src], #0x10")
211 __ASM_EMIT("add %[dst], %[dst], #0x10")
212 __ASM_EMIT("4:")
213 // Tail: 1x-3x block
214 __ASM_EMIT("adds %[count], %[count], #4")
215 __ASM_EMIT("bls 12f")
216 __ASM_EMIT("tst %[count], #1")
217 __ASM_EMIT("b.eq 6f")
218 __ASM_EMIT("ld1 {v0.s}[0], [%[src]]")
219 __ASM_EMIT("add %[src], %[src], #0x04")
220 __ASM_EMIT("6:")
221 __ASM_EMIT("tst %[count], #2")
222 __ASM_EMIT("b.eq 8f")
223 __ASM_EMIT("ld1 {v0.d}[1], [%[src]]")
224 __ASM_EMIT("8:")
225 EXP_CORE_X4_NOLOAD
226 __ASM_EMIT("tst %[count], #1")
227 __ASM_EMIT("b.eq 10f")
228 __ASM_EMIT("st1 {v0.s}[0], [%[dst]]")
229 __ASM_EMIT("add %[dst], %[dst], #0x04")
230 __ASM_EMIT("10:")
231 __ASM_EMIT("tst %[count], #2")
232 __ASM_EMIT("b.eq 12f")
233 __ASM_EMIT("st1 {v0.d}[1], [%[dst]]")
234 __ASM_EMIT("12:")
235
236 : [dst] "+r" (dst), [src] "+r" (src), [count] "+r" (count)
237 : [E2C] "r" (&EXP2_CONST[0]),
238 [LOG2E] "r" (&EXP_LOG2E[0])
239 : "cc", "memory",
240 "v0", "v1", "v2", "v3",
241 "v4", "v5", "v6", "v7",
242 "v8", "v9", "v10", "v11",
243 "v16", "v17", "v18", "v19",
244 "v20", "v21", "v22", "v23",
245 "v24", "v25", "v26", "v27"
246 );
247 }
248
249 void exp1(float *dst, size_t count)
250 {
251 ARCH_AARCH64_ASM(
252 EXP_CORE_LOAD
253 // x8 blocks
254 __ASM_EMIT("subs %[count], %[count], #8")
255 __ASM_EMIT("b.lo 2f")
256 __ASM_EMIT("1:")
257 __ASM_EMIT("ldp q0, q1, [%[dst]]")
258 EXP_CORE_X8
259 __ASM_EMIT("subs %[count], %[count], #8")
260 __ASM_EMIT("stp q0, q1, [%[dst]]")
261 __ASM_EMIT("add %[dst], %[dst], #0x20")
262 __ASM_EMIT("bhs 1b")
263 __ASM_EMIT("2:")
264 // x4 block
265 __ASM_EMIT("adds %[count], %[count], #4")
266 __ASM_EMIT("blt 4f")
267 __ASM_EMIT("ldr q0, [%[dst]]")
268 EXP_CORE_X4
269 __ASM_EMIT("sub %[count], %[count], #4")
270 __ASM_EMIT("str q0, [%[dst]]")
271 __ASM_EMIT("add %[dst], %[dst], #0x10")
272 __ASM_EMIT("4:")
273 // Tail: 1x-3x block
274 __ASM_EMIT("adds %[count], %[count], #4")
275 __ASM_EMIT("bls 12f")
276 __ASM_EMIT("tst %[count], #1")
277 __ASM_EMIT("b.eq 6f")
278 __ASM_EMIT("ld1 {v0.s}[0], [%[dst]]")
279 __ASM_EMIT("add %[dst], %[dst], #0x04")
280 __ASM_EMIT("6:")
281 __ASM_EMIT("tst %[count], #2")
282 __ASM_EMIT("b.eq 8f")
283 __ASM_EMIT("ld1 {v0.d}[1], [%[dst]]")
284 __ASM_EMIT("8:")
285 EXP_CORE_X4
286 __ASM_EMIT("tst %[count], #1")
287 __ASM_EMIT("b.eq 10f")
288 __ASM_EMIT("sub %[dst], %[dst], #0x04")
289 __ASM_EMIT("st1 {v0.s}[0], [%[dst]]")
290 __ASM_EMIT("add %[dst], %[dst], #0x04")
291 __ASM_EMIT("10:")
292 __ASM_EMIT("tst %[count], #2")
293 __ASM_EMIT("b.eq 12f")
294 __ASM_EMIT("st1 {v0.d}[1], [%[dst]]")
295 __ASM_EMIT("12:")
296
297 : [dst] "+r" (dst), [count] "+r" (count)
298 : [E2C] "r" (&EXP2_CONST[0]),
299 [LOG2E] "r" (&EXP_LOG2E[0])
300 : "cc", "memory",
301 "v0", "v1", "v2", "v3",
302 "v4", "v5", "v6", "v7",
303 "v8", "v9", "v10", "v11",
304 "v16", "v17", "v18", "v19",
305 "v20", "v21", "v22", "v23",
306 "v24", "v25", "v26", "v27"
307 );
308 }
309 } /* namespace asimd */
310 } /* namespace lsp */
311
312 #endif /* PRIVATE_DSP_ARCH_AARCH64_ASIMD_PMATH_EXP_H_ */