Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/aarch64/asimd/pmath/log.h

raw
1 /*
2 * Copyright (C) 2025 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2025 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 31 мар. 2020 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_AARCH64_ASIMD_PMATH_LOG_H_
23 #define PRIVATE_DSP_ARCH_AARCH64_ASIMD_PMATH_LOG_H_
24
25 #ifndef PRIVATE_DSP_ARCH_AARCH64_ASIMD_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_AARCH64_ASIMD_IMPL */
28
29 namespace lsp
30 {
31 namespace asimd
32 {
33 IF_ARCH_AARCH64(
34 static const uint32_t LOG2_CONST[] __lsp_aligned16 =
35 {
36 LSP_DSP_VEC4(0x007fffff), // MM = frac
37 LSP_DSP_VEC4(0x0000007f), // ME = 127
38 LSP_DSP_VEC4(0x3dba2e8c), // C0 = 1/11 = 0.0909090936183929
39 LSP_DSP_VEC4(0x3de38e39), // C1 = 1/9 = 0.1111111119389534
40 LSP_DSP_VEC4(0x3e124925), // C2 = 1/7 = 0.1428571492433548
41 LSP_DSP_VEC4(0x3e4ccccd), // C3 = 1/5 = 0.2000000029802322
42 LSP_DSP_VEC4(0x3eaaaaab), // C4 = 1/3 = 0.3333333432674408
43 LSP_DSP_VEC4(0x3f800000), // C5 = 1.0f
44 };
45
46 static const float LOGB_C[] __lsp_aligned16 =
47 {
48 LSP_DSP_VEC4(2.0f * float(M_LOG2E)),
49 LSP_DSP_VEC4(2.0f * float(M_LOG2E))
50 };
51
52 static const float LOGE_C[] __lsp_aligned16 =
53 {
54 LSP_DSP_VEC4(1.0f / float(M_LOG2E)),
55 LSP_DSP_VEC4(1.0f / float(M_LOG2E))
56 };
57
58 static const float LOGD_C[] __lsp_aligned16 =
59 {
60 LSP_DSP_VEC4(2.0f * float(M_LOG10E)),
61 LSP_DSP_VEC4(0.301029995663981f) // 1/log2(10)
62 };
63 )
64
65 #define LOGN_CORE_X8(MM, ME, C0, C1, C2, C3, C4, C5) \
66 /* v0 = x */ \
67 __ASM_EMIT("ushr v2.4s, v0.4s, #23") /* v2 = ilog2(x) + 127 */ \
68 __ASM_EMIT("ushr v3.4s, v1.4s, #23") \
69 __ASM_EMIT("and v0.16b, v0.16b, " MM ".16b") /* v0 = x & MM */ \
70 __ASM_EMIT("and v1.16b, v1.16b, " MM ".16b") \
71 __ASM_EMIT("sub v2.4s, v2.4s, " ME ".4s") /* v2 = r - ME = ilog2(x) */ \
72 __ASM_EMIT("sub v3.4s, v3.4s, " ME ".4s") \
73 __ASM_EMIT("orr v0.16b, v0.16b, " C5 ".16b") /* v0 = X = (x & MM) | (C7 = 1.0f) */ \
74 __ASM_EMIT("orr v1.16b, v1.16b, " C5 ".16b") \
75 __ASM_EMIT("scvtf v2.4s, v2.4s") /* v2 = R = float(r) */ \
76 __ASM_EMIT("scvtf v3.4s, v3.4s") \
77 __ASM_EMIT("fadd v4.4s, v0.4s, " C5 ".4s") /* v4 = XB = X + (C7 = 1) */ \
78 __ASM_EMIT("fadd v5.4s, v1.4s, " C5 ".4s") \
79 __ASM_EMIT("fsub v0.4s, v0.4s, " C5 ".4s") /* v0 = XT = X - (C7 = 1) */ \
80 __ASM_EMIT("fsub v1.4s, v1.4s, " C5 ".4s") \
81 __ASM_EMIT("frecpe v6.4s, v4.4s") /* v6 = xb */ \
82 __ASM_EMIT("frecpe v7.4s, v5.4s") \
83 __ASM_EMIT("frecps v8.4s, v6.4s, v4.4s") /* v8 = (2 - XB*xb) */ \
84 __ASM_EMIT("frecps v9.4s, v7.4s, v5.4s") \
85 __ASM_EMIT("fmul v6.4s, v8.4s, v6.4s") /* v6 = xb' = xb * (2 - XB*xb) */ \
86 __ASM_EMIT("fmul v7.4s, v9.4s, v7.4s") \
87 __ASM_EMIT("frecps v8.4s, v6.4s, v4.4s") /* v8 = (2 - XB*xb') */ \
88 __ASM_EMIT("frecps v9.4s, v7.4s, v5.4s") \
89 __ASM_EMIT("fmul v4.4s, v8.4s, v6.4s") /* v4 = 1/(X+1) = xb" = xb' * (2 - XB*xb) */ \
90 __ASM_EMIT("fmul v5.4s, v9.4s, v7.4s") \
91 __ASM_EMIT("fmul v0.4s, v0.4s, v4.4s") /* v0 = y = (X-1)/(X+1) */ \
92 __ASM_EMIT("fmul v1.4s, v1.4s, v5.4s") \
93 __ASM_EMIT("fmul v4.4s, v0.4s, v0.4s") /* v4 = Y = y*y */ \
94 __ASM_EMIT("fmul v5.4s, v1.4s, v1.4s") \
95 /* v0 = y, v2 = R, v4 = Y */ \
96 __ASM_EMIT("fmul v6.4s, v4.4s, " C0 ".4s") /* v6 = C0*Y */ \
97 __ASM_EMIT("fmul v7.4s, v5.4s, " C0 ".4s") \
98 __ASM_EMIT("fadd v6.4s, v6.4s, " C1 ".4s") /* v6 = C1+C0*Y */ \
99 __ASM_EMIT("fadd v7.4s, v7.4s, " C1 ".4s") \
100 __ASM_EMIT("fmul v6.4s, v6.4s, v4.4s") /* v6 = Y*(C1+C0*Y) */ \
101 __ASM_EMIT("fmul v7.4s, v7.4s, v5.4s") \
102 __ASM_EMIT("fadd v6.4s, v6.4s, " C2 ".4s") /* v6 = C2+Y*(C1+C0*Y) */ \
103 __ASM_EMIT("fadd v7.4s, v7.4s, " C2 ".4s") \
104 __ASM_EMIT("fmul v6.4s, v6.4s, v4.4s") /* v6 = Y*(C2+Y*(C1+C0*Y)) */ \
105 __ASM_EMIT("fmul v7.4s, v7.4s, v5.4s") \
106 __ASM_EMIT("fadd v6.4s, v6.4s, " C3 ".4s") /* v6 = C3+Y*(C2+Y*(C1+C0*Y)) */ \
107 __ASM_EMIT("fadd v7.4s, v7.4s, " C3 ".4s") \
108 __ASM_EMIT("fmul v6.4s, v6.4s, v4.4s") /* v6 = Y*(C3+Y*(C2+Y*(C1+C0*Y))) */ \
109 __ASM_EMIT("fmul v7.4s, v7.4s, v5.4s") \
110 __ASM_EMIT("fadd v6.4s, v6.4s, " C4 ".4s") /* v6 = C4+Y*(C3+Y*(C2+Y*(C1+C0*Y))) */ \
111 __ASM_EMIT("fadd v7.4s, v7.4s, " C4 ".4s") \
112 __ASM_EMIT("fmul v6.4s, v6.4s, v4.4s") /* v6 = Y*(C4+Y*(C3+Y*(C2+Y*(C1+C0*Y)))) */ \
113 __ASM_EMIT("fmul v7.4s, v7.4s, v5.4s") \
114 __ASM_EMIT("fadd v6.4s, v6.4s, " C5 ".4s") /* v6 = C5+Y*(C4+Y*(C3+Y*(C2+Y*(C1+C0*Y)))) */ \
115 __ASM_EMIT("fadd v7.4s, v7.4s, " C5 ".4s") \
116 __ASM_EMIT("fmul v0.4s, v0.4s, v6.4s") /* v0 = y*(C5+Y*(C4+Y*(C3+Y*(C2+Y*(C1+C0*Y))))) */ \
117 __ASM_EMIT("fmul v1.4s, v1.4s, v7.4s") \
118 /* v0 = y*L, v2 = R */
119
120 #define LOGN_CORE_X4(MM, ME, C0, C1, C2, C3, C4, C5) \
121 /* v0 = x */ \
122 __ASM_EMIT("ushr v2.4s, v0.4s, #23") /* v2 = ilog2(x) + 127 */ \
123 __ASM_EMIT("and v0.16b, v0.16b, " MM ".16b") /* v0 = x & MM */ \
124 __ASM_EMIT("sub v2.4s, v2.4s, " ME ".4s") /* v2 = r - ME = ilog2(x) */ \
125 __ASM_EMIT("orr v0.16b, v0.16b, " C5 ".16b") /* v0 = X = (x & MM) | (C7 = 1.0f) */ \
126 __ASM_EMIT("scvtf v2.4s, v2.4s") /* v2 = R = float(r) */ \
127 __ASM_EMIT("fadd v4.4s, v0.4s, " C5 ".4s") /* v4 = XB = X + (C7 = 1) */ \
128 __ASM_EMIT("fsub v0.4s, v0.4s, " C5 ".4s") /* v0 = XT = X - (C7 = 1) */ \
129 __ASM_EMIT("frecpe v6.4s, v4.4s") /* v6 = xb */ \
130 __ASM_EMIT("frecps v8.4s, v6.4s, v4.4s") /* v8 = (2 - XB*xb) */ \
131 __ASM_EMIT("fmul v6.4s, v8.4s, v6.4s") /* v6 = xb' = xb * (2 - XB*xb) */ \
132 __ASM_EMIT("frecps v8.4s, v6.4s, v4.4s") /* v8 = (2 - XB*xb') */ \
133 __ASM_EMIT("fmul v4.4s, v8.4s, v6.4s") /* v4 = 1/(X+1) = xb" = xb' * (2 - XB*xb) */ \
134 __ASM_EMIT("fmul v0.4s, v0.4s, v4.4s") /* v0 = y = (X-1)/(X+1) */ \
135 __ASM_EMIT("fmul v4.4s, v0.4s, v0.4s") /* v4 = Y = y*y */ \
136 /* v0 = y, v2 = R, v4 = Y */ \
137 __ASM_EMIT("fmul v6.4s, v4.4s, " C0 ".4s") /* v6 = C0*Y */ \
138 __ASM_EMIT("fadd v6.4s, v6.4s, " C1 ".4s") /* v6 = C1+C0*Y */ \
139 __ASM_EMIT("fmul v6.4s, v6.4s, v4.4s") /* v6 = Y*(C1+C0*Y) */ \
140 __ASM_EMIT("fadd v6.4s, v6.4s, " C2 ".4s") /* v6 = C2+Y*(C1+C0*Y) */ \
141 __ASM_EMIT("fmul v6.4s, v6.4s, v4.4s") /* v6 = Y*(C2+Y*(C1+C0*Y)) */ \
142 __ASM_EMIT("fadd v6.4s, v6.4s, " C3 ".4s") /* v6 = C3+Y*(C2+Y*(C1+C0*Y)) */ \
143 __ASM_EMIT("fmul v6.4s, v6.4s, v4.4s") /* v6 = Y*(C3+Y*(C2+Y*(C1+C0*Y))) */ \
144 __ASM_EMIT("fadd v6.4s, v6.4s, " C4 ".4s") /* v6 = C4+Y*(C3+Y*(C2+Y*(C1+C0*Y))) */ \
145 __ASM_EMIT("fmul v6.4s, v6.4s, v4.4s") /* v6 = Y*(C4+Y*(C3+Y*(C2+Y*(C1+C0*Y)))) */ \
146 __ASM_EMIT("fadd v6.4s, v6.4s, " C5 ".4s") /* v6 = C5+Y*(C4+Y*(C3+Y*(C2+Y*(C1+C0*Y)))) */ \
147 __ASM_EMIT("fmul v0.4s, v0.4s, v6.4s") /* v0 = y*(C5+Y*(C4+Y*(C3+Y*(C2+Y*(C1+C0*Y))))) */ \
148 /* v0 = y*L, v2 = R */
149
150 #define LOGN_CORE_LOAD \
151 __ASM_EMIT("ldp q26, q27, [%[LOGC]]") \
152 __ASM_EMIT("ldp q16, q17, [%[L2C], #0x00]") /* v16 = MM, v17 = ME */ \
153 __ASM_EMIT("ldp q18, q19, [%[L2C], #0x20]") /* v18 = C0, v19 = C1 */ \
154 __ASM_EMIT("ldp q20, q21, [%[L2C], #0x40]") /* v20 = C2, v21 = C3 */ \
155 __ASM_EMIT("ldp q22, q23, [%[L2C], #0x60]") /* v22 = C4, v23 = C5 */
156
157 #define LOGB_CORE_X8_NOLOAD \
158 /* in: v0 = x1, v1 = x2 */ \
159 LOGN_CORE_X8("v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23") \
160 __ASM_EMIT("fmla v2.4s, v0.4s, v26.4s") /* v2 = R + 2*y*L*M_LOG2E */ \
161 __ASM_EMIT("fmla v3.4s, v1.4s, v27.4s") \
162 /* out: v0 = logb(x0), v1 = logb(x1) */
163
164 #define LOGB_CORE_X4_NOLOAD \
165 /* in: v0 = x1 */ \
166 LOGN_CORE_X4("v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23") \
167 __ASM_EMIT("fmla v2.4s, v0.4s, v26.4s") /* v2 = R + 2*y*L*M_LOG2E */ \
168 /* out: v0 = logb(x0) */
169
170 #define LOGB_CORE_X8 \
171 /* in: v0 = x1, v1 = x2 */ \
172 LOGN_CORE_LOAD \
173 LOGB_CORE_X8_NOLOAD \
174 /* out: v0 = logb(x0), v1 = logb(x1) */
175
176 #define LOGB_CORE_X4 \
177 /* in: v0 = x1 */ \
178 LOGN_CORE_LOAD \
179 LOGB_CORE_X4_NOLOAD \
180 /* out: v0 = logb(x0) */
181
182 #define LOGE_CORE_X8_NOLOAD \
183 /* in: v0 = x1, v1 = x2 */ \
184 LOGN_CORE_X8("v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23") \
185 __ASM_EMIT("fadd v0.4s, v0.4s, v0.4s") /* v0 = 2*y*L */ \
186 __ASM_EMIT("fadd v1.4s, v1.4s, v1.4s") \
187 __ASM_EMIT("fmla v0.4s, v2.4s, v26.4s") /* v0 = 2*y*L + R/log2(E) */ \
188 __ASM_EMIT("fmla v1.4s, v3.4s, v27.4s") \
189 /* out: v0 = loge(x0), v1 = loge(x1) */
190
191 #define LOGE_CORE_X4_NOLOAD \
192 /* in: v0 = x1 */ \
193 LOGN_CORE_X4("v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23") \
194 __ASM_EMIT("fadd v0.4s, v0.4s, v0.4s") /* v0 = 2*y*L */ \
195 __ASM_EMIT("fmla v0.4s, v2.4s, v26.4s") /* v0 = 2*y*L + R/log2(E) */ \
196 /* out: v0 = loge(x0) */
197
198 #define LOGE_CORE_X8 \
199 /* in: v0 = x1, v1 = x2 */ \
200 LOGN_CORE_LOAD \
201 LOGE_CORE_X8_NOLOAD \
202 /* out: v0 = loge(x0), v1 = loge(x1) */
203
204 #define LOGE_CORE_X4 \
205 /* in: v0 = x1 */ \
206 LOGN_CORE_LOAD \
207 LOGE_CORE_X4_NOLOAD \
208 /* out: v0 = loge(x0) */
209
210 #define LOGD_CORE_X8_NOLOAD \
211 /* in: v0 = x1, v1 = x2 */ \
212 LOGN_CORE_X8("v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23") \
213 __ASM_EMIT("fmul v0.4s, v0.4s, v26.4s") /* v0 = 2*y*L*log10(E) */ \
214 __ASM_EMIT("fmul v1.4s, v1.4s, v26.4s") \
215 __ASM_EMIT("fmla v0.4s, v2.4s, v27.4s") /* v0 = 2*y*L*log10(E) + R/log2(10) */ \
216 __ASM_EMIT("fmla v1.4s, v3.4s, v27.4s") \
217 /* out: v0 = logd(x0), v1 = logd(x1) */
218
219 #define LOGD_CORE_X4_NOLOAD \
220 /* in: v0 = x1 */ \
221 LOGN_CORE_X4("v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23") \
222 __ASM_EMIT("fmul v0.4s, v0.4s, v26.4s") /* v0 = 2*y*L*log10(E) */ \
223 __ASM_EMIT("fmla v0.4s, v2.4s, v27.4s") /* v0 = 2*y*L*log10(E) + R/log2(10) */ \
224 /* out: v0 = logd(x0) */
225
226 #define LOGD_CORE_X8 \
227 /* in: v0 = x1, v1 = x2 */ \
228 LOGN_CORE_LOAD \
229 LOGD_CORE_X8_NOLOAD \
230 /* out: v0 = logd(x0), v1 = logd(x1) */
231
232 #define LOGD_CORE_X4 \
233 /* in: v0 = x1 */ \
234 LOGN_CORE_LOAD \
235 LOGD_CORE_X4_NOLOAD \
236 /* out: v0 = logd(x0) */
237
238 void logb2(float *dst, const float *src, size_t count)
239 {
240 ARCH_AARCH64_ASM(
241 // prepare
242 LOGN_CORE_LOAD
243 // x8 blocks
244 __ASM_EMIT("subs %[count], %[count], #8")
245 __ASM_EMIT("b.lo 2f")
246 __ASM_EMIT("1:")
247 __ASM_EMIT("ldp q0, q1, [%[src]]")
248 LOGB_CORE_X8_NOLOAD
249 __ASM_EMIT("subs %[count], %[count], #8")
250 __ASM_EMIT("stp q2, q3, [%[dst]]")
251 __ASM_EMIT("add %[src], %[src], #0x20")
252 __ASM_EMIT("add %[dst], %[dst], #0x20")
253 __ASM_EMIT("b.hs 1b")
254 __ASM_EMIT("2:")
255 // x4 block
256 __ASM_EMIT("adds %[count], %[count] , #4")
257 __ASM_EMIT("b.lt 4f")
258 __ASM_EMIT("ldr q0, [%[src]]")
259 LOGB_CORE_X4_NOLOAD
260 __ASM_EMIT("sub %[count], %[count], #4")
261 __ASM_EMIT("str q2, [%[dst]]")
262 __ASM_EMIT("add %[src], %[src], #0x10")
263 __ASM_EMIT("add %[dst], %[dst], #0x10")
264 __ASM_EMIT("4:")
265 // Tail: 1x-3x block
266 __ASM_EMIT("adds %[count], %[count], #4")
267 __ASM_EMIT("b.ls 12f")
268 __ASM_EMIT("tst %[count], #1")
269 __ASM_EMIT("b.eq 6f")
270 __ASM_EMIT("ld1 {v0.s}[0], [%[src]]")
271 __ASM_EMIT("add %[src], %[src], #0x04")
272 __ASM_EMIT("6:")
273 __ASM_EMIT("tst %[count], #2")
274 __ASM_EMIT("b.eq 8f")
275 __ASM_EMIT("ld1 {v0.d}[1], [%[src]]")
276 __ASM_EMIT("8:")
277 LOGB_CORE_X4_NOLOAD
278 __ASM_EMIT("tst %[count], #1")
279 __ASM_EMIT("b.eq 10f")
280 __ASM_EMIT("st1 {v2.s}[0], [%[dst]]")
281 __ASM_EMIT("add %[dst], %[dst], #0x04")
282 __ASM_EMIT("10:")
283 __ASM_EMIT("tst %[count], #2")
284 __ASM_EMIT("b.eq 12f")
285 __ASM_EMIT("st1 {v2.d}[1], [%[dst]]")
286 __ASM_EMIT("12:")
287
288 : [dst] "+r" (dst), [src] "+r" (src), [count] "+r" (count)
289 : [L2C] "r" (&LOG2_CONST[0]),
290 [LOGC] "r" (&LOGB_C[0])
291 : "cc", "memory",
292 "v0", "v1", "v2", "v3",
293 "v4", "v5", "v6", "v7",
294 "v8", "v9",
295 "v16", "v17", "v18", "v19",
296 "v20", "v21", "v22", "v23",
297 "v24", "v25", "v26", "v27"
298 );
299 }
300
301 void logb1(float *dst, size_t count)
302 {
303 ARCH_AARCH64_ASM(
304 // prepare
305 LOGN_CORE_LOAD
306 // x8 blocks
307 __ASM_EMIT("subs %[count], %[count], #8")
308 __ASM_EMIT("b.lo 2f")
309 __ASM_EMIT("1:")
310 __ASM_EMIT("ldp q0, q1, [%[dst]]")
311 LOGB_CORE_X8_NOLOAD
312 __ASM_EMIT("subs %[count], %[count], #8")
313 __ASM_EMIT("stp q2, q3, [%[dst]]")
314 __ASM_EMIT("add %[dst], %[dst], #0x20")
315 __ASM_EMIT("b.hs 1b")
316 __ASM_EMIT("2:")
317 // x4 block
318 __ASM_EMIT("adds %[count], %[count] , #4")
319 __ASM_EMIT("b.lt 4f")
320 __ASM_EMIT("ldr q0, [%[dst]]")
321 LOGB_CORE_X4_NOLOAD
322 __ASM_EMIT("sub %[count], %[count], #4")
323 __ASM_EMIT("str q2, [%[dst]]")
324 __ASM_EMIT("add %[dst], %[dst], #0x10")
325 __ASM_EMIT("4:")
326 // Tail: 1x-3x block
327 __ASM_EMIT("adds %[count], %[count], #4")
328 __ASM_EMIT("b.ls 12f")
329 __ASM_EMIT("tst %[count], #1")
330 __ASM_EMIT("b.eq 6f")
331 __ASM_EMIT("ld1 {v0.s}[0], [%[dst]]")
332 __ASM_EMIT("add %[dst], %[dst], #0x04")
333 __ASM_EMIT("6:")
334 __ASM_EMIT("tst %[count], #2")
335 __ASM_EMIT("b.eq 8f")
336 __ASM_EMIT("ld1 {v0.d}[1], [%[dst]]")
337 __ASM_EMIT("8:")
338 LOGB_CORE_X4_NOLOAD
339 __ASM_EMIT("tst %[count], #1")
340 __ASM_EMIT("b.eq 10f")
341 __ASM_EMIT("sub %[dst], %[dst], #0x04")
342 __ASM_EMIT("st1 {v2.s}[0], [%[dst]]")
343 __ASM_EMIT("add %[dst], %[dst], #0x04")
344 __ASM_EMIT("10:")
345 __ASM_EMIT("tst %[count], #2")
346 __ASM_EMIT("b.eq 12f")
347 __ASM_EMIT("st1 {v2.d}[1], [%[dst]]")
348 __ASM_EMIT("12:")
349
350 : [dst] "+r" (dst), [count] "+r" (count)
351 : [L2C] "r" (&LOG2_CONST[0]),
352 [LOGC] "r" (&LOGB_C[0])
353 : "cc", "memory",
354 "v0", "v1", "v2", "v3",
355 "v4", "v5", "v6", "v7",
356 "v8", "v9",
357 "v16", "v17", "v18", "v19",
358 "v20", "v21", "v22", "v23",
359 "v24", "v25", "v26", "v27"
360 );
361 }
362
363 void loge2(float *dst, const float *src, size_t count)
364 {
365 ARCH_AARCH64_ASM(
366 // prepare
367 LOGN_CORE_LOAD
368 // x8 blocks
369 __ASM_EMIT("subs %[count], %[count], #8")
370 __ASM_EMIT("b.lo 2f")
371 __ASM_EMIT("1:")
372 __ASM_EMIT("ldp q0, q1, [%[src]]")
373 LOGE_CORE_X8_NOLOAD
374 __ASM_EMIT("subs %[count], %[count], #8")
375 __ASM_EMIT("stp q0, q1, [%[dst]]")
376 __ASM_EMIT("add %[src], %[src], #0x20")
377 __ASM_EMIT("add %[dst], %[dst], #0x20")
378 __ASM_EMIT("b.hs 1b")
379 __ASM_EMIT("2:")
380 // x4 block
381 __ASM_EMIT("adds %[count], %[count] , #4")
382 __ASM_EMIT("b.lt 4f")
383 __ASM_EMIT("ldr q0, [%[src]]")
384 LOGE_CORE_X4_NOLOAD
385 __ASM_EMIT("sub %[count], %[count], #4")
386 __ASM_EMIT("str q0, [%[dst]]")
387 __ASM_EMIT("add %[src], %[src], #0x10")
388 __ASM_EMIT("add %[dst], %[dst], #0x10")
389 __ASM_EMIT("4:")
390 // Tail: 1x-3x block
391 __ASM_EMIT("adds %[count], %[count], #4")
392 __ASM_EMIT("b.ls 12f")
393 __ASM_EMIT("tst %[count], #1")
394 __ASM_EMIT("b.eq 6f")
395 __ASM_EMIT("ld1 {v0.s}[0], [%[src]]")
396 __ASM_EMIT("add %[src], %[src], #0x04")
397 __ASM_EMIT("6:")
398 __ASM_EMIT("tst %[count], #2")
399 __ASM_EMIT("b.eq 8f")
400 __ASM_EMIT("ld1 {v0.d}[1], [%[src]]")
401 __ASM_EMIT("8:")
402 LOGE_CORE_X4_NOLOAD
403 __ASM_EMIT("tst %[count], #1")
404 __ASM_EMIT("b.eq 10f")
405 __ASM_EMIT("st1 {v0.s}[0], [%[dst]]")
406 __ASM_EMIT("add %[dst], %[dst], #0x04")
407 __ASM_EMIT("10:")
408 __ASM_EMIT("tst %[count], #2")
409 __ASM_EMIT("b.eq 12f")
410 __ASM_EMIT("st1 {v0.d}[1], [%[dst]]")
411 __ASM_EMIT("12:")
412
413 : [dst] "+r" (dst), [src] "+r" (src), [count] "+r" (count)
414 : [L2C] "r" (&LOG2_CONST[0]),
415 [LOGC] "r" (&LOGE_C[0])
416 : "cc", "memory",
417 "v0", "v1", "v2", "v3",
418 "v4", "v5", "v6", "v7",
419 "v8", "v9",
420 "v16", "v17", "v18", "v19",
421 "v20", "v21", "v22", "v23",
422 "v24", "v25", "v26", "v27"
423 );
424 }
425
426 void loge1(float *dst, size_t count)
427 {
428 ARCH_AARCH64_ASM(
429 // prepare
430 LOGN_CORE_LOAD
431 // x8 blocks
432 __ASM_EMIT("subs %[count], %[count], #8")
433 __ASM_EMIT("b.lo 2f")
434 __ASM_EMIT("1:")
435 __ASM_EMIT("ldp q0, q1, [%[dst]]")
436 LOGE_CORE_X8_NOLOAD
437 __ASM_EMIT("subs %[count], %[count], #8")
438 __ASM_EMIT("stp q0, q1, [%[dst]]")
439 __ASM_EMIT("add %[dst], %[dst], #0x20")
440 __ASM_EMIT("b.hs 1b")
441 __ASM_EMIT("2:")
442 // x4 block
443 __ASM_EMIT("adds %[count], %[count] , #4")
444 __ASM_EMIT("b.lt 4f")
445 __ASM_EMIT("ldr q0, [%[dst]]")
446 LOGE_CORE_X4_NOLOAD
447 __ASM_EMIT("sub %[count], %[count], #4")
448 __ASM_EMIT("str q0, [%[dst]]")
449 __ASM_EMIT("add %[dst], %[dst], #0x10")
450 __ASM_EMIT("4:")
451 // Tail: 1x-3x block
452 __ASM_EMIT("adds %[count], %[count], #4")
453 __ASM_EMIT("b.ls 12f")
454 __ASM_EMIT("tst %[count], #1")
455 __ASM_EMIT("b.eq 6f")
456 __ASM_EMIT("ld1 {v0.s}[0], [%[dst]]")
457 __ASM_EMIT("add %[dst], %[dst], #0x04")
458 __ASM_EMIT("6:")
459 __ASM_EMIT("tst %[count], #2")
460 __ASM_EMIT("b.eq 8f")
461 __ASM_EMIT("ld1 {v0.d}[1], [%[dst]]")
462 __ASM_EMIT("8:")
463 LOGE_CORE_X4_NOLOAD
464 __ASM_EMIT("tst %[count], #1")
465 __ASM_EMIT("b.eq 10f")
466 __ASM_EMIT("sub %[dst], %[dst], #0x04")
467 __ASM_EMIT("st1 {v0.s}[0], [%[dst]]")
468 __ASM_EMIT("add %[dst], %[dst], #0x04")
469 __ASM_EMIT("10:")
470 __ASM_EMIT("tst %[count], #2")
471 __ASM_EMIT("b.eq 12f")
472 __ASM_EMIT("st1 {v0.d}[1], [%[dst]]")
473 __ASM_EMIT("12:")
474
475 : [dst] "+r" (dst), [count] "+r" (count)
476 : [L2C] "r" (&LOG2_CONST[0]),
477 [LOGC] "r" (&LOGE_C[0])
478 : "cc", "memory",
479 "v0", "v1", "v2", "v3",
480 "v4", "v5", "v6", "v7",
481 "v8", "v9",
482 "v16", "v17", "v18", "v19",
483 "v20", "v21", "v22", "v23",
484 "v24", "v25", "v26", "v27"
485 );
486 }
487
488 void logd2(float *dst, const float *src, size_t count)
489 {
490 ARCH_AARCH64_ASM(
491 // prepare
492 LOGN_CORE_LOAD
493 // x8 blocks
494 __ASM_EMIT("subs %[count], %[count], #8")
495 __ASM_EMIT("b.lo 2f")
496 __ASM_EMIT("1:")
497 __ASM_EMIT("ldp q0, q1, [%[src]]")
498 LOGD_CORE_X8_NOLOAD
499 __ASM_EMIT("subs %[count], %[count], #8")
500 __ASM_EMIT("stp q0, q1, [%[dst]]")
501 __ASM_EMIT("add %[src], %[src], #0x20")
502 __ASM_EMIT("add %[dst], %[dst], #0x20")
503 __ASM_EMIT("b.hs 1b")
504 // x4 block
505 __ASM_EMIT("2:")
506 __ASM_EMIT("adds %[count], %[count] , #4")
507 __ASM_EMIT("b.lt 4f")
508 __ASM_EMIT("ldr q0, [%[src]]")
509 LOGD_CORE_X4_NOLOAD
510 __ASM_EMIT("sub %[count], %[count], #4")
511 __ASM_EMIT("str q0, [%[dst]]")
512 __ASM_EMIT("add %[src], %[src], #0x10")
513 __ASM_EMIT("add %[dst], %[dst], #0x10")
514 __ASM_EMIT("4:")
515 // Tail: 1x-3x block
516 __ASM_EMIT("adds %[count], %[count], #4")
517 __ASM_EMIT("b.ls 12f")
518 __ASM_EMIT("tst %[count], #1")
519 __ASM_EMIT("b.eq 6f")
520 __ASM_EMIT("ld1 {v0.s}[0], [%[src]]")
521 __ASM_EMIT("add %[src], %[src], #0x04")
522 __ASM_EMIT("6:")
523 __ASM_EMIT("tst %[count], #2")
524 __ASM_EMIT("b.eq 8f")
525 __ASM_EMIT("ld1 {v0.d}[1], [%[src]]")
526 __ASM_EMIT("8:")
527 LOGD_CORE_X4_NOLOAD
528 __ASM_EMIT("tst %[count], #1")
529 __ASM_EMIT("b.eq 10f")
530 __ASM_EMIT("st1 {v0.s}[0], [%[dst]]")
531 __ASM_EMIT("add %[dst], %[dst], #0x04")
532 __ASM_EMIT("10:")
533 __ASM_EMIT("tst %[count], #2")
534 __ASM_EMIT("b.eq 12f")
535 __ASM_EMIT("st1 {v0.d}[1], [%[dst]]")
536 __ASM_EMIT("12:")
537
538 : [dst] "+r" (dst), [src] "+r" (src), [count] "+r" (count)
539 : [L2C] "r" (&LOG2_CONST[0]),
540 [LOGC] "r" (&LOGD_C[0])
541 : "cc", "memory",
542 "v0", "v1", "v2", "v3",
543 "v4", "v5", "v6", "v7",
544 "v8", "v9",
545 "v16", "v17", "v18", "v19",
546 "v20", "v21", "v22", "v23",
547 "v24", "v25", "v26", "v27"
548 );
549 }
550
551 void logd1(float *dst, size_t count)
552 {
553 ARCH_AARCH64_ASM(
554 // prepare
555 LOGN_CORE_LOAD
556 // x8 blocks
557 __ASM_EMIT("subs %[count], %[count], #8")
558 __ASM_EMIT("b.lo 2f")
559 __ASM_EMIT("1:")
560 __ASM_EMIT("ldp q0, q1, [%[dst]]")
561 LOGD_CORE_X8_NOLOAD
562 __ASM_EMIT("subs %[count], %[count], #8")
563 __ASM_EMIT("stp q0, q1, [%[dst]]")
564 __ASM_EMIT("add %[dst], %[dst], #0x20")
565 __ASM_EMIT("b.hs 1b")
566 __ASM_EMIT("2:")
567 // x4 block
568 __ASM_EMIT("adds %[count], %[count] , #4")
569 __ASM_EMIT("b.lt 4f")
570 __ASM_EMIT("ldr q0, [%[dst]]")
571 LOGD_CORE_X4_NOLOAD
572 __ASM_EMIT("sub %[count], %[count], #4")
573 __ASM_EMIT("str q0, [%[dst]]")
574 __ASM_EMIT("add %[dst], %[dst], #0x10")
575 // Tail: 1x-3x block
576 __ASM_EMIT("4:")
577 __ASM_EMIT("adds %[count], %[count], #4")
578 __ASM_EMIT("b.ls 12f")
579 __ASM_EMIT("tst %[count], #1")
580 __ASM_EMIT("b.eq 6f")
581 __ASM_EMIT("ld1 {v0.s}[0], [%[dst]]")
582 __ASM_EMIT("add %[dst], %[dst], #0x04")
583 __ASM_EMIT("6:")
584 __ASM_EMIT("tst %[count], #2")
585 __ASM_EMIT("b.eq 8f")
586 __ASM_EMIT("ld1 {v0.d}[1], [%[dst]]")
587 __ASM_EMIT("8:")
588 LOGD_CORE_X4_NOLOAD
589 __ASM_EMIT("tst %[count], #1")
590 __ASM_EMIT("b.eq 10f")
591 __ASM_EMIT("sub %[dst], %[dst], #0x04")
592 __ASM_EMIT("st1 {v0.s}[0], [%[dst]]")
593 __ASM_EMIT("add %[dst], %[dst], #0x04")
594 __ASM_EMIT("10:")
595 __ASM_EMIT("tst %[count], #2")
596 __ASM_EMIT("b.eq 12f")
597 __ASM_EMIT("st1 {v0.d}[1], [%[dst]]")
598 __ASM_EMIT("12:")
599
600 : [dst] "+r" (dst), [count] "+r" (count)
601 : [L2C] "r" (&LOG2_CONST[0]),
602 [LOGC] "r" (&LOGD_C[0])
603 : "cc", "memory",
604 "v0", "v1", "v2", "v3",
605 "v4", "v5", "v6", "v7",
606 "v8", "v9",
607 "v16", "v17", "v18", "v19",
608 "v20", "v21", "v22", "v23",
609 "v24", "v25", "v26", "v27"
610 );
611 }
612 } /* namespace asimd */
613 } /* namespace lsp */
614
615 #endif /* PRIVATE_DSP_ARCH_AARCH64_ASIMD_PMATH_LOG_H_ */