Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/aarch64/asimd/hmath/hdotp.h

raw
1 /*
2 * Copyright (C) 2020 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2020 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 31 мар. 2020 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_AARCH64_ASIMD_HMATH_HDOTP_H_
23 #define PRIVATE_DSP_ARCH_AARCH64_ASIMD_HMATH_HDOTP_H_
24
25 #ifndef PRIVATE_DSP_ARCH_AARCH64_ASIMD_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_AARCH64_ASIMD_IMPL */
28
29 namespace lsp
30 {
31 namespace asimd
32 {
33 float h_dotp(const float *a, const float *b, size_t count)
34 {
35 float res;
36 ARCH_AARCH64_ASM
37 (
38 __ASM_EMIT("eor v16.16b, v16.16b, v16.16b")
39 __ASM_EMIT("eor v17.16b, v17.16b, v17.16b")
40 __ASM_EMIT("subs %[count], %[count], #16")
41 __ASM_EMIT("b.lt 2f")
42 /* 16x block */
43 __ASM_EMIT("1:")
44 __ASM_EMIT("ldp q0, q1, [%[a], #0x00]")
45 __ASM_EMIT("ldp q2, q3, [%[a], #0x20]")
46 __ASM_EMIT("ldp q4, q5, [%[b], #0x00]")
47 __ASM_EMIT("ldp q6, q7, [%[b], #0x20]")
48 __ASM_EMIT("fmla v16.4s, v0.4s, v4.4s")
49 __ASM_EMIT("fmla v17.4s, v1.4s, v5.4s")
50 __ASM_EMIT("fmla v16.4s, v2.4s, v6.4s")
51 __ASM_EMIT("fmla v17.4s, v3.4s, v7.4s")
52 __ASM_EMIT("subs %[count], %[count], #16")
53 __ASM_EMIT("add %[a], %[a], #0x40")
54 __ASM_EMIT("add %[b], %[b], #0x40")
55 __ASM_EMIT("b.hs 1b")
56 /* 8x block */
57 __ASM_EMIT("2:")
58 __ASM_EMIT("adds %[count], %[count], #8")
59 __ASM_EMIT("b.lt 4f")
60 __ASM_EMIT("ldp q0, q1, [%[a], #0x00]")
61 __ASM_EMIT("ldp q4, q5, [%[b], #0x00]")
62 __ASM_EMIT("fmla v16.4s, v0.4s, v4.4s")
63 __ASM_EMIT("fmla v17.4s, v1.4s, v5.4s")
64 __ASM_EMIT("sub %[count], %[count], #8")
65 __ASM_EMIT("add %[a], %[a], #0x20")
66 __ASM_EMIT("add %[b], %[b], #0x20")
67 /* 4x block */
68 __ASM_EMIT("4:")
69 __ASM_EMIT("adds %[count], %[count], #4")
70 __ASM_EMIT("fadd v16.4s, v16.4s, v17.4s")
71 __ASM_EMIT("b.lt 6f")
72 __ASM_EMIT("ldr q0, [%[a], #0x00]")
73 __ASM_EMIT("ldr q4, [%[b], #0x00]")
74 __ASM_EMIT("fmla v16.4s, v0.4s, v4.4s")
75 __ASM_EMIT("sub %[count], %[count], #4")
76 __ASM_EMIT("add %[a], %[a], #0x10")
77 __ASM_EMIT("add %[b], %[b], #0x10")
78 /* 1x block */
79 __ASM_EMIT("6:")
80 __ASM_EMIT("ext v17.16b, v16.16b, v16.16b, #8")
81 __ASM_EMIT("fadd v16.4s, v16.4s, v17.4s")
82 __ASM_EMIT("ext v17.16b, v16.16b, v16.16b, #4")
83 __ASM_EMIT("fadd %[res].4s, v16.4s, v17.4s")
84 __ASM_EMIT("adds %[count], %[count], #3")
85 __ASM_EMIT("b.lt 8f")
86 __ASM_EMIT("7:")
87 __ASM_EMIT("ld1 {v1.s}[0], [%[a]]")
88 __ASM_EMIT("ld1 {v4.s}[0], [%[b]]")
89 __ASM_EMIT("fmla %[res].4s, v1.4s, v4.4s")
90 __ASM_EMIT("subs %[count], %[count], #1")
91 __ASM_EMIT("add %[a], %[a], #0x04")
92 __ASM_EMIT("add %[b], %[b], #0x04")
93 __ASM_EMIT("b.ge 7b")
94 /* end of sum */
95 __ASM_EMIT("8:")
96 : [res] "=w" (res),
97 [a] "+r" (a), [b] "+r" (b), [count] "+r" (count)
98 :
99 : "cc", "memory",
100 "v1", "v2", "v3", "v4", "v5", "v6", "v7",
101 "v16", "v17"
102 );
103
104 return res;
105 }
106
107 float h_sqr_dotp(const float *a, const float *b, size_t count)
108 {
109 float res;
110 ARCH_AARCH64_ASM
111 (
112 __ASM_EMIT("eor v16.16b, v16.16b, v16.16b")
113 __ASM_EMIT("eor v17.16b, v17.16b, v17.16b")
114 __ASM_EMIT("subs %[count], %[count], #16")
115 __ASM_EMIT("b.lt 2f")
116 /* 16x block */
117 __ASM_EMIT("1:")
118 __ASM_EMIT("ldp q0, q1, [%[a], #0x00]")
119 __ASM_EMIT("ldp q2, q3, [%[a], #0x20]")
120 __ASM_EMIT("ldp q4, q5, [%[b], #0x00]")
121 __ASM_EMIT("ldp q6, q7, [%[b], #0x20]")
122 __ASM_EMIT("fmul v0.4s, v0.4s, v0.4s")
123 __ASM_EMIT("fmul v1.4s, v1.4s, v1.4s")
124 __ASM_EMIT("fmul v2.4s, v2.4s, v2.4s")
125 __ASM_EMIT("fmul v3.4s, v3.4s, v3.4s")
126 __ASM_EMIT("fmul v4.4s, v4.4s, v4.4s")
127 __ASM_EMIT("fmul v5.4s, v5.4s, v5.4s")
128 __ASM_EMIT("fmul v6.4s, v6.4s, v6.4s")
129 __ASM_EMIT("fmul v7.4s, v7.4s, v7.4s")
130 __ASM_EMIT("fmla v16.4s, v0.4s, v4.4s")
131 __ASM_EMIT("fmla v17.4s, v1.4s, v5.4s")
132 __ASM_EMIT("fmla v16.4s, v2.4s, v6.4s")
133 __ASM_EMIT("fmla v17.4s, v3.4s, v7.4s")
134 __ASM_EMIT("subs %[count], %[count], #16")
135 __ASM_EMIT("add %[a], %[a], #0x40")
136 __ASM_EMIT("add %[b], %[b], #0x40")
137 __ASM_EMIT("b.hs 1b")
138 /* 8x block */
139 __ASM_EMIT("2:")
140 __ASM_EMIT("adds %[count], %[count], #8")
141 __ASM_EMIT("b.lt 4f")
142 __ASM_EMIT("ldp q0, q1, [%[a], #0x00]")
143 __ASM_EMIT("ldp q4, q5, [%[b], #0x00]")
144 __ASM_EMIT("fmul v0.4s, v0.4s, v0.4s")
145 __ASM_EMIT("fmul v1.4s, v1.4s, v1.4s")
146 __ASM_EMIT("fmul v4.4s, v4.4s, v4.4s")
147 __ASM_EMIT("fmul v5.4s, v5.4s, v5.4s")
148 __ASM_EMIT("fmla v16.4s, v0.4s, v4.4s")
149 __ASM_EMIT("fmla v17.4s, v1.4s, v5.4s")
150 __ASM_EMIT("sub %[count], %[count], #8")
151 __ASM_EMIT("add %[a], %[a], #0x20")
152 __ASM_EMIT("add %[b], %[b], #0x20")
153 /* 4x block */
154 __ASM_EMIT("4:")
155 __ASM_EMIT("adds %[count], %[count], #4")
156 __ASM_EMIT("fadd v16.4s, v16.4s, v17.4s")
157 __ASM_EMIT("b.lt 6f")
158 __ASM_EMIT("ldr q0, [%[a], #0x00]")
159 __ASM_EMIT("ldr q4, [%[b], #0x00]")
160 __ASM_EMIT("fmul v0.4s, v0.4s, v0.4s")
161 __ASM_EMIT("fmul v4.4s, v4.4s, v4.4s")
162 __ASM_EMIT("fmla v16.4s, v0.4s, v4.4s")
163 __ASM_EMIT("sub %[count], %[count], #4")
164 __ASM_EMIT("add %[a], %[a], #0x10")
165 __ASM_EMIT("add %[b], %[b], #0x10")
166 /* 1x block */
167 __ASM_EMIT("6:")
168 __ASM_EMIT("ext v17.16b, v16.16b, v16.16b, #8")
169 __ASM_EMIT("fadd v16.4s, v16.4s, v17.4s")
170 __ASM_EMIT("ext v17.16b, v16.16b, v16.16b, #4")
171 __ASM_EMIT("fadd %[res].4s, v16.4s, v17.4s")
172 __ASM_EMIT("adds %[count], %[count], #3")
173 __ASM_EMIT("b.lt 8f")
174 __ASM_EMIT("7:")
175 __ASM_EMIT("ld1 {v1.s}[0], [%[a]]")
176 __ASM_EMIT("ld1 {v4.s}[0], [%[b]]")
177 __ASM_EMIT("fmul v1.4s, v1.4s, v1.4s")
178 __ASM_EMIT("fmul v4.4s, v4.4s, v4.4s")
179 __ASM_EMIT("fmla %[res].4s, v1.4s, v4.4s")
180 __ASM_EMIT("subs %[count], %[count], #1")
181 __ASM_EMIT("add %[a], %[a], #0x04")
182 __ASM_EMIT("add %[b], %[b], #0x04")
183 __ASM_EMIT("b.ge 7b")
184 /* end of sum */
185 __ASM_EMIT("8:")
186
187 : [res] "=w" (res),
188 [a] "+r" (a), [b] "+r" (b), [count] "+r" (count)
189 :
190 : "cc", "memory",
191 "v1", "v2", "v3", "v4", "v5", "v6", "v7",
192 "v16", "v17"
193 );
194
195 return res;
196 }
197
198 float h_abs_dotp(const float *a, const float *b, size_t count)
199 {
200 IF_ARCH_AARCH64(float res);
201 ARCH_AARCH64_ASM
202 (
203 __ASM_EMIT("eor v16.16b, v16.16b, v16.16b")
204 __ASM_EMIT("eor v17.16b, v17.16b, v17.16b")
205 __ASM_EMIT("subs %[count], %[count], #16")
206 __ASM_EMIT("b.lt 2f")
207 /* 16x block */
208 __ASM_EMIT("1:")
209 __ASM_EMIT("ldp q0, q1, [%[a], #0x00]")
210 __ASM_EMIT("ldp q2, q3, [%[a], #0x20]")
211 __ASM_EMIT("ldp q4, q5, [%[b], #0x00]")
212 __ASM_EMIT("ldp q6, q7, [%[b], #0x20]")
213 __ASM_EMIT("fabs v0.4s, v0.4s")
214 __ASM_EMIT("fabs v1.4s, v1.4s")
215 __ASM_EMIT("fabs v2.4s, v2.4s")
216 __ASM_EMIT("fabs v3.4s, v3.4s")
217 __ASM_EMIT("fabs v4.4s, v4.4s")
218 __ASM_EMIT("fabs v5.4s, v5.4s")
219 __ASM_EMIT("fabs v6.4s, v6.4s")
220 __ASM_EMIT("fabs v7.4s, v7.4s")
221 __ASM_EMIT("fmla v16.4s, v0.4s, v4.4s")
222 __ASM_EMIT("fmla v17.4s, v1.4s, v5.4s")
223 __ASM_EMIT("fmla v16.4s, v2.4s, v6.4s")
224 __ASM_EMIT("fmla v17.4s, v3.4s, v7.4s")
225 __ASM_EMIT("subs %[count], %[count], #16")
226 __ASM_EMIT("add %[a], %[a], #0x40")
227 __ASM_EMIT("add %[b], %[b], #0x40")
228 __ASM_EMIT("b.hs 1b")
229 /* 8x block */
230 __ASM_EMIT("2:")
231 __ASM_EMIT("adds %[count], %[count], #8")
232 __ASM_EMIT("b.lt 4f")
233 __ASM_EMIT("ldp q0, q1, [%[a], #0x00]")
234 __ASM_EMIT("ldp q4, q5, [%[b], #0x00]")
235 __ASM_EMIT("fabs v0.4s, v0.4s")
236 __ASM_EMIT("fabs v1.4s, v1.4s")
237 __ASM_EMIT("fabs v4.4s, v4.4s")
238 __ASM_EMIT("fabs v5.4s, v5.4s")
239 __ASM_EMIT("fmla v16.4s, v0.4s, v4.4s")
240 __ASM_EMIT("fmla v17.4s, v1.4s, v5.4s")
241 __ASM_EMIT("sub %[count], %[count], #8")
242 __ASM_EMIT("add %[a], %[a], #0x20")
243 __ASM_EMIT("add %[b], %[b], #0x20")
244 /* 4x block */
245 __ASM_EMIT("4:")
246 __ASM_EMIT("adds %[count], %[count], #4")
247 __ASM_EMIT("fadd v16.4s, v16.4s, v17.4s")
248 __ASM_EMIT("b.lt 6f")
249 __ASM_EMIT("ldr q0, [%[a], #0x00]")
250 __ASM_EMIT("ldr q4, [%[b], #0x00]")
251 __ASM_EMIT("fabs v0.4s, v0.4s")
252 __ASM_EMIT("fabs v4.4s, v4.4s")
253 __ASM_EMIT("fmla v16.4s, v0.4s, v4.4s")
254 __ASM_EMIT("sub %[count], %[count], #4")
255 __ASM_EMIT("add %[a], %[a], #0x10")
256 __ASM_EMIT("add %[b], %[b], #0x10")
257 /* 1x block */
258 __ASM_EMIT("6:")
259 __ASM_EMIT("ext v17.16b, v16.16b, v16.16b, #8")
260 __ASM_EMIT("fadd v16.4s, v16.4s, v17.4s")
261 __ASM_EMIT("ext v17.16b, v16.16b, v16.16b, #4")
262 __ASM_EMIT("fadd %[res].4s, v16.4s, v17.4s")
263 __ASM_EMIT("adds %[count], %[count], #3")
264 __ASM_EMIT("b.lt 8f")
265 __ASM_EMIT("7:")
266 __ASM_EMIT("ld1 {v1.s}[0], [%[a]]")
267 __ASM_EMIT("ld1 {v4.s}[0], [%[b]]")
268 __ASM_EMIT("fabs v1.4s, v1.4s")
269 __ASM_EMIT("fabs v4.4s, v4.4s")
270 __ASM_EMIT("fmla %[res].4s, v1.4s, v4.4s")
271 __ASM_EMIT("subs %[count], %[count], #1")
272 __ASM_EMIT("add %[a], %[a], #0x04")
273 __ASM_EMIT("add %[b], %[b], #0x04")
274 __ASM_EMIT("b.ge 7b")
275 /* end of sum */
276 __ASM_EMIT("8:")
277
278 : [res] "=w" (res),
279 [a] "+r" (a), [b] "+r" (b), [count] "+r" (count)
280 :
281 : "cc", "memory",
282 "v1", "v2", "v3", "v4", "v5", "v6", "v7",
283 "v16", "v17"
284 );
285
286 return res;
287 }
288 }
289 }
290
291 #endif /* PRIVATE_DSP_ARCH_AARCH64_ASIMD_HMATH_HDOTP_H_ */