Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/aarch64/asimd/float.h

raw
1 /*
2 * Copyright (C) 2020 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2020 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 31 мар. 2020 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_AARCH64_ASIMD_FLOAT_H_
23 #define PRIVATE_DSP_ARCH_AARCH64_ASIMD_FLOAT_H_
24
25 #ifndef PRIVATE_DSP_ARCH_AARCH64_ASIMD_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_AARCH64_ASIMD_IMPL */
28
29 namespace lsp
30 {
31 namespace asimd
32 {
33 #define SEL_DST(a, b) a
34 #define SEL_NODST(a, b) b
35
36 IF_ARCH_AARCH64(
37 static uint32_t SAT_IARGS[] __lsp_aligned16 =
38 {
39 LSP_DSP_VEC4(0x7f800000), // X_P_INF
40 LSP_DSP_VEC4(0xff800000), // X_N_INF
41 LSP_DSP_VEC4(LSP_DSP_FLOAT_SAT_P_NAN_I), // SX_P_NAN
42 LSP_DSP_VEC4(LSP_DSP_FLOAT_SAT_P_INF_I), // SX_P_INF
43 LSP_DSP_VEC4(LSP_DSP_FLOAT_SAT_N_NAN_I), // SX_N_NAN
44 LSP_DSP_VEC4(LSP_DSP_FLOAT_SAT_N_INF_I) // SX_N_INF
45 };
46 )
47
48 #define MULTIPLE_SAT_BODY(DST, SRC, SEL) \
49 __ASM_EMIT("ldp q16, q17, [%[IARGS], #0x00]") /* v16 = +inf, v17 = -inf */ \
50 __ASM_EMIT("ldp q18, q19, [%[IARGS], #0x20]") /* v18 = X_P_NAN, v19 = X_P_INF */ \
51 __ASM_EMIT("ldp q20, q21, [%[IARGS], #0x40]") /* v20 = X_N_NAN, v21 = X_N_INF */ \
52 /* x8 blocks */ \
53 __ASM_EMIT("subs %[count], %[count], #8") \
54 __ASM_EMIT("b.lo 2f") \
55 __ASM_EMIT("1:") \
56 __ASM_EMIT("ldp q0, q1, [%[" SRC "]]") /* v0 = s0..s3, v1 = s4..s7 */ \
57 __ASM_EMIT("cmgt v2.4s, v0.4s, v16.4s") /* v2 = [s > +inf] */ \
58 __ASM_EMIT("cmgt v3.4s, v1.4s, v16.4s") \
59 __ASM_EMIT("cmeq v4.4s, v0.4s, v16.4s") /* v4 = [s == +inf] */ \
60 __ASM_EMIT("cmeq v5.4s, v1.4s, v16.4s") \
61 __ASM_EMIT("bit v0.16b, v18.16b, v2.16b") /* v0 = s' = X_P_NAN * [s > +inf] | s * [s <= +inf] */ \
62 __ASM_EMIT("bit v1.16b, v18.16b, v3.16b") \
63 __ASM_EMIT("bit v0.16b, v19.16b, v4.16b") /* v0 = X_P_INF * [s == +inf] | s' * [s != +inf] */ \
64 __ASM_EMIT("bit v1.16b, v19.16b, v5.16b") \
65 __ASM_EMIT("cmhi v2.4s, v0.4s, v17.4s") /* v2 = [s > -inf] */ \
66 __ASM_EMIT("cmhi v3.4s, v1.4s, v17.4s") \
67 __ASM_EMIT("cmeq v4.4s, v0.4s, v17.4s") /* v4 = [s == -inf] */ \
68 __ASM_EMIT("cmeq v5.4s, v1.4s, v17.4s") \
69 __ASM_EMIT("bit v0.16b, v20.16b, v2.16b") /* v0 = s' = X_N_NAN * [s > -inf] | s * [s <= -inf] */ \
70 __ASM_EMIT("bit v1.16b, v20.16b, v3.16b") \
71 __ASM_EMIT("bit v0.16b, v21.16b, v4.16b") /* v0 = X_N_INF * [s == +inf] | s' * [s != +inf] */ \
72 __ASM_EMIT("bit v1.16b, v21.16b, v5.16b") \
73 __ASM_EMIT("stp q0, q1, [%[" DST "]]") \
74 __ASM_EMIT("subs %[count], %[count], #8") \
75 __ASM_EMIT("add %[" SRC "], %[" SRC "], #0x20") \
76 __ASM_EMIT(SEL("add %[" DST "], %[" DST "], #0x20", "")) \
77 __ASM_EMIT("b.hs 1b") \
78 /* x4 block */ \
79 __ASM_EMIT("2:") \
80 __ASM_EMIT("adds %[count], %[count], #4") \
81 __ASM_EMIT("b.lt 4f") \
82 __ASM_EMIT("ldr q0, [%[" SRC "]]") /* v0 = s0..s3 */ \
83 __ASM_EMIT("cmgt v2.4s, v0.4s, v16.4s") /* v2 = [s > +inf] */ \
84 __ASM_EMIT("cmeq v4.4s, v0.4s, v16.4s") /* v4 = [s == +inf] */ \
85 __ASM_EMIT("bit v0.16b, v18.16b, v2.16b") /* v0 = s' = X_P_NAN * [s > +inf] | s * [s <= +inf] */ \
86 __ASM_EMIT("bit v0.16b, v19.16b, v4.16b") /* v0 = X_P_INF * [s == +inf] | s' * [s != +inf] */ \
87 __ASM_EMIT("cmhi v2.4s, v0.4s, v17.4s") /* v2 = [s > -inf] */ \
88 __ASM_EMIT("cmeq v4.4s, v0.4s, v17.4s") /* v4 = [s == -inf] */ \
89 __ASM_EMIT("bit v0.16b, v20.16b, v2.16b") /* v0 = s' = X_N_NAN * [s > -inf] | s * [s <= -inf] */ \
90 __ASM_EMIT("bit v0.16b, v21.16b, v4.16b") /* v0 = X_N_INF * [s == +inf] | s' * [s != +inf] */ \
91 __ASM_EMIT("str q0, [%[" DST "]]") \
92 __ASM_EMIT("sub %[count], %[count], #4") \
93 __ASM_EMIT("add %[" SRC "], %[" SRC "], #0x10") \
94 __ASM_EMIT(SEL("add %[" DST "], %[" DST "], #0x10", "")) \
95 __ASM_EMIT("4:") \
96 /* x1 blocks */ \
97 __ASM_EMIT("adds %[count], %[count], #3") \
98 __ASM_EMIT("b.lt 6f") \
99 __ASM_EMIT("5:") \
100 __ASM_EMIT("ld1r {v0.4s}, [%[" SRC "]]") /* v0 = s */ \
101 __ASM_EMIT("cmgt v2.4s, v0.4s, v16.4s") /* v2 = [s > +inf] */ \
102 __ASM_EMIT("cmeq v4.4s, v0.4s, v16.4s") /* v4 = [s == +inf] */ \
103 __ASM_EMIT("bit v0.16b, v18.16b, v2.16b") /* v0 = s' = X_P_NAN * [s > +inf] | s * [s <= +inf] */ \
104 __ASM_EMIT("bit v0.16b, v19.16b, v4.16b") /* v0 = X_P_INF * [s == +inf] | s' * [s != +inf] */ \
105 __ASM_EMIT("cmhi v2.4s, v0.4s, v17.4s") /* v2 = [s > -inf] */ \
106 __ASM_EMIT("cmeq v4.4s, v0.4s, v17.4s") /* v4 = [s == -inf] */ \
107 __ASM_EMIT("bit v0.16b, v20.16b, v2.16b") /* v0 = s' = X_N_NAN * [s > -inf] | s * [s <= -inf] */ \
108 __ASM_EMIT("bit v0.16b, v21.16b, v4.16b") /* v0 = X_N_INF * [s == +inf] | s' * [s != +inf] */ \
109 __ASM_EMIT("st1 {v0.s}[0], [%[" DST "]]") \
110 __ASM_EMIT("subs %[count], %[count], #1") \
111 __ASM_EMIT("add %[" SRC "], %[" SRC "], #0x04") \
112 __ASM_EMIT(SEL("add %[" DST "], %[" DST "], #0x04", "")) \
113 __ASM_EMIT("b.ge 5b") \
114 __ASM_EMIT("6:")
115
116 void copy_saturated(float *dst, const float *src, size_t count)
117 {
118 ARCH_AARCH64_ASM (
119 MULTIPLE_SAT_BODY("dst", "src", SEL_DST)
120 : [dst] "+r" (dst), [src] "+r" (src),
121 [count] "+r" (count)
122 : [IARGS] "r" (&SAT_IARGS[0])
123 : "cc", "memory",
124 "v0", "v1", "v2", "v3",
125 "v4", "v5",
126 "v16", "v17", "v18", "v19",
127 "v20", "v21"
128 );
129 }
130
131 void saturate(float *dst, size_t count)
132 {
133 ARCH_AARCH64_ASM(
134 MULTIPLE_SAT_BODY("dst", "dst", SEL_NODST)
135 : [dst] "+r" (dst), [count] "+r" (count)
136 : [IARGS] "r" (&SAT_IARGS[0])
137 : "cc", "memory",
138 "v0", "v1", "v2", "v3",
139 "v4", "v5",
140 "v16", "v17", "v18", "v19",
141 "v20", "v21"
142 );
143 }
144
145 #undef MULTIPLE_SAT_BODY
146
147 #define LIMIT_SAT_BODY(DST, SRC, SEL) \
148 __ASM_EMIT("ldp q16, q17, [%[XC], #0x00]") /* v16 = abs, v17 = +1 */ \
149 __ASM_EMIT("ldp q18, q19, [%[XC], #0x20]") /* v18 = sign, v19 = +Inf */ \
150 /* x8 blocks */ \
151 __ASM_EMIT("subs %[count], %[count], #8") \
152 __ASM_EMIT("b.lo 2f") \
153 __ASM_EMIT("1:") \
154 __ASM_EMIT("ldp q0, q1, [%[" SRC "]]") /* v0 = s0, v1 = s1 */ \
155 __ASM_EMIT("and v6.16b, v0.16b, v16.16b") /* v6 = abs(s) */ \
156 __ASM_EMIT("and v7.16b, v1.16b, v16.16b") \
157 __ASM_EMIT("fcmge v2.4s, v17.4s, v6.4s") /* v2 = [ abs(s) <= +1 ] */ \
158 __ASM_EMIT("fcmge v3.4s, v17.4s, v7.4s") \
159 __ASM_EMIT("and v4.16b, v0.16b, v18.16b") /* v4 = sign(s) */ \
160 __ASM_EMIT("and v5.16b, v1.16b, v18.16b") \
161 __ASM_EMIT("cmge v6.4s, v19.4s, v6.4s") /* v6 = [ abs(s) <= +Inf ] */ \
162 __ASM_EMIT("cmge v7.4s, v19.4s, v7.4s") \
163 __ASM_EMIT("orr v4.16b, v4.16b, v17.16b") /* v4 = +1 * sign(s) */ \
164 __ASM_EMIT("orr v5.16b, v5.16b, v17.16b") \
165 __ASM_EMIT("and v4.16b, v4.16b, v6.16b") /* v4 = r = +1 * sign(s) & [ abs(s) <= +Inf ] */ \
166 __ASM_EMIT("and v5.16b, v5.16b, v7.16b") \
167 __ASM_EMIT("bif v0.16b, v4.16b, v2.16b") /* v0 = ( s & [ abs(s) <= +1 ]) | ( r & [ abs(s) > +1 ]) */ \
168 __ASM_EMIT("bif v1.16b, v5.16b, v3.16b") \
169 __ASM_EMIT("stp q0, q1, [%[" DST "]]") \
170 __ASM_EMIT("subs %[count], %[count], #8") \
171 __ASM_EMIT("add %[" SRC "], %[" SRC "], #0x20") \
172 __ASM_EMIT(SEL("add %[" DST "], %[" DST "], #0x20", "")) \
173 __ASM_EMIT("b.hs 1b") \
174 __ASM_EMIT("2:") \
175 /* x4 block */ \
176 __ASM_EMIT("adds %[count], %[count], #4") \
177 __ASM_EMIT("b.lt 4f") \
178 __ASM_EMIT("ldr q0, [%[" SRC "]]") /* v0 = s0, v1 = s1 */ \
179 __ASM_EMIT("and v6.16b, v0.16b, v16.16b") /* v6 = abs(s) */ \
180 __ASM_EMIT("fcmge v2.4s, v17.4s, v6.4s") /* v2 = [ abs(s) <= +1 ] */ \
181 __ASM_EMIT("and v4.16b, v0.16b, v18.16b") /* v4 = sign(s) */ \
182 __ASM_EMIT("cmge v6.4s, v19.4s, v6.4s") /* v6 = [ abs(s) <= +Inf ] */ \
183 __ASM_EMIT("orr v4.16b, v4.16b, v17.16b") /* v4 = +1 * sign(s) */ \
184 __ASM_EMIT("and v4.16b, v4.16b, v6.16b") /* v4 = r = +1 * sign(s) & [ abs(s) <= +Inf ] */ \
185 __ASM_EMIT("bif v0.16b, v4.16b, v2.16b") /* v0 = ( s & [ abs(s) <= +1 ]) | ( r & [ abs(s) > +1 ]) */ \
186 __ASM_EMIT("str q0, [%[" DST "]]") \
187 __ASM_EMIT("sub %[count], %[count], #4") \
188 __ASM_EMIT("add %[" SRC "], %[" SRC "], #0x10") \
189 __ASM_EMIT(SEL("add %[" DST "], %[" DST "], #0x10", "")) \
190 __ASM_EMIT("4:") \
191 /* 1x blocks */ \
192 __ASM_EMIT("adds %[count], %[count], #3") \
193 __ASM_EMIT("b.lt 6f") \
194 __ASM_EMIT("5:") \
195 __ASM_EMIT("ld1r {v0.4s}, [%[" SRC "]]") /* v0 = s0, v1 = s1 */ \
196 __ASM_EMIT("and v6.16b, v0.16b, v16.16b") /* v6 = abs(s) */ \
197 __ASM_EMIT("fcmge v2.4s, v17.4s, v6.4s") /* v2 = [ abs(s) <= +1 ] */ \
198 __ASM_EMIT("and v4.16b, v0.16b, v18.16b") /* v4 = sign(s) */ \
199 __ASM_EMIT("cmge v6.4s, v19.4s, v6.4s") /* v6 = [ abs(s) <= +Inf ] */ \
200 __ASM_EMIT("orr v4.16b, v4.16b, v17.16b") /* v4 = +1 * sign(s) */ \
201 __ASM_EMIT("and v4.16b, v4.16b, v6.16b") /* v4 = r = +1 * sign(s) & [ abs(s) <= +Inf ] */ \
202 __ASM_EMIT("bif v0.16b, v4.16b, v2.16b") /* v0 = ( s & [ abs(s) <= +1 ]) | ( r & [ abs(s) > +1 ]) */ \
203 __ASM_EMIT("st1 {v0.s}[0], [%[" DST "]]") \
204 __ASM_EMIT("subs %[count], %[count], #1") \
205 __ASM_EMIT("add %[" SRC "], %[" SRC "], #0x04") \
206 __ASM_EMIT(SEL("add %[" DST "], %[" DST "], #0x04", "")) \
207 __ASM_EMIT("b.ge 5b") \
208 __ASM_EMIT("6:")
209
210 IF_ARCH_AARCH64(
211 static uint32_t XLIM_SAT[] __lsp_aligned16 =
212 {
213 LSP_DSP_VEC4(0x7fffffff), // abs
214 LSP_DSP_VEC4(0x3f800000), // +1
215 LSP_DSP_VEC4(0x80000000), // sign
216 LSP_DSP_VEC4(0x7f800000) // +Inf
217 };
218 )
219
220 void limit_saturate1(float *dst, size_t count)
221 {
222 ARCH_AARCH64_ASM(
223 LIMIT_SAT_BODY("dst", "dst", SEL_NODST)
224 : [dst] "+r" (dst), [count] "+r" (count)
225 : [XC] "r" (&XLIM_SAT[0])
226 : "cc", "memory",
227 "v0", "v1", "v2", "v3",
228 "v4", "v5", "v6", "v7",
229 "v16", "v17", "v18", "v19"
230 );
231 }
232
233 void limit_saturate2(float *dst, const float *src, size_t count)
234 {
235 ARCH_AARCH64_ASM(
236 LIMIT_SAT_BODY("dst", "src", SEL_DST)
237 : [dst] "+r" (dst), [src] "+r" (src),
238 [count] "+r" (count)
239 : [XC] "r" (&XLIM_SAT[0])
240 : "cc", "memory",
241 "v0", "v1", "v2", "v3",
242 "v4", "v5", "v6", "v7",
243 "v16", "v17", "v18", "v19"
244 );
245 }
246
247 #undef LIMIT_SAT_BODY
248
249 #define LIMIT_BODY(DST, SRC, SEL) \
250 __ASM_EMIT("dup v16.4s, %[min].s[0]") /* v16 = min */ \
251 __ASM_EMIT("dup v17.4s, %[max].s[0]") /* v17 = max */ \
252 /* x16 blocks */ \
253 __ASM_EMIT("subs %[count], %[count], #16") \
254 __ASM_EMIT("b.lo 2f") \
255 __ASM_EMIT("1:") \
256 __ASM_EMIT("ldp q0, q1, [%[" SRC "], #0x00]") /* v0 = s0, v1 = s1 */ \
257 __ASM_EMIT("ldp q2, q3, [%[" SRC "], #0x20]") /* v0 = s0, v1 = s1 */ \
258 __ASM_EMIT("fcmge v4.4s, v0.4s, v16.4s") /* v4 = c = [ (s >= min) && (!isnan(s)) ] */ \
259 __ASM_EMIT("fcmge v5.4s, v1.4s, v16.4s") \
260 __ASM_EMIT("fcmge v6.4s, v2.4s, v16.4s") \
261 __ASM_EMIT("fcmge v7.4s, v3.4s, v16.4s") \
262 __ASM_EMIT("bif v0.16b, v16.16b, v4.16b") /* v0 = s*c | min*(!c) */ \
263 __ASM_EMIT("bif v1.16b, v16.16b, v5.16b") \
264 __ASM_EMIT("bif v2.16b, v16.16b, v6.16b") \
265 __ASM_EMIT("bif v3.16b, v16.16b, v7.16b") \
266 __ASM_EMIT("fcmge v4.4s, v17.4s, v0.4s") /* v4 = c = [ (s <= max) && (!isnan(s)) ] */ \
267 __ASM_EMIT("fcmge v5.4s, v17.4s, v1.4s") \
268 __ASM_EMIT("fcmge v6.4s, v17.4s, v2.4s") \
269 __ASM_EMIT("fcmge v7.4s, v17.4s, v3.4s") \
270 __ASM_EMIT("bif v0.16b, v17.16b, v4.16b") /* v0 = s*c | min*(!c) */ \
271 __ASM_EMIT("bif v1.16b, v17.16b, v5.16b") \
272 __ASM_EMIT("bif v2.16b, v17.16b, v6.16b") \
273 __ASM_EMIT("bif v3.16b, v17.16b, v7.16b") \
274 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
275 __ASM_EMIT("stp q2, q3, [%[" DST "], #0x20]") \
276 __ASM_EMIT("subs %[count], %[count], #16") \
277 __ASM_EMIT("add %[" SRC "], %[" SRC "], #0x40") \
278 __ASM_EMIT(SEL("add %[" DST "], %[" DST "], #0x40", "")) \
279 __ASM_EMIT("b.hs 1b") \
280 __ASM_EMIT("2:") \
281 /* x8 block */ \
282 __ASM_EMIT("adds %[count], %[count], #8") \
283 __ASM_EMIT("b.lt 4f") \
284 __ASM_EMIT("ldp q0, q1, [%[" SRC "], #0x00]") /* v0 = s0, v1 = s1 */ \
285 __ASM_EMIT("fcmge v4.4s, v0.4s, v16.4s") /* v4 = c = [ (s >= min) && (!isnan(s)) ] */ \
286 __ASM_EMIT("fcmge v5.4s, v1.4s, v16.4s") \
287 __ASM_EMIT("bif v0.16b, v16.16b, v4.16b") /* v0 = s*c | min*(!c) */ \
288 __ASM_EMIT("bif v1.16b, v16.16b, v5.16b") \
289 __ASM_EMIT("fcmge v4.4s, v17.4s, v0.4s") /* v4 = c = [ (s <= max) && (!isnan(s)) ] */ \
290 __ASM_EMIT("fcmge v5.4s, v17.4s, v1.4s") \
291 __ASM_EMIT("bif v0.16b, v17.16b, v4.16b") /* v0 = s*c | min*(!c) */ \
292 __ASM_EMIT("bif v1.16b, v17.16b, v5.16b") \
293 __ASM_EMIT("stp q0, q1, [%[" DST "], #0x00]") \
294 __ASM_EMIT("sub %[count], %[count], #8") \
295 __ASM_EMIT("add %[" SRC "], %[" SRC "], #0x20") \
296 __ASM_EMIT(SEL("add %[" DST "], %[" DST "], #0x20", "")) \
297 __ASM_EMIT("4:") \
298 /* x4 block */ \
299 __ASM_EMIT("adds %[count], %[count], #4") \
300 __ASM_EMIT("b.lt 6f") \
301 __ASM_EMIT("ldr q0, [%[" SRC "]]") /* v0 = s0 */ \
302 __ASM_EMIT("fcmge v4.4s, v0.4s, v16.4s") /* v4 = c = [ (s >= min) && (!isnan(s)) ] */ \
303 __ASM_EMIT("bif v0.16b, v16.16b, v4.16b") /* v0 = s*c | min*(!c) */ \
304 __ASM_EMIT("fcmge v4.4s, v17.4s, v0.4s") /* v4 = c = [ (s <= max) && (!isnan(s)) ] */ \
305 __ASM_EMIT("bif v0.16b, v17.16b, v4.16b") /* v0 = s*c | min*(!c) */ \
306 __ASM_EMIT("str q0, [%[" DST "]]") \
307 __ASM_EMIT("sub %[count], %[count], #4") \
308 __ASM_EMIT("add %[" SRC "], %[" SRC "], #0x10") \
309 __ASM_EMIT(SEL("add %[" DST "], %[" DST "], #0x10", "")) \
310 __ASM_EMIT("6:") \
311 /* 1x blocks */ \
312 __ASM_EMIT("adds %[count], %[count], #3") \
313 __ASM_EMIT("b.lt 8f") \
314 __ASM_EMIT("7:") \
315 __ASM_EMIT("ld1r {v0.4s}, [%[" SRC "]]") /* v0 = s0 */ \
316 __ASM_EMIT("fcmge v4.4s, v0.4s, v16.4s") /* v4 = c = [ (s >= min) && (!isnan(s)) ] */ \
317 __ASM_EMIT("bif v0.16b, v16.16b, v4.16b") /* v0 = s*c | min*(!c) */ \
318 __ASM_EMIT("fcmge v4.4s, v17.4s, v0.4s") /* v4 = c = [ (s <= max) && (!isnan(s)) ] */ \
319 __ASM_EMIT("bif v0.16b, v17.16b, v4.16b") /* v0 = s*c | min*(!c) */ \
320 __ASM_EMIT("st1 {v0.s}[0], [%[" DST "]]") \
321 __ASM_EMIT("subs %[count], %[count], #1") \
322 __ASM_EMIT("add %[" SRC "], %[" SRC "], #0x04") \
323 __ASM_EMIT(SEL("add %[" DST "], %[" DST "], #0x04", "")) \
324 __ASM_EMIT("b.ge 7b") \
325 __ASM_EMIT("8:")
326
327 void limit1(float *dst, float min, float max, size_t count)
328 {
329 ARCH_AARCH64_ASM(
330 LIMIT_BODY("dst", "dst", SEL_NODST)
331 : [dst] "+r" (dst), [count] "+r" (count),
332 [min] "+w" (min), [max] "+w" (max)
333 :
334 : "cc", "memory",
335 "v0", "v1", "v2", "v3",
336 "v4", "v5", "v6", "v7",
337 "v16", "v17"
338 );
339 }
340
341 void limit2(float *dst, const float *src, float min, float max, size_t count)
342 {
343 ARCH_AARCH64_ASM(
344 LIMIT_BODY("dst", "src", SEL_DST)
345 : [dst] "+r" (dst), [src] "+r" (src), [count] "+r" (count),
346 [min] "+w" (min), [max] "+w" (max)
347 :
348 : "cc", "memory",
349 "v0", "v1", "v2", "v3",
350 "v4", "v5", "v6", "v7",
351 "v16", "v17"
352 );
353 }
354
355 #undef LIMIT_BODY
356
357 #define SANITIZE_BODY(DST, SRC, SEL) \
358 __ASM_EMIT("ldp q16, q17, [%[CVAL], #0x00]") \
359 __ASM_EMIT("ldp q18, q19, [%[CVAL], #0x20]") \
360 __ASM_EMIT("subs %[count], %[count], #8") \
361 __ASM_EMIT("b.lo 2f") \
362 /* 8x blocks */ \
363 __ASM_EMIT("1:") \
364 __ASM_EMIT("ldp q0, q1, [%[" SRC "]]") /* v0 = s */ \
365 __ASM_EMIT("and v2.16b, v0.16b, v16.16b") /* v2 = abs(s) */ \
366 __ASM_EMIT("and v3.16b, v1.16b, v16.16b") \
367 __ASM_EMIT("and v4.16b, v0.16b, v17.16b") /* v4 = sign(s) */ \
368 __ASM_EMIT("and v5.16b, v1.16b, v17.16b") \
369 __ASM_EMIT("cmge v6.4s, v18.4s, v2.4s") /* v6 = abs(s) <= X_MAX */ \
370 __ASM_EMIT("cmge v7.4s, v18.4s, v3.4s") \
371 __ASM_EMIT("cmgt v2.4s, v2.4s, v19.4s") /* v2 = abs(s) > X_MIN */ \
372 __ASM_EMIT("cmgt v3.4s, v3.4s, v19.4s") \
373 __ASM_EMIT("and v2.16b, v2.16b, v6.16b") /* v2 = (abs(s) > X_MIN) & (abs(s) <= X_MIN) */ \
374 __ASM_EMIT("and v3.16b, v3.16b, v7.16b") \
375 __ASM_EMIT("bif v0.16b, v4.16b, v2.16b") /* v0 = ((abs(s) > X_MIN) & (abs(s) <= X_MIN)) ? s : sign(s) */ \
376 __ASM_EMIT("bif v1.16b, v5.16b, v3.16b") \
377 __ASM_EMIT("stp q0, q1, [%[" DST "]]") /* q0 = s */ \
378 __ASM_EMIT("subs %[count], %[count], #8") \
379 __ASM_EMIT("add %[" SRC "], %[" SRC "], #0x20") \
380 __ASM_EMIT(SEL("add %[" DST "], %[" DST "], #0x20", "")) \
381 __ASM_EMIT("b.hs 1b") \
382 /* 4x block */ \
383 __ASM_EMIT("2:") \
384 __ASM_EMIT("adds %[count], %[count], #4") \
385 __ASM_EMIT("b.lt 4f") \
386 __ASM_EMIT("ldr q0, [%[" SRC "]]") /* v0 = s */ \
387 __ASM_EMIT("and v2.16b, v0.16b, v16.16b") /* v2 = abs(s) */ \
388 __ASM_EMIT("and v4.16b, v0.16b, v17.16b") /* v4 = sign(s) */ \
389 __ASM_EMIT("cmge v6.4s, v18.4s, v2.4s") /* v6 = abs(s) <= X_MAX */ \
390 __ASM_EMIT("cmgt v2.4s, v2.4s, v19.4s") /* v2 = abs(s) > X_MIN */ \
391 __ASM_EMIT("and v2.16b, v2.16b, v6.16b") /* v2 = (abs(s) > X_MIN) & (abs(s) <= X_MIN) */ \
392 __ASM_EMIT("bif v0.16b, v4.16b, v2.16b") /* v0 = ((abs(s) > X_MIN) & (abs(s) <= X_MIN)) ? s : sign(s) */ \
393 __ASM_EMIT("str q0, [%[" DST "]]") /* q0 = s */ \
394 __ASM_EMIT("sub %[count], %[count], #4") \
395 __ASM_EMIT("add %[" SRC "], %[" SRC "], #0x10") \
396 __ASM_EMIT(SEL("add %[" DST "], %[" DST "], #0x10", "")) \
397 /* 1x blocks */ \
398 __ASM_EMIT("4:") \
399 __ASM_EMIT("adds %[count], %[count], #3") \
400 __ASM_EMIT("b.lt 6f") \
401 __ASM_EMIT("5:") \
402 __ASM_EMIT("ld1r {v0.4s}, [%[" SRC "]]") /* v0 = s */ \
403 __ASM_EMIT("and v2.16b, v0.16b, v16.16b") /* v2 = abs(s) */ \
404 __ASM_EMIT("and v4.16b, v0.16b, v17.16b") /* v4 = sign(s) */ \
405 __ASM_EMIT("cmge v6.4s, v18.4s, v2.4s") /* v6 = abs(s) <= X_MAX */ \
406 __ASM_EMIT("cmgt v2.4s, v2.4s, v19.4s") /* v2 = abs(s) > X_MIN */ \
407 __ASM_EMIT("and v2.16b, v2.16b, v6.16b") /* v2 = (abs(s) > X_MIN) & (abs(s) <= X_MIN) */ \
408 __ASM_EMIT("bif v0.16b, v4.16b, v2.16b") /* v0 = ((abs(s) > X_MIN) & (abs(s) <= X_MIN)) ? s : sign(s) */ \
409 __ASM_EMIT("st1 {v0.s}[0], [%[" DST "]]") /* q0 = s */ \
410 __ASM_EMIT("subs %[count], %[count], #1") \
411 __ASM_EMIT("add %[" SRC "], %[" SRC "], #0x04") \
412 __ASM_EMIT(SEL("add %[" DST "], %[" DST "], #0x04", "")) \
413 __ASM_EMIT("b.ge 5b") \
414 /* end */ \
415 __ASM_EMIT("6:")
416
417 IF_ARCH_AARCH64(
418 static uint32_t SANITIZE_CVAL[] __lsp_aligned16 =
419 {
420 LSP_DSP_VEC4(0x7fffffff), // X_ABS
421 LSP_DSP_VEC4(0x80000000), // X_SIGN
422 LSP_DSP_VEC4(0x7f7fffff), // X_MAX
423 LSP_DSP_VEC4(0x007fffff) // X_MIN
424 };
425 )
426
427 void sanitize1(float *dst, size_t count)
428 {
429 ARCH_AARCH64_ASM
430 (
431 SANITIZE_BODY("dst", "dst", SEL_NODST)
432 : [dst] "+r" (dst),
433 [count] "+r" (count)
434 : [CVAL] "r" (&SANITIZE_CVAL[0])
435 : "cc", "memory",
436 "v0", "v1", "v2", "v3",
437 "v4", "v5", "v6", "v7",
438 "v16", "v17", "v18", "v19"
439 );
440 }
441
442 void sanitize2(float *dst, const float *src, size_t count)
443 {
444 ARCH_AARCH64_ASM
445 (
446 SANITIZE_BODY("dst", "src", SEL_DST)
447 : [dst] "+r" (dst), [src] "+r" (src),
448 [count] "+r" (count)
449 : [CVAL] "r" (&SANITIZE_CVAL[0])
450 : "cc", "memory",
451 "v0", "v1", "v2", "v3",
452 "v4", "v5", "v6", "v7",
453 "v16", "v17", "v18", "v19"
454 );
455 }
456
457 #undef SANITIZE_BODY
458
459 #undef SEL_DST
460 #undef SEL_NODST
461 }
462 }
463
464 #endif /* PRIVATE_DSP_ARCH_AARCH64_ASIMD_FLOAT_H_ */