Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/aarch64/asimd/search/iminmax.h

raw
1 /*
2 * Copyright (C) 2020 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2020 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 31 мар. 2020 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_AARCH64_ASIMD_SEARCH_IMINMAX_H_
23 #define PRIVATE_DSP_ARCH_AARCH64_ASIMD_SEARCH_IMINMAX_H_
24
25 #ifndef PRIVATE_DSP_ARCH_AARCH64_ASIMD_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_AARCH64_ASIMD_IMPL */
28
29 namespace lsp
30 {
31 namespace asimd
32 {
33 IF_ARCH_AARCH64(
34 static uint32_t search_indexes[] __lsp_aligned16 =
35 {
36 // indexes
37 0, 1, 2, 3,
38 4, 5, 6, 7,
39 8, 9, 10, 11,
40 12, 13, 14, 15,
41 // steps
42 16, 16, 16, 16,
43 8, 8, 8, 8,
44 4, 4, 4, 4,
45 1, 1, 1, 1
46 };
47 )
48
49 /* Register allocation:
50 v0 = cind[0..3] // current index
51 v1 = cind[4..7]
52 v2 = cind[8..11]
53 v3 = cind[12..15]
54 v4 = cval[0..3] // current value
55 v5 = cval[4..7]
56 v6 = cval[8..11]
57 v7 = cval[12..15]
58 v8 = nind[0..3] // new index
59 v9 = nind[4..7]
60 v10 = nind[8..11]
61 v11 = nind[12..15]
62 v12 = nval[0..3] // new value
63 v13 = nval[4..7]
64 v14 = nval[8..11]
65 v15 = nval[12..15]
66 v16 = xcmp[0..3] // comparison result
67 v17 = xcmp[4..7]
68 v18 = xcmp[8..11]
69 v19 = xcmp[12..15]
70 v30 = step[0..3] // bulk step value
71 v31 = step[4..7]
72 */
73
74 #define IOP_SEARCH_CORE(DST, SRC, CMP, UPD) \
75 __ASM_EMIT("cmp %[count], #1") \
76 __ASM_EMIT("b.lo 200f") \
77 __ASM_EMIT("ld1r {v4.4s}, [%[" SRC "]]") /* v4 = cval[0..3] */ \
78 __ASM_EMIT("ldp q8, q9, [%[mask], #0x00]") /* v8, v9 = nind[0..7] */ \
79 __ASM_EMIT("eor v0.16b, v0.16b, v0.16b") /* v0 = cind[0..3] */ \
80 __ASM_EMIT("subs %[count], %[count], #8") \
81 __ASM_EMIT("b.lt 8f") \
82 __ASM_EMIT("ldp q30, q31, [%[mask], #0x40]") /* v30, v31 = step[0..7] */ \
83 __ASM_EMIT("eor v1.16b, v1.16b, v1.16b") /* v1 = cind[4..7] */ \
84 __ASM_EMIT("mov v5.16b, v4.16b") /* v5 = cval[4..7] */ \
85 /* 16x blocks */ \
86 __ASM_EMIT("subs %[count], %[count], #8") \
87 __ASM_EMIT("b.lt 2f") \
88 __ASM_EMIT("ldp q10, q11, [%[mask], #0x20]") /* v10, v11 = nind[8..15] */ \
89 __ASM_EMIT("eor v2.16b, v2.16b, v2.16b") /* v2 = cind[8..11] */ \
90 __ASM_EMIT("mov v6.16b, v4.16b") /* v6 = cval[8..11] */ \
91 __ASM_EMIT("eor v3.16b, v3.16b, v3.16b") /* v3 = cind[12..15] */ \
92 __ASM_EMIT("mov v7.16b, v5.16b") /* v7 = cval[12..15] */ \
93 __ASM_EMIT("1:") \
94 __ASM_EMIT("ldp q12, q13, [%[" SRC "], #0x00]") /* v12, v13 = nval[0..7] */ \
95 __ASM_EMIT("ldp q14, q15, [%[" SRC "], #0x20]") \
96 __ASM_EMIT(CMP " v16.4s, v12.4s, v4.4s") /* v16 = xcmp[0..3] */ \
97 __ASM_EMIT(CMP " v17.4s, v13.4s, v5.4s") \
98 __ASM_EMIT(CMP " v18.4s, v14.4s, v6.4s") \
99 __ASM_EMIT(CMP " v19.4s, v15.4s, v7.4s") \
100 __ASM_EMIT(UPD " v0.16b, v8.16b, v16.16b") /* v0 = upd(cind[0..3], nind[0..3], xcmp[0..3]) */ \
101 __ASM_EMIT(UPD " v1.16b, v9.16b, v17.16b") \
102 __ASM_EMIT(UPD " v2.16b, v10.16b, v18.16b") \
103 __ASM_EMIT(UPD " v3.16b, v11.16b, v19.16b") \
104 __ASM_EMIT(UPD " v4.16b, v12.16b, v16.16b") /* v4 = upd(cval[0..3], nval[0..3], xcmp[0..3]) */ \
105 __ASM_EMIT(UPD " v5.16b, v13.16b, v17.16b") \
106 __ASM_EMIT(UPD " v6.16b, v14.16b, v18.16b") \
107 __ASM_EMIT(UPD " v7.16b, v15.16b, v19.16b") \
108 __ASM_EMIT("add v8.4s, v8.4s, v30.4s") /* v8 = nind[0..3] += (step = 16) */ \
109 __ASM_EMIT("add v9.4s, v9.4s, v30.4s") \
110 __ASM_EMIT("add v10.4s, v10.4s, v30.4s") \
111 __ASM_EMIT("add v11.4s, v11.4s, v30.4s") \
112 __ASM_EMIT("subs %[count], %[count], #16") \
113 __ASM_EMIT("add %[" SRC "], %[" SRC "], #0x40") \
114 __ASM_EMIT("b.hs 1b") \
115 /* reduce 16 -> 8 */ \
116 __ASM_EMIT(CMP " v16.4s, v5.4s, v4.4s") \
117 __ASM_EMIT(CMP " v17.4s, v7.4s, v6.4s") \
118 __ASM_EMIT(UPD " v0.16b, v1.16b, v16.16b") \
119 __ASM_EMIT(UPD " v2.16b, v3.16b, v17.16b") \
120 __ASM_EMIT(UPD " v4.16b, v5.16b, v16.16b") \
121 __ASM_EMIT(UPD " v6.16b, v7.16b, v17.16b") \
122 __ASM_EMIT("mov v1.16b, v2.16b") \
123 __ASM_EMIT("mov v5.16b, v6.16b") \
124 __ASM_EMIT("2:") \
125 /* 8x block */ \
126 __ASM_EMIT("adds %[count], %[count], #8") \
127 __ASM_EMIT("b.lt 6f") \
128 __ASM_EMIT("ldp q12, q13, [%[" SRC "], #0x00]") /* v12, v13 = nval[0..7] */ \
129 __ASM_EMIT(CMP " v16.4s, v12.4s, v4.4s") /* v16 = xcmp[0..3] */ \
130 __ASM_EMIT(CMP " v17.4s, v13.4s, v5.4s") \
131 __ASM_EMIT(UPD " v0.16b, v8.16b, v16.16b") /* v0 = upd(cind[0..3], nind[0..3], xcmp[0..3]) */ \
132 __ASM_EMIT(UPD " v1.16b, v9.16b, v17.16b") \
133 __ASM_EMIT(UPD " v4.16b, v12.16b, v16.16b") /* v4 = upd(cval[0..3], nval[0..3], xcmp[0..3]) */ \
134 __ASM_EMIT(UPD " v5.16b, v13.16b, v17.16b") \
135 __ASM_EMIT("add v8.4s, v8.4s, v31.4s") /* v8 = nind[0..3] += (step = 8) */ \
136 __ASM_EMIT("sub %[count], %[count], #8") \
137 __ASM_EMIT("add %[" SRC "], %[" SRC "], #0x20") \
138 __ASM_EMIT("6:") \
139 /* reduce 8 -> 4 */ \
140 __ASM_EMIT(CMP " v16.4s, v5.4s, v4.4s") \
141 __ASM_EMIT(UPD " v0.16b, v1.16b, v16.16b") \
142 __ASM_EMIT(UPD " v4.16b, v5.16b, v16.16b") \
143 /* 4x block */ \
144 __ASM_EMIT("8:") \
145 __ASM_EMIT("ldp q30, q31, [%[mask], #0x60]") /* v30, v31 = step[8..15] */ \
146 __ASM_EMIT("adds %[count], %[count], #4") \
147 __ASM_EMIT("b.lt 8f") \
148 __ASM_EMIT("ldr q12, [%[" SRC "], #0x00]") /* v12, v13 = nval[0..7] */ \
149 __ASM_EMIT(CMP " v16.4s, v12.4s, v4.4s") /* v16 = xcmp[0..3] */ \
150 __ASM_EMIT(UPD " v0.16b, v8.16b, v16.16b") /* v0 = upd(cind[0..3], nind[0..3], xcmp[0..3]) */ \
151 __ASM_EMIT(UPD " v4.16b, v12.16b, v16.16b") /* v4 = upd(cval[0..3], nval[0..3], xcmp[0..3]) */ \
152 __ASM_EMIT("add v8.4s, v8.4s, v30.4s") /* v8 = nind[0..3] += (step = 4) */ \
153 __ASM_EMIT("sub %[count], %[count], #4") \
154 __ASM_EMIT("add %[" SRC "], %[" SRC "], #0x10") \
155 __ASM_EMIT("8:") \
156 /* reduce 4->1 , step 1 */ \
157 __ASM_EMIT("ext v1.16b, v0.16b, v0.16b, #12") /* v0 = i0 i1 i2 i3, v1 = i1 i2 i3 i0 */ \
158 __ASM_EMIT("ext v5.16b, v4.16b, v4.16b, #12") /* v4 = x0 x1 x2 x3, v5 = x1 x2 x3 x0 */ \
159 __ASM_EMIT(CMP " v16.4s, v5.4s, v4.4s") \
160 __ASM_EMIT(UPD " v0.16b, v1.16b, v16.16b") \
161 __ASM_EMIT(UPD " v4.16b, v5.16b, v16.16b") \
162 /* reduce 4->1 , step 2 */ \
163 __ASM_EMIT("ext v1.16b, v0.16b, v0.16b, #8") /* v0 = i0 i1 i2 i3, v1 = i2 i3 i0 i1 */ \
164 __ASM_EMIT("ext v5.16b, v4.16b, v4.16b, #8") /* v4 = x0 x1 x2 x3, v5 = x2 x3 x0 x1 */ \
165 __ASM_EMIT(CMP " v16.4s, v5.4s, v4.4s") \
166 __ASM_EMIT(UPD " v0.16b, v1.16b, v16.16b") \
167 __ASM_EMIT(UPD " v4.16b, v5.16b, v16.16b") \
168 /* 1x blocks */ \
169 __ASM_EMIT("adds %[count], %[count], #3") \
170 __ASM_EMIT("b.lt 10f") \
171 __ASM_EMIT("9:") \
172 __ASM_EMIT("ld1r {v12.4s}, [%[" SRC "]]") \
173 __ASM_EMIT(CMP " v16.4s, v12.4s, v4.4s") /* v16 = xcmp[0..3] */ \
174 __ASM_EMIT(UPD " v0.16b, v8.16b, v16.16b") /* v0 = upd(cind[0..3], nind[0..3], xcmp[0..3]) */ \
175 __ASM_EMIT(UPD " v4.16b, v12.16b, v16.16b") /* v4 = upd(cval[0..3], nval[0..3], xcmp[0..3]) */ \
176 __ASM_EMIT("add v8.4s, v8.4s, v31.4s") /* v8 = nind[0..3] += (step = 1) */ \
177 __ASM_EMIT("subs %[count], %[count], #1") \
178 __ASM_EMIT("add %[" SRC "], %[" SRC "], #0x04") \
179 __ASM_EMIT("b.ge 9b") \
180 __ASM_EMIT("10:") \
181 /* end */ \
182 __ASM_EMIT("mov %w[" DST "], v0.s[0]") \
183 __ASM_EMIT("200:")
184
185 size_t min_index(const float *src, size_t count)
186 {
187 uint32_t res = 0;
188
189 ARCH_AARCH64_ASM(
190 IOP_SEARCH_CORE("res", "src", "fcmge", "bif")
191 : [src] "+r" (src), [count] "+r" (count),
192 [res] "+r" (res)
193 : [mask] "r" (&search_indexes[0])
194 : "cc", "memory",
195 "v0", "v1", "v2", "v3",
196 "v4", "v5", "v6", "v7",
197 "v8", "v9", "v10", "v11",
198 "v12", "v13", "v14", "v15",
199 "v16", "v17", "v18", "v19",
200 "v30", "v31"
201 );
202
203 return res;
204 }
205
206 size_t max_index(const float *src, size_t count)
207 {
208 uint32_t res = 0;
209
210 ARCH_AARCH64_ASM(
211 IOP_SEARCH_CORE("res", "src", "fcmgt", "bit")
212 : [src] "+r" (src), [count] "+r" (count),
213 [res] "+r" (res)
214 : [mask] "r" (&search_indexes[0])
215 : "cc", "memory",
216 "v0", "v1", "v2", "v3",
217 "v4", "v5", "v6", "v7",
218 "v8", "v9", "v10", "v11",
219 "v12", "v13", "v14", "v15",
220 "v16", "v17", "v18", "v19",
221 "v30", "v31"
222 );
223
224 return res;
225 }
226
227 size_t abs_min_index(const float *src, size_t count)
228 {
229 uint32_t res = 0;
230
231 ARCH_AARCH64_ASM(
232 IOP_SEARCH_CORE("res", "src", "facge", "bif")
233 : [src] "+r" (src), [count] "+r" (count),
234 [res] "+r" (res)
235 : [mask] "r" (&search_indexes[0])
236 : "cc", "memory",
237 "v0", "v1", "v2", "v3",
238 "v4", "v5", "v6", "v7",
239 "v8", "v9", "v10", "v11",
240 "v12", "v13", "v14", "v15",
241 "v16", "v17", "v18", "v19",
242 "v30", "v31"
243 );
244
245 return res;
246 }
247
248 size_t abs_max_index(const float *src, size_t count)
249 {
250 uint32_t res = 0;
251
252 ARCH_AARCH64_ASM(
253 IOP_SEARCH_CORE("res", "src", "facgt", "bit")
254 : [src] "+r" (src), [count] "+r" (count),
255 [res] "+r" (res)
256 : [mask] "r" (&search_indexes[0])
257 : "cc", "memory",
258 "v0", "v1", "v2", "v3",
259 "v4", "v5", "v6", "v7",
260 "v8", "v9", "v10", "v11",
261 "v12", "v13", "v14", "v15",
262 "v16", "v17", "v18", "v19",
263 "v30", "v31"
264 );
265
266 return res;
267 }
268
269 #undef IOP_SEARCH_CORE
270
271 /* Register allocation:
272 v0 = imin[0..3] // index of minimum
273 v1 = imin[4..7]
274 v2 = imin[8..11]
275 v3 = imin[12..15]
276 v4 = vmin[0..3] // value of minimum
277 v5 = vmin[4..7]
278 v6 = vmin[8..11]
279 v7 = vmin[12..15]
280 v8 = imax[0..3] // index of maximum
281 v9 = imax[4..7]
282 v10 = imax[8..11]
283 v11 = imax[12..15]
284 v12 = vmax[0..3] // value of maximum
285 v13 = vmax[4..7]
286 v14 = vmax[8..11]
287 v15 = vmax[12..15]
288 v16 = cind[0..3] // current index
289 v17 = cind[4..7]
290 v18 = cind[8..11]
291 v19 = cind[12..15]
292 v20 = cval[0..3] // current value
293 v21 = cval[4..7]
294 v22 = cval[8..11]
295 v23 = cval[12..15]
296 v24 = xcmp[0..3] // comparison mask
297 v25 = xcmp[4..7]
298 v26 = xcmp[8..11]
299 v27 = xcmp[12..15]
300 v30 = step[0..3] // bulk step value
301 v31 = step[4..7]
302 */
303
304 #define IOP_MINMAX_CORE(DST1, DST2, SRC, CMP1, UPD1, CMP2, UPD2) \
305 __ASM_EMIT("eor v0.16b, v0.16b, v0.16b") /* v0 = imin[0..3] */ \
306 __ASM_EMIT("eor v8.16b, v8.16b, v8.16b") /* v8 = imax[0..3] */ \
307 __ASM_EMIT("cmp %[count], #1") \
308 __ASM_EMIT("b.lo 200f") \
309 __ASM_EMIT("ld1r {v4.4s}, [%[" SRC "]]") /* v4 = vmin[0..3] */ \
310 __ASM_EMIT("ldp q16, q17, [%[mask], #0x00]") /* v16, v17 = cind[0..7] */ \
311 __ASM_EMIT("mov v12.16b, v4.16b") /* v12 = vmax[0..3] */ \
312 __ASM_EMIT("subs %[count], %[count], #8") \
313 __ASM_EMIT("b.lt 8f") \
314 __ASM_EMIT("ldp q30, q31, [%[mask], #0x40]") /* v30, v31 = step[0..7] */ \
315 __ASM_EMIT("mov v5.16b, v4.16b") /* v5 = vmin[4..7] */ \
316 __ASM_EMIT("mov v13.16b, v12.16b") /* v13 = vmax[4..7] */ \
317 __ASM_EMIT("eor v1.16b, v1.16b, v1.16b") /* v1 = imin[4..7] */ \
318 __ASM_EMIT("eor v9.16b, v9.16b, v9.16b") /* v9 = imax[4..7] */ \
319 /* 16x blocks */ \
320 __ASM_EMIT("subs %[count], %[count], #8") \
321 __ASM_EMIT("b.lt 2f") \
322 __ASM_EMIT("ldp q18, q19, [%[mask], #0x20]") /* v18, v19 = cind[8..15] */ \
323 __ASM_EMIT("eor v2.16b, v2.16b, v2.16b") /* v2 = imin[8..11] */ \
324 __ASM_EMIT("eor v3.16b, v3.16b, v3.16b") /* v3 = imin[12..15] */ \
325 __ASM_EMIT("eor v10.16b, v10.16b, v10.16b") /* v10 = imax[8..11] */ \
326 __ASM_EMIT("eor v11.16b, v11.16b, v11.16b") /* v11 = imax[12..15] */ \
327 __ASM_EMIT("mov v6.16b, v4.16b") /* v6 = vmin[8..11] */ \
328 __ASM_EMIT("mov v7.16b, v5.16b") /* v7 = vmin[12..15] */ \
329 __ASM_EMIT("mov v14.16b, v12.16b") /* v14 = vmax[8..11] */ \
330 __ASM_EMIT("mov v15.16b, v13.16b") /* v15 = vmax[12..15] */ \
331 __ASM_EMIT("1:") \
332 __ASM_EMIT("ldp q20, q21, [%[" SRC "], #0x00]") /* v20, v21 = cval[0..7] */ \
333 __ASM_EMIT("ldp q22, q23, [%[" SRC "], #0x20]") \
334 __ASM_EMIT(CMP1 " v24.4s, v20.4s, v4.4s") /* v24 = xcmp[0..3] = (cval[0..3] <=> vmin[0..3]) */ \
335 __ASM_EMIT(CMP1 " v25.4s, v21.4s, v5.4s") \
336 __ASM_EMIT(UPD1 " v0.16b, v16.16b, v24.16b") /* v0 = upd(imin[0..3], cind[0..3], xcmp[0..3]) */ \
337 __ASM_EMIT(CMP1 " v26.4s, v22.4s, v6.4s") \
338 __ASM_EMIT(UPD1 " v1.16b, v17.16b, v25.16b") \
339 __ASM_EMIT(CMP1 " v27.4s, v23.4s, v7.4s") \
340 __ASM_EMIT(UPD1 " v2.16b, v18.16b, v26.16b") \
341 __ASM_EMIT(UPD1 " v3.16b, v19.16b, v27.16b") \
342 __ASM_EMIT(UPD1 " v4.16b, v20.16b, v24.16b") /* vv = upd(vmin[0..3], cval[0..3], xcmp[0..3]) */ \
343 __ASM_EMIT(UPD1 " v5.16b, v21.16b, v25.16b") \
344 __ASM_EMIT(CMP2 " v24.4s, v20.4s, v12.4s") /* v24 = xcmp[0..3] = (cval[0..3] <=> vmax[0..3]) */ \
345 __ASM_EMIT(UPD1 " v6.16b, v22.16b, v26.16b") \
346 __ASM_EMIT(CMP2 " v25.4s, v21.4s, v13.4s") \
347 __ASM_EMIT(UPD1 " v7.16b, v23.16b, v27.16b") \
348 __ASM_EMIT(CMP2 " v26.4s, v22.4s, v14.4s") \
349 __ASM_EMIT(UPD2 " v8.16b, v16.16b, v24.16b") /* v0 = upd(imin[0..3], cind[0..3], xcmp[0..3]) */ \
350 __ASM_EMIT(CMP2 " v27.4s, v23.4s, v15.4s") \
351 __ASM_EMIT(UPD2 " v9.16b, v17.16b, v25.16b") \
352 __ASM_EMIT(UPD2 " v10.16b, v18.16b, v26.16b") \
353 __ASM_EMIT(UPD2 " v11.16b, v19.16b, v27.16b") \
354 __ASM_EMIT(UPD2 " v12.16b, v20.16b, v24.16b") /* v12 = upd(vmin[0..3], cval[0..3], xcmp[0..3]) */ \
355 __ASM_EMIT("add v16.4s, v16.4s, v30.4s") /* v16 = nind[0..3] += (step = 16) */ \
356 __ASM_EMIT(UPD2 " v13.16b, v21.16b, v25.16b") \
357 __ASM_EMIT("add v17.4s, v17.4s, v30.4s") \
358 __ASM_EMIT(UPD2 " v14.16b, v22.16b, v26.16b") \
359 __ASM_EMIT("add v18.4s, v18.4s, v30.4s") \
360 __ASM_EMIT(UPD2 " v15.16b, v23.16b, v27.16b") \
361 __ASM_EMIT("subs %[count], %[count], #16") \
362 __ASM_EMIT("add v19.4s, v19.4s, v30.4s") \
363 __ASM_EMIT("add %[" SRC "], %[" SRC "], #0x40") \
364 __ASM_EMIT("b.hs 1b") \
365 /* reduce 16 -> 8 */ \
366 __ASM_EMIT(CMP1 " v24.4s, v5.4s, v4.4s") \
367 __ASM_EMIT(CMP1 " v25.4s, v7.4s, v6.4s") \
368 __ASM_EMIT(CMP2 " v26.4s, v13.4s, v12.4s") \
369 __ASM_EMIT(CMP2 " v27.4s, v15.4s, v14.4s") \
370 __ASM_EMIT(UPD1 " v0.16b, v1.16b, v24.16b") \
371 __ASM_EMIT(UPD1 " v2.16b, v3.16b, v25.16b") \
372 __ASM_EMIT(UPD2 " v8.16b, v9.16b, v26.16b") \
373 __ASM_EMIT(UPD2 " v10.16b, v11.16b, v27.16b") \
374 __ASM_EMIT(UPD1 " v4.16b, v5.16b, v24.16b") \
375 __ASM_EMIT(UPD1 " v6.16b, v7.16b, v25.16b") \
376 __ASM_EMIT(UPD2 " v12.16b, v13.16b, v26.16b") \
377 __ASM_EMIT(UPD2 " v14.16b, v15.16b, v27.16b") \
378 __ASM_EMIT("mov v1.16b, v2.16b") \
379 __ASM_EMIT("mov v5.16b, v6.16b") \
380 __ASM_EMIT("mov v9.16b, v10.16b") \
381 __ASM_EMIT("mov v13.16b, v14.16b") \
382 __ASM_EMIT("2:") \
383 /* 8x block */ \
384 __ASM_EMIT("adds %[count], %[count], #8") \
385 __ASM_EMIT("b.lt 6f") \
386 __ASM_EMIT("ldp q20, q21, [%[" SRC "], #0x00]") /* v12, v13 = nval[0..7] */ \
387 __ASM_EMIT(CMP1 " v24.4s, v20.4s, v4.4s") /* v24 = xcmp[0..3] = (cval[0..3] <=> vmin[0..3]) */ \
388 __ASM_EMIT(CMP1 " v25.4s, v21.4s, v5.4s") \
389 __ASM_EMIT(UPD1 " v0.16b, v16.16b, v24.16b") /* v0 = upd(imin[0..3], cind[0..3], xcmp[0..3]) */ \
390 __ASM_EMIT(UPD1 " v1.16b, v17.16b, v25.16b") \
391 __ASM_EMIT(UPD1 " v4.16b, v20.16b, v24.16b") /* vv = upd(vmin[0..3], cval[0..3], xcmp[0..3]) */ \
392 __ASM_EMIT(UPD1 " v5.16b, v21.16b, v25.16b") \
393 __ASM_EMIT(CMP2 " v24.4s, v20.4s, v12.4s") /* v24 = xcmp[0..3] = (cval[0..3] <=> vmax[0..3]) */ \
394 __ASM_EMIT(CMP2 " v25.4s, v21.4s, v13.4s") \
395 __ASM_EMIT(UPD2 " v8.16b, v16.16b, v24.16b") /* v0 = upd(imin[0..3], cind[0..3], xcmp[0..3]) */ \
396 __ASM_EMIT(UPD2 " v9.16b, v17.16b, v25.16b") \
397 __ASM_EMIT(UPD2 " v12.16b, v20.16b, v24.16b") /* v12 = upd(vmin[0..3], cval[0..3], xcmp[0..3]) */ \
398 __ASM_EMIT(UPD2 " v13.16b, v21.16b, v25.16b") \
399 __ASM_EMIT("add v16.4s, v16.4s, v31.4s") /* v16 = nind[0..3] += (step = 16) */ \
400 __ASM_EMIT("sub %[count], %[count], #8") \
401 __ASM_EMIT("add %[" SRC "], %[" SRC "], #0x20") \
402 __ASM_EMIT("6:") \
403 /* reduce 8 -> 4 */ \
404 __ASM_EMIT(CMP1 " v24.4s, v5.4s, v4.4s") \
405 __ASM_EMIT(CMP2 " v26.4s, v13.4s, v12.4s") \
406 __ASM_EMIT(UPD1 " v0.16b, v1.16b, v24.16b") \
407 __ASM_EMIT(UPD2 " v8.16b, v9.16b, v26.16b") \
408 __ASM_EMIT(UPD1 " v4.16b, v5.16b, v24.16b") \
409 __ASM_EMIT(UPD2 " v12.16b, v13.16b, v26.16b") \
410 /* 4x block */ \
411 __ASM_EMIT("8:") \
412 __ASM_EMIT("ldp q30, q31, [%[mask], #0x60]") /* v30, v31 = step[8..15] */ \
413 __ASM_EMIT("adds %[count], %[count], #4") \
414 __ASM_EMIT("b.lt 8f") \
415 __ASM_EMIT("ldr q20, [%[" SRC "], #0x00]") /* v12, v13 = nval[0..7] */ \
416 __ASM_EMIT(CMP1 " v24.4s, v20.4s, v4.4s") /* v24 = xcmp[0..3] = (cval[0..3] <=> vmin[0..3]) */ \
417 __ASM_EMIT(CMP2 " v25.4s, v20.4s, v12.4s") /* v25 = xcmp[0..3] = (cval[0..3] <=> vmax[0..3]) */ \
418 __ASM_EMIT(UPD1 " v0.16b, v16.16b, v24.16b") /* v0 = upd(imin[0..3], cind[0..3], xcmp[0..3]) */ \
419 __ASM_EMIT(UPD2 " v8.16b, v16.16b, v25.16b") /* v0 = upd(imin[0..3], cind[0..3], xcmp[0..3]) */ \
420 __ASM_EMIT(UPD1 " v4.16b, v20.16b, v24.16b") /* vv = upd(vmin[0..3], cval[0..3], xcmp[0..3]) */ \
421 __ASM_EMIT(UPD2 " v12.16b, v20.16b, v25.16b") /* v12 = upd(vmin[0..3], cval[0..3], xcmp[0..3]) */ \
422 __ASM_EMIT("add v16.4s, v16.4s, v30.4s") /* v16 = nind[0..3] += (step = 4) */ \
423 __ASM_EMIT("sub %[count], %[count], #4") \
424 __ASM_EMIT("add %[" SRC "], %[" SRC "], #0x10") \
425 __ASM_EMIT("8:") \
426 /* reduce 4->1 , step 1 */ \
427 __ASM_EMIT("ext v1.16b, v0.16b, v0.16b, #12") /* v0 = i0 i1 i2 i3, v1 = i1 i2 i3 i0 */ \
428 __ASM_EMIT("ext v9.16b, v8.16b, v8.16b, #12") \
429 __ASM_EMIT("ext v5.16b, v4.16b, v4.16b, #12") /* v4 = x0 x1 x2 x3, v5 = x1 x2 x3 x0 */ \
430 __ASM_EMIT("ext v13.16b, v12.16b, v12.16b, #12") \
431 __ASM_EMIT(CMP1 " v24.4s, v5.4s, v4.4s") \
432 __ASM_EMIT(CMP2 " v25.4s, v13.4s, v12.4s") \
433 __ASM_EMIT(UPD1 " v0.16b, v1.16b, v24.16b") \
434 __ASM_EMIT(UPD2 " v8.16b, v9.16b, v25.16b") \
435 __ASM_EMIT(UPD1 " v4.16b, v5.16b, v24.16b") \
436 __ASM_EMIT(UPD2 " v12.16b, v13.16b, v25.16b") \
437 /* reduce 4->1 , step 2 */ \
438 __ASM_EMIT("ext v1.16b, v0.16b, v0.16b, #8") /* v0 = i0 i1 i2 i3, v1 = i1 i2 i3 i0 */ \
439 __ASM_EMIT("ext v9.16b, v8.16b, v8.16b, #8") \
440 __ASM_EMIT("ext v5.16b, v4.16b, v4.16b, #8") /* v4 = x0 x1 x2 x3, v5 = x1 x2 x3 x0 */ \
441 __ASM_EMIT("ext v13.16b, v12.16b, v12.16b, #8") \
442 __ASM_EMIT(CMP1 " v24.4s, v5.4s, v4.4s") \
443 __ASM_EMIT(CMP2 " v25.4s, v13.4s, v12.4s") \
444 __ASM_EMIT(UPD1 " v0.16b, v1.16b, v24.16b") \
445 __ASM_EMIT(UPD2 " v8.16b, v9.16b, v25.16b") \
446 __ASM_EMIT(UPD1 " v4.16b, v5.16b, v24.16b") \
447 __ASM_EMIT(UPD2 " v12.16b, v13.16b, v25.16b") \
448 /* 1x blocks */ \
449 __ASM_EMIT("adds %[count], %[count], #3") \
450 __ASM_EMIT("b.lt 10f") \
451 __ASM_EMIT("9:") \
452 __ASM_EMIT("ld1r {v20.4s}, [%[" SRC "]]") \
453 __ASM_EMIT(CMP1 " v24.4s, v20.4s, v4.4s") /* v24 = xcmp[0..3] = (cval[0..3] <=> vmin[0..3]) */ \
454 __ASM_EMIT(CMP2 " v25.4s, v20.4s, v12.4s") /* v25 = xcmp[0..3] = (cval[0..3] <=> vmax[0..3]) */ \
455 __ASM_EMIT(UPD1 " v0.16b, v16.16b, v24.16b") /* v0 = upd(imin[0..3], cind[0..3], xcmp[0..3]) */ \
456 __ASM_EMIT(UPD2 " v8.16b, v16.16b, v25.16b") /* v0 = upd(imin[0..3], cind[0..3], xcmp[0..3]) */ \
457 __ASM_EMIT(UPD1 " v4.16b, v20.16b, v24.16b") /* vv = upd(vmin[0..3], cval[0..3], xcmp[0..3]) */ \
458 __ASM_EMIT(UPD2 " v12.16b, v20.16b, v25.16b") /* v12 = upd(vmin[0..3], cval[0..3], xcmp[0..3]) */ \
459 __ASM_EMIT("add v16.4s, v16.4s, v31.4s") /* v16 = nind[0..3] += (step = 1) */ \
460 __ASM_EMIT("subs %[count], %[count], #1") \
461 __ASM_EMIT("add %[" SRC "], %[" SRC "], #0x04") \
462 __ASM_EMIT("b.ge 9b") \
463 __ASM_EMIT("10:") \
464 /* end */ \
465 __ASM_EMIT("200:") \
466 __ASM_EMIT("st1 {v0.s}[0], [%[" DST1 "]]") \
467 __ASM_EMIT("st1 {v8.s}[0], [%[" DST2 "]]")
468
469 void minmax_index(const float *src, size_t count, size_t *min, size_t *max)
470 {
471 *min = 0;
472 *max = 0;
473 ARCH_AARCH64_ASM(
474 IOP_MINMAX_CORE("min", "max", "src", "fcmge", "bif", "fcmgt", "bit")
475 : [src] "+r" (src), [count] "+r" (count)
476 : [min] "r" (min), [max] "r" (max),
477 [mask] "r" (&search_indexes[0])
478 : "cc", "memory",
479 "v0", "v1", "v2", "v3",
480 "v4", "v5", "v6", "v7",
481 "v8", "v9", "v10", "v11",
482 "v12", "v13", "v14", "v15",
483 "v16", "v17", "v18", "v19",
484 "v20", "v21", "v22", "v23",
485 "v24", "v25", "v26", "v27",
486 "v30", "v31"
487 );
488 }
489
490 void abs_minmax_index(const float *src, size_t count, size_t *min, size_t *max)
491 {
492 *min = 0;
493 *max = 0;
494 ARCH_AARCH64_ASM(
495 IOP_MINMAX_CORE("min", "max", "src", "facge", "bif", "facgt", "bit")
496 : [src] "+r" (src), [count] "+r" (count)
497 : [min] "r" (min), [max] "r" (max),
498 [mask] "r" (&search_indexes[0])
499 : "cc", "memory",
500 "v0", "v1", "v2", "v3",
501 "v4", "v5", "v6", "v7",
502 "v8", "v9", "v10", "v11",
503 "v12", "v13", "v14", "v15",
504 "v16", "v17", "v18", "v19",
505 "v20", "v21", "v22", "v23",
506 "v24", "v25", "v26", "v27",
507 "v30", "v31"
508 );
509 }
510 }
511 }
512
513 #endif /* PRIVATE_DSP_ARCH_AARCH64_ASIMD_SEARCH_IMINMAX_H_ */