Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/x86/avx2/search/iminmax.h

raw
1 /*
2 * Copyright (C) 2020 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2020 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 31 мар. 2020 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_X86_AVX2_SEARCH_IMINMAX_H_
23 #define PRIVATE_DSP_ARCH_X86_AVX2_SEARCH_IMINMAX_H_
24
25 #ifndef PRIVATE_DSP_ARCH_X86_AVX2_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_X86_AVX2_IMPL */
28
29 namespace lsp
30 {
31 namespace avx2
32 {
33 IF_ARCH_X86(
34 static const uint32_t minmax_incr[] __lsp_aligned64 =
35 {
36 0, 1, 2, 3, // Index
37 4, 5, 6, 7,
38 8, 8, 8, 8, // Increment
39 8, 8, 8, 8,
40 4, 4, 4, 4,
41 1, 1, 1, 1,
42 0x7fffffff, 0x7fffffff, 0x7fffffff, 0x7fffffff, // Abs mask
43 0x7fffffff, 0x7fffffff, 0x7fffffff, 0x7fffffff
44 }
45 );
46
47 #define SEL_NONE(x, y) x
48 #define SEL_ABS(x, y) y
49
50 /*
51 * Register allocation:
52 * ymm0 = imin
53 * ymm1 = vmin
54 * ymm2 = cind
55 * ymm3 = cval
56 * ymm4 = cmask
57 * ymm5 = temp
58 * ymm6 = amask
59 * ymm7 = incr
60 */
61 #define IMINMAX_CORE(COND, SEL) \
62 __ASM_EMIT("vpxor %%ymm0, %%ymm0, %%ymm0") \
63 __ASM_EMIT("test %[count], %[count]") \
64 __ASM_EMIT("jz 6f") \
65 /* 8x block */ \
66 __ASM_EMIT("vbroadcastss 0x00(%[src]), %%ymm1") \
67 __ASM_EMIT(SEL("", "vmovaps 0x60 + %[INCR], %%ymm6")) \
68 __ASM_EMIT("vmovdqu 0x00 + %[INCR], %%ymm2") \
69 __ASM_EMIT(SEL("", "vandps %%ymm6, %%ymm1, %%ymm1")) \
70 __ASM_EMIT("vmovdqu 0x20 + %[INCR], %%ymm7") \
71 __ASM_EMIT("sub $8, %[count]") \
72 __ASM_EMIT("jb 2f") \
73 __ASM_EMIT("1:") \
74 __ASM_EMIT(SEL("vmovups 0x00(%[src]), %%ymm3", "vandps 0x00(%[src]), %%ymm6, %%ymm3")) \
75 __ASM_EMIT("vcmpps " COND ", %%ymm1, %%ymm3, %%ymm4") \
76 __ASM_EMIT("vpblendvb %%ymm4, %%ymm2, %%ymm0, %%ymm0") \
77 __ASM_EMIT("vblendvps %%ymm4, %%ymm3, %%ymm1, %%ymm1") \
78 __ASM_EMIT("vpaddd %%ymm7, %%ymm2, %%ymm2") \
79 __ASM_EMIT("add $0x20, %[src]") \
80 __ASM_EMIT("sub $8, %[count]") \
81 __ASM_EMIT("jae 1b") \
82 __ASM_EMIT("vextracti128 $1, %%ymm0, %%xmm5") \
83 __ASM_EMIT("vextractf128 $1, %%ymm1, %%xmm3") \
84 __ASM_EMIT("vcmpps " COND ", %%xmm1, %%xmm3, %%xmm4") \
85 __ASM_EMIT("vpblendvb %%xmm4, %%xmm5, %%xmm0, %%xmm0") \
86 __ASM_EMIT("vblendvps %%xmm4, %%xmm3, %%xmm1, %%xmm1") \
87 __ASM_EMIT("2:") \
88 /* 4x block */ \
89 __ASM_EMIT("add $4, %[count]") \
90 __ASM_EMIT("jl 4f") \
91 __ASM_EMIT(SEL("vmovups 0x00(%[src]), %%xmm3", "vandps 0x00(%[src]), %%xmm6, %%xmm3")) \
92 __ASM_EMIT("vcmpps " COND ", %%xmm1, %%xmm3, %%xmm4") \
93 __ASM_EMIT("vpblendvb %%xmm4, %%xmm2, %%xmm0, %%xmm0") \
94 __ASM_EMIT("vblendvps %%xmm4, %%xmm3, %%xmm1, %%xmm1") \
95 __ASM_EMIT("vpaddd 0x40 + %[INCR], %%xmm2, %%xmm2") \
96 __ASM_EMIT("sub $4, %[count]") \
97 __ASM_EMIT("add $0x10, %[src]") \
98 __ASM_EMIT("4:") \
99 __ASM_EMIT("vpshufd $0x39, %%xmm0, %%xmm5") \
100 __ASM_EMIT("vshufps $0x39, %%xmm1, %%xmm1, %%xmm3") \
101 __ASM_EMIT("vcmpps " COND ", %%xmm1, %%xmm3, %%xmm4") \
102 __ASM_EMIT("vpblendvb %%xmm4, %%xmm5, %%xmm0, %%xmm0") \
103 __ASM_EMIT("vblendvps %%xmm4, %%xmm3, %%xmm1, %%xmm1") \
104 __ASM_EMIT("vpunpckhdq %%xmm0, %%xmm0, %%xmm5") \
105 __ASM_EMIT("vunpckhps %%xmm1, %%xmm1, %%xmm3") \
106 __ASM_EMIT("vcmpps " COND ", %%xmm1, %%xmm3, %%xmm4") \
107 __ASM_EMIT("vpblendvb %%xmm4, %%xmm5, %%xmm0, %%xmm0") \
108 __ASM_EMIT("vblendvps %%xmm4, %%xmm3, %%xmm1, %%xmm1") \
109 /* 1x block */ \
110 __ASM_EMIT("add $3, %[count]") \
111 __ASM_EMIT("jl 6f") \
112 __ASM_EMIT("5:") \
113 __ASM_EMIT("movss 0x00(%[src]), %%xmm3") \
114 __ASM_EMIT(SEL("", "vandps %%xmm6, %%xmm3, %%xmm3")) \
115 __ASM_EMIT("vcmpps " COND ", %%xmm1, %%xmm3, %%xmm4") \
116 __ASM_EMIT("vpblendvb %%xmm4, %%xmm2, %%xmm0, %%xmm0") \
117 __ASM_EMIT("vblendvps %%xmm4, %%xmm3, %%xmm1, %%xmm1") \
118 __ASM_EMIT("vpaddd 0x50 + %[INCR], %%xmm2, %%xmm2") \
119 __ASM_EMIT("add $0x04, %[src]") \
120 __ASM_EMIT("dec %[count]") \
121 __ASM_EMIT("jge 5b") \
122 /* end */ \
123 __ASM_EMIT("6:")
124
125
126 size_t min_index(const float *src, size_t count)
127 {
128 uint32_t index;
129
130 ARCH_X86_ASM(
131 IMINMAX_CORE("$1", SEL_NONE)
132 : [src] "+r" (src), [count] "+r" (count),
133 [index] "=Yz" (index)
134 : [INCR] "o" (minmax_incr)
135 : "cc",
136 "%xmm1", "%xmm2", "%xmm3",
137 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
138 );
139
140 return index;
141 }
142
143 size_t max_index(const float *src, size_t count)
144 {
145 uint32_t index;
146
147 ARCH_X86_ASM(
148 IMINMAX_CORE("$6", SEL_NONE)
149 : [src] "+r" (src), [count] "+r" (count),
150 [index] "=Yz" (index)
151 : [INCR] "o" (minmax_incr)
152 : "cc",
153 "%xmm1", "%xmm2", "%xmm3",
154 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
155 );
156
157 return index;
158 }
159
160 size_t abs_min_index(const float *src, size_t count)
161 {
162 uint32_t index;
163
164 ARCH_X86_ASM(
165 IMINMAX_CORE("$1", SEL_ABS)
166 : [src] "+r" (src), [count] "+r" (count),
167 [index] "=Yz" (index)
168 : [INCR] "o" (minmax_incr)
169 : "cc",
170 "%xmm1", "%xmm2", "%xmm3",
171 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
172 );
173
174 return index;
175 }
176
177 size_t abs_max_index(const float *src, size_t count)
178 {
179 uint32_t index;
180
181 ARCH_X86_ASM(
182 IMINMAX_CORE("$6", SEL_ABS)
183 : [src] "+r" (src), [count] "+r" (count),
184 [index] "=Yz" (index)
185 : [INCR] "o" (minmax_incr)
186 : "cc",
187 "%xmm1", "%xmm2", "%xmm3",
188 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
189 );
190
191 return index;
192 }
193
194 #undef IMINMAX_CORE
195
196 /*
197 * Register allocation:
198 * ymm0 = imin
199 * ymm1 = vmin
200 * ymm2 = imax
201 * ymm3 = vmax
202 * ymm4 = cind
203 * ymm5 = cval
204 * ymm6 = cmask
205 * ymm7 = temp
206 */
207 #define IMINMAX2_CORE(SEL) \
208 __ASM_EMIT("vpxor %%ymm0, %%ymm0, %%ymm0") /* imin */ \
209 __ASM_EMIT("vpxor %%ymm2, %%ymm2, %%ymm2") /* imax */ \
210 __ASM_EMIT("test %[count], %[count]") \
211 __ASM_EMIT("jz 6f") \
212 /* 8x block */ \
213 __ASM_EMIT("vbroadcastss 0x00(%[src]), %%ymm1") /* vmin */ \
214 __ASM_EMIT("vmovdqu 0x00 + %[INCR], %%ymm4") /* cind */ \
215 __ASM_EMIT(SEL("", "vandps 0x60 + %[INCR], %%ymm1, %%ymm1")) \
216 __ASM_EMIT("vmovdqa %%ymm1, %%ymm3") /* vmax */ \
217 __ASM_EMIT("sub $8, %[count]") \
218 __ASM_EMIT("jb 2f") \
219 __ASM_EMIT("1:") \
220 __ASM_EMIT("vmovups 0x00(%[src]), %%ymm5") \
221 __ASM_EMIT(SEL("", "vandps 0x60 + %[INCR], %%ymm5, %%ymm5")) \
222 __ASM_EMIT("vcmpps $1, %%ymm1, %%ymm5, %%ymm6") /* cmpltps */ \
223 __ASM_EMIT("vcmpps $6, %%ymm3, %%ymm5, %%ymm7") /* cmpgtps */ \
224 __ASM_EMIT("vpblendvb %%ymm6, %%ymm4, %%ymm0, %%ymm0") \
225 __ASM_EMIT("vblendvps %%ymm6, %%ymm5, %%ymm1, %%ymm1") \
226 __ASM_EMIT("vpblendvb %%ymm7, %%ymm4, %%ymm2, %%ymm2") \
227 __ASM_EMIT("vblendvps %%ymm7, %%ymm5, %%ymm3, %%ymm3") \
228 __ASM_EMIT("vpaddd 0x20 + %[INCR], %%ymm4, %%ymm4") \
229 __ASM_EMIT("add $0x20, %[src]") \
230 __ASM_EMIT("sub $8, %[count]") \
231 __ASM_EMIT("jae 1b") \
232 __ASM_EMIT("vextracti128 $1, %%ymm0, %%xmm7") \
233 __ASM_EMIT("vextractf128 $1, %%ymm1, %%xmm5") \
234 __ASM_EMIT("vcmpps $1, %%xmm1, %%xmm5, %%xmm6") \
235 __ASM_EMIT("vpblendvb %%xmm6, %%xmm7, %%xmm0, %%xmm0") \
236 __ASM_EMIT("vblendvps %%xmm6, %%xmm5, %%xmm1, %%xmm1") \
237 __ASM_EMIT("vextracti128 $1, %%ymm2, %%xmm7") \
238 __ASM_EMIT("vextractf128 $1, %%ymm3, %%xmm5") \
239 __ASM_EMIT("vcmpps $6, %%xmm3, %%xmm5, %%xmm6") \
240 __ASM_EMIT("vpblendvb %%xmm6, %%xmm7, %%xmm2, %%xmm2") \
241 __ASM_EMIT("vblendvps %%xmm6, %%xmm5, %%xmm3, %%xmm3") \
242 __ASM_EMIT("2:") \
243 /* 4x block */ \
244 __ASM_EMIT("add $4, %[count]") \
245 __ASM_EMIT("jl 4f") \
246 __ASM_EMIT("vmovups 0x00(%[src]), %%xmm5") \
247 __ASM_EMIT(SEL("", "vandps 0x60 + %[INCR], %%xmm5, %%xmm5")) \
248 __ASM_EMIT("vcmpps $1, %%xmm1, %%xmm5, %%xmm6") /* cmpltps */ \
249 __ASM_EMIT("vcmpps $6, %%xmm3, %%xmm5, %%xmm7") /* cmpgtps */ \
250 __ASM_EMIT("vpblendvb %%xmm6, %%xmm4, %%xmm0, %%xmm0") \
251 __ASM_EMIT("vblendvps %%xmm6, %%xmm5, %%xmm1, %%xmm1") \
252 __ASM_EMIT("vpblendvb %%xmm7, %%xmm4, %%xmm2, %%xmm2") \
253 __ASM_EMIT("vblendvps %%xmm7, %%xmm5, %%xmm3, %%xmm3") \
254 __ASM_EMIT("vpaddd 0x40 + %[INCR], %%xmm4, %%xmm4") \
255 __ASM_EMIT("sub $4, %[count]") \
256 __ASM_EMIT("add $0x10, %[src]") \
257 __ASM_EMIT("4:") \
258 __ASM_EMIT("vpshufd $0x39, %%xmm0, %%xmm7") \
259 __ASM_EMIT("vshufps $0x39, %%xmm1, %%xmm1, %%xmm5") \
260 __ASM_EMIT("vcmpps $1, %%xmm1, %%xmm5, %%xmm6") \
261 __ASM_EMIT("vpblendvb %%xmm6, %%xmm7, %%xmm0, %%xmm0") \
262 __ASM_EMIT("vblendvps %%xmm6, %%xmm5, %%xmm1, %%xmm1") \
263 __ASM_EMIT("vpunpckhdq %%xmm0, %%xmm0, %%xmm7") \
264 __ASM_EMIT("vunpckhps %%xmm1, %%xmm1, %%xmm5") \
265 __ASM_EMIT("vcmpps $1, %%xmm1, %%xmm5, %%xmm6") \
266 __ASM_EMIT("vpblendvb %%xmm6, %%xmm7, %%xmm0, %%xmm0") \
267 __ASM_EMIT("vblendvps %%xmm6, %%xmm5, %%xmm1, %%xmm1") \
268 __ASM_EMIT("vpshufd $0x39, %%xmm2, %%xmm7") \
269 __ASM_EMIT("vshufps $0x39, %%xmm3, %%xmm3, %%xmm5") \
270 __ASM_EMIT("vcmpps $6, %%xmm3, %%xmm5, %%xmm6") \
271 __ASM_EMIT("vpblendvb %%xmm6, %%xmm7, %%xmm2, %%xmm2") \
272 __ASM_EMIT("vblendvps %%xmm6, %%xmm5, %%xmm3, %%xmm3") \
273 __ASM_EMIT("vpunpckhdq %%xmm2, %%xmm2, %%xmm7") \
274 __ASM_EMIT("vunpckhps %%xmm3, %%xmm3, %%xmm5") \
275 __ASM_EMIT("vcmpps $6, %%xmm3, %%xmm5, %%xmm6") \
276 __ASM_EMIT("vpblendvb %%xmm6, %%xmm7, %%xmm2, %%xmm2") \
277 __ASM_EMIT("vblendvps %%xmm6, %%xmm5, %%xmm3, %%xmm3") \
278 /* 1x block */ \
279 __ASM_EMIT("add $3, %[count]") \
280 __ASM_EMIT("jl 6f") \
281 __ASM_EMIT("5:") \
282 __ASM_EMIT("movss 0x00(%[src]), %%xmm5") \
283 __ASM_EMIT(SEL("", "vandps 0x60 + %[INCR], %%xmm5, %%xmm5")) \
284 __ASM_EMIT("vcmpps $1, %%xmm1, %%xmm5, %%xmm6") /* cmpltps */ \
285 __ASM_EMIT("vcmpps $6, %%xmm3, %%xmm5, %%xmm7") /* cmpgtps */ \
286 __ASM_EMIT("vpblendvb %%xmm6, %%xmm4, %%xmm0, %%xmm0") \
287 __ASM_EMIT("vblendvps %%xmm6, %%xmm5, %%xmm1, %%xmm1") \
288 __ASM_EMIT("vpblendvb %%xmm7, %%xmm4, %%xmm2, %%xmm2") \
289 __ASM_EMIT("vblendvps %%xmm7, %%xmm5, %%xmm3, %%xmm3") \
290 __ASM_EMIT("vpaddd 0x50 + %[INCR], %%xmm4, %%xmm4") \
291 __ASM_EMIT("add $0x04, %[src]") \
292 __ASM_EMIT("dec %[count]") \
293 __ASM_EMIT("jge 5b") \
294 /* end */ \
295 __ASM_EMIT("6:") \
296 __ASM_EMIT("movss %%xmm0, (%[min])") \
297 __ASM_EMIT("movss %%xmm2, (%[max])")
298
299 void minmax_index(const float *src, size_t count, size_t *min, size_t *max)
300 {
301 *min = 0;
302 *max = 0;
303 ARCH_X86_ASM(
304 IMINMAX2_CORE(SEL_NONE)
305 : [src] "+r" (src), [count] "+r" (count)
306 : [min] "r" (min), [max] "r" (max),
307 [INCR] "o" (minmax_incr)
308 : "cc", "memory",
309 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
310 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
311 );
312 }
313
314 void abs_minmax_index(const float *src, size_t count, size_t *min, size_t *max)
315 {
316 *min = 0;
317 *max = 0;
318 ARCH_X86_ASM(
319 IMINMAX2_CORE(SEL_ABS)
320 : [src] "+r" (src), [count] "+r" (count)
321 : [min] "r" (min), [max] "r" (max),
322 [INCR] "o" (minmax_incr)
323 : "cc", "memory",
324 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
325 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
326 );
327 }
328
329 #undef IMINMAX2_CORE
330 #undef SEL_NONE
331 #undef SEL_ABS
332 } /* namespace avx2 */
333 } /* namespace lsp */
334
335 #endif /* PRIVATE_DSP_ARCH_X86_AVX2_SEARCH_IMINMAX_H_ */