Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/x86/avx512/search/iminmax.h

raw
1 /*
2 * Copyright (C) 2024 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2024 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 29 июл. 2024 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_X86_AVX512_SEARCH_IMINMAX_H_
23 #define PRIVATE_DSP_ARCH_X86_AVX512_SEARCH_IMINMAX_H_
24
25 #ifndef PRIVATE_DSP_ARCH_X86_AVX512_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_X86_AVX512_IMPL */
28
29 namespace lsp
30 {
31 namespace avx512
32 {
33 IF_ARCH_X86(
34 static const uint32_t minmax_incr[] __lsp_aligned64 =
35 {
36 // +0x00 Index
37 0, 1, 2, 3,
38 4, 5, 6, 7,
39 8, 9, 10, 11,
40 12, 13, 14, 15,
41 // Increments
42 LSP_DSP_VEC16(16), // +0x40
43 LSP_DSP_VEC8(8), // +0x80
44 LSP_DSP_VEC4(4), // +0xa0
45 LSP_DSP_VEC4(1), // +0xb0
46 // Absolute mask
47 LSP_DSP_VEC16(0x7fffffff) // +0xc0
48 }
49 );
50
51 #define SEL_NONE(x, y) x
52 #define SEL_ABS(x, y) y
53
54 /*
55 * Register allocation:
56 * zmm0 = imin
57 * zmm1 = vmin
58 * zmm2 = cind
59 * zmm3 = cval
60 * zmm4/k4 = cmask
61 * zmm5 = temp
62 * zmm6 = amask
63 * zmm7 = incr
64 */
65 #define IMINMAX_CORE(COND, SEL) \
66 __ASM_EMIT("vpxord %%zmm0, %%zmm0, %%zmm0") \
67 __ASM_EMIT("test %[count], %[count]") \
68 __ASM_EMIT("jz 8f") \
69 /* 16x block */ \
70 __ASM_EMIT("vbroadcastss 0x00(%[src]), %%zmm1") \
71 __ASM_EMIT(SEL("", "vmovaps 0xc0 + %[INCR], %%zmm6")) \
72 __ASM_EMIT("vmovdqa32 0x00 + %[INCR], %%zmm2") \
73 __ASM_EMIT(SEL("", "vandps %%zmm6, %%zmm1, %%zmm1")) \
74 __ASM_EMIT("vmovdqa32 0x40 + %[INCR], %%zmm7") \
75 __ASM_EMIT("sub $16, %[count]") \
76 __ASM_EMIT("jb 2f") \
77 __ASM_EMIT("1:") \
78 __ASM_EMIT(SEL("vmovups 0x00(%[src]), %%zmm3", "vandps 0x00(%[src]), %%zmm6, %%zmm3")) \
79 __ASM_EMIT("vcmpps " COND ", %%zmm1, %%zmm3, %%k4") \
80 __ASM_EMIT("vmovdqa32 %%zmm2, %%zmm0 %{%%k4%}") \
81 __ASM_EMIT("vmovaps %%zmm3, %%zmm1 %{%%k4%}") \
82 __ASM_EMIT("vpaddd %%zmm7, %%zmm2, %%zmm2") \
83 __ASM_EMIT("add $0x40, %[src]") \
84 __ASM_EMIT("sub $16, %[count]") \
85 __ASM_EMIT("jae 1b") \
86 __ASM_EMIT("vextracti64x4 $1, %%zmm0, %%ymm5") \
87 __ASM_EMIT("vextractf64x4 $1, %%zmm1, %%ymm3") \
88 __ASM_EMIT("vcmpps " COND ", %%ymm1, %%ymm3, %%ymm4") \
89 __ASM_EMIT("vpblendvb %%ymm4, %%ymm5, %%ymm0, %%ymm0") \
90 __ASM_EMIT("vblendvps %%ymm4, %%ymm3, %%ymm1, %%ymm1") \
91 __ASM_EMIT("2:") \
92 /* 8x block */ \
93 __ASM_EMIT("add $8, %[count]") \
94 __ASM_EMIT("jl 4f") \
95 __ASM_EMIT("1:") \
96 __ASM_EMIT(SEL("vmovups 0x00(%[src]), %%ymm3", "vandps 0x00(%[src]), %%ymm6, %%ymm3")) \
97 __ASM_EMIT("vcmpps " COND ", %%ymm1, %%ymm3, %%ymm4") \
98 __ASM_EMIT("vpblendvb %%ymm4, %%ymm2, %%ymm0, %%ymm0") \
99 __ASM_EMIT("vblendvps %%ymm4, %%ymm3, %%ymm1, %%ymm1") \
100 __ASM_EMIT("vpaddd 0x80 + %[INCR], %%ymm2, %%ymm2") \
101 __ASM_EMIT("add $0x20, %[src]") \
102 __ASM_EMIT("sub $8, %[count]") \
103 __ASM_EMIT("4:") \
104 __ASM_EMIT("vextracti128 $1, %%ymm0, %%xmm5") \
105 __ASM_EMIT("vextractf128 $1, %%ymm1, %%xmm3") \
106 __ASM_EMIT("vcmpps " COND ", %%xmm1, %%xmm3, %%xmm4") \
107 __ASM_EMIT("vpblendvb %%xmm4, %%xmm5, %%xmm0, %%xmm0") \
108 __ASM_EMIT("vblendvps %%xmm4, %%xmm3, %%xmm1, %%xmm1") \
109 /* 4x block */ \
110 __ASM_EMIT("add $4, %[count]") \
111 __ASM_EMIT("jl 6f") \
112 __ASM_EMIT(SEL("vmovups 0x00(%[src]), %%xmm3", "vandps 0x00(%[src]), %%xmm6, %%xmm3")) \
113 __ASM_EMIT("vcmpps " COND ", %%xmm1, %%xmm3, %%xmm4") \
114 __ASM_EMIT("vpblendvb %%xmm4, %%xmm2, %%xmm0, %%xmm0") \
115 __ASM_EMIT("vblendvps %%xmm4, %%xmm3, %%xmm1, %%xmm1") \
116 __ASM_EMIT("vpaddd 0xa0 + %[INCR], %%xmm2, %%xmm2") \
117 __ASM_EMIT("sub $4, %[count]") \
118 __ASM_EMIT("add $0x10, %[src]") \
119 __ASM_EMIT("6:") \
120 __ASM_EMIT("vpshufd $0x39, %%xmm0, %%xmm5") \
121 __ASM_EMIT("vshufps $0x39, %%xmm1, %%xmm1, %%xmm3") \
122 __ASM_EMIT("vcmpps " COND ", %%xmm1, %%xmm3, %%xmm4") \
123 __ASM_EMIT("vpblendvb %%xmm4, %%xmm5, %%xmm0, %%xmm0") \
124 __ASM_EMIT("vblendvps %%xmm4, %%xmm3, %%xmm1, %%xmm1") \
125 __ASM_EMIT("vpunpckhdq %%xmm0, %%xmm0, %%xmm5") \
126 __ASM_EMIT("vunpckhps %%xmm1, %%xmm1, %%xmm3") \
127 __ASM_EMIT("vcmpps " COND ", %%xmm1, %%xmm3, %%xmm4") \
128 __ASM_EMIT("vpblendvb %%xmm4, %%xmm5, %%xmm0, %%xmm0") \
129 __ASM_EMIT("vblendvps %%xmm4, %%xmm3, %%xmm1, %%xmm1") \
130 /* 1x block */ \
131 __ASM_EMIT("add $3, %[count]") \
132 __ASM_EMIT("jl 8f") \
133 __ASM_EMIT("7:") \
134 __ASM_EMIT("movss 0x00(%[src]), %%xmm3") \
135 __ASM_EMIT(SEL("", "vandps %%xmm6, %%xmm3, %%xmm3")) \
136 __ASM_EMIT("vcmpps " COND ", %%xmm1, %%xmm3, %%xmm4") \
137 __ASM_EMIT("vpblendvb %%xmm4, %%xmm2, %%xmm0, %%xmm0") \
138 __ASM_EMIT("vblendvps %%xmm4, %%xmm3, %%xmm1, %%xmm1") \
139 __ASM_EMIT("vpaddd 0xb0 + %[INCR], %%xmm2, %%xmm2") \
140 __ASM_EMIT("add $0x04, %[src]") \
141 __ASM_EMIT("dec %[count]") \
142 __ASM_EMIT("jge 7b") \
143 __ASM_EMIT("8:") \
144 /* end */
145
146
147 size_t min_index(const float *src, size_t count)
148 {
149 uint32_t index;
150
151 ARCH_X86_ASM(
152 IMINMAX_CORE("$1", SEL_NONE)
153 : [src] "+r" (src), [count] "+r" (count),
154 [index] "=Yz" (index)
155 : [INCR] "o" (minmax_incr)
156 : "cc",
157 "%xmm1", "%xmm2", "%xmm3",
158 "%xmm4", "%xmm5", "%xmm6", "%xmm7",
159 "%k4"
160 );
161
162 return index;
163 }
164
165 size_t max_index(const float *src, size_t count)
166 {
167 uint32_t index;
168
169 ARCH_X86_ASM(
170 IMINMAX_CORE("$6", SEL_NONE)
171 : [src] "+r" (src), [count] "+r" (count),
172 [index] "=Yz" (index)
173 : [INCR] "o" (minmax_incr)
174 : "cc",
175 "%xmm1", "%xmm2", "%xmm3",
176 "%xmm4", "%xmm5", "%xmm6", "%xmm7",
177 "%k4"
178 );
179
180 return index;
181 }
182
183 size_t abs_min_index(const float *src, size_t count)
184 {
185 uint32_t index;
186
187 ARCH_X86_ASM(
188 IMINMAX_CORE("$1", SEL_ABS)
189 : [src] "+r" (src), [count] "+r" (count),
190 [index] "=Yz" (index)
191 : [INCR] "o" (minmax_incr)
192 : "cc",
193 "%xmm1", "%xmm2", "%xmm3",
194 "%xmm4", "%xmm5", "%xmm6", "%xmm7",
195 "%k4"
196 );
197
198 return index;
199 }
200
201 size_t abs_max_index(const float *src, size_t count)
202 {
203 uint32_t index;
204
205 ARCH_X86_ASM(
206 IMINMAX_CORE("$6", SEL_ABS)
207 : [src] "+r" (src), [count] "+r" (count),
208 [index] "=Yz" (index)
209 : [INCR] "o" (minmax_incr)
210 : "cc",
211 "%xmm1", "%xmm2", "%xmm3",
212 "%xmm4", "%xmm5", "%xmm6", "%xmm7",
213 "%k4"
214 );
215
216 return index;
217 }
218
219 #undef IMINMAX_CORE
220
221 /*
222 * Register allocation:
223 * ymm0 = imin
224 * ymm1 = vmin
225 * ymm2 = imax
226 * ymm3 = vmax
227 * ymm4 = cind
228 * ymm5 = cval
229 * ymm6/k6 = cmask
230 * ymm7/k7 = temp
231 */
232 #define IMINMAX2_CORE(SEL) \
233 __ASM_EMIT("vpxord %%zmm0, %%zmm0, %%zmm0") /* imin */ \
234 __ASM_EMIT("vpxord %%zmm2, %%zmm2, %%zmm2") /* imax */ \
235 __ASM_EMIT("test %[count], %[count]") \
236 __ASM_EMIT("jz 8f") \
237 /* 16x block */ \
238 __ASM_EMIT("vbroadcastss 0x00(%[src]), %%zmm1") /* vmin */ \
239 __ASM_EMIT("vmovdqa32 0x00 + %[INCR], %%zmm4") /* cind */ \
240 __ASM_EMIT(SEL("", "vandps 0xc0 + %[INCR], %%zmm1, %%zmm1")) \
241 __ASM_EMIT("vmovdqa32 %%zmm1, %%zmm3") /* vmax */ \
242 __ASM_EMIT("sub $16, %[count]") \
243 __ASM_EMIT("jb 2f") \
244 __ASM_EMIT("1:") \
245 __ASM_EMIT("vmovups 0x00(%[src]), %%zmm5") \
246 __ASM_EMIT(SEL("", "vandps 0xc0 + %[INCR], %%zmm5, %%zmm5")) \
247 __ASM_EMIT("vcmpps $1, %%zmm1, %%zmm5, %%k6") /* cmpltps */ \
248 __ASM_EMIT("vcmpps $6, %%zmm3, %%zmm5, %%k7") /* cmpgtps */ \
249 __ASM_EMIT("vmovdqa32 %%zmm4, %%zmm0 %{%%k6%}") \
250 __ASM_EMIT("vmovaps %%zmm5, %%zmm1 %{%%k6%}") \
251 __ASM_EMIT("vmovdqa32 %%zmm4, %%zmm2 %{%%k7%}") \
252 __ASM_EMIT("vmovaps %%zmm5, %%zmm3 %{%%k7%}") \
253 __ASM_EMIT("vpaddd 0x40 + %[INCR], %%zmm4, %%zmm4") \
254 __ASM_EMIT("add $0x40, %[src]") \
255 __ASM_EMIT("sub $16, %[count]") \
256 __ASM_EMIT("jae 1b") \
257 __ASM_EMIT("vextracti64x4 $1, %%zmm0, %%ymm7") \
258 __ASM_EMIT("vextractf64x4 $1, %%zmm1, %%ymm5") \
259 __ASM_EMIT("vcmpps $1, %%ymm1, %%ymm5, %%ymm6") \
260 __ASM_EMIT("vpblendvb %%ymm6, %%ymm7, %%ymm0, %%ymm0") \
261 __ASM_EMIT("vblendvps %%ymm6, %%ymm5, %%ymm1, %%ymm1") \
262 __ASM_EMIT("vextracti64x4 $1, %%zmm2, %%ymm7") \
263 __ASM_EMIT("vextractf64x4 $1, %%zmm3, %%ymm5") \
264 __ASM_EMIT("vcmpps $6, %%ymm3, %%ymm5, %%ymm6") \
265 __ASM_EMIT("vpblendvb %%ymm6, %%ymm7, %%ymm2, %%ymm2") \
266 __ASM_EMIT("vblendvps %%ymm6, %%ymm5, %%ymm3, %%ymm3") \
267 __ASM_EMIT("2:") \
268 /* 8x block */ \
269 __ASM_EMIT("add $8, %[count]") \
270 __ASM_EMIT("jl 4f") \
271 __ASM_EMIT("vmovups 0x00(%[src]), %%ymm5") \
272 __ASM_EMIT(SEL("", "vandps 0xc0 + %[INCR], %%ymm5, %%ymm5")) \
273 __ASM_EMIT("vcmpps $1, %%ymm1, %%ymm5, %%ymm6") /* cmpltps */ \
274 __ASM_EMIT("vcmpps $6, %%ymm3, %%ymm5, %%ymm7") /* cmpgtps */ \
275 __ASM_EMIT("vpblendvb %%ymm6, %%ymm4, %%ymm0, %%ymm0") \
276 __ASM_EMIT("vblendvps %%ymm6, %%ymm5, %%ymm1, %%ymm1") \
277 __ASM_EMIT("vpblendvb %%ymm7, %%ymm4, %%ymm2, %%ymm2") \
278 __ASM_EMIT("vblendvps %%ymm7, %%ymm5, %%ymm3, %%ymm3") \
279 __ASM_EMIT("vpaddd 0x80 + %[INCR], %%ymm4, %%ymm4") \
280 __ASM_EMIT("add $0x20, %[src]") \
281 __ASM_EMIT("sub $8, %[count]") \
282 __ASM_EMIT("4:") \
283 __ASM_EMIT("vextracti128 $1, %%ymm0, %%xmm7") \
284 __ASM_EMIT("vextractf128 $1, %%ymm1, %%xmm5") \
285 __ASM_EMIT("vcmpps $1, %%xmm1, %%xmm5, %%xmm6") \
286 __ASM_EMIT("vpblendvb %%xmm6, %%xmm7, %%xmm0, %%xmm0") \
287 __ASM_EMIT("vblendvps %%xmm6, %%xmm5, %%xmm1, %%xmm1") \
288 __ASM_EMIT("vextracti128 $1, %%ymm2, %%xmm7") \
289 __ASM_EMIT("vextractf128 $1, %%ymm3, %%xmm5") \
290 __ASM_EMIT("vcmpps $6, %%xmm3, %%xmm5, %%xmm6") \
291 __ASM_EMIT("vpblendvb %%xmm6, %%xmm7, %%xmm2, %%xmm2") \
292 __ASM_EMIT("vblendvps %%xmm6, %%xmm5, %%xmm3, %%xmm3") \
293 /* 4x block */ \
294 __ASM_EMIT("add $4, %[count]") \
295 __ASM_EMIT("jl 6f") \
296 __ASM_EMIT("vmovups 0x00(%[src]), %%xmm5") \
297 __ASM_EMIT(SEL("", "vandps 0xc0 + %[INCR], %%xmm5, %%xmm5")) \
298 __ASM_EMIT("vcmpps $1, %%xmm1, %%xmm5, %%xmm6") /* cmpltps */ \
299 __ASM_EMIT("vcmpps $6, %%xmm3, %%xmm5, %%xmm7") /* cmpgtps */ \
300 __ASM_EMIT("vpblendvb %%xmm6, %%xmm4, %%xmm0, %%xmm0") \
301 __ASM_EMIT("vblendvps %%xmm6, %%xmm5, %%xmm1, %%xmm1") \
302 __ASM_EMIT("vpblendvb %%xmm7, %%xmm4, %%xmm2, %%xmm2") \
303 __ASM_EMIT("vblendvps %%xmm7, %%xmm5, %%xmm3, %%xmm3") \
304 __ASM_EMIT("vpaddd 0xa0 + %[INCR], %%xmm4, %%xmm4") \
305 __ASM_EMIT("sub $4, %[count]") \
306 __ASM_EMIT("add $0x10, %[src]") \
307 __ASM_EMIT("6:") \
308 __ASM_EMIT("vpshufd $0x39, %%xmm0, %%xmm7") \
309 __ASM_EMIT("vshufps $0x39, %%xmm1, %%xmm1, %%xmm5") \
310 __ASM_EMIT("vcmpps $1, %%xmm1, %%xmm5, %%xmm6") \
311 __ASM_EMIT("vpblendvb %%xmm6, %%xmm7, %%xmm0, %%xmm0") \
312 __ASM_EMIT("vblendvps %%xmm6, %%xmm5, %%xmm1, %%xmm1") \
313 __ASM_EMIT("vpunpckhdq %%xmm0, %%xmm0, %%xmm7") \
314 __ASM_EMIT("vunpckhps %%xmm1, %%xmm1, %%xmm5") \
315 __ASM_EMIT("vcmpps $1, %%xmm1, %%xmm5, %%xmm6") \
316 __ASM_EMIT("vpblendvb %%xmm6, %%xmm7, %%xmm0, %%xmm0") \
317 __ASM_EMIT("vblendvps %%xmm6, %%xmm5, %%xmm1, %%xmm1") \
318 __ASM_EMIT("vpshufd $0x39, %%xmm2, %%xmm7") \
319 __ASM_EMIT("vshufps $0x39, %%xmm3, %%xmm3, %%xmm5") \
320 __ASM_EMIT("vcmpps $6, %%xmm3, %%xmm5, %%xmm6") \
321 __ASM_EMIT("vpblendvb %%xmm6, %%xmm7, %%xmm2, %%xmm2") \
322 __ASM_EMIT("vblendvps %%xmm6, %%xmm5, %%xmm3, %%xmm3") \
323 __ASM_EMIT("vpunpckhdq %%xmm2, %%xmm2, %%xmm7") \
324 __ASM_EMIT("vunpckhps %%xmm3, %%xmm3, %%xmm5") \
325 __ASM_EMIT("vcmpps $6, %%xmm3, %%xmm5, %%xmm6") \
326 __ASM_EMIT("vpblendvb %%xmm6, %%xmm7, %%xmm2, %%xmm2") \
327 __ASM_EMIT("vblendvps %%xmm6, %%xmm5, %%xmm3, %%xmm3") \
328 /* 1x block */ \
329 __ASM_EMIT("add $3, %[count]") \
330 __ASM_EMIT("jl 8f") \
331 __ASM_EMIT("7:") \
332 __ASM_EMIT("movss 0x00(%[src]), %%xmm5") \
333 __ASM_EMIT(SEL("", "vandps 0xc0 + %[INCR], %%xmm5, %%xmm5")) \
334 __ASM_EMIT("vcmpps $1, %%xmm1, %%xmm5, %%xmm6") /* cmpltps */ \
335 __ASM_EMIT("vcmpps $6, %%xmm3, %%xmm5, %%xmm7") /* cmpgtps */ \
336 __ASM_EMIT("vpblendvb %%xmm6, %%xmm4, %%xmm0, %%xmm0") \
337 __ASM_EMIT("vblendvps %%xmm6, %%xmm5, %%xmm1, %%xmm1") \
338 __ASM_EMIT("vpblendvb %%xmm7, %%xmm4, %%xmm2, %%xmm2") \
339 __ASM_EMIT("vblendvps %%xmm7, %%xmm5, %%xmm3, %%xmm3") \
340 __ASM_EMIT("vpaddd 0xb0 + %[INCR], %%xmm4, %%xmm4") \
341 __ASM_EMIT("add $0x04, %[src]") \
342 __ASM_EMIT("dec %[count]") \
343 __ASM_EMIT("jge 7b") \
344 __ASM_EMIT("8:") \
345 /* end */ \
346 __ASM_EMIT("movss %%xmm0, (%[min])") \
347 __ASM_EMIT("movss %%xmm2, (%[max])")
348
349 void minmax_index(const float *src, size_t count, size_t *min, size_t *max)
350 {
351 ARCH_X86_ASM(
352 IMINMAX2_CORE(SEL_NONE)
353 : [src] "+r" (src), [count] "+r" (count)
354 : [min] "r" (min), [max] "r" (max),
355 [INCR] "o" (minmax_incr)
356 : "cc", "memory",
357 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
358 "%xmm4", "%xmm5", "%xmm6", "%xmm7",
359 "%k6", "%k7"
360 );
361 }
362
363 void abs_minmax_index(const float *src, size_t count, size_t *min, size_t *max)
364 {
365 ARCH_X86_ASM(
366 IMINMAX2_CORE(SEL_ABS)
367 : [src] "+r" (src), [count] "+r" (count)
368 : [min] "r" (min), [max] "r" (max),
369 [INCR] "o" (minmax_incr)
370 : "cc", "memory",
371 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
372 "%xmm4", "%xmm5", "%xmm6", "%xmm7",
373 "%k6", "%k7"
374 );
375 }
376
377 #undef IMINMAX2_CORE
378 #undef SEL_NONE
379 #undef SEL_ABS
380 } /* namespace avx512 */
381 } /* namespace lsp */
382
383
384 #endif /* PRIVATE_DSP_ARCH_X86_AVX512_SEARCH_IMINMAX_H_ */