Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/x86/sse2/search/iminmax.h

raw
1 /*
2 * Copyright (C) 2020 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2020 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 31 мар. 2020 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_X86_SSE2_SEARCH_IMINMAX_H_
23 #define PRIVATE_DSP_ARCH_X86_SSE2_SEARCH_IMINMAX_H_
24
25 #ifndef PRIVATE_DSP_ARCH_X86_SSE2_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_X86_SSE2_IMPL */
28
29 namespace lsp
30 {
31 namespace sse2
32 {
33 IF_ARCH_X86(
34 static const uint32_t indexes[] __lsp_aligned16 =
35 {
36 0, 1, 2, 3, // indexes
37 4, 4, 4, 4, // step
38 1, 1, 1, 1
39 };
40
41 static const uint32_t iminmax_const[] __lsp_aligned16 =
42 {
43 LSP_DSP_VEC4(0x7fffffff)
44 };
45 )
46
47 #define CMPLTPS "$1"
48 #define CMPLEPS "$2"
49 #define CMPGEPS "$5"
50 #define CMPGTPS "$6"
51
52 /* Register allocation:
53 * xmm0 = rind - result indexes
54 * xmm1 = cind - current indexes
55 * xmm2 = rval - result values
56 * xmm3 = cval - current values
57 * xmm4 = xcmp - comparison mask
58 * xmm5 = temp
59 * xmm6 = temp
60 * xmm7 = incr - increment
61 */
62 #define SEARCH_CORE(op) \
63 __ASM_EMIT("pxor %%xmm0, %%xmm0") \
64 __ASM_EMIT("test %[count], %[count]") \
65 __ASM_EMIT("jz 4f") \
66 __ASM_EMIT("movss 0x00(%[src]), %%xmm2") /* x2 = s = s0 */ \
67 __ASM_EMIT("movdqa 0x00(%[IDXS]), %%xmm1") /* x1 = I[k] = 0 1 2 3 */ \
68 __ASM_EMIT("shufps $0x00, %%xmm2, %%xmm2") /* x2 = s[k] = s0 s0 s0 s0 */ \
69 __ASM_EMIT("sub $0x04, %[count]") \
70 __ASM_EMIT("jb 2f") \
71 __ASM_EMIT("movdqa 0x10(%[IDXS]), %%xmm7") /* x7 = D = 4 4 4 4 */ \
72 \
73 /* x4 blocks */ \
74 __ASM_EMIT("1:") \
75 __ASM_EMIT("movaps %%xmm2, %%xmm4") /* x4 = s[k] */ \
76 __ASM_EMIT("movups 0x00(%[src]), %%xmm3") /* x3 = S[k] */ \
77 __ASM_EMIT("cmpps " op ", %%xmm3, %%xmm4") /* x4 = s[k] <=> S[k] */ \
78 __ASM_EMIT("movdqa %%xmm4, %%xmm5") /* x5 = s[k] <=> S[k] */ \
79 __ASM_EMIT("pand %%xmm5, %%xmm0") /* x0 = i[k] & (s[k] <=> S[k]) */ \
80 __ASM_EMIT("andps %%xmm4, %%xmm2") /* x2 = s[k] & (s[k] <=> S[k]) */ \
81 __ASM_EMIT("pandn %%xmm1, %%xmm5") /* x5 = I[k] & !(s[k] <=> S[k]) */ \
82 __ASM_EMIT("andnps %%xmm3, %%xmm4") /* x4 = S[k] & !(s[k] <=> S[k]) */ \
83 __ASM_EMIT("por %%xmm5, %%xmm0") /* x0 = (i[k] & (s[k] <=> S[k])) | (I[k] & !(s[k] <=> S[k])) */ \
84 __ASM_EMIT("orps %%xmm4, %%xmm2") /* x2 = (s[k] & (s[k] <=> S[k])) | (S[k] & !(s[k] <=> S[k])) */ \
85 __ASM_EMIT("paddd %%xmm7, %%xmm1") /* x1 = I[k] = i[k]+D */ \
86 __ASM_EMIT("add $0x10, %[src]") \
87 __ASM_EMIT("sub $4, %[count]") \
88 __ASM_EMIT("jae 1b") \
89 /* Post-process, step 1 */ \
90 __ASM_EMIT("movdqa %%xmm0, %%xmm6") /* x5 = rind = i0 i1 i2 i3 */ \
91 __ASM_EMIT("movaps %%xmm2, %%xmm3") /* x3 = rval = x0 x1 x2 x3 */ \
92 __ASM_EMIT("pshufd $0x39, %%xmm6, %%xmm6") /* x6 = cind = i1 i2 i3 i0 */ \
93 __ASM_EMIT("shufps $0x39, %%xmm3, %%xmm3") /* x3 = cval = x1 x2 x3 x0 */ \
94 __ASM_EMIT("movaps %%xmm2, %%xmm4") /* x4 = x0 x1 x2 x3 */ \
95 __ASM_EMIT("cmpps " op ", %%xmm3, %%xmm4") /* x4 = cmp(cval, nval) */ \
96 __ASM_EMIT("movdqa %%xmm4, %%xmm5") \
97 __ASM_EMIT("pand %%xmm5, %%xmm0") \
98 __ASM_EMIT("andps %%xmm4, %%xmm2") \
99 __ASM_EMIT("pandn %%xmm6, %%xmm5") \
100 __ASM_EMIT("andnps %%xmm3, %%xmm4") \
101 __ASM_EMIT("por %%xmm5, %%xmm0") \
102 __ASM_EMIT("orps %%xmm4, %%xmm2") \
103 /* Post-process, step 2 */ \
104 __ASM_EMIT("movdqa %%xmm0, %%xmm6") /* x5 = rind = i0 i1 i2 i3 */ \
105 __ASM_EMIT("movaps %%xmm2, %%xmm3") /* x3 = rval = x0 x1 x2 x3 */ \
106 __ASM_EMIT("punpckhdq %%xmm6, %%xmm6") /* x6 = cind = i2 i2 i3 i3 */ \
107 __ASM_EMIT("unpckhps %%xmm3, %%xmm3") /* x3 = cval = x2 x3 x2 x3 */ \
108 __ASM_EMIT("movaps %%xmm2, %%xmm4") /* x4 = x0 x1 x2 x3 */ \
109 __ASM_EMIT("cmpps " op ", %%xmm3, %%xmm4") /* x4 = cmp(cval, nval) */ \
110 __ASM_EMIT("movdqa %%xmm4, %%xmm5") \
111 __ASM_EMIT("pand %%xmm5, %%xmm0") \
112 __ASM_EMIT("andps %%xmm4, %%xmm2") \
113 __ASM_EMIT("pandn %%xmm6, %%xmm5") \
114 __ASM_EMIT("andnps %%xmm3, %%xmm4") \
115 __ASM_EMIT("por %%xmm5, %%xmm0") \
116 __ASM_EMIT("orps %%xmm4, %%xmm2") \
117 /* x1 blocks */ \
118 __ASM_EMIT("2:") \
119 __ASM_EMIT("add $3, %[count]") \
120 __ASM_EMIT("jl 4f") \
121 __ASM_EMIT("movdqa 0x20(%[IDXS]), %%xmm7") /* x7 = D = 1 1 1 1 */ \
122 __ASM_EMIT("3:") \
123 __ASM_EMIT("movaps %%xmm2, %%xmm4") \
124 __ASM_EMIT("movss 0x00(%[src]), %%xmm3") \
125 __ASM_EMIT("cmpss " op ", %%xmm3, %%xmm4") \
126 __ASM_EMIT("pand %%xmm4, %%xmm0") \
127 __ASM_EMIT("movdqa %%xmm4, %%xmm5") \
128 __ASM_EMIT("andps %%xmm4, %%xmm2") \
129 __ASM_EMIT("pandn %%xmm1, %%xmm5") \
130 __ASM_EMIT("andnps %%xmm3, %%xmm4") \
131 __ASM_EMIT("por %%xmm5, %%xmm0") \
132 __ASM_EMIT("orps %%xmm4, %%xmm2") \
133 __ASM_EMIT("paddd %%xmm7, %%xmm1") \
134 __ASM_EMIT("add $0x04, %[src]") \
135 __ASM_EMIT("dec %[count]") \
136 __ASM_EMIT("jge 3b") \
137 /* End */ \
138 __ASM_EMIT("4:")
139
140
141 size_t min_index(const float *src, size_t count)
142 {
143 uint32_t index;
144
145 ARCH_X86_ASM(
146 SEARCH_CORE(CMPLEPS)
147 : [src] "+r" (src), [count] "+r" (count),
148 [index] "=Yz" (index)
149 : [IDXS] "r" (indexes)
150 : "cc",
151 "%xmm1", "%xmm2", "%xmm3",
152 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
153 );
154
155 return index;
156 }
157
158 size_t max_index(const float *src, size_t count)
159 {
160 uint32_t index;
161
162 ARCH_X86_ASM(
163 SEARCH_CORE(CMPGEPS)
164 : [src] "+r" (src), [count] "+r" (count),
165 [index] "=Yz" (index)
166 : [IDXS] "r" (indexes)
167 : "cc",
168 "%xmm1", "%xmm2", "%xmm3",
169 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
170 );
171
172 return index;
173 }
174
175 #undef SEARCH_CORE
176
177 /* Register allocation:
178 * xmm0 = rind - result indexes
179 * xmm1 = cind - current indexes
180 * xmm2 = rval - result values
181 * xmm3 = cval - current values
182 * xmm4 = xcmp - comparison mask
183 * xmm5 = temp
184 * xmm6 = sign - sign mask
185 * xmm7 = incr - increment
186 */
187 #define ABS_SEARCH_CORE(op) \
188 __ASM_EMIT("pxor %%xmm0, %%xmm0") \
189 __ASM_EMIT("test %[count], %[count]") \
190 __ASM_EMIT("jz 4f") \
191 __ASM_EMIT("movss 0x00(%[src]), %%xmm2") /* x2 = s = s0 */ \
192 __ASM_EMIT("movdqa 0x00(%[IDXS]), %%xmm1") /* x1 = I[k] = 0 1 2 3 */ \
193 __ASM_EMIT("movaps %[X_SIGN], %%xmm6") /* x6 = SIGN */ \
194 __ASM_EMIT("shufps $0x00, %%xmm2, %%xmm2") /* x2 = s[k] = s0 s0 s0 s0 */ \
195 __ASM_EMIT("sub $0x04, %[count]") \
196 __ASM_EMIT("andps %%xmm6, %%xmm2") /* x2 = abs(s[k]) */ \
197 __ASM_EMIT("jb 2f") \
198 __ASM_EMIT("movdqa 0x10(%[IDXS]), %%xmm7") /* x7 = D = 4 4 4 4 */ \
199 \
200 /* x4 blocks */ \
201 __ASM_EMIT("1:") \
202 __ASM_EMIT("movaps %%xmm2, %%xmm4") /* x4 = s[k] */ \
203 __ASM_EMIT("movups 0x00(%[src]), %%xmm3") /* x3 = S[k] */ \
204 __ASM_EMIT("andps %%xmm6, %%xmm3") /* x3 = abs(S[k]) */ \
205 __ASM_EMIT("cmpps " op ", %%xmm3, %%xmm4") /* x4 = s[k] <=> S[k] */ \
206 __ASM_EMIT("movdqa %%xmm4, %%xmm5") /* x5 = s[k] <=> S[k] */ \
207 __ASM_EMIT("pand %%xmm5, %%xmm0") /* x0 = i[k] & (s[k] <=> S[k]) */ \
208 __ASM_EMIT("andps %%xmm4, %%xmm2") /* x2 = s[k] & (s[k] <=> S[k]) */ \
209 __ASM_EMIT("pandn %%xmm1, %%xmm5") /* x5 = I[k] & !(s[k] <=> S[k]) */ \
210 __ASM_EMIT("andnps %%xmm3, %%xmm4") /* x4 = S[k] & !(s[k] <=> S[k]) */ \
211 __ASM_EMIT("por %%xmm5, %%xmm0") /* x0 = (i[k] & (s[k] <=> S[k])) | (I[k] & !(s[k] <=> S[k])) */ \
212 __ASM_EMIT("orps %%xmm4, %%xmm2") /* x2 = (s[k] & (s[k] <=> S[k])) | (S[k] & !(s[k] <=> S[k])) */ \
213 __ASM_EMIT("paddd %%xmm7, %%xmm1") /* x1 = I[k] = i[k]+D */ \
214 __ASM_EMIT("add $0x10, %[src]") \
215 __ASM_EMIT("sub $4, %[count]") \
216 __ASM_EMIT("jae 1b") \
217 /* Post-process, step 1 */ \
218 __ASM_EMIT("movdqa %%xmm0, %%xmm6") /* x5 = rind = i0 i1 i2 i3 */ \
219 __ASM_EMIT("movaps %%xmm2, %%xmm3") /* x3 = rval = x0 x1 x2 x3 */ \
220 __ASM_EMIT("pshufd $0x39, %%xmm6, %%xmm6") /* x6 = cind = i1 i2 i3 i0 */ \
221 __ASM_EMIT("shufps $0x39, %%xmm3, %%xmm3") /* x3 = cval = x1 x2 x3 x0 */ \
222 __ASM_EMIT("movaps %%xmm2, %%xmm4") /* x4 = x0 x1 x2 x3 */ \
223 __ASM_EMIT("cmpps " op ", %%xmm3, %%xmm4") /* x4 = cmp(cval, nval) */ \
224 __ASM_EMIT("movdqa %%xmm4, %%xmm5") \
225 __ASM_EMIT("pand %%xmm5, %%xmm0") \
226 __ASM_EMIT("andps %%xmm4, %%xmm2") \
227 __ASM_EMIT("pandn %%xmm6, %%xmm5") \
228 __ASM_EMIT("andnps %%xmm3, %%xmm4") \
229 __ASM_EMIT("por %%xmm5, %%xmm0") \
230 __ASM_EMIT("orps %%xmm4, %%xmm2") \
231 /* Post-process, step 2 */ \
232 __ASM_EMIT("movdqa %%xmm0, %%xmm6") /* x5 = rind = i0 i1 i2 i3 */ \
233 __ASM_EMIT("movaps %%xmm2, %%xmm3") /* x3 = rval = x0 x1 x2 x3 */ \
234 __ASM_EMIT("punpckhdq %%xmm6, %%xmm6") /* x6 = cind = i2 i2 i3 i3 */ \
235 __ASM_EMIT("unpckhps %%xmm3, %%xmm3") /* x3 = cval = x2 x3 x2 x3 */ \
236 __ASM_EMIT("movaps %%xmm2, %%xmm4") /* x4 = x0 x1 x2 x3 */ \
237 __ASM_EMIT("cmpps " op ", %%xmm3, %%xmm4") /* x4 = cmp(cval, nval) */ \
238 __ASM_EMIT("movdqa %%xmm4, %%xmm5") \
239 __ASM_EMIT("pand %%xmm5, %%xmm0") \
240 __ASM_EMIT("andps %%xmm4, %%xmm2") \
241 __ASM_EMIT("pandn %%xmm6, %%xmm5") \
242 __ASM_EMIT("andnps %%xmm3, %%xmm4") \
243 __ASM_EMIT("por %%xmm5, %%xmm0") \
244 __ASM_EMIT("orps %%xmm4, %%xmm2") \
245 __ASM_EMIT("movaps %[X_SIGN], %%xmm6") /* x6 = SIGN (restore) */ \
246 /* x1 blocks */ \
247 __ASM_EMIT("2:") \
248 __ASM_EMIT("add $3, %[count]") \
249 __ASM_EMIT("jl 4f") \
250 __ASM_EMIT("movdqa 0x20(%[IDXS]), %%xmm7") /* x7 = D = 1 1 1 1 */ \
251 __ASM_EMIT("3:") \
252 __ASM_EMIT("movaps %%xmm2, %%xmm4") \
253 __ASM_EMIT("movss 0x00(%[src]), %%xmm3") \
254 __ASM_EMIT("andps %%xmm6, %%xmm3") /* x3 = abs(S[k]) */ \
255 __ASM_EMIT("cmpps " op ", %%xmm3, %%xmm4") \
256 __ASM_EMIT("movdqa %%xmm4, %%xmm5") /* x5 = s[k] <=> S[k] */ \
257 __ASM_EMIT("pand %%xmm5, %%xmm0") /* x0 = i[k] & (s[k] <=> S[k]) */ \
258 __ASM_EMIT("andps %%xmm4, %%xmm2") /* x2 = s[k] & (s[k] <=> S[k]) */ \
259 __ASM_EMIT("pandn %%xmm1, %%xmm5") /* x5 = I[k] & !(s[k] <=> S[k]) */ \
260 __ASM_EMIT("andnps %%xmm3, %%xmm4") /* x4 = S[k] & !(s[k] <=> S[k]) */ \
261 __ASM_EMIT("por %%xmm5, %%xmm0") /* x0 = (i[k] & (s[k] <=> S[k])) | (I[k] & !(s[k] <=> S[k])) */ \
262 __ASM_EMIT("orps %%xmm4, %%xmm2") /* x2 = (s[k] & (s[k] <=> S[k])) | (S[k] & !(s[k] <=> S[k])) */ \
263 __ASM_EMIT("paddd %%xmm7, %%xmm1") /* x1 = I[k] = i[k]+D */ \
264 __ASM_EMIT("add $0x04, %[src]") \
265 __ASM_EMIT("dec %[count]") \
266 __ASM_EMIT("jge 3b") \
267 /* End */ \
268 __ASM_EMIT("4:")
269
270 size_t abs_min_index(const float *src, size_t count)
271 {
272 uint32_t index;
273
274 ARCH_X86_ASM(
275 ABS_SEARCH_CORE(CMPLEPS)
276 : [src] "+r" (src), [count] "+r" (count),
277 [index] "=Yz" (index)
278 : [IDXS] "r" (indexes),
279 [X_SIGN] "m" (iminmax_const)
280 : "cc",
281 "%xmm1", "%xmm2", "%xmm3",
282 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
283 );
284
285 return index;
286 }
287
288 size_t abs_max_index(const float *src, size_t count)
289 {
290 uint32_t index;
291
292 ARCH_X86_ASM(
293 ABS_SEARCH_CORE(CMPGEPS)
294 : [src] "+r" (src), [count] "+r" (count),
295 [index] "=Yz" (index)
296 : [IDXS] "r" (indexes),
297 [X_SIGN] "m" (iminmax_const)
298 : "cc",
299 "%xmm1", "%xmm2", "%xmm3",
300 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
301 );
302
303 return index;
304 }
305
306 #undef ABS_SEARCH_CORE
307
308 void minmax_index(const float *src, size_t count, size_t *min, size_t *max)
309 {
310 uint32_t counters[4] __lsp_aligned16;
311
312 ARCH_X86_ASM(
313 __ASM_EMIT("pxor %%xmm0, %%xmm0") // x0 = idx_min
314 __ASM_EMIT("pxor %%xmm1, %%xmm1") // x1 = idx_max
315 __ASM_EMIT("test %[count], %[count]")
316 __ASM_EMIT("jz 4f")
317
318 __ASM_EMIT("movss 0x00(%[src]), %%xmm2") // x2 = min
319 __ASM_EMIT("shufps $0x00, %%xmm2, %%xmm2") // x2 = min
320 __ASM_EMIT("movdqa 0x00(%[IDXS]), %%xmm4") // x4 = idx_new
321 __ASM_EMIT("movaps %%xmm2, %%xmm3") // x3 = max
322 __ASM_EMIT("sub $0x04, %[count]")
323 __ASM_EMIT("jb 2f")
324
325 // x4 blocks
326 __ASM_EMIT("1:")
327 __ASM_EMIT("movups 0x00(%[src]), %%xmm5") // x5 = sample
328 // Find minimum
329 __ASM_EMIT("movaps %%xmm2, %%xmm6") // x6 = min
330 __ASM_EMIT("cmpps " CMPLEPS ", %%xmm5, %%xmm6") // x6 = min <= sample
331 __ASM_EMIT("movdqa %%xmm6, %%xmm7") // x7 = min <= sample
332 __ASM_EMIT("pand %%xmm6, %%xmm0") // x0 = idx_min & (min <= sample)
333 __ASM_EMIT("andps %%xmm7, %%xmm2") // x2 = min & (min <= sample)
334 __ASM_EMIT("pandn %%xmm4, %%xmm6") // x6 = idx_new & !(min <= sample)
335 __ASM_EMIT("andnps %%xmm5, %%xmm7") // x7 = sample & !(min <= sample)
336 __ASM_EMIT("por %%xmm6, %%xmm0") // x0 = idx_min & (min <= sample) | idx_new & !(min <= sample)
337 __ASM_EMIT("orps %%xmm7, %%xmm2") // x2 = min & (min <= sample) | sample & !(min <= sample)
338 // Find maximum
339 __ASM_EMIT("movaps %%xmm3, %%xmm6") // x6 = max
340 __ASM_EMIT("cmpps " CMPGEPS ", %%xmm5, %%xmm6") // x6 = max >= sample
341 __ASM_EMIT("movdqa %%xmm6, %%xmm7") // x7 = max >= sample
342 __ASM_EMIT("pand %%xmm6, %%xmm1") // x1 = idx_max & (max >= sample)
343 __ASM_EMIT("andps %%xmm7, %%xmm3") // x3 = max & (max >= sample)
344 __ASM_EMIT("pandn %%xmm4, %%xmm6") // x6 = idx_new & !(max >= sample)
345 __ASM_EMIT("andnps %%xmm5, %%xmm7") // x7 = sample & !(max >= sample)
346 __ASM_EMIT("por %%xmm6, %%xmm1") // x1 = idx_max & (max >= sample) | idx_new & !(max >= sample)
347 __ASM_EMIT("orps %%xmm7, %%xmm3") // x3 = max & (max >= sample) | sample & !(max >= sample)
348 // Next loop
349 __ASM_EMIT("paddd 0x10(%[IDXS]), %%xmm4") // x4 = idx_new + 4
350 __ASM_EMIT("add $0x10, %[src]") // src += 4
351 __ASM_EMIT("sub $4, %[count]") // count -= 4
352 __ASM_EMIT("jae 1b")
353
354 // Post-process
355 __ASM_EMIT("movdqa %%xmm4, %[COUNTERS]") // Save current index
356 // Step 1
357 __ASM_EMIT("movhlps %%xmm0, %%xmm4")
358 __ASM_EMIT("movhlps %%xmm2, %%xmm5")
359 __ASM_EMIT("movaps %%xmm2, %%xmm6")
360 __ASM_EMIT("cmpps " CMPLEPS ", %%xmm5, %%xmm6")
361 __ASM_EMIT("movdqa %%xmm6, %%xmm7")
362 __ASM_EMIT("pand %%xmm6, %%xmm0")
363 __ASM_EMIT("andps %%xmm7, %%xmm2")
364 __ASM_EMIT("pandn %%xmm4, %%xmm6")
365 __ASM_EMIT("andnps %%xmm5, %%xmm7")
366 __ASM_EMIT("por %%xmm6, %%xmm0")
367 __ASM_EMIT("orps %%xmm7, %%xmm2")
368
369 __ASM_EMIT("movhlps %%xmm1, %%xmm4")
370 __ASM_EMIT("movhlps %%xmm3, %%xmm5")
371 __ASM_EMIT("movaps %%xmm3, %%xmm6")
372 __ASM_EMIT("cmpps " CMPGEPS ", %%xmm5, %%xmm6")
373 __ASM_EMIT("movdqa %%xmm6, %%xmm7")
374 __ASM_EMIT("pand %%xmm6, %%xmm1")
375 __ASM_EMIT("andps %%xmm7, %%xmm3")
376 __ASM_EMIT("pandn %%xmm4, %%xmm6")
377 __ASM_EMIT("andnps %%xmm5, %%xmm7")
378 __ASM_EMIT("por %%xmm6, %%xmm1")
379 __ASM_EMIT("orps %%xmm7, %%xmm3")
380
381 // Step 2
382 __ASM_EMIT("unpcklps %%xmm0, %%xmm0")
383 __ASM_EMIT("unpcklps %%xmm2, %%xmm2")
384 __ASM_EMIT("movhlps %%xmm0, %%xmm4")
385 __ASM_EMIT("movhlps %%xmm2, %%xmm5")
386 __ASM_EMIT("movaps %%xmm2, %%xmm6")
387 __ASM_EMIT("cmpps " CMPLEPS ", %%xmm5, %%xmm6")
388 __ASM_EMIT("movdqa %%xmm6, %%xmm7")
389 __ASM_EMIT("pand %%xmm6, %%xmm0")
390 __ASM_EMIT("andps %%xmm7, %%xmm2")
391 __ASM_EMIT("pandn %%xmm4, %%xmm6")
392 __ASM_EMIT("andnps %%xmm5, %%xmm7")
393 __ASM_EMIT("por %%xmm6, %%xmm0")
394 __ASM_EMIT("orps %%xmm7, %%xmm2")
395
396 __ASM_EMIT("unpcklps %%xmm1, %%xmm1")
397 __ASM_EMIT("unpcklps %%xmm3, %%xmm3")
398 __ASM_EMIT("movhlps %%xmm1, %%xmm4")
399 __ASM_EMIT("movhlps %%xmm3, %%xmm5")
400 __ASM_EMIT("movaps %%xmm3, %%xmm6")
401 __ASM_EMIT("cmpps " CMPGEPS ", %%xmm5, %%xmm6")
402 __ASM_EMIT("movdqa %%xmm6, %%xmm7")
403 __ASM_EMIT("pand %%xmm6, %%xmm1")
404 __ASM_EMIT("andps %%xmm7, %%xmm3")
405 __ASM_EMIT("pandn %%xmm4, %%xmm6")
406 __ASM_EMIT("andnps %%xmm5, %%xmm7")
407 __ASM_EMIT("por %%xmm6, %%xmm1")
408 __ASM_EMIT("orps %%xmm7, %%xmm3")
409
410 __ASM_EMIT("movdqa %[COUNTERS], %%xmm4") // Restore current index
411
412 __ASM_EMIT("2:")
413 __ASM_EMIT("add $3, %[count]")
414 __ASM_EMIT("jl 4f")
415
416 // x1 blocks
417 __ASM_EMIT("3:")
418 __ASM_EMIT("movss 0x00(%[src]), %%xmm5") // x5 = sample
419 // Find minimum
420 __ASM_EMIT("movaps %%xmm2, %%xmm6") // x6 = min
421 __ASM_EMIT("cmpss " CMPLEPS ", %%xmm5, %%xmm6") // x6 = min <= sample
422 __ASM_EMIT("movdqa %%xmm6, %%xmm7") // x7 = min <= sample
423 __ASM_EMIT("pand %%xmm6, %%xmm0") // x0 = idx_min & (min <= sample)
424 __ASM_EMIT("andps %%xmm7, %%xmm2") // x2 = min & (min <= sample)
425 __ASM_EMIT("pandn %%xmm4, %%xmm6") // x6 = idx_new & !(min <= sample)
426 __ASM_EMIT("andnps %%xmm5, %%xmm7") // x7 = sample & !(min <= sample)
427 __ASM_EMIT("por %%xmm6, %%xmm0") // x0 = idx_min & (min <= sample) | idx_new & !(min <= sample)
428 __ASM_EMIT("orps %%xmm7, %%xmm2") // x2 = min & (min <= sample) | sample & !(min <= sample)
429 // Find maximum
430 __ASM_EMIT("movaps %%xmm3, %%xmm6") // x6 = max
431 __ASM_EMIT("cmpss " CMPGEPS ", %%xmm5, %%xmm6") // x6 = max >= sample
432 __ASM_EMIT("movdqa %%xmm6, %%xmm7") // x7 = max >= sample
433 __ASM_EMIT("pand %%xmm6, %%xmm1") // x1 = idx_max & (max >= sample)
434 __ASM_EMIT("andps %%xmm7, %%xmm3") // x3 = max & (max >= sample)
435 __ASM_EMIT("pandn %%xmm4, %%xmm6") // x6 = idx_new & !(max >= sample)
436 __ASM_EMIT("andnps %%xmm5, %%xmm7") // x7 = sample & !(max >= sample)
437 __ASM_EMIT("por %%xmm6, %%xmm1") // x1 = idx_max & (max >= sample) | idx_new & !(max >= sample)
438 __ASM_EMIT("orps %%xmm7, %%xmm3") // x3 = max & (max >= sample) | sample & !(max >= sample)
439 // Next loop
440 __ASM_EMIT("paddd 0x20(%[IDXS]), %%xmm4") // x4 = idx_new + 1
441 __ASM_EMIT("add $0x04, %[src]") // src += 4
442 __ASM_EMIT("dec %[count]") // count --
443 __ASM_EMIT("jge 3b")
444
445 __ASM_EMIT("4:")
446 __ASM_EMIT("movd %%xmm0, (%[min])")
447 __ASM_EMIT("movd %%xmm1, (%[max])")
448 __ASM_EMIT64("movl $0, 0x04(%[min])")
449 __ASM_EMIT64("movl $0, 0x04(%[max])")
450 : [src] "+r" (src), [count] "+r" (count)
451 : [min] "r" (min), [max] "r" (max),
452 [IDXS] "r" (indexes),
453 [COUNTERS] "m" (counters)
454 );
455 }
456
457 void abs_minmax_index(const float *src, size_t count, size_t *min, size_t *max)
458 {
459 uint32_t counters[4] __lsp_aligned16;
460
461 ARCH_X86_ASM(
462 __ASM_EMIT("pxor %%xmm0, %%xmm0") // x0 = idx_min
463 __ASM_EMIT("pxor %%xmm1, %%xmm1") // x1 = idx_max
464 __ASM_EMIT("test %[count], %[count]")
465 __ASM_EMIT("jz 4f")
466
467 __ASM_EMIT("movss 0x00(%[src]), %%xmm2") // x2 = min
468 __ASM_EMIT("shufps $0x00, %%xmm2, %%xmm2") // x2 = min
469 __ASM_EMIT("movdqa 0x00(%[IDXS]), %%xmm4") // x4 = idx_new
470 __ASM_EMIT("andps %[X_SIGN], %%xmm2") // x5 = abs(sample)
471 __ASM_EMIT("movaps %%xmm2, %%xmm3") // x3 = max = abs(sample)
472 __ASM_EMIT("sub $0x04, %[count]")
473 __ASM_EMIT("jb 2f")
474
475 // x4 blocks
476 __ASM_EMIT("1:")
477 __ASM_EMIT("movups 0x00(%[src]), %%xmm5") // x5 = sample
478 // Find minimum
479 __ASM_EMIT("movaps %%xmm2, %%xmm6") // x6 = min
480 __ASM_EMIT("andps %[X_SIGN], %%xmm5") // x5 = abs(sample)
481 __ASM_EMIT("cmpps " CMPLEPS ", %%xmm5, %%xmm6") // x6 = min <= sample
482 __ASM_EMIT("movdqa %%xmm6, %%xmm7") // x7 = min <= sample
483 __ASM_EMIT("pand %%xmm6, %%xmm0") // x0 = idx_min & (min <= sample)
484 __ASM_EMIT("andps %%xmm7, %%xmm2") // x2 = min & (min <= sample)
485 __ASM_EMIT("pandn %%xmm4, %%xmm6") // x6 = idx_new & !(min <= sample)
486 __ASM_EMIT("andnps %%xmm5, %%xmm7") // x7 = sample & !(min <= sample)
487 __ASM_EMIT("por %%xmm6, %%xmm0") // x0 = idx_min & (min <= sample) | idx_new & !(min <= sample)
488 __ASM_EMIT("orps %%xmm7, %%xmm2") // x2 = min & (min <= sample) | sample & !(min <= sample)
489 // Find maximum
490 __ASM_EMIT("movaps %%xmm3, %%xmm6") // x6 = max
491 __ASM_EMIT("cmpps " CMPGEPS ", %%xmm5, %%xmm6") // x6 = max >= sample
492 __ASM_EMIT("movdqa %%xmm6, %%xmm7") // x7 = max >= sample
493 __ASM_EMIT("pand %%xmm6, %%xmm1") // x1 = idx_max & (max >= sample)
494 __ASM_EMIT("andps %%xmm7, %%xmm3") // x3 = max & (max >= sample)
495 __ASM_EMIT("pandn %%xmm4, %%xmm6") // x6 = idx_new & !(max >= sample)
496 __ASM_EMIT("andnps %%xmm5, %%xmm7") // x7 = sample & !(max >= sample)
497 __ASM_EMIT("por %%xmm6, %%xmm1") // x1 = idx_max & (max >= sample) | idx_new & !(max >= sample)
498 __ASM_EMIT("orps %%xmm7, %%xmm3") // x3 = max & (max >= sample) | sample & !(max >= sample)
499 // Next loop
500 __ASM_EMIT("paddd 0x10(%[IDXS]), %%xmm4") // x4 = idx_new + 4
501 __ASM_EMIT("add $0x10, %[src]") // src += 4
502 __ASM_EMIT("sub $4, %[count]") // count -= 4
503 __ASM_EMIT("jae 1b")
504
505 // Post-process
506 __ASM_EMIT("movdqa %%xmm4, %[COUNTERS]") // Save current index
507 // Step 1
508 __ASM_EMIT("movhlps %%xmm0, %%xmm4")
509 __ASM_EMIT("movhlps %%xmm2, %%xmm5")
510 __ASM_EMIT("movaps %%xmm2, %%xmm6")
511 __ASM_EMIT("cmpps " CMPLEPS ", %%xmm5, %%xmm6")
512 __ASM_EMIT("movdqa %%xmm6, %%xmm7")
513 __ASM_EMIT("pand %%xmm6, %%xmm0")
514 __ASM_EMIT("andps %%xmm7, %%xmm2")
515 __ASM_EMIT("pandn %%xmm4, %%xmm6")
516 __ASM_EMIT("andnps %%xmm5, %%xmm7")
517 __ASM_EMIT("por %%xmm6, %%xmm0")
518 __ASM_EMIT("orps %%xmm7, %%xmm2")
519
520 __ASM_EMIT("movhlps %%xmm1, %%xmm4")
521 __ASM_EMIT("movhlps %%xmm3, %%xmm5")
522 __ASM_EMIT("movaps %%xmm3, %%xmm6")
523 __ASM_EMIT("cmpps " CMPGEPS ", %%xmm5, %%xmm6")
524 __ASM_EMIT("movdqa %%xmm6, %%xmm7")
525 __ASM_EMIT("pand %%xmm6, %%xmm1")
526 __ASM_EMIT("andps %%xmm7, %%xmm3")
527 __ASM_EMIT("pandn %%xmm4, %%xmm6")
528 __ASM_EMIT("andnps %%xmm5, %%xmm7")
529 __ASM_EMIT("por %%xmm6, %%xmm1")
530 __ASM_EMIT("orps %%xmm7, %%xmm3")
531
532 // Step 2
533 __ASM_EMIT("unpcklps %%xmm0, %%xmm0")
534 __ASM_EMIT("unpcklps %%xmm2, %%xmm2")
535 __ASM_EMIT("movhlps %%xmm0, %%xmm4")
536 __ASM_EMIT("movhlps %%xmm2, %%xmm5")
537 __ASM_EMIT("movaps %%xmm2, %%xmm6")
538 __ASM_EMIT("cmpps " CMPLEPS ", %%xmm5, %%xmm6")
539 __ASM_EMIT("movdqa %%xmm6, %%xmm7")
540 __ASM_EMIT("pand %%xmm6, %%xmm0")
541 __ASM_EMIT("andps %%xmm7, %%xmm2")
542 __ASM_EMIT("pandn %%xmm4, %%xmm6")
543 __ASM_EMIT("andnps %%xmm5, %%xmm7")
544 __ASM_EMIT("por %%xmm6, %%xmm0")
545 __ASM_EMIT("orps %%xmm7, %%xmm2")
546
547 __ASM_EMIT("unpcklps %%xmm1, %%xmm1")
548 __ASM_EMIT("unpcklps %%xmm3, %%xmm3")
549 __ASM_EMIT("movhlps %%xmm1, %%xmm4")
550 __ASM_EMIT("movhlps %%xmm3, %%xmm5")
551 __ASM_EMIT("movaps %%xmm3, %%xmm6")
552 __ASM_EMIT("cmpps " CMPGEPS ", %%xmm5, %%xmm6")
553 __ASM_EMIT("movdqa %%xmm6, %%xmm7")
554 __ASM_EMIT("pand %%xmm6, %%xmm1")
555 __ASM_EMIT("andps %%xmm7, %%xmm3")
556 __ASM_EMIT("pandn %%xmm4, %%xmm6")
557 __ASM_EMIT("andnps %%xmm5, %%xmm7")
558 __ASM_EMIT("por %%xmm6, %%xmm1")
559 __ASM_EMIT("orps %%xmm7, %%xmm3")
560
561 __ASM_EMIT("movdqa %[COUNTERS], %%xmm4") // Restore current index
562
563 __ASM_EMIT("2:")
564 __ASM_EMIT("add $3, %[count]")
565 __ASM_EMIT("jl 4f")
566
567 // x1 blocks
568 __ASM_EMIT("3:")
569 __ASM_EMIT("movss 0x00(%[src]), %%xmm5") // x5 = sample
570 // Find minimum
571 __ASM_EMIT("movaps %%xmm2, %%xmm6") // x6 = min
572 __ASM_EMIT("andps %[X_SIGN], %%xmm5") // x5 = abs(sample)
573 __ASM_EMIT("cmpss " CMPLEPS ", %%xmm5, %%xmm6") // x6 = min <= sample
574 __ASM_EMIT("movdqa %%xmm6, %%xmm7") // x7 = min <= sample
575 __ASM_EMIT("pand %%xmm6, %%xmm0") // x0 = idx_min & (min <= sample)
576 __ASM_EMIT("andps %%xmm7, %%xmm2") // x2 = min & (min <= sample)
577 __ASM_EMIT("pandn %%xmm4, %%xmm6") // x6 = idx_new & !(min <= sample)
578 __ASM_EMIT("andnps %%xmm5, %%xmm7") // x7 = sample & !(min <= sample)
579 __ASM_EMIT("por %%xmm6, %%xmm0") // x0 = idx_min & (min <= sample) | idx_new & !(min <= sample)
580 __ASM_EMIT("orps %%xmm7, %%xmm2") // x2 = min & (min <= sample) | sample & !(min <= sample)
581 // Find maximum
582 __ASM_EMIT("movaps %%xmm3, %%xmm6") // x6 = max
583 __ASM_EMIT("cmpss " CMPGEPS ", %%xmm5, %%xmm6") // x6 = max >= sample
584 __ASM_EMIT("movdqa %%xmm6, %%xmm7") // x7 = max >= sample
585 __ASM_EMIT("pand %%xmm6, %%xmm1") // x1 = idx_max & (max >= sample)
586 __ASM_EMIT("andps %%xmm7, %%xmm3") // x3 = max & (max >= sample)
587 __ASM_EMIT("pandn %%xmm4, %%xmm6") // x6 = idx_new & !(max >= sample)
588 __ASM_EMIT("andnps %%xmm5, %%xmm7") // x7 = sample & !(max >= sample)
589 __ASM_EMIT("por %%xmm6, %%xmm1") // x1 = idx_max & (max >= sample) | idx_new & !(max >= sample)
590 __ASM_EMIT("orps %%xmm7, %%xmm3") // x3 = max & (max >= sample) | sample & !(max >= sample)
591 // Next loop
592 __ASM_EMIT("paddd 0x20(%[IDXS]), %%xmm4") // x4 = idx_new + 1
593 __ASM_EMIT("add $0x04, %[src]") // src += 4
594 __ASM_EMIT("dec %[count]") // count --
595 __ASM_EMIT("jge 3b")
596
597 __ASM_EMIT("4:")
598 __ASM_EMIT("movd %%xmm0, (%[min])")
599 __ASM_EMIT("movd %%xmm1, (%[max])")
600 __ASM_EMIT64("movl $0, 0x04(%[min])")
601 __ASM_EMIT64("movl $0, 0x04(%[max])")
602 : [src] "+r" (src), [count] "+r" (count)
603 : [min] "r" (min), [max] "r" (max),
604 [IDXS] "r" (indexes),
605 [X_SIGN] "m" (iminmax_const),
606 [COUNTERS] "m" (counters)
607 );
608 }
609
610 #undef CMPGTPS
611 #undef CMPGEPS
612 #undef CMPLTPS
613 #undef CMPLEPS
614 }
615 }
616
617 #endif /* PRIVATE_DSP_ARCH_X86_SSE2_SEARCH_IMINMAX_H_ */