/* * Copyright (C) 2020 Linux Studio Plugins Project * (C) 2020 Vladimir Sadovnikov * * This file is part of lsp-dsp-lib * Created on: 31 мар. 2020 г. * * lsp-dsp-lib is free software: you can redistribute it and/or modify * it under the terms of the GNU Lesser General Public License as published by * the Free Software Foundation, either version 3 of the License, or * any later version. * * lsp-dsp-lib is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public License * along with lsp-dsp-lib. If not, see . */ #ifndef PRIVATE_DSP_ARCH_X86_SSE2_SEARCH_IMINMAX_H_ #define PRIVATE_DSP_ARCH_X86_SSE2_SEARCH_IMINMAX_H_ #ifndef PRIVATE_DSP_ARCH_X86_SSE2_IMPL #error "This header should not be included directly" #endif /* PRIVATE_DSP_ARCH_X86_SSE2_IMPL */ namespace lsp { namespace sse2 { IF_ARCH_X86( static const uint32_t indexes[] __lsp_aligned16 = { 0, 1, 2, 3, // indexes 4, 4, 4, 4, // step 1, 1, 1, 1 }; static const uint32_t iminmax_const[] __lsp_aligned16 = { LSP_DSP_VEC4(0x7fffffff) }; ) #define CMPLTPS "$1" #define CMPLEPS "$2" #define CMPGEPS "$5" #define CMPGTPS "$6" /* Register allocation: * xmm0 = rind - result indexes * xmm1 = cind - current indexes * xmm2 = rval - result values * xmm3 = cval - current values * xmm4 = xcmp - comparison mask * xmm5 = temp * xmm6 = temp * xmm7 = incr - increment */ #define SEARCH_CORE(op) \ __ASM_EMIT("pxor %%xmm0, %%xmm0") \ __ASM_EMIT("test %[count], %[count]") \ __ASM_EMIT("jz 4f") \ __ASM_EMIT("movss 0x00(%[src]), %%xmm2") /* x2 = s = s0 */ \ __ASM_EMIT("movdqa 0x00(%[IDXS]), %%xmm1") /* x1 = I[k] = 0 1 2 3 */ \ __ASM_EMIT("shufps $0x00, %%xmm2, %%xmm2") /* x2 = s[k] = s0 s0 s0 s0 */ \ __ASM_EMIT("sub $0x04, %[count]") \ __ASM_EMIT("jb 2f") \ __ASM_EMIT("movdqa 0x10(%[IDXS]), %%xmm7") /* x7 = D = 4 4 4 4 */ \ \ /* x4 blocks */ \ __ASM_EMIT("1:") \ __ASM_EMIT("movaps %%xmm2, %%xmm4") /* x4 = s[k] */ \ __ASM_EMIT("movups 0x00(%[src]), %%xmm3") /* x3 = S[k] */ \ __ASM_EMIT("cmpps " op ", %%xmm3, %%xmm4") /* x4 = s[k] <=> S[k] */ \ __ASM_EMIT("movdqa %%xmm4, %%xmm5") /* x5 = s[k] <=> S[k] */ \ __ASM_EMIT("pand %%xmm5, %%xmm0") /* x0 = i[k] & (s[k] <=> S[k]) */ \ __ASM_EMIT("andps %%xmm4, %%xmm2") /* x2 = s[k] & (s[k] <=> S[k]) */ \ __ASM_EMIT("pandn %%xmm1, %%xmm5") /* x5 = I[k] & !(s[k] <=> S[k]) */ \ __ASM_EMIT("andnps %%xmm3, %%xmm4") /* x4 = S[k] & !(s[k] <=> S[k]) */ \ __ASM_EMIT("por %%xmm5, %%xmm0") /* x0 = (i[k] & (s[k] <=> S[k])) | (I[k] & !(s[k] <=> S[k])) */ \ __ASM_EMIT("orps %%xmm4, %%xmm2") /* x2 = (s[k] & (s[k] <=> S[k])) | (S[k] & !(s[k] <=> S[k])) */ \ __ASM_EMIT("paddd %%xmm7, %%xmm1") /* x1 = I[k] = i[k]+D */ \ __ASM_EMIT("add $0x10, %[src]") \ __ASM_EMIT("sub $4, %[count]") \ __ASM_EMIT("jae 1b") \ /* Post-process, step 1 */ \ __ASM_EMIT("movdqa %%xmm0, %%xmm6") /* x5 = rind = i0 i1 i2 i3 */ \ __ASM_EMIT("movaps %%xmm2, %%xmm3") /* x3 = rval = x0 x1 x2 x3 */ \ __ASM_EMIT("pshufd $0x39, %%xmm6, %%xmm6") /* x6 = cind = i1 i2 i3 i0 */ \ __ASM_EMIT("shufps $0x39, %%xmm3, %%xmm3") /* x3 = cval = x1 x2 x3 x0 */ \ __ASM_EMIT("movaps %%xmm2, %%xmm4") /* x4 = x0 x1 x2 x3 */ \ __ASM_EMIT("cmpps " op ", %%xmm3, %%xmm4") /* x4 = cmp(cval, nval) */ \ __ASM_EMIT("movdqa %%xmm4, %%xmm5") \ __ASM_EMIT("pand %%xmm5, %%xmm0") \ __ASM_EMIT("andps %%xmm4, %%xmm2") \ __ASM_EMIT("pandn %%xmm6, %%xmm5") \ __ASM_EMIT("andnps %%xmm3, %%xmm4") \ __ASM_EMIT("por %%xmm5, %%xmm0") \ __ASM_EMIT("orps %%xmm4, %%xmm2") \ /* Post-process, step 2 */ \ __ASM_EMIT("movdqa %%xmm0, %%xmm6") /* x5 = rind = i0 i1 i2 i3 */ \ __ASM_EMIT("movaps %%xmm2, %%xmm3") /* x3 = rval = x0 x1 x2 x3 */ \ __ASM_EMIT("punpckhdq %%xmm6, %%xmm6") /* x6 = cind = i2 i2 i3 i3 */ \ __ASM_EMIT("unpckhps %%xmm3, %%xmm3") /* x3 = cval = x2 x3 x2 x3 */ \ __ASM_EMIT("movaps %%xmm2, %%xmm4") /* x4 = x0 x1 x2 x3 */ \ __ASM_EMIT("cmpps " op ", %%xmm3, %%xmm4") /* x4 = cmp(cval, nval) */ \ __ASM_EMIT("movdqa %%xmm4, %%xmm5") \ __ASM_EMIT("pand %%xmm5, %%xmm0") \ __ASM_EMIT("andps %%xmm4, %%xmm2") \ __ASM_EMIT("pandn %%xmm6, %%xmm5") \ __ASM_EMIT("andnps %%xmm3, %%xmm4") \ __ASM_EMIT("por %%xmm5, %%xmm0") \ __ASM_EMIT("orps %%xmm4, %%xmm2") \ /* x1 blocks */ \ __ASM_EMIT("2:") \ __ASM_EMIT("add $3, %[count]") \ __ASM_EMIT("jl 4f") \ __ASM_EMIT("movdqa 0x20(%[IDXS]), %%xmm7") /* x7 = D = 1 1 1 1 */ \ __ASM_EMIT("3:") \ __ASM_EMIT("movaps %%xmm2, %%xmm4") \ __ASM_EMIT("movss 0x00(%[src]), %%xmm3") \ __ASM_EMIT("cmpss " op ", %%xmm3, %%xmm4") \ __ASM_EMIT("pand %%xmm4, %%xmm0") \ __ASM_EMIT("movdqa %%xmm4, %%xmm5") \ __ASM_EMIT("andps %%xmm4, %%xmm2") \ __ASM_EMIT("pandn %%xmm1, %%xmm5") \ __ASM_EMIT("andnps %%xmm3, %%xmm4") \ __ASM_EMIT("por %%xmm5, %%xmm0") \ __ASM_EMIT("orps %%xmm4, %%xmm2") \ __ASM_EMIT("paddd %%xmm7, %%xmm1") \ __ASM_EMIT("add $0x04, %[src]") \ __ASM_EMIT("dec %[count]") \ __ASM_EMIT("jge 3b") \ /* End */ \ __ASM_EMIT("4:") size_t min_index(const float *src, size_t count) { uint32_t index; ARCH_X86_ASM( SEARCH_CORE(CMPLEPS) : [src] "+r" (src), [count] "+r" (count), [index] "=Yz" (index) : [IDXS] "r" (indexes) : "cc", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7" ); return index; } size_t max_index(const float *src, size_t count) { uint32_t index; ARCH_X86_ASM( SEARCH_CORE(CMPGEPS) : [src] "+r" (src), [count] "+r" (count), [index] "=Yz" (index) : [IDXS] "r" (indexes) : "cc", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7" ); return index; } #undef SEARCH_CORE /* Register allocation: * xmm0 = rind - result indexes * xmm1 = cind - current indexes * xmm2 = rval - result values * xmm3 = cval - current values * xmm4 = xcmp - comparison mask * xmm5 = temp * xmm6 = sign - sign mask * xmm7 = incr - increment */ #define ABS_SEARCH_CORE(op) \ __ASM_EMIT("pxor %%xmm0, %%xmm0") \ __ASM_EMIT("test %[count], %[count]") \ __ASM_EMIT("jz 4f") \ __ASM_EMIT("movss 0x00(%[src]), %%xmm2") /* x2 = s = s0 */ \ __ASM_EMIT("movdqa 0x00(%[IDXS]), %%xmm1") /* x1 = I[k] = 0 1 2 3 */ \ __ASM_EMIT("movaps %[X_SIGN], %%xmm6") /* x6 = SIGN */ \ __ASM_EMIT("shufps $0x00, %%xmm2, %%xmm2") /* x2 = s[k] = s0 s0 s0 s0 */ \ __ASM_EMIT("sub $0x04, %[count]") \ __ASM_EMIT("andps %%xmm6, %%xmm2") /* x2 = abs(s[k]) */ \ __ASM_EMIT("jb 2f") \ __ASM_EMIT("movdqa 0x10(%[IDXS]), %%xmm7") /* x7 = D = 4 4 4 4 */ \ \ /* x4 blocks */ \ __ASM_EMIT("1:") \ __ASM_EMIT("movaps %%xmm2, %%xmm4") /* x4 = s[k] */ \ __ASM_EMIT("movups 0x00(%[src]), %%xmm3") /* x3 = S[k] */ \ __ASM_EMIT("andps %%xmm6, %%xmm3") /* x3 = abs(S[k]) */ \ __ASM_EMIT("cmpps " op ", %%xmm3, %%xmm4") /* x4 = s[k] <=> S[k] */ \ __ASM_EMIT("movdqa %%xmm4, %%xmm5") /* x5 = s[k] <=> S[k] */ \ __ASM_EMIT("pand %%xmm5, %%xmm0") /* x0 = i[k] & (s[k] <=> S[k]) */ \ __ASM_EMIT("andps %%xmm4, %%xmm2") /* x2 = s[k] & (s[k] <=> S[k]) */ \ __ASM_EMIT("pandn %%xmm1, %%xmm5") /* x5 = I[k] & !(s[k] <=> S[k]) */ \ __ASM_EMIT("andnps %%xmm3, %%xmm4") /* x4 = S[k] & !(s[k] <=> S[k]) */ \ __ASM_EMIT("por %%xmm5, %%xmm0") /* x0 = (i[k] & (s[k] <=> S[k])) | (I[k] & !(s[k] <=> S[k])) */ \ __ASM_EMIT("orps %%xmm4, %%xmm2") /* x2 = (s[k] & (s[k] <=> S[k])) | (S[k] & !(s[k] <=> S[k])) */ \ __ASM_EMIT("paddd %%xmm7, %%xmm1") /* x1 = I[k] = i[k]+D */ \ __ASM_EMIT("add $0x10, %[src]") \ __ASM_EMIT("sub $4, %[count]") \ __ASM_EMIT("jae 1b") \ /* Post-process, step 1 */ \ __ASM_EMIT("movdqa %%xmm0, %%xmm6") /* x5 = rind = i0 i1 i2 i3 */ \ __ASM_EMIT("movaps %%xmm2, %%xmm3") /* x3 = rval = x0 x1 x2 x3 */ \ __ASM_EMIT("pshufd $0x39, %%xmm6, %%xmm6") /* x6 = cind = i1 i2 i3 i0 */ \ __ASM_EMIT("shufps $0x39, %%xmm3, %%xmm3") /* x3 = cval = x1 x2 x3 x0 */ \ __ASM_EMIT("movaps %%xmm2, %%xmm4") /* x4 = x0 x1 x2 x3 */ \ __ASM_EMIT("cmpps " op ", %%xmm3, %%xmm4") /* x4 = cmp(cval, nval) */ \ __ASM_EMIT("movdqa %%xmm4, %%xmm5") \ __ASM_EMIT("pand %%xmm5, %%xmm0") \ __ASM_EMIT("andps %%xmm4, %%xmm2") \ __ASM_EMIT("pandn %%xmm6, %%xmm5") \ __ASM_EMIT("andnps %%xmm3, %%xmm4") \ __ASM_EMIT("por %%xmm5, %%xmm0") \ __ASM_EMIT("orps %%xmm4, %%xmm2") \ /* Post-process, step 2 */ \ __ASM_EMIT("movdqa %%xmm0, %%xmm6") /* x5 = rind = i0 i1 i2 i3 */ \ __ASM_EMIT("movaps %%xmm2, %%xmm3") /* x3 = rval = x0 x1 x2 x3 */ \ __ASM_EMIT("punpckhdq %%xmm6, %%xmm6") /* x6 = cind = i2 i2 i3 i3 */ \ __ASM_EMIT("unpckhps %%xmm3, %%xmm3") /* x3 = cval = x2 x3 x2 x3 */ \ __ASM_EMIT("movaps %%xmm2, %%xmm4") /* x4 = x0 x1 x2 x3 */ \ __ASM_EMIT("cmpps " op ", %%xmm3, %%xmm4") /* x4 = cmp(cval, nval) */ \ __ASM_EMIT("movdqa %%xmm4, %%xmm5") \ __ASM_EMIT("pand %%xmm5, %%xmm0") \ __ASM_EMIT("andps %%xmm4, %%xmm2") \ __ASM_EMIT("pandn %%xmm6, %%xmm5") \ __ASM_EMIT("andnps %%xmm3, %%xmm4") \ __ASM_EMIT("por %%xmm5, %%xmm0") \ __ASM_EMIT("orps %%xmm4, %%xmm2") \ __ASM_EMIT("movaps %[X_SIGN], %%xmm6") /* x6 = SIGN (restore) */ \ /* x1 blocks */ \ __ASM_EMIT("2:") \ __ASM_EMIT("add $3, %[count]") \ __ASM_EMIT("jl 4f") \ __ASM_EMIT("movdqa 0x20(%[IDXS]), %%xmm7") /* x7 = D = 1 1 1 1 */ \ __ASM_EMIT("3:") \ __ASM_EMIT("movaps %%xmm2, %%xmm4") \ __ASM_EMIT("movss 0x00(%[src]), %%xmm3") \ __ASM_EMIT("andps %%xmm6, %%xmm3") /* x3 = abs(S[k]) */ \ __ASM_EMIT("cmpps " op ", %%xmm3, %%xmm4") \ __ASM_EMIT("movdqa %%xmm4, %%xmm5") /* x5 = s[k] <=> S[k] */ \ __ASM_EMIT("pand %%xmm5, %%xmm0") /* x0 = i[k] & (s[k] <=> S[k]) */ \ __ASM_EMIT("andps %%xmm4, %%xmm2") /* x2 = s[k] & (s[k] <=> S[k]) */ \ __ASM_EMIT("pandn %%xmm1, %%xmm5") /* x5 = I[k] & !(s[k] <=> S[k]) */ \ __ASM_EMIT("andnps %%xmm3, %%xmm4") /* x4 = S[k] & !(s[k] <=> S[k]) */ \ __ASM_EMIT("por %%xmm5, %%xmm0") /* x0 = (i[k] & (s[k] <=> S[k])) | (I[k] & !(s[k] <=> S[k])) */ \ __ASM_EMIT("orps %%xmm4, %%xmm2") /* x2 = (s[k] & (s[k] <=> S[k])) | (S[k] & !(s[k] <=> S[k])) */ \ __ASM_EMIT("paddd %%xmm7, %%xmm1") /* x1 = I[k] = i[k]+D */ \ __ASM_EMIT("add $0x04, %[src]") \ __ASM_EMIT("dec %[count]") \ __ASM_EMIT("jge 3b") \ /* End */ \ __ASM_EMIT("4:") size_t abs_min_index(const float *src, size_t count) { uint32_t index; ARCH_X86_ASM( ABS_SEARCH_CORE(CMPLEPS) : [src] "+r" (src), [count] "+r" (count), [index] "=Yz" (index) : [IDXS] "r" (indexes), [X_SIGN] "m" (iminmax_const) : "cc", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7" ); return index; } size_t abs_max_index(const float *src, size_t count) { uint32_t index; ARCH_X86_ASM( ABS_SEARCH_CORE(CMPGEPS) : [src] "+r" (src), [count] "+r" (count), [index] "=Yz" (index) : [IDXS] "r" (indexes), [X_SIGN] "m" (iminmax_const) : "cc", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7" ); return index; } #undef ABS_SEARCH_CORE void minmax_index(const float *src, size_t count, size_t *min, size_t *max) { uint32_t counters[4] __lsp_aligned16; ARCH_X86_ASM( __ASM_EMIT("pxor %%xmm0, %%xmm0") // x0 = idx_min __ASM_EMIT("pxor %%xmm1, %%xmm1") // x1 = idx_max __ASM_EMIT("test %[count], %[count]") __ASM_EMIT("jz 4f") __ASM_EMIT("movss 0x00(%[src]), %%xmm2") // x2 = min __ASM_EMIT("shufps $0x00, %%xmm2, %%xmm2") // x2 = min __ASM_EMIT("movdqa 0x00(%[IDXS]), %%xmm4") // x4 = idx_new __ASM_EMIT("movaps %%xmm2, %%xmm3") // x3 = max __ASM_EMIT("sub $0x04, %[count]") __ASM_EMIT("jb 2f") // x4 blocks __ASM_EMIT("1:") __ASM_EMIT("movups 0x00(%[src]), %%xmm5") // x5 = sample // Find minimum __ASM_EMIT("movaps %%xmm2, %%xmm6") // x6 = min __ASM_EMIT("cmpps " CMPLEPS ", %%xmm5, %%xmm6") // x6 = min <= sample __ASM_EMIT("movdqa %%xmm6, %%xmm7") // x7 = min <= sample __ASM_EMIT("pand %%xmm6, %%xmm0") // x0 = idx_min & (min <= sample) __ASM_EMIT("andps %%xmm7, %%xmm2") // x2 = min & (min <= sample) __ASM_EMIT("pandn %%xmm4, %%xmm6") // x6 = idx_new & !(min <= sample) __ASM_EMIT("andnps %%xmm5, %%xmm7") // x7 = sample & !(min <= sample) __ASM_EMIT("por %%xmm6, %%xmm0") // x0 = idx_min & (min <= sample) | idx_new & !(min <= sample) __ASM_EMIT("orps %%xmm7, %%xmm2") // x2 = min & (min <= sample) | sample & !(min <= sample) // Find maximum __ASM_EMIT("movaps %%xmm3, %%xmm6") // x6 = max __ASM_EMIT("cmpps " CMPGEPS ", %%xmm5, %%xmm6") // x6 = max >= sample __ASM_EMIT("movdqa %%xmm6, %%xmm7") // x7 = max >= sample __ASM_EMIT("pand %%xmm6, %%xmm1") // x1 = idx_max & (max >= sample) __ASM_EMIT("andps %%xmm7, %%xmm3") // x3 = max & (max >= sample) __ASM_EMIT("pandn %%xmm4, %%xmm6") // x6 = idx_new & !(max >= sample) __ASM_EMIT("andnps %%xmm5, %%xmm7") // x7 = sample & !(max >= sample) __ASM_EMIT("por %%xmm6, %%xmm1") // x1 = idx_max & (max >= sample) | idx_new & !(max >= sample) __ASM_EMIT("orps %%xmm7, %%xmm3") // x3 = max & (max >= sample) | sample & !(max >= sample) // Next loop __ASM_EMIT("paddd 0x10(%[IDXS]), %%xmm4") // x4 = idx_new + 4 __ASM_EMIT("add $0x10, %[src]") // src += 4 __ASM_EMIT("sub $4, %[count]") // count -= 4 __ASM_EMIT("jae 1b") // Post-process __ASM_EMIT("movdqa %%xmm4, %[COUNTERS]") // Save current index // Step 1 __ASM_EMIT("movhlps %%xmm0, %%xmm4") __ASM_EMIT("movhlps %%xmm2, %%xmm5") __ASM_EMIT("movaps %%xmm2, %%xmm6") __ASM_EMIT("cmpps " CMPLEPS ", %%xmm5, %%xmm6") __ASM_EMIT("movdqa %%xmm6, %%xmm7") __ASM_EMIT("pand %%xmm6, %%xmm0") __ASM_EMIT("andps %%xmm7, %%xmm2") __ASM_EMIT("pandn %%xmm4, %%xmm6") __ASM_EMIT("andnps %%xmm5, %%xmm7") __ASM_EMIT("por %%xmm6, %%xmm0") __ASM_EMIT("orps %%xmm7, %%xmm2") __ASM_EMIT("movhlps %%xmm1, %%xmm4") __ASM_EMIT("movhlps %%xmm3, %%xmm5") __ASM_EMIT("movaps %%xmm3, %%xmm6") __ASM_EMIT("cmpps " CMPGEPS ", %%xmm5, %%xmm6") __ASM_EMIT("movdqa %%xmm6, %%xmm7") __ASM_EMIT("pand %%xmm6, %%xmm1") __ASM_EMIT("andps %%xmm7, %%xmm3") __ASM_EMIT("pandn %%xmm4, %%xmm6") __ASM_EMIT("andnps %%xmm5, %%xmm7") __ASM_EMIT("por %%xmm6, %%xmm1") __ASM_EMIT("orps %%xmm7, %%xmm3") // Step 2 __ASM_EMIT("unpcklps %%xmm0, %%xmm0") __ASM_EMIT("unpcklps %%xmm2, %%xmm2") __ASM_EMIT("movhlps %%xmm0, %%xmm4") __ASM_EMIT("movhlps %%xmm2, %%xmm5") __ASM_EMIT("movaps %%xmm2, %%xmm6") __ASM_EMIT("cmpps " CMPLEPS ", %%xmm5, %%xmm6") __ASM_EMIT("movdqa %%xmm6, %%xmm7") __ASM_EMIT("pand %%xmm6, %%xmm0") __ASM_EMIT("andps %%xmm7, %%xmm2") __ASM_EMIT("pandn %%xmm4, %%xmm6") __ASM_EMIT("andnps %%xmm5, %%xmm7") __ASM_EMIT("por %%xmm6, %%xmm0") __ASM_EMIT("orps %%xmm7, %%xmm2") __ASM_EMIT("unpcklps %%xmm1, %%xmm1") __ASM_EMIT("unpcklps %%xmm3, %%xmm3") __ASM_EMIT("movhlps %%xmm1, %%xmm4") __ASM_EMIT("movhlps %%xmm3, %%xmm5") __ASM_EMIT("movaps %%xmm3, %%xmm6") __ASM_EMIT("cmpps " CMPGEPS ", %%xmm5, %%xmm6") __ASM_EMIT("movdqa %%xmm6, %%xmm7") __ASM_EMIT("pand %%xmm6, %%xmm1") __ASM_EMIT("andps %%xmm7, %%xmm3") __ASM_EMIT("pandn %%xmm4, %%xmm6") __ASM_EMIT("andnps %%xmm5, %%xmm7") __ASM_EMIT("por %%xmm6, %%xmm1") __ASM_EMIT("orps %%xmm7, %%xmm3") __ASM_EMIT("movdqa %[COUNTERS], %%xmm4") // Restore current index __ASM_EMIT("2:") __ASM_EMIT("add $3, %[count]") __ASM_EMIT("jl 4f") // x1 blocks __ASM_EMIT("3:") __ASM_EMIT("movss 0x00(%[src]), %%xmm5") // x5 = sample // Find minimum __ASM_EMIT("movaps %%xmm2, %%xmm6") // x6 = min __ASM_EMIT("cmpss " CMPLEPS ", %%xmm5, %%xmm6") // x6 = min <= sample __ASM_EMIT("movdqa %%xmm6, %%xmm7") // x7 = min <= sample __ASM_EMIT("pand %%xmm6, %%xmm0") // x0 = idx_min & (min <= sample) __ASM_EMIT("andps %%xmm7, %%xmm2") // x2 = min & (min <= sample) __ASM_EMIT("pandn %%xmm4, %%xmm6") // x6 = idx_new & !(min <= sample) __ASM_EMIT("andnps %%xmm5, %%xmm7") // x7 = sample & !(min <= sample) __ASM_EMIT("por %%xmm6, %%xmm0") // x0 = idx_min & (min <= sample) | idx_new & !(min <= sample) __ASM_EMIT("orps %%xmm7, %%xmm2") // x2 = min & (min <= sample) | sample & !(min <= sample) // Find maximum __ASM_EMIT("movaps %%xmm3, %%xmm6") // x6 = max __ASM_EMIT("cmpss " CMPGEPS ", %%xmm5, %%xmm6") // x6 = max >= sample __ASM_EMIT("movdqa %%xmm6, %%xmm7") // x7 = max >= sample __ASM_EMIT("pand %%xmm6, %%xmm1") // x1 = idx_max & (max >= sample) __ASM_EMIT("andps %%xmm7, %%xmm3") // x3 = max & (max >= sample) __ASM_EMIT("pandn %%xmm4, %%xmm6") // x6 = idx_new & !(max >= sample) __ASM_EMIT("andnps %%xmm5, %%xmm7") // x7 = sample & !(max >= sample) __ASM_EMIT("por %%xmm6, %%xmm1") // x1 = idx_max & (max >= sample) | idx_new & !(max >= sample) __ASM_EMIT("orps %%xmm7, %%xmm3") // x3 = max & (max >= sample) | sample & !(max >= sample) // Next loop __ASM_EMIT("paddd 0x20(%[IDXS]), %%xmm4") // x4 = idx_new + 1 __ASM_EMIT("add $0x04, %[src]") // src += 4 __ASM_EMIT("dec %[count]") // count -- __ASM_EMIT("jge 3b") __ASM_EMIT("4:") __ASM_EMIT("movd %%xmm0, (%[min])") __ASM_EMIT("movd %%xmm1, (%[max])") __ASM_EMIT64("movl $0, 0x04(%[min])") __ASM_EMIT64("movl $0, 0x04(%[max])") : [src] "+r" (src), [count] "+r" (count) : [min] "r" (min), [max] "r" (max), [IDXS] "r" (indexes), [COUNTERS] "m" (counters) ); } void abs_minmax_index(const float *src, size_t count, size_t *min, size_t *max) { uint32_t counters[4] __lsp_aligned16; ARCH_X86_ASM( __ASM_EMIT("pxor %%xmm0, %%xmm0") // x0 = idx_min __ASM_EMIT("pxor %%xmm1, %%xmm1") // x1 = idx_max __ASM_EMIT("test %[count], %[count]") __ASM_EMIT("jz 4f") __ASM_EMIT("movss 0x00(%[src]), %%xmm2") // x2 = min __ASM_EMIT("shufps $0x00, %%xmm2, %%xmm2") // x2 = min __ASM_EMIT("movdqa 0x00(%[IDXS]), %%xmm4") // x4 = idx_new __ASM_EMIT("andps %[X_SIGN], %%xmm2") // x5 = abs(sample) __ASM_EMIT("movaps %%xmm2, %%xmm3") // x3 = max = abs(sample) __ASM_EMIT("sub $0x04, %[count]") __ASM_EMIT("jb 2f") // x4 blocks __ASM_EMIT("1:") __ASM_EMIT("movups 0x00(%[src]), %%xmm5") // x5 = sample // Find minimum __ASM_EMIT("movaps %%xmm2, %%xmm6") // x6 = min __ASM_EMIT("andps %[X_SIGN], %%xmm5") // x5 = abs(sample) __ASM_EMIT("cmpps " CMPLEPS ", %%xmm5, %%xmm6") // x6 = min <= sample __ASM_EMIT("movdqa %%xmm6, %%xmm7") // x7 = min <= sample __ASM_EMIT("pand %%xmm6, %%xmm0") // x0 = idx_min & (min <= sample) __ASM_EMIT("andps %%xmm7, %%xmm2") // x2 = min & (min <= sample) __ASM_EMIT("pandn %%xmm4, %%xmm6") // x6 = idx_new & !(min <= sample) __ASM_EMIT("andnps %%xmm5, %%xmm7") // x7 = sample & !(min <= sample) __ASM_EMIT("por %%xmm6, %%xmm0") // x0 = idx_min & (min <= sample) | idx_new & !(min <= sample) __ASM_EMIT("orps %%xmm7, %%xmm2") // x2 = min & (min <= sample) | sample & !(min <= sample) // Find maximum __ASM_EMIT("movaps %%xmm3, %%xmm6") // x6 = max __ASM_EMIT("cmpps " CMPGEPS ", %%xmm5, %%xmm6") // x6 = max >= sample __ASM_EMIT("movdqa %%xmm6, %%xmm7") // x7 = max >= sample __ASM_EMIT("pand %%xmm6, %%xmm1") // x1 = idx_max & (max >= sample) __ASM_EMIT("andps %%xmm7, %%xmm3") // x3 = max & (max >= sample) __ASM_EMIT("pandn %%xmm4, %%xmm6") // x6 = idx_new & !(max >= sample) __ASM_EMIT("andnps %%xmm5, %%xmm7") // x7 = sample & !(max >= sample) __ASM_EMIT("por %%xmm6, %%xmm1") // x1 = idx_max & (max >= sample) | idx_new & !(max >= sample) __ASM_EMIT("orps %%xmm7, %%xmm3") // x3 = max & (max >= sample) | sample & !(max >= sample) // Next loop __ASM_EMIT("paddd 0x10(%[IDXS]), %%xmm4") // x4 = idx_new + 4 __ASM_EMIT("add $0x10, %[src]") // src += 4 __ASM_EMIT("sub $4, %[count]") // count -= 4 __ASM_EMIT("jae 1b") // Post-process __ASM_EMIT("movdqa %%xmm4, %[COUNTERS]") // Save current index // Step 1 __ASM_EMIT("movhlps %%xmm0, %%xmm4") __ASM_EMIT("movhlps %%xmm2, %%xmm5") __ASM_EMIT("movaps %%xmm2, %%xmm6") __ASM_EMIT("cmpps " CMPLEPS ", %%xmm5, %%xmm6") __ASM_EMIT("movdqa %%xmm6, %%xmm7") __ASM_EMIT("pand %%xmm6, %%xmm0") __ASM_EMIT("andps %%xmm7, %%xmm2") __ASM_EMIT("pandn %%xmm4, %%xmm6") __ASM_EMIT("andnps %%xmm5, %%xmm7") __ASM_EMIT("por %%xmm6, %%xmm0") __ASM_EMIT("orps %%xmm7, %%xmm2") __ASM_EMIT("movhlps %%xmm1, %%xmm4") __ASM_EMIT("movhlps %%xmm3, %%xmm5") __ASM_EMIT("movaps %%xmm3, %%xmm6") __ASM_EMIT("cmpps " CMPGEPS ", %%xmm5, %%xmm6") __ASM_EMIT("movdqa %%xmm6, %%xmm7") __ASM_EMIT("pand %%xmm6, %%xmm1") __ASM_EMIT("andps %%xmm7, %%xmm3") __ASM_EMIT("pandn %%xmm4, %%xmm6") __ASM_EMIT("andnps %%xmm5, %%xmm7") __ASM_EMIT("por %%xmm6, %%xmm1") __ASM_EMIT("orps %%xmm7, %%xmm3") // Step 2 __ASM_EMIT("unpcklps %%xmm0, %%xmm0") __ASM_EMIT("unpcklps %%xmm2, %%xmm2") __ASM_EMIT("movhlps %%xmm0, %%xmm4") __ASM_EMIT("movhlps %%xmm2, %%xmm5") __ASM_EMIT("movaps %%xmm2, %%xmm6") __ASM_EMIT("cmpps " CMPLEPS ", %%xmm5, %%xmm6") __ASM_EMIT("movdqa %%xmm6, %%xmm7") __ASM_EMIT("pand %%xmm6, %%xmm0") __ASM_EMIT("andps %%xmm7, %%xmm2") __ASM_EMIT("pandn %%xmm4, %%xmm6") __ASM_EMIT("andnps %%xmm5, %%xmm7") __ASM_EMIT("por %%xmm6, %%xmm0") __ASM_EMIT("orps %%xmm7, %%xmm2") __ASM_EMIT("unpcklps %%xmm1, %%xmm1") __ASM_EMIT("unpcklps %%xmm3, %%xmm3") __ASM_EMIT("movhlps %%xmm1, %%xmm4") __ASM_EMIT("movhlps %%xmm3, %%xmm5") __ASM_EMIT("movaps %%xmm3, %%xmm6") __ASM_EMIT("cmpps " CMPGEPS ", %%xmm5, %%xmm6") __ASM_EMIT("movdqa %%xmm6, %%xmm7") __ASM_EMIT("pand %%xmm6, %%xmm1") __ASM_EMIT("andps %%xmm7, %%xmm3") __ASM_EMIT("pandn %%xmm4, %%xmm6") __ASM_EMIT("andnps %%xmm5, %%xmm7") __ASM_EMIT("por %%xmm6, %%xmm1") __ASM_EMIT("orps %%xmm7, %%xmm3") __ASM_EMIT("movdqa %[COUNTERS], %%xmm4") // Restore current index __ASM_EMIT("2:") __ASM_EMIT("add $3, %[count]") __ASM_EMIT("jl 4f") // x1 blocks __ASM_EMIT("3:") __ASM_EMIT("movss 0x00(%[src]), %%xmm5") // x5 = sample // Find minimum __ASM_EMIT("movaps %%xmm2, %%xmm6") // x6 = min __ASM_EMIT("andps %[X_SIGN], %%xmm5") // x5 = abs(sample) __ASM_EMIT("cmpss " CMPLEPS ", %%xmm5, %%xmm6") // x6 = min <= sample __ASM_EMIT("movdqa %%xmm6, %%xmm7") // x7 = min <= sample __ASM_EMIT("pand %%xmm6, %%xmm0") // x0 = idx_min & (min <= sample) __ASM_EMIT("andps %%xmm7, %%xmm2") // x2 = min & (min <= sample) __ASM_EMIT("pandn %%xmm4, %%xmm6") // x6 = idx_new & !(min <= sample) __ASM_EMIT("andnps %%xmm5, %%xmm7") // x7 = sample & !(min <= sample) __ASM_EMIT("por %%xmm6, %%xmm0") // x0 = idx_min & (min <= sample) | idx_new & !(min <= sample) __ASM_EMIT("orps %%xmm7, %%xmm2") // x2 = min & (min <= sample) | sample & !(min <= sample) // Find maximum __ASM_EMIT("movaps %%xmm3, %%xmm6") // x6 = max __ASM_EMIT("cmpss " CMPGEPS ", %%xmm5, %%xmm6") // x6 = max >= sample __ASM_EMIT("movdqa %%xmm6, %%xmm7") // x7 = max >= sample __ASM_EMIT("pand %%xmm6, %%xmm1") // x1 = idx_max & (max >= sample) __ASM_EMIT("andps %%xmm7, %%xmm3") // x3 = max & (max >= sample) __ASM_EMIT("pandn %%xmm4, %%xmm6") // x6 = idx_new & !(max >= sample) __ASM_EMIT("andnps %%xmm5, %%xmm7") // x7 = sample & !(max >= sample) __ASM_EMIT("por %%xmm6, %%xmm1") // x1 = idx_max & (max >= sample) | idx_new & !(max >= sample) __ASM_EMIT("orps %%xmm7, %%xmm3") // x3 = max & (max >= sample) | sample & !(max >= sample) // Next loop __ASM_EMIT("paddd 0x20(%[IDXS]), %%xmm4") // x4 = idx_new + 1 __ASM_EMIT("add $0x04, %[src]") // src += 4 __ASM_EMIT("dec %[count]") // count -- __ASM_EMIT("jge 3b") __ASM_EMIT("4:") __ASM_EMIT("movd %%xmm0, (%[min])") __ASM_EMIT("movd %%xmm1, (%[max])") __ASM_EMIT64("movl $0, 0x04(%[min])") __ASM_EMIT64("movl $0, 0x04(%[max])") : [src] "+r" (src), [count] "+r" (count) : [min] "r" (min), [max] "r" (max), [IDXS] "r" (indexes), [X_SIGN] "m" (iminmax_const), [COUNTERS] "m" (counters) ); } #undef CMPGTPS #undef CMPGEPS #undef CMPLTPS #undef CMPLEPS } } #endif /* PRIVATE_DSP_ARCH_X86_SSE2_SEARCH_IMINMAX_H_ */