Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/x86/sse/correlation.h

raw
1 /*
2 * Copyright (C) 2024 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2024 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 9 мар. 2024 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_X86_SSE_CORRELATION_H_
23 #define PRIVATE_DSP_ARCH_X86_SSE_CORRELATION_H_
24
25 #ifndef PRIVATE_DSP_ARCH_X86_SSE_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_X86_SSE_IMPL */
28
29 namespace lsp
30 {
31 namespace sse
32 {
33
34 void corr_init(dsp::correlation_t *corr, const float *a, const float *b, size_t count)
35 {
36 IF_ARCH_X86(
37 size_t off;
38 );
39
40 ARCH_X86_ASM
41 (
42 __ASM_EMIT("xor %[off], %[off]")
43 __ASM_EMIT("xorps %%xmm0, %%xmm0") /* xv = 0 */
44 __ASM_EMIT("xorps %%xmm1, %%xmm1") /* xa = 0 */
45 __ASM_EMIT("xorps %%xmm2, %%xmm2") /* xb = 0 */
46 /* 8x blocks */
47 __ASM_EMIT("sub $8, %[count]")
48 __ASM_EMIT("jb 2f")
49 __ASM_EMIT("1:")
50 __ASM_EMIT("movups 0x00(%[a], %[off]), %%xmm3") /* xmm3 = a0 */
51 __ASM_EMIT("movups 0x10(%[a], %[off]), %%xmm4") /* xmm4 = a1 */
52 __ASM_EMIT("movaps %%xmm3, %%xmm7") /* xmm7 = a0 */
53 __ASM_EMIT("movups 0x00(%[b], %[off]), %%xmm5") /* xmm5 = b0 */
54 __ASM_EMIT("movups 0x10(%[b], %[off]), %%xmm6") /* xmm6 = b1 */
55 __ASM_EMIT("mulps %%xmm5, %%xmm7") /* xmm7 = a0*b0 */
56 __ASM_EMIT("mulps %%xmm3, %%xmm3") /* xmm3 = a0*a0 */
57 __ASM_EMIT("mulps %%xmm5, %%xmm5") /* xmm5 = b0*b0 */
58 __ASM_EMIT("addps %%xmm7, %%xmm0") /* xmm0 = xv + a0*b0 */
59 __ASM_EMIT("addps %%xmm3, %%xmm1") /* xmm1 = xa + a0*a0 */
60 __ASM_EMIT("movaps %%xmm4, %%xmm7") /* xmm7 = a1 */
61 __ASM_EMIT("addps %%xmm5, %%xmm2") /* xmm2 = xb + b0*b0 */
62 __ASM_EMIT("mulps %%xmm6, %%xmm7") /* xmm7 = a1*b1 */
63 __ASM_EMIT("mulps %%xmm4, %%xmm4") /* xmm4 = a1*a1 */
64 __ASM_EMIT("mulps %%xmm6, %%xmm6") /* xmm6 = b1*b1 */
65 __ASM_EMIT("addps %%xmm7, %%xmm0") /* xmm0 = xv + a1*b1 */
66 __ASM_EMIT("addps %%xmm4, %%xmm1") /* xmm1 = xa + a1*a1 */
67 __ASM_EMIT("addps %%xmm6, %%xmm2") /* xmm2 = xb + b1*b1 */
68 __ASM_EMIT("add $0x20, %[off]") /* ++off */
69 __ASM_EMIT("sub $8, %[count]")
70 __ASM_EMIT("jae 1b")
71 __ASM_EMIT("2:")
72 /* 4x block */
73 __ASM_EMIT("add $4, %[count]")
74 __ASM_EMIT("jl 4f")
75 __ASM_EMIT("movups 0x00(%[a], %[off]), %%xmm3") /* xmm3 = a0 */
76 __ASM_EMIT("movups 0x00(%[b], %[off]), %%xmm5") /* xmm5 = b0 */
77 __ASM_EMIT("movaps %%xmm3, %%xmm7") /* xmm7 = a0 */
78 __ASM_EMIT("mulps %%xmm3, %%xmm3") /* xmm3 = a0*a0 */
79 __ASM_EMIT("mulps %%xmm5, %%xmm7") /* xmm7 = a0*b0 */
80 __ASM_EMIT("mulps %%xmm5, %%xmm5") /* xmm5 = b0*b0 */
81 __ASM_EMIT("addps %%xmm7, %%xmm0") /* xmm0 = xv + a0*b0 */
82 __ASM_EMIT("addps %%xmm3, %%xmm1") /* xmm1 = xa + a0*a0 */
83 __ASM_EMIT("addps %%xmm5, %%xmm2") /* xmm2 = xb + b0*b0 */
84 __ASM_EMIT("add $0x10, %[off]") /* ++off */
85 __ASM_EMIT("sub $4, %[count]")
86 __ASM_EMIT("4:")
87 /* Do horizontal sum */
88 __ASM_EMIT("movhlps %%xmm0, %%xmm4") /* xmm4 = xv2 xv3 ? ? */
89 __ASM_EMIT("movhlps %%xmm1, %%xmm5") /* xmm5 = xa2 xa3 ? ? */
90 __ASM_EMIT("movhlps %%xmm2, %%xmm6") /* xmm6 = xb2 xb3 ? ? */
91 __ASM_EMIT("addps %%xmm4, %%xmm0") /* xmm0 = xv0+xv2 xv1+xv3 ? ? */
92 __ASM_EMIT("addps %%xmm5, %%xmm1") /* xmm1 = xa0+xa2 xa1+xa3 ? ? */
93 __ASM_EMIT("addps %%xmm6, %%xmm2") /* xmm2 = xb0+xb2 xb1+xb3 ? ? */
94 __ASM_EMIT("unpcklps %%xmm4, %%xmm0") /* xmm0 = xv0+xv2 xv2 xv1+xv3 xv3 */
95 __ASM_EMIT("unpcklps %%xmm5, %%xmm1") /* xmm1 = xa0+xa2 xa2 xa1+xa3 xa3 */
96 __ASM_EMIT("unpcklps %%xmm6, %%xmm2") /* xmm2 = xb0+xb2 xb2 xb1+xb3 xb3 */
97 __ASM_EMIT("movhlps %%xmm0, %%xmm4") /* xmm4 = xv1+xv3 xv3 ? ? */
98 __ASM_EMIT("movhlps %%xmm1, %%xmm5") /* xmm5 = xa1+xa3 xa3 ? ? */
99 __ASM_EMIT("movhlps %%xmm2, %%xmm6") /* xmm6 = xb1+xb3 xb3 ? ? */
100 __ASM_EMIT("addss %%xmm4, %%xmm0") /* xmm0 = xv0+xv1+xv2+xv3 xv2+xv3 ? ? */
101 __ASM_EMIT("addss %%xmm5, %%xmm1") /* xmm1 = xa0+xa1+xa2+xa3 xa2+xa3 ? ? */
102 __ASM_EMIT("addss %%xmm6, %%xmm2") /* xmm2 = xb0+xb1+xb2+xb3 xb2+xb3 ? ? */
103 /* 1x blocks */
104 __ASM_EMIT("add $3, %[count]")
105 __ASM_EMIT("jl 6f")
106 __ASM_EMIT("5:")
107 __ASM_EMIT("movss 0x00(%[a], %[off]), %%xmm3") /* xmm3 = a0 */
108 __ASM_EMIT("movss 0x00(%[b], %[off]), %%xmm5") /* xmm5 = b0 */
109 __ASM_EMIT("movaps %%xmm3, %%xmm7") /* xmm7 = a0 */
110 __ASM_EMIT("mulss %%xmm3, %%xmm3") /* xmm3 = a0*a0 */
111 __ASM_EMIT("mulss %%xmm5, %%xmm7") /* xmm7 = a0*b0 */
112 __ASM_EMIT("mulss %%xmm5, %%xmm5") /* xmm5 = b0*b0 */
113 __ASM_EMIT("addss %%xmm7, %%xmm0") /* xmm0 = xv + a0*b0 */
114 __ASM_EMIT("addss %%xmm3, %%xmm1") /* xmm1 = xa + a0*a0 */
115 __ASM_EMIT("addss %%xmm5, %%xmm2") /* xmm2 = xb + b0*b0 */
116 __ASM_EMIT("add $0x04, %[off]") /* ++off */
117 __ASM_EMIT("dec %[count]")
118 __ASM_EMIT("jge 5b")
119 __ASM_EMIT("6:")
120 /* Store result */
121 __ASM_EMIT("movss 0x00(%[corr]), %%xmm4")
122 __ASM_EMIT("movss 0x04(%[corr]), %%xmm5")
123 __ASM_EMIT("movss 0x08(%[corr]), %%xmm6")
124 __ASM_EMIT("addss %%xmm4, %%xmm0")
125 __ASM_EMIT("addss %%xmm5, %%xmm1")
126 __ASM_EMIT("addss %%xmm6, %%xmm2")
127 __ASM_EMIT("movss %%xmm0, 0x00(%[corr])")
128 __ASM_EMIT("movss %%xmm1, 0x04(%[corr])")
129 __ASM_EMIT("movss %%xmm2, 0x08(%[corr])")
130
131 : [corr] "+r" (corr), [off] "=&r" (off), [count] "+r" (count)
132 : [a] "r" (a), [b] "r" (b)
133 : "cc", "memory",
134 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
135 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
136 );
137 }
138
139 static const float corr_const[] __lsp_aligned16 =
140 {
141 LSP_DSP_VEC4(1e-18f)
142 };
143
144 void corr_incr(dsp::correlation_t *corr, float *dst,
145 const float *a_head, const float *b_head,
146 const float *a_tail, const float *b_tail,
147 size_t count)
148 {
149 IF_ARCH_I386(
150 void *ptr;
151 );
152
153 ARCH_X86_ASM
154 (
155 /* 4x blocks */
156 __ASM_EMIT32("subl $4, %[count]")
157 __ASM_EMIT64("sub $4, %[count]")
158 __ASM_EMIT("jb 2f")
159 __ASM_EMIT("1:")
160 __ASM_EMIT("movups 0x00(%[a_head]), %%xmm0") /* xmm0 = ah */
161 __ASM_EMIT("movups 0x00(%[b_head]), %%xmm1") /* xmm1 = bh */
162 __ASM_EMIT("movaps %%xmm0, %%xmm2") /* xmm2 = ah */
163 __ASM_EMIT("movups 0x00(%[a_tail]), %%xmm3") /* xmm3 = at */
164 __ASM_EMIT("movups 0x00(%[b_tail]), %%xmm4") /* xmm4 = bt */
165 __ASM_EMIT("movaps %%xmm3, %%xmm5") /* xmm5 = at */
166 __ASM_EMIT("mulps %%xmm1, %%xmm2") /* xmm2 = ah*bh */
167 __ASM_EMIT("mulps %%xmm0, %%xmm0") /* xmm0 = ah*ah */
168 __ASM_EMIT("mulps %%xmm1, %%xmm1") /* xmm1 = bh*bh */
169 __ASM_EMIT("mulps %%xmm4, %%xmm5") /* xmm5 = at*bt */
170 __ASM_EMIT("mulps %%xmm3, %%xmm3") /* xmm3 = at*at */
171 __ASM_EMIT("mulps %%xmm4, %%xmm4") /* xmm4 = bt*bt */
172 __ASM_EMIT("subps %%xmm3, %%xmm0") /* xmm0 = DA = ah*ah - at*at */
173 __ASM_EMIT("subps %%xmm4, %%xmm1") /* xmm1 = DB = bh*bh - bt*bt */
174 __ASM_EMIT("subps %%xmm5, %%xmm2") /* xmm2 = DV = ah*bh - at*bt */
175
176 __ASM_EMIT("xorps %%xmm3, %%xmm3") /* xmm3 = 0 0 0 0 */
177 __ASM_EMIT("xorps %%xmm4, %%xmm4") /* xmm4 = 0 0 0 0 */
178 __ASM_EMIT("xorps %%xmm5, %%xmm5") /* xmm5 = 0 0 0 0 */
179 __ASM_EMIT("movlhps %%xmm0, %%xmm3") /* xmm3 = 0 0 DA[0] DA[1] */
180 __ASM_EMIT("movlhps %%xmm1, %%xmm4") /* xmm4 = 0 0 DB[0] DB[1] */
181 __ASM_EMIT("movlhps %%xmm2, %%xmm5") /* xmm5 = 0 0 DV[0] DV[1] */
182 __ASM_EMIT("addps %%xmm3, %%xmm0") /* xmm0 = DA[0] DA[1] DA[0]+DA[2] DA[1]+DA[3] */
183 __ASM_EMIT("addps %%xmm4, %%xmm1") /* xmm1 = DB[0] DB[1] DB[0]+DB[2] DB[1]+DB[3] */
184 __ASM_EMIT("addps %%xmm5, %%xmm2") /* xmm2 = DV[0] DV[1] DV[0]+DV[2] DV[1]+DV[3] */
185 __ASM_EMIT("shufps $0x99, %%xmm0, %%xmm3") /* xmm3 = 0 DA[0] DA[1] DA[0]+DA[2] */
186 __ASM_EMIT("shufps $0x99, %%xmm1, %%xmm4") /* xmm4 = 0 DB[0] DB[1] DB[0]+DB[2] */
187 __ASM_EMIT("shufps $0x99, %%xmm2, %%xmm5") /* xmm5 = 0 DV[0] DV[1] DV[0]+DV[2] */
188 __ASM_EMIT("addps %%xmm0, %%xmm3") /* xmm3 = DA[0] DA[0]+DA[1] DA[0]+DA[1]+DA[2] DA[0]+DA[1]+DA[2]+DA[3] */
189 __ASM_EMIT("addps %%xmm1, %%xmm4") /* xmm4 = DB[0] DB[0]+DB[1] DB[0]+DB[1]+DB[2] DB[0]+DB[1]+DB[2]+DB[3] */
190 __ASM_EMIT("addps %%xmm2, %%xmm5") /* xmm5 = DV[0] DV[0]+DV[1] DV[0]+DV[1]+DV[2] DV[0]+DV[1]+DV[2]+DV[3] */
191
192 __ASM_EMIT32("mov %[corr], %[ptr]")
193 __ASM_EMIT32("movss 0x00(%[ptr]), %%xmm0")
194 __ASM_EMIT32("movss 0x04(%[ptr]), %%xmm1")
195 __ASM_EMIT32("movss 0x08(%[ptr]), %%xmm2")
196 __ASM_EMIT64("movss 0x00(%[corr]), %%xmm0")
197 __ASM_EMIT64("movss 0x04(%[corr]), %%xmm1")
198 __ASM_EMIT64("movss 0x08(%[corr]), %%xmm2")
199
200 __ASM_EMIT("shufps $0x00, %%xmm0, %%xmm0") /* xmm0 = xv xv xv xv */
201 __ASM_EMIT("shufps $0x00, %%xmm1, %%xmm1") /* xmm1 = xa xa xa xa */
202 __ASM_EMIT("shufps $0x00, %%xmm2, %%xmm2") /* xmm2 = xb xb xb xb */
203 __ASM_EMIT("addps %%xmm3, %%xmm1") /* xmm1 = BA = xa+DA[0] xa+DA[0]+DA[1] xa+DA[0]+DA[1]+DA[2] xa+DA[0]+DA[1]+DA[2]+DA[3] */
204 __ASM_EMIT("addps %%xmm4, %%xmm2") /* xmm2 = BB = xb+DV[0] xb+DV[0]+DV[1] xb+DV[0]+DV[1]+DV[2] xb+DV[0]+DV[1]+DV[2]+DV[3] */
205 __ASM_EMIT("movaps %%xmm1, %%xmm6") /* xmm6 = BA */
206 __ASM_EMIT("addps %%xmm5, %%xmm0") /* xmm0 = T = xv+DV[0] xv+DV[0]+DV[1] xv+DV[0]+DV[1]+DV[2] xv+DV[0]+DV[1]+DV[2]+DV[3] */
207 __ASM_EMIT("mulps %%xmm2, %%xmm1") /* xmm1 = B = BA*BB */
208 __ASM_EMIT("movaps %%xmm0, %%xmm5") /* xmm5 = T */
209 __ASM_EMIT("shufps $0xff, %%xmm6, %%xmm6") /* xmm6 = BA[3] BA[3] BA[3] BA[3] */
210 __ASM_EMIT("shufps $0xff, %%xmm5, %%xmm5") /* xmm5 = T[3] T[3] T[3] T[3] */
211 __ASM_EMIT("shufps $0xff, %%xmm2, %%xmm2") /* xmm7 = BB[3] BB[3] BB[3] BB[3] */
212
213 __ASM_EMIT32("movss %%xmm5, 0x00(%[ptr])")
214 __ASM_EMIT32("movss %%xmm6, 0x04(%[ptr])")
215 __ASM_EMIT32("movss %%xmm2, 0x08(%[ptr])")
216 __ASM_EMIT64("movss %%xmm5, 0x00(%[corr])")
217 __ASM_EMIT64("movss %%xmm6, 0x04(%[corr])")
218 __ASM_EMIT64("movss %%xmm2, 0x08(%[corr])")
219
220 __ASM_EMIT("sqrtps %%xmm1, %%xmm7") /* xmm7 = sqrtf(B) */
221 __ASM_EMIT("cmpps $5, %[CORR_CC], %%xmm1") /* xmm1 = B >= threshold */
222 __ASM_EMIT("divps %%xmm7, %%xmm0") /* xmm0 = T/sqrtf(B) */
223 __ASM_EMIT32("mov %[dst], %[ptr]")
224 __ASM_EMIT("andps %%xmm1, %%xmm0") /* xmm0 = (B >= threshold) ? T/sqrtf(B) : 0 */
225 __ASM_EMIT("add $0x10, %[a_head]") /* ++a_head */
226 __ASM_EMIT("add $0x10, %[b_head]") /* ++b_head */
227 __ASM_EMIT32("movups %%xmm0, 0x00(%[ptr])")
228 __ASM_EMIT64("movups %%xmm0, 0x00(%[dst])")
229 __ASM_EMIT("add $0x10, %[a_tail]") /* ++a_tail */
230 __ASM_EMIT("add $0x10, %[b_tail]") /* ++b_tail */
231 __ASM_EMIT32("add $0x10, %[ptr]")
232 __ASM_EMIT64("add $0x10, %[dst]")
233 __ASM_EMIT32("mov %[ptr], %[dst]")
234 __ASM_EMIT32("subl $4, %[count]")
235 __ASM_EMIT64("sub $4, %[count]")
236 __ASM_EMIT("jae 1b")
237 __ASM_EMIT("2:")
238 /* 1x blocks */
239 __ASM_EMIT32("addl $3, %[count]")
240 __ASM_EMIT64("add $3, %[count]")
241 __ASM_EMIT("jl 4f")
242 __ASM_EMIT("3:")
243 __ASM_EMIT("movss 0x00(%[a_head]), %%xmm0") /* xmm0 = ah */
244 __ASM_EMIT("movss 0x00(%[b_head]), %%xmm1") /* xmm1 = bh */
245 __ASM_EMIT("movss %%xmm0, %%xmm2") /* xmm2 = ah */
246 __ASM_EMIT("movss 0x00(%[a_tail]), %%xmm3") /* xmm3 = at */
247 __ASM_EMIT("movss 0x00(%[b_tail]), %%xmm4") /* xmm4 = bt */
248 __ASM_EMIT("movss %%xmm3, %%xmm5") /* xmm5 = at */
249 __ASM_EMIT("mulss %%xmm1, %%xmm2") /* xmm2 = ah*bh */
250 __ASM_EMIT("mulss %%xmm0, %%xmm0") /* xmm0 = ah*ah */
251 __ASM_EMIT("mulss %%xmm1, %%xmm1") /* xmm1 = bh*bh */
252 __ASM_EMIT("mulss %%xmm4, %%xmm5") /* xmm5 = at*bt */
253 __ASM_EMIT("mulss %%xmm3, %%xmm3") /* xmm3 = at*at */
254 __ASM_EMIT("mulss %%xmm4, %%xmm4") /* xmm4 = bt*bt */
255 __ASM_EMIT("subss %%xmm3, %%xmm0") /* xmm0 = DA = ah*ah - at*at */
256 __ASM_EMIT("subss %%xmm4, %%xmm1") /* xmm1 = DB = bh*bh - bt*bt */
257 __ASM_EMIT("subss %%xmm5, %%xmm2") /* xmm2 = DV = ah*bh - at*bt */
258
259 __ASM_EMIT32("mov %[corr], %[ptr]")
260 __ASM_EMIT32("movss 0x00(%[ptr]), %%xmm3")
261 __ASM_EMIT32("movss 0x04(%[ptr]), %%xmm4")
262 __ASM_EMIT32("movss 0x08(%[ptr]), %%xmm5")
263 __ASM_EMIT64("movss 0x00(%[corr]), %%xmm3")
264 __ASM_EMIT64("movss 0x04(%[corr]), %%xmm4")
265 __ASM_EMIT64("movss 0x08(%[corr]), %%xmm5")
266
267 __ASM_EMIT("addss %%xmm4, %%xmm0") /* xmm0 = BA = xa+DA */
268 __ASM_EMIT("addss %%xmm3, %%xmm2") /* xmm2 = T = xv+DV */
269 __ASM_EMIT("movaps %%xmm0, %%xmm3") /* xmm3 = BA */
270 __ASM_EMIT("addss %%xmm5, %%xmm1") /* xmm1 = BB = xb+DB */
271 __ASM_EMIT("mulss %%xmm1, %%xmm3") /* xmm3 = B = BA*BB */
272
273 __ASM_EMIT32("movss %%xmm2, 0x00(%[ptr])")
274 __ASM_EMIT32("movss %%xmm0, 0x04(%[ptr])")
275 __ASM_EMIT32("movss %%xmm1, 0x08(%[ptr])")
276 __ASM_EMIT64("movss %%xmm2, 0x00(%[corr])")
277 __ASM_EMIT64("movss %%xmm0, 0x04(%[corr])")
278 __ASM_EMIT64("movss %%xmm1, 0x08(%[corr])")
279
280 __ASM_EMIT("sqrtss %%xmm3, %%xmm7") /* xmm7 = sqrtf(B) */
281 __ASM_EMIT("cmpss $5, %[CORR_CC], %%xmm3") /* xmm3 = B >= threshold */
282 __ASM_EMIT("divss %%xmm7, %%xmm2") /* xmm2 = T/sqrtf(B) */
283 __ASM_EMIT32("mov %[dst], %[ptr]")
284 __ASM_EMIT("andps %%xmm3, %%xmm2") /* xmm0 = (B >= threshold) ? T/sqrtf(B) : 0 */
285 __ASM_EMIT("add $0x04, %[a_head]") /* ++a_head */
286 __ASM_EMIT("add $0x04, %[b_head]") /* ++b_head */
287 __ASM_EMIT32("movss %%xmm2, 0x00(%[ptr])")
288 __ASM_EMIT64("movss %%xmm2, 0x00(%[dst])")
289 __ASM_EMIT32("add $0x04, %[ptr]")
290 __ASM_EMIT64("add $0x04, %[dst]")
291 __ASM_EMIT("add $0x04, %[a_tail]") /* ++a_tail */
292 __ASM_EMIT("add $0x04, %[b_tail]") /* ++b_tail */
293 __ASM_EMIT32("mov %[ptr], %[dst]")
294 __ASM_EMIT32("decl %[count]")
295 __ASM_EMIT64("dec %[count]")
296 __ASM_EMIT("jge 3b")
297 __ASM_EMIT("4:")
298
299 : __IF_32(
300 [ptr] "=&r" (ptr),
301 [corr] "+m" (corr), [dst] "+m" (dst),
302 [a_head] "+r" (a_head), [b_head] "+r" (b_head),
303 [a_tail] "+r" (a_tail), [b_tail] "+r" (b_tail),
304 [count] "+m" (count)
305 )
306 __IF_64(
307 [dst] "+r" (dst),
308 [a_head] "+r" (a_head), [b_head] "+r" (b_head),
309 [a_tail] "+r" (a_tail), [b_tail] "+r" (b_tail),
310 [count] "+r" (count)
311 )
312 : __IF_64( [corr] "r" (corr), )
313 [CORR_CC] "o" (corr_const)
314 : "cc", "memory",
315 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
316 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
317 );
318 }
319
320 } /* namespace sse */
321 } /* namespace lsp */
322
323
324
325 #endif /* PRIVATE_DSP_ARCH_X86_SSE_CORRELATION_H_ */