Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/x86/avx/msmatrix.h

raw
1 /*
2 * Copyright (C) 2024 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2024 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 31 мар. 2020 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_X86_AVX_MSMATRIX_H_
23 #define PRIVATE_DSP_ARCH_X86_AVX_MSMATRIX_H_
24
25 #ifndef PRIVATE_DSP_ARCH_X86_AVX_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_X86_AVX_IMPL */
28
29 namespace lsp
30 {
31 namespace avx
32 {
33 IF_ARCH_X86(
34 static const float msmatrix_const[] __lsp_aligned32 =
35 {
36 LSP_DSP_VEC8(0.5f)
37 };
38 )
39
40 void lr_to_ms(float *m, float *s, const float *l, const float *r, size_t count)
41 {
42 IF_ARCH_X86(size_t off);
43 ARCH_X86_ASM(
44 __ASM_EMIT("xor %[off], %[off]")
45 __ASM_EMIT("vmovaps %[X_HALF], %%ymm7")
46 // 16x blocks
47 __ASM_EMIT32("subl $16, %[count]")
48 __ASM_EMIT64("sub $16, %[count]")
49 __ASM_EMIT("vmovaps %%ymm7, %%ymm6")
50 __ASM_EMIT("jb 2f")
51 __ASM_EMIT("1:")
52 __ASM_EMIT("vmulps 0x00(%[left], %[off]), %%ymm6, %%ymm0") // ymm0 = l*0.5
53 __ASM_EMIT("vmulps 0x20(%[left], %[off]), %%ymm7, %%ymm1")
54 __ASM_EMIT("vmulps 0x00(%[right], %[off]), %%ymm6, %%ymm4") // ymm4 = r*0.5
55 __ASM_EMIT("vmulps 0x20(%[right], %[off]), %%ymm7, %%ymm5")
56 __ASM_EMIT("vsubps %%ymm4, %%ymm0, %%ymm2") // ymm2 = s = (l-r)*0.5
57 __ASM_EMIT("vsubps %%ymm5, %%ymm1, %%ymm3")
58 __ASM_EMIT("vaddps %%ymm4, %%ymm0, %%ymm0") // ymm0 = m = (l+r)*0.5
59 __ASM_EMIT("vaddps %%ymm5, %%ymm1, %%ymm1")
60 __ASM_EMIT("vmovups %%ymm0, 0x00(%[mid], %[off])")
61 __ASM_EMIT("vmovups %%ymm1, 0x20(%[mid], %[off])")
62 __ASM_EMIT("vmovups %%ymm2, 0x00(%[side], %[off])")
63 __ASM_EMIT("vmovups %%ymm3, 0x20(%[side], %[off])")
64 __ASM_EMIT("add $0x40, %[off]")
65 __ASM_EMIT32("subl $16, %[count]")
66 __ASM_EMIT64("sub $16, %[count]")
67 __ASM_EMIT("jae 1b")
68 __ASM_EMIT("2:")
69 // 8x block
70 __ASM_EMIT32("addl $8, %[count]")
71 __ASM_EMIT64("add $8, %[count]")
72 __ASM_EMIT("jl 4f")
73 __ASM_EMIT("vmulps 0x00(%[left], %[off]), %%ymm6, %%ymm0") // ymm0 = l*0.5
74 __ASM_EMIT("vmulps 0x00(%[right], %[off]), %%ymm7, %%ymm4") // ymm4 = r*0.5
75 __ASM_EMIT("vsubps %%ymm4, %%ymm0, %%ymm2") // ymm2 = s = (l-r)*0.5
76 __ASM_EMIT("vaddps %%ymm4, %%ymm0, %%ymm0") // ymm0 = m = (l+r)*0.5
77 __ASM_EMIT("vmovups %%ymm0, 0x00(%[mid], %[off])")
78 __ASM_EMIT("vmovups %%ymm2, 0x00(%[side], %[off])")
79 __ASM_EMIT("add $0x20, %[off]")
80 __ASM_EMIT32("subl $8, %[count]")
81 __ASM_EMIT64("sub $8, %[count]")
82 __ASM_EMIT("4:")
83 // 4x block
84 __ASM_EMIT32("addl $4, %[count]")
85 __ASM_EMIT64("add $4, %[count]")
86 __ASM_EMIT("jl 6f")
87 __ASM_EMIT("vmulps 0x00(%[left], %[off]), %%xmm6, %%xmm0") // xmm0 = l*0.5
88 __ASM_EMIT("vmulps 0x00(%[right], %[off]), %%xmm7, %%xmm4") // xmm4 = r*0.5
89 __ASM_EMIT("vsubps %%xmm4, %%xmm0, %%xmm2") // xmm2 = s = (l-r)*0.5
90 __ASM_EMIT("vaddps %%xmm4, %%xmm0, %%xmm0") // xmm0 = m = (l+r)*0.5
91 __ASM_EMIT("vmovups %%xmm0, 0x00(%[mid], %[off])")
92 __ASM_EMIT("vmovups %%xmm2, 0x00(%[side], %[off])")
93 __ASM_EMIT("add $0x10, %[off]")
94 __ASM_EMIT32("subl $4, %[count]")
95 __ASM_EMIT64("sub $4, %[count]")
96 __ASM_EMIT("6:")
97 // 1x blocks
98 __ASM_EMIT32("addl $3, %[count]")
99 __ASM_EMIT64("add $3, %[count]")
100 __ASM_EMIT("jl 8f")
101 __ASM_EMIT("7:")
102 __ASM_EMIT("vmulss 0x00(%[left], %[off]), %%xmm6, %%xmm0") // xmm0 = l*0.5
103 __ASM_EMIT("vmulss 0x00(%[right], %[off]), %%xmm7, %%xmm4") // xmm4 = r*0.5
104 __ASM_EMIT("vsubss %%xmm4, %%xmm0, %%xmm2") // xmm2 = s = (l-r)*0.5
105 __ASM_EMIT("vaddss %%xmm4, %%xmm0, %%xmm0") // xmm0 = m = (l+r)*0.5
106 __ASM_EMIT("vmovss %%xmm0, 0x00(%[mid], %[off])")
107 __ASM_EMIT("vmovss %%xmm2, 0x00(%[side], %[off])")
108 __ASM_EMIT("add $0x04, %[off]")
109 __ASM_EMIT32("decl %[count]")
110 __ASM_EMIT64("dec %[count]")
111 __ASM_EMIT("jge 7b")
112 __ASM_EMIT("8:")
113
114 : [off] "=&r" (off),
115 [count] X86_PGREG (count)
116 : [left] "r"(l), [right] "r" (r),
117 [mid] "r" (m), [side] "r" (s),
118 [X_HALF] "m" (msmatrix_const)
119 : "cc", "memory",
120 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
121 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
122 );
123 }
124
125 #define LR_TO_PART(P, L, R, OP) \
126 __ASM_EMIT("xor %[off], %[off]") \
127 __ASM_EMIT("vmovaps %[X_HALF], %%ymm7") \
128 /* 32x blocks */ \
129 __ASM_EMIT("sub $32, %[count]") \
130 __ASM_EMIT("vmovaps %%ymm7, %%ymm6") \
131 __ASM_EMIT("jb 2f") \
132 __ASM_EMIT("1:") \
133 __ASM_EMIT("vmovups 0x00(%[" L "], %[off]), %%ymm0") /* ymm0 = l */ \
134 __ASM_EMIT("vmovups 0x20(%[" L "], %[off]), %%ymm1") \
135 __ASM_EMIT("vmovups 0x40(%[" L "], %[off]), %%ymm2") \
136 __ASM_EMIT("vmovups 0x60(%[" L "], %[off]), %%ymm3") \
137 __ASM_EMIT(OP "ps 0x00(%[" R "], %[off]), %%ymm0, %%ymm0") /* ymm0 = l op r */ \
138 __ASM_EMIT(OP "ps 0x20(%[" R "], %[off]), %%ymm1, %%ymm1") \
139 __ASM_EMIT(OP "ps 0x40(%[" R "], %[off]), %%ymm2, %%ymm2") \
140 __ASM_EMIT(OP "ps 0x60(%[" R "], %[off]), %%ymm3, %%ymm3") \
141 __ASM_EMIT("vmulps %%ymm6, %%ymm0, %%ymm0") /* ymm0 = (l op r) * 0.5 */ \
142 __ASM_EMIT("vmulps %%ymm7, %%ymm1, %%ymm1") \
143 __ASM_EMIT("vmulps %%ymm6, %%ymm2, %%ymm2") \
144 __ASM_EMIT("vmulps %%ymm7, %%ymm3, %%ymm3") \
145 __ASM_EMIT("vmovups %%ymm0, 0x00(%[" P "], %[off])") \
146 __ASM_EMIT("vmovups %%ymm1, 0x20(%[" P "], %[off])") \
147 __ASM_EMIT("vmovups %%ymm2, 0x40(%[" P "], %[off])") \
148 __ASM_EMIT("vmovups %%ymm3, 0x60(%[" P "], %[off])") \
149 __ASM_EMIT("add $0x80, %[off]") \
150 __ASM_EMIT("sub $32, %[count]") \
151 __ASM_EMIT("jae 1b") \
152 __ASM_EMIT("2:") \
153 /* 16x block */ \
154 __ASM_EMIT("add $16, %[count]") \
155 __ASM_EMIT("jl 4f") \
156 __ASM_EMIT("vmovups 0x00(%[" L "], %[off]), %%ymm0") /* ymm0 = l */ \
157 __ASM_EMIT("vmovups 0x20(%[" L "], %[off]), %%ymm1") \
158 __ASM_EMIT(OP "ps 0x00(%[" R "], %[off]), %%ymm0, %%ymm0") /* ymm0 = l op r */ \
159 __ASM_EMIT(OP "ps 0x20(%[" R "], %[off]), %%ymm1, %%ymm1") \
160 __ASM_EMIT("vmulps %%ymm6, %%ymm0, %%ymm0") /* ymm0 = (l op r) * 0.5 */ \
161 __ASM_EMIT("vmulps %%ymm7, %%ymm1, %%ymm1") \
162 __ASM_EMIT("vmovups %%ymm0, 0x00(%[" P "], %[off])") \
163 __ASM_EMIT("vmovups %%ymm1, 0x20(%[" P "], %[off])") \
164 __ASM_EMIT("sub $16, %[count]") \
165 __ASM_EMIT("add $0x40, %[off]") \
166 __ASM_EMIT("4:") \
167 /* 8x block */ \
168 __ASM_EMIT("add $8, %[count]") \
169 __ASM_EMIT("jl 6f") \
170 __ASM_EMIT("vmovups 0x00(%[" L "], %[off]), %%ymm0") /* ymm0 = l */ \
171 __ASM_EMIT(OP "ps 0x00(%[" R "], %[off]), %%ymm0, %%ymm0") /* ymm0 = l op r */ \
172 __ASM_EMIT("vmulps %%ymm6, %%ymm0, %%ymm0") /* ymm0 = (l op r) * 0.5 */ \
173 __ASM_EMIT("vmovups %%ymm0, 0x00(%[" P "], %[off])") \
174 __ASM_EMIT("sub $8, %[count]") \
175 __ASM_EMIT("add $0x20, %[off]") \
176 __ASM_EMIT("6:") \
177 /* 4x block */ \
178 __ASM_EMIT("add $4, %[count]") \
179 __ASM_EMIT("jl 8f") \
180 __ASM_EMIT("vmovups 0x00(%[" L "], %[off]), %%xmm0") /* xmm0 = l */ \
181 __ASM_EMIT(OP "ps 0x00(%[" R "], %[off]), %%xmm0, %%xmm0") /* xmm0 = l op r */ \
182 __ASM_EMIT("vmulps %%xmm6, %%xmm0, %%xmm0") /* xmm0 = (l op r) * 0.5 */ \
183 __ASM_EMIT("vmovups %%xmm0, 0x00(%[" P "], %[off])") \
184 __ASM_EMIT("sub $4, %[count]") \
185 __ASM_EMIT("add $0x10, %[off]") \
186 __ASM_EMIT("8:") \
187 /* 1x blocks */ \
188 __ASM_EMIT("add $3, %[count]") \
189 __ASM_EMIT("jl 10f") \
190 __ASM_EMIT("9:") \
191 __ASM_EMIT("vmovss 0x00(%[" L "], %[off]), %%xmm0") /* xmm0 = l */ \
192 __ASM_EMIT(OP "ss 0x00(%[" R "], %[off]), %%xmm0, %%xmm0") /* xmm0 = l op r */ \
193 __ASM_EMIT("vmulss %%xmm6, %%xmm0, %%xmm0") /* xmm0 = (l op r) * 0.5 */ \
194 __ASM_EMIT("vmovss %%xmm0, 0x00(%[" P "], %[off])") \
195 __ASM_EMIT("add $0x04, %[off]") \
196 __ASM_EMIT("dec %[count]") \
197 __ASM_EMIT("jge 9b") \
198 __ASM_EMIT("10:")
199
200 void lr_to_mid(float *m, const float *l, const float *r, size_t count)
201 {
202 IF_ARCH_X86(size_t off);
203 ARCH_X86_ASM (
204 LR_TO_PART("mid", "left", "right", "vadd")
205 : [off] "=&r" (off), [count] "+r" (count)
206 : [left] "r" (l), [right] "r" (r),
207 [mid] "r" (m),
208 [X_HALF] "m" (msmatrix_const)
209 : "cc", "memory",
210 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
211 "%xmm6", "%xmm7"
212 );
213 }
214
215 void lr_to_side(float *s, const float *l, const float *r, size_t count)
216 {
217 IF_ARCH_X86(size_t off);
218 ARCH_X86_ASM (
219 LR_TO_PART("side", "left", "right", "vsub")
220 : [off] "=&r" (off), [count] "+r" (count)
221 : [left] "r" (l), [right] "r" (r),
222 [side] "r" (s),
223 [X_HALF] "m" (msmatrix_const)
224 : "cc", "memory",
225 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
226 "%xmm6", "%xmm7"
227 );
228 }
229
230 #undef LR_TO_PART
231
232 void ms_to_lr(float *l, float *r, const float *m, const float *s, size_t count)
233 {
234 IF_ARCH_X86(size_t off);
235 ARCH_X86_ASM(
236 __ASM_EMIT("xor %[off], %[off]")
237 // 32x blocks
238 __ASM_EMIT32("subl $32, %[count]")
239 __ASM_EMIT64("sub $32, %[count]")
240 __ASM_EMIT("jb 2f")
241 __ASM_EMIT("1:")
242 __ASM_EMIT("vmovups 0x00(%[mid], %[off]), %%ymm0") // ymm0 = m
243 __ASM_EMIT("vmovups 0x20(%[mid], %[off]), %%ymm1")
244 __ASM_EMIT("vmovups 0x40(%[mid], %[off]), %%ymm2")
245 __ASM_EMIT("vmovups 0x60(%[mid], %[off]), %%ymm3")
246 __ASM_EMIT("vsubps 0x00(%[side], %[off]), %%ymm0, %%ymm4") // ymm4 = m - s
247 __ASM_EMIT("vsubps 0x20(%[side], %[off]), %%ymm1, %%ymm5")
248 __ASM_EMIT("vsubps 0x40(%[side], %[off]), %%ymm2, %%ymm6")
249 __ASM_EMIT("vsubps 0x60(%[side], %[off]), %%ymm3, %%ymm7")
250 __ASM_EMIT("vaddps 0x00(%[side], %[off]), %%ymm0, %%ymm0") // ymm0 = m + s
251 __ASM_EMIT("vaddps 0x20(%[side], %[off]), %%ymm1, %%ymm1")
252 __ASM_EMIT("vaddps 0x40(%[side], %[off]), %%ymm2, %%ymm2")
253 __ASM_EMIT("vaddps 0x60(%[side], %[off]), %%ymm3, %%ymm3")
254 __ASM_EMIT("vmovups %%ymm0, 0x00(%[left], %[off])")
255 __ASM_EMIT("vmovups %%ymm1, 0x20(%[left], %[off])")
256 __ASM_EMIT("vmovups %%ymm2, 0x40(%[left], %[off])")
257 __ASM_EMIT("vmovups %%ymm3, 0x60(%[left], %[off])")
258 __ASM_EMIT("vmovups %%ymm4, 0x00(%[right], %[off])")
259 __ASM_EMIT("vmovups %%ymm5, 0x20(%[right], %[off])")
260 __ASM_EMIT("vmovups %%ymm6, 0x40(%[right], %[off])")
261 __ASM_EMIT("vmovups %%ymm7, 0x60(%[right], %[off])")
262 __ASM_EMIT("add $0x80, %[off]")
263 __ASM_EMIT32("subl $32, %[count]")
264 __ASM_EMIT64("sub $32, %[count]")
265 __ASM_EMIT("jae 1b")
266 __ASM_EMIT("2:")
267 // 16x block
268 __ASM_EMIT32("addl $16, %[count]")
269 __ASM_EMIT64("add $16, %[count]")
270 __ASM_EMIT("jl 4f")
271 __ASM_EMIT("vmovups 0x00(%[mid], %[off]), %%ymm0") // ymm0 = m
272 __ASM_EMIT("vmovups 0x20(%[mid], %[off]), %%ymm1")
273 __ASM_EMIT("vsubps 0x00(%[side], %[off]), %%ymm0, %%ymm4") // ymm4 = m - s
274 __ASM_EMIT("vsubps 0x20(%[side], %[off]), %%ymm1, %%ymm5")
275 __ASM_EMIT("vaddps 0x00(%[side], %[off]), %%ymm0, %%ymm0") // ymm0 = m + s
276 __ASM_EMIT("vaddps 0x20(%[side], %[off]), %%ymm1, %%ymm1")
277 __ASM_EMIT("vmovups %%ymm0, 0x00(%[left], %[off])")
278 __ASM_EMIT("vmovups %%ymm1, 0x20(%[left], %[off])")
279 __ASM_EMIT("vmovups %%ymm4, 0x00(%[right], %[off])")
280 __ASM_EMIT("vmovups %%ymm5, 0x20(%[right], %[off])")
281 __ASM_EMIT32("subl $16, %[count]")
282 __ASM_EMIT64("sub $16, %[count]")
283 __ASM_EMIT("add $0x40, %[off]")
284 __ASM_EMIT("4:")
285 // 8x block
286 __ASM_EMIT32("addl $8, %[count]")
287 __ASM_EMIT64("add $8, %[count]")
288 __ASM_EMIT("jl 6f")
289 __ASM_EMIT("vmovups 0x00(%[mid], %[off]), %%ymm0") // ymm0 = m
290 __ASM_EMIT("vsubps 0x00(%[side], %[off]), %%ymm0, %%ymm4") // ymm4 = m - s
291 __ASM_EMIT("vaddps 0x00(%[side], %[off]), %%ymm0, %%ymm0") // ymm0 = m + s
292 __ASM_EMIT("vmovups %%ymm0, 0x00(%[left], %[off])")
293 __ASM_EMIT("vmovups %%ymm4, 0x00(%[right], %[off])")
294 __ASM_EMIT32("subl $8, %[count]")
295 __ASM_EMIT64("sub $8, %[count]")
296 __ASM_EMIT("add $0x20, %[off]")
297 __ASM_EMIT("6:")
298 // 4x block
299 __ASM_EMIT32("addl $4, %[count]")
300 __ASM_EMIT64("add $4, %[count]")
301 __ASM_EMIT("jl 8f")
302 __ASM_EMIT("vmovups 0x00(%[mid], %[off]), %%xmm0") // xmm0 = m
303 __ASM_EMIT("vsubps 0x00(%[side], %[off]), %%xmm0, %%xmm4") // xmm4 = m - s
304 __ASM_EMIT("vaddps 0x00(%[side], %[off]), %%xmm0, %%xmm0") // xmm0 = m + s
305 __ASM_EMIT("vmovups %%xmm0, 0x00(%[left], %[off])")
306 __ASM_EMIT("vmovups %%xmm4, 0x00(%[right], %[off])")
307 __ASM_EMIT32("subl $4, %[count]")
308 __ASM_EMIT64("sub $4, %[count]")
309 __ASM_EMIT("add $0x10, %[off]")
310 __ASM_EMIT("8:")
311 // 1x blocks
312 __ASM_EMIT32("addl $3, %[count]")
313 __ASM_EMIT64("add $3, %[count]")
314 __ASM_EMIT("jl 10f")
315 __ASM_EMIT("9:")
316 __ASM_EMIT("vmovss 0x00(%[mid], %[off]), %%xmm0") // xmm0 = m
317 __ASM_EMIT("vsubss 0x00(%[side], %[off]), %%xmm0, %%xmm4") // xmm4 = m - s
318 __ASM_EMIT("vaddss 0x00(%[side], %[off]), %%xmm0, %%xmm0") // xmm0 = m + s
319 __ASM_EMIT("vmovss %%xmm0, 0x00(%[left], %[off])")
320 __ASM_EMIT("vmovss %%xmm4, 0x00(%[right], %[off])")
321 __ASM_EMIT("add $0x04, %[off]")
322 __ASM_EMIT32("decl %[count]")
323 __ASM_EMIT64("dec %[count]")
324 __ASM_EMIT("jge 9b")
325 __ASM_EMIT("10:")
326
327 : [off] "=&r" (off),
328 [count] X86_PGREG (count)
329 : [left] "r"(l), [right] "r" (r),
330 [mid] "r" (m), [side] "r" (s)
331 : "cc", "memory",
332 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
333 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
334 );
335 }
336
337 #define MS_TO_PART(P, M, S, OP) \
338 __ASM_EMIT("xor %[off], %[off]") \
339 /* 32x blocks */ \
340 __ASM_EMIT("sub $32, %[count]") \
341 __ASM_EMIT("jb 2f") \
342 __ASM_EMIT("1:") \
343 __ASM_EMIT("vmovups 0x00(%[" M "], %[off]), %%ymm0") /* ymm0 = m */ \
344 __ASM_EMIT("vmovups 0x20(%[" M "], %[off]), %%ymm1") \
345 __ASM_EMIT("vmovups 0x40(%[" M "], %[off]), %%ymm2") \
346 __ASM_EMIT("vmovups 0x60(%[" M "], %[off]), %%ymm3") \
347 __ASM_EMIT(OP "ps 0x00(%[" S "], %[off]), %%ymm0, %%ymm0") /* ymm4 = m - s */ \
348 __ASM_EMIT(OP "ps 0x20(%[" S "], %[off]), %%ymm1, %%ymm1") \
349 __ASM_EMIT(OP "ps 0x40(%[" S "], %[off]), %%ymm2, %%ymm2") \
350 __ASM_EMIT(OP "ps 0x60(%[" S "], %[off]), %%ymm3, %%ymm3") \
351 __ASM_EMIT("vmovups %%ymm0, 0x00(%[" P "], %[off])") \
352 __ASM_EMIT("vmovups %%ymm1, 0x20(%[" P "], %[off])") \
353 __ASM_EMIT("vmovups %%ymm2, 0x40(%[" P "], %[off])") \
354 __ASM_EMIT("vmovups %%ymm3, 0x60(%[" P "], %[off])") \
355 __ASM_EMIT("add $0x80, %[off]") \
356 __ASM_EMIT("sub $32, %[count]") \
357 __ASM_EMIT("jae 1b") \
358 __ASM_EMIT("2:") \
359 /* 16x block */ \
360 __ASM_EMIT("add $16, %[count]") \
361 __ASM_EMIT("jl 4f") \
362 __ASM_EMIT("vmovups 0x00(%[" M "], %[off]), %%ymm0") /* ymm0 = m */ \
363 __ASM_EMIT("vmovups 0x20(%[" M "], %[off]), %%ymm1") \
364 __ASM_EMIT(OP "ps 0x00(%[" S "], %[off]), %%ymm0, %%ymm0") /* ymm4 = m - s */ \
365 __ASM_EMIT(OP "ps 0x20(%[" S "], %[off]), %%ymm1, %%ymm1") \
366 __ASM_EMIT("vmovups %%ymm0, 0x00(%[" P "], %[off])") \
367 __ASM_EMIT("vmovups %%ymm1, 0x20(%[" P "], %[off])") \
368 __ASM_EMIT("sub $16, %[count]") \
369 __ASM_EMIT("add $0x40, %[off]") \
370 __ASM_EMIT("4:") \
371 /* 8x block */ \
372 __ASM_EMIT("add $8, %[count]") \
373 __ASM_EMIT("jl 6f") \
374 __ASM_EMIT("vmovups 0x00(%[" M "], %[off]), %%ymm0") /* ymm0 = m */ \
375 __ASM_EMIT(OP "ps 0x00(%[" S "], %[off]), %%ymm0, %%ymm0") /* ymm4 = m - s */ \
376 __ASM_EMIT("vmovups %%ymm0, 0x00(%[" P "], %[off])") \
377 __ASM_EMIT("sub $8, %[count]") \
378 __ASM_EMIT("add $0x20, %[off]") \
379 __ASM_EMIT("6:") \
380 /* 4x block */ \
381 __ASM_EMIT("add $4, %[count]") \
382 __ASM_EMIT("jl 8f") \
383 __ASM_EMIT("vmovups 0x00(%[" M "], %[off]), %%xmm0") /* xmm0 = m */ \
384 __ASM_EMIT(OP "ps 0x00(%[" S "], %[off]), %%xmm0, %%xmm0") /* xmm4 = m - s */ \
385 __ASM_EMIT("vmovups %%xmm0, 0x00(%[" P "], %[off])") \
386 __ASM_EMIT("sub $4, %[count]") \
387 __ASM_EMIT("add $0x10, %[off]") \
388 __ASM_EMIT("8:") \
389 /* 1x blocks */ \
390 __ASM_EMIT("add $3, %[count]") \
391 __ASM_EMIT("jl 10f") \
392 __ASM_EMIT("9:") \
393 __ASM_EMIT("vmovss 0x00(%[" M "], %[off]), %%xmm0") /* xmm0 = m */ \
394 __ASM_EMIT(OP "ss 0x00(%[" S "], %[off]), %%xmm0, %%xmm0") /* xmm4 = m - s */ \
395 __ASM_EMIT("vmovss %%xmm0, 0x00(%[" P "], %[off])") \
396 __ASM_EMIT("add $0x04, %[off]") \
397 __ASM_EMIT("dec %[count]") \
398 __ASM_EMIT("jge 9b") \
399 __ASM_EMIT("10:")
400
401 void ms_to_left(float *l, const float *m, const float *s, size_t count)
402 {
403 size_t off;
404
405 ARCH_X86_ASM (
406 MS_TO_PART("left", "mid", "side", "vadd")
407 : [off] "=&r" (off), [count] "+r" (count)
408 : [left] "r" (l),
409 [mid] "r" (m), [side] "r" (s)
410 : "cc", "memory",
411 "%xmm0", "%xmm1", "%xmm2", "%xmm3"
412 );
413 }
414
415 void ms_to_right(float *r, const float *m, const float *s, size_t count)
416 {
417 size_t off;
418
419 ARCH_X86_ASM (
420 MS_TO_PART("right", "mid", "side", "vsub")
421 : [off] "=&r" (off), [count] "+r" (count)
422 : [right] "r" (r),
423 [mid] "r" (m), [side] "r" (s)
424 : "cc", "memory",
425 "%xmm0", "%xmm1", "%xmm2", "%xmm3"
426 );
427 }
428
429 #undef MS_TO_PART
430 } /* namespace avx */
431 } /* namespace lsp */
432
433 #endif /* PRIVATE_DSP_ARCH_X86_AVX_MSMATRIX_H_ */