Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/x86/sse/msmatrix.h

raw
1 /*
2 * Copyright (C) 2020 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2020 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 31 мар. 2020 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_X86_SSE_MSMATRIX_H_
23 #define PRIVATE_DSP_ARCH_X86_SSE_MSMATRIX_H_
24
25 #ifndef PRIVATE_DSP_ARCH_X86_SSE_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_X86_SSE_IMPL */
28
29 namespace lsp
30 {
31 namespace sse
32 {
33 IF_ARCH_X86(
34 static const float msmatrix_const[] __lsp_aligned16 =
35 {
36 LSP_DSP_VEC4(0.5f)
37 };
38 );
39
40 void lr_to_ms(float *m, float *s, const float *l, const float *r, size_t count)
41 {
42 size_t off;
43
44 ARCH_X86_ASM
45 (
46 __ASM_EMIT("xor %[off], %[off]")
47 __ASM_EMIT("movaps %[X_HALF], %%xmm7")
48 __ASM_EMIT64("sub $8, %[count]")
49 __ASM_EMIT32("subl $8, %[count]")
50 __ASM_EMIT("movaps %%xmm7, %%xmm6")
51 __ASM_EMIT("jb 2f")
52 // 8x blocks
53 __ASM_EMIT("1:")
54 __ASM_EMIT("movups 0x00(%[left], %[off]), %%xmm0") // xmm0 = l
55 __ASM_EMIT("movups 0x10(%[left], %[off]), %%xmm3")
56 __ASM_EMIT("movups 0x00(%[right], %[off]), %%xmm2") // xmm2 = r
57 __ASM_EMIT("movups 0x10(%[right], %[off]), %%xmm5")
58 __ASM_EMIT("movaps %%xmm0, %%xmm1") // xmm1 = l
59 __ASM_EMIT("movaps %%xmm3, %%xmm4")
60 __ASM_EMIT("addps %%xmm2, %%xmm0") // xmm0 = l + r
61 __ASM_EMIT("addps %%xmm5, %%xmm3")
62 __ASM_EMIT("subps %%xmm2, %%xmm1") // xmm1 = l - r
63 __ASM_EMIT("subps %%xmm5, %%xmm4")
64 __ASM_EMIT("mulps %%xmm6, %%xmm0") // xmm0 = (l + r) * 0.5f
65 __ASM_EMIT("mulps %%xmm7, %%xmm3")
66 __ASM_EMIT("mulps %%xmm6, %%xmm1") // xmm1 = (l - r) * 0.5f
67 __ASM_EMIT("mulps %%xmm7, %%xmm4")
68 __ASM_EMIT("movups %%xmm0, 0x00(%[mid], %[off])")
69 __ASM_EMIT("movups %%xmm3, 0x10(%[mid], %[off])")
70 __ASM_EMIT("movups %%xmm1, 0x00(%[side], %[off])")
71 __ASM_EMIT("movups %%xmm4, 0x10(%[side], %[off])")
72 __ASM_EMIT("add $0x20, %[off]")
73 __ASM_EMIT64("sub $8, %[count]")
74 __ASM_EMIT32("subl $8, %[count]")
75 __ASM_EMIT("jae 1b")
76 // 4x block
77 __ASM_EMIT("2:")
78 __ASM_EMIT64("add $4, %[count]")
79 __ASM_EMIT32("addl $4, %[count]")
80 __ASM_EMIT("jl 4f")
81 __ASM_EMIT("movups 0x00(%[left], %[off]), %%xmm0") // xmm0 = l
82 __ASM_EMIT("movups 0x00(%[right], %[off]), %%xmm2") // xmm2 = r
83 __ASM_EMIT("movaps %%xmm0, %%xmm1") // xmm1 = l
84 __ASM_EMIT("addps %%xmm2, %%xmm0") // xmm0 = l + r
85 __ASM_EMIT("subps %%xmm2, %%xmm1") // xmm1 = l - r
86 __ASM_EMIT("mulps %%xmm6, %%xmm0") // xmm0 = (l + r) * 0.5f
87 __ASM_EMIT("mulps %%xmm7, %%xmm1") // xmm1 = (l - r) * 0.5f
88 __ASM_EMIT("movups %%xmm0, 0x00(%[mid], %[off])")
89 __ASM_EMIT("movups %%xmm1, 0x00(%[side], %[off])")
90 __ASM_EMIT64("sub $4, %[count]")
91 __ASM_EMIT32("subl $4, %[count]")
92 __ASM_EMIT("add $0x10, %[off]")
93 // 1x blocks
94 __ASM_EMIT("4:")
95 __ASM_EMIT64("add $3, %[count]")
96 __ASM_EMIT32("addl $3, %[count]")
97 __ASM_EMIT("jl 6f")
98 __ASM_EMIT("5:")
99 __ASM_EMIT("movss 0x00(%[left], %[off]), %%xmm0") // xmm0 = l
100 __ASM_EMIT("movss 0x00(%[right], %[off]), %%xmm2") // xmm2 = r
101 __ASM_EMIT("movaps %%xmm0, %%xmm1") // xmm1 = l
102 __ASM_EMIT("addss %%xmm2, %%xmm0") // xmm0 = l + r
103 __ASM_EMIT("subss %%xmm2, %%xmm1") // xmm1 = l - r
104 __ASM_EMIT("mulss %%xmm6, %%xmm0") // xmm0 = (l + r) * 0.5f
105 __ASM_EMIT("mulss %%xmm7, %%xmm1") // xmm1 = (l - r) * 0.5f
106 __ASM_EMIT("movss %%xmm0, 0x00(%[mid], %[off])")
107 __ASM_EMIT("movss %%xmm1, 0x00(%[side], %[off])")
108 __ASM_EMIT("add $0x04, %[off]")
109 __ASM_EMIT64("dec %[count]")
110 __ASM_EMIT32("decl %[count]")
111 __ASM_EMIT("jge 5b")
112 __ASM_EMIT("6:")
113
114 : [off] "=&r" (off),
115 [count] X86_PGREG (count)
116 : [left] "r"(l), [right] "r" (r),
117 [mid] "r" (m), [side] "r" (s),
118 [X_HALF] "m" (msmatrix_const)
119 : "cc", "memory",
120 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
121 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
122 );
123 }
124
125 void ms_to_lr(float *l, float *r, const float *m, const float *s, size_t count)
126 {
127 size_t off;
128
129 ARCH_X86_ASM
130 (
131 __ASM_EMIT("xor %[off], %[off]")
132 __ASM_EMIT64("sub $8, %[count]")
133 __ASM_EMIT32("subl $8, %[count]")
134 __ASM_EMIT("jb 2f")
135 // 8x blocks
136 __ASM_EMIT("1:")
137 __ASM_EMIT("movups 0x00(%[mid], %[off]), %%xmm0") // xmm0 = m
138 __ASM_EMIT("movups 0x10(%[mid], %[off]), %%xmm4")
139 __ASM_EMIT("movups 0x00(%[side], %[off]), %%xmm2") // xmm2 = s
140 __ASM_EMIT("movups 0x10(%[side], %[off]), %%xmm6")
141 __ASM_EMIT("movaps %%xmm0, %%xmm1") // xmm1 = m
142 __ASM_EMIT("movaps %%xmm4, %%xmm5")
143 __ASM_EMIT("addps %%xmm2, %%xmm0") // xmm0 = m + s
144 __ASM_EMIT("addps %%xmm6, %%xmm4")
145 __ASM_EMIT("subps %%xmm2, %%xmm1") // xmm1 = m - s
146 __ASM_EMIT("subps %%xmm6, %%xmm5")
147 __ASM_EMIT("movups %%xmm0, 0x00(%[left], %[off])")
148 __ASM_EMIT("movups %%xmm4, 0x10(%[left], %[off])")
149 __ASM_EMIT("movups %%xmm1, 0x00(%[right], %[off])")
150 __ASM_EMIT("movups %%xmm5, 0x10(%[right], %[off])")
151 __ASM_EMIT("add $0x20, %[off]")
152 __ASM_EMIT64("sub $8, %[count]")
153 __ASM_EMIT32("subl $8, %[count]")
154 __ASM_EMIT("jae 1b")
155 // 4x block
156 __ASM_EMIT("2:")
157 __ASM_EMIT64("add $4, %[count]")
158 __ASM_EMIT32("addl $4, %[count]")
159 __ASM_EMIT("jl 4f")
160 __ASM_EMIT("movups 0x00(%[mid], %[off]), %%xmm0") // xmm0 = m
161 __ASM_EMIT("movups 0x00(%[side], %[off]), %%xmm2") // xmm2 = s
162 __ASM_EMIT("movaps %%xmm0, %%xmm1") // xmm1 = m
163 __ASM_EMIT("addps %%xmm2, %%xmm0") // xmm0 = m + s
164 __ASM_EMIT("subps %%xmm2, %%xmm1") // xmm1 = m - s
165 __ASM_EMIT("movups %%xmm0, 0x00(%[left], %[off])")
166 __ASM_EMIT("movups %%xmm1, 0x00(%[right], %[off])")
167 __ASM_EMIT64("sub $4, %[count]")
168 __ASM_EMIT32("subl $4, %[count]")
169 __ASM_EMIT("add $0x10, %[off]")
170 // 1x blocks
171 __ASM_EMIT("4:")
172 __ASM_EMIT64("add $3, %[count]")
173 __ASM_EMIT32("addl $3, %[count]")
174 __ASM_EMIT("jl 6f")
175 __ASM_EMIT("5:")
176 __ASM_EMIT("movss 0x00(%[mid], %[off]), %%xmm0") // xmm0 = m
177 __ASM_EMIT("movss 0x00(%[side], %[off]), %%xmm2") // xmm2 = s
178 __ASM_EMIT("movaps %%xmm0, %%xmm1") // xmm1 = m
179 __ASM_EMIT("addss %%xmm2, %%xmm0") // xmm0 = m + s
180 __ASM_EMIT("subss %%xmm2, %%xmm1") // xmm1 = m - s
181 __ASM_EMIT("movss %%xmm0, 0x00(%[left], %[off])")
182 __ASM_EMIT("movss %%xmm1, 0x00(%[right], %[off])")
183 __ASM_EMIT("add $0x04, %[off]")
184 __ASM_EMIT64("dec %[count]")
185 __ASM_EMIT32("decl %[count]")
186 __ASM_EMIT("jge 5b")
187 __ASM_EMIT("6:")
188
189 : [off] "=&r" (off),
190 [count] X86_PGREG (count)
191 : [left] "r"(l), [right] "r" (r),
192 [mid] "r" (m), [side] "r" (s)
193 : "cc", "memory",
194 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
195 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
196 );
197 }
198
199 #define LR_CVT_BODY(d, l, r, op) \
200 __ASM_EMIT("xor %[off], %[off]") \
201 __ASM_EMIT("movaps %[X_HALF], %%xmm7") \
202 __ASM_EMIT("sub $12, %[count]") \
203 __ASM_EMIT("movaps %%xmm7, %%xmm6") \
204 __ASM_EMIT("jb 2f") \
205 /* x12 blocks */ \
206 __ASM_EMIT("1:") \
207 __ASM_EMIT("movups 0x00(%[" l "], %[off]), %%xmm0") \
208 __ASM_EMIT("movups 0x10(%[" l "], %[off]), %%xmm1") \
209 __ASM_EMIT("movups 0x20(%[" l "], %[off]), %%xmm2") \
210 __ASM_EMIT("movups 0x00(%[" r "], %[off]), %%xmm3") \
211 __ASM_EMIT("movups 0x10(%[" r "], %[off]), %%xmm4") \
212 __ASM_EMIT("movups 0x20(%[" r "], %[off]), %%xmm5") \
213 __ASM_EMIT(op "ps %%xmm3, %%xmm0") /* xmm0 = l <+-> r */ \
214 __ASM_EMIT(op "ps %%xmm4, %%xmm1") \
215 __ASM_EMIT(op "ps %%xmm5, %%xmm2") \
216 __ASM_EMIT("mulps %%xmm7, %%xmm0") /* xmm0 = (l <+-> r) * 0.5f */ \
217 __ASM_EMIT("mulps %%xmm6, %%xmm1") \
218 __ASM_EMIT("mulps %%xmm7, %%xmm2") \
219 __ASM_EMIT("movups %%xmm0, 0x00(%[" d "], %[off])") \
220 __ASM_EMIT("movups %%xmm1, 0x10(%[" d "], %[off])") \
221 __ASM_EMIT("movups %%xmm2, 0x20(%[" d "], %[off])") \
222 \
223 __ASM_EMIT("add $0x30, %[off]") \
224 __ASM_EMIT("sub $12, %[count]") \
225 __ASM_EMIT("jae 1b") \
226 /* x8 block */ \
227 __ASM_EMIT("2:") \
228 __ASM_EMIT("add $4, %[count]") \
229 __ASM_EMIT("jl 4f") \
230 __ASM_EMIT("movups 0x00(%[" l "], %[off]), %%xmm0") \
231 __ASM_EMIT("movups 0x10(%[" l "], %[off]), %%xmm1") \
232 __ASM_EMIT("movups 0x00(%[" r "], %[off]), %%xmm3") \
233 __ASM_EMIT("movups 0x10(%[" r "], %[off]), %%xmm4") \
234 __ASM_EMIT(op "ps %%xmm3, %%xmm0") /* xmm0 = l <+-> r */ \
235 __ASM_EMIT(op "ps %%xmm4, %%xmm1") \
236 __ASM_EMIT("mulps %%xmm6, %%xmm0") /* xmm0 = (l <+-> r) * 0.5f */ \
237 __ASM_EMIT("mulps %%xmm7, %%xmm1") \
238 __ASM_EMIT("movups %%xmm0, 0x00(%[" d "], %[off])") \
239 __ASM_EMIT("movups %%xmm1, 0x10(%[" d "], %[off])") \
240 __ASM_EMIT("sub $8, %[count]") \
241 __ASM_EMIT("add $0x20, %[off]") \
242 /* x4 block */ \
243 __ASM_EMIT("4:") \
244 __ASM_EMIT("add $4, %[count]") \
245 __ASM_EMIT("jl 6f") \
246 __ASM_EMIT("movups 0x00(%[" l "], %[off]), %%xmm0") \
247 __ASM_EMIT("movups 0x00(%[" r "], %[off]), %%xmm3") \
248 __ASM_EMIT(op "ps %%xmm3, %%xmm0") /* xmm0 = l <+-> r */ \
249 __ASM_EMIT("mulps %%xmm7, %%xmm0") /* xmm0 = (l <+-> r) * 0.5f */ \
250 __ASM_EMIT("movups %%xmm0, 0x00(%[" d "], %[off])") \
251 __ASM_EMIT("sub $4, %[count]") \
252 __ASM_EMIT("add $0x10, %[off]") \
253 /* x1 blocks */ \
254 __ASM_EMIT("6:") \
255 __ASM_EMIT("add $3, %[count]") \
256 __ASM_EMIT("jl 8f") \
257 __ASM_EMIT("7:") \
258 __ASM_EMIT("movss 0x00(%[" l "], %[off]), %%xmm0") \
259 __ASM_EMIT("movss 0x00(%[" r "], %[off]), %%xmm3") \
260 __ASM_EMIT(op "ss %%xmm3, %%xmm0") /* xmm0 = l <+-> r */ \
261 __ASM_EMIT("mulss %%xmm7, %%xmm0") /* xmm0 = (l <+-> r) * 0.5f */ \
262 __ASM_EMIT("movss %%xmm0, 0x00(%[" d "], %[off])") \
263 __ASM_EMIT("add $0x04, %[off]") \
264 __ASM_EMIT("dec %[count]") \
265 __ASM_EMIT("jge 7b") \
266 \
267 __ASM_EMIT("8:")
268
269 void lr_to_mid(float *m, const float *l, const float *r, size_t count)
270 {
271 size_t off;
272
273 ARCH_X86_ASM
274 (
275 LR_CVT_BODY("mid", "left", "right", "add")
276 : [off] "=&r" (off), [count] "+r" (count)
277 : [left] "r"(l), [right] "r" (r),
278 [mid] "r" (m),
279 [X_HALF] "m" (msmatrix_const)
280 : "cc", "memory",
281 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
282 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
283 );
284 }
285
286 void lr_to_side(float *s, const float *l, const float *r, size_t count)
287 {
288 size_t off;
289
290 ARCH_X86_ASM
291 (
292 LR_CVT_BODY("side", "left", "right", "sub")
293 : [off] "=&r" (off), [count] "+r" (count)
294 : [left] "r"(l), [right] "r" (r),
295 [side] "r" (s),
296 [X_HALF] "m" (msmatrix_const)
297 : "cc", "memory",
298 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
299 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
300 );
301 }
302
303 #undef LR_CVT_BODY
304
305 #define MS_CVT_BODY(d, m, s, op) \
306 __ASM_EMIT("xor %[off], %[off]") \
307 __ASM_EMIT("sub $16, %[count]") \
308 __ASM_EMIT("jb 2f") \
309 \
310 /* x16 blocks */ \
311 __ASM_EMIT("1:") \
312 __ASM_EMIT("movups 0x00(%[" m "], %[off]), %%xmm0") \
313 __ASM_EMIT("movups 0x10(%[" m "], %[off]), %%xmm1") \
314 __ASM_EMIT("movups 0x20(%[" m "], %[off]), %%xmm2") \
315 __ASM_EMIT("movups 0x30(%[" m "], %[off]), %%xmm3") \
316 __ASM_EMIT("movups 0x00(%[" s "], %[off]), %%xmm4") \
317 __ASM_EMIT("movups 0x10(%[" s "], %[off]), %%xmm5") \
318 __ASM_EMIT("movups 0x20(%[" s "], %[off]), %%xmm6") \
319 __ASM_EMIT("movups 0x30(%[" s "], %[off]), %%xmm7") \
320 __ASM_EMIT(op "ps %%xmm4, %%xmm0") /* xmm0 = m <+-> s */ \
321 __ASM_EMIT(op "ps %%xmm5, %%xmm1") \
322 __ASM_EMIT(op "ps %%xmm6, %%xmm2") \
323 __ASM_EMIT(op "ps %%xmm7, %%xmm3") \
324 __ASM_EMIT("movups %%xmm0, 0x00(%[" d "], %[off])") \
325 __ASM_EMIT("movups %%xmm1, 0x10(%[" d "], %[off])") \
326 __ASM_EMIT("movups %%xmm2, 0x20(%[" d "], %[off])") \
327 __ASM_EMIT("movups %%xmm3, 0x30(%[" d "], %[off])") \
328 \
329 __ASM_EMIT("add $0x40, %[off]") \
330 __ASM_EMIT("sub $16, %[count]") \
331 __ASM_EMIT("jae 1b") \
332 /* x8 block */ \
333 __ASM_EMIT("2:") \
334 __ASM_EMIT("add $8, %[count]") \
335 __ASM_EMIT("jl 4f") \
336 __ASM_EMIT("movups 0x00(%[" m "], %[off]), %%xmm0") \
337 __ASM_EMIT("movups 0x10(%[" m "], %[off]), %%xmm1") \
338 __ASM_EMIT("movups 0x00(%[" s "], %[off]), %%xmm4") \
339 __ASM_EMIT("movups 0x10(%[" s "], %[off]), %%xmm5") \
340 __ASM_EMIT(op "ps %%xmm4, %%xmm0") /* xmm0 = m <+-> s */ \
341 __ASM_EMIT(op "ps %%xmm5, %%xmm1") \
342 __ASM_EMIT("movups %%xmm0, 0x00(%[" d "], %[off])") \
343 __ASM_EMIT("movups %%xmm1, 0x10(%[" d "], %[off])") \
344 __ASM_EMIT("sub $8, %[count]") \
345 __ASM_EMIT("add $0x20, %[off]") \
346 /* x4 block */ \
347 __ASM_EMIT("4:") \
348 __ASM_EMIT("add $4, %[count]") \
349 __ASM_EMIT("jl 6f") \
350 __ASM_EMIT("movups 0x00(%[" m "], %[off]), %%xmm0") \
351 __ASM_EMIT("movups 0x00(%[" s "], %[off]), %%xmm4") \
352 __ASM_EMIT(op "ps %%xmm4, %%xmm0") /* xmm0 = m <+-> s */ \
353 __ASM_EMIT("movups %%xmm0, 0x00(%[" d "], %[off])") \
354 __ASM_EMIT("sub $4, %[count]") \
355 __ASM_EMIT("add $0x10, %[off]") \
356 /* x1 blocks */ \
357 __ASM_EMIT("6:") \
358 __ASM_EMIT("add $3, %[count]") \
359 __ASM_EMIT("jl 8f") \
360 __ASM_EMIT("7:") \
361 __ASM_EMIT("movss 0x00(%[" m "], %[off]), %%xmm0") \
362 __ASM_EMIT("movss 0x00(%[" s "], %[off]), %%xmm4") \
363 __ASM_EMIT(op "ss %%xmm4, %%xmm0") /* xmm0 = m <+-> s */ \
364 __ASM_EMIT("movss %%xmm0, 0x00(%[" d "], %[off])") \
365 __ASM_EMIT("add $0x04, %[off]") \
366 __ASM_EMIT("dec %[count]") \
367 __ASM_EMIT("jge 7b") \
368 \
369 __ASM_EMIT("8:")
370
371 void ms_to_left(float *l, const float *m, const float *s, size_t count)
372 {
373 size_t off;
374
375 ARCH_X86_ASM
376 (
377 MS_CVT_BODY("left", "mid", "side", "add")
378 : [off] "=&r" (off), [count] "+r" (count)
379 : [left] "r" (l),
380 [mid] "r" (m), [side] "r" (s)
381 : "cc", "memory",
382 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
383 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
384 );
385 }
386
387 void ms_to_right(float *r, const float *m, const float *s, size_t count)
388 {
389 size_t off;
390
391 ARCH_X86_ASM
392 (
393 MS_CVT_BODY("right", "mid", "side", "sub")
394 : [off] "=&r" (off), [count] "+r" (count)
395 : [right] "r" (r),
396 [mid] "r" (m), [side] "r" (s)
397 : "cc", "memory",
398 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
399 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
400 );
401 }
402
403 #undef MS_CVT_BODY
404 } /* namespace sse */
405 } /* namespace lsp */
406
407 #endif /* PRIVATE_DSP_ARCH_X86_SSE_MSMATRIX_H_ */