Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/x86/avx512/msmatrix.h

raw
1 /*
2 * Copyright (C) 2024 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2024 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 9 сент. 2023 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_X86_AVX512_MSMATRIX_H_
23 #define PRIVATE_DSP_ARCH_X86_AVX512_MSMATRIX_H_
24
25 namespace lsp
26 {
27 namespace avx512
28 {
29 IF_ARCH_X86(
30 static const float msmatrix_const[] __lsp_aligned64 =
31 {
32 LSP_DSP_VEC16(0.5f)
33 };
34 )
35
36 void lr_to_ms(float *m, float *s, const float *l, const float *r, size_t count)
37 {
38 IF_ARCH_X86(size_t off);
39 ARCH_X86_ASM(
40 __ASM_EMIT("xor %[off], %[off]")
41 __ASM_EMIT("vmovaps %[X_HALF], %%zmm7")
42 // 32x blocks
43 __ASM_EMIT32("subl $32, %[count]")
44 __ASM_EMIT64("sub $32, %[count]")
45 __ASM_EMIT("vmovaps %%zmm7, %%zmm6")
46 __ASM_EMIT("jb 2f")
47 __ASM_EMIT("1:")
48 __ASM_EMIT("vmulps 0x00(%[left], %[off]), %%zmm6, %%zmm0") // zmm0 = l*0.5
49 __ASM_EMIT("vmulps 0x40(%[left], %[off]), %%zmm7, %%zmm1")
50 __ASM_EMIT("vmulps 0x00(%[right], %[off]), %%zmm6, %%zmm4") // zmm4 = r*0.5
51 __ASM_EMIT("vmulps 0x40(%[right], %[off]), %%zmm7, %%zmm5")
52 __ASM_EMIT("vsubps %%zmm4, %%zmm0, %%zmm2") // zmm2 = s = (l-r)*0.5
53 __ASM_EMIT("vsubps %%zmm5, %%zmm1, %%zmm3")
54 __ASM_EMIT("vaddps %%zmm4, %%zmm0, %%zmm0") // zmm0 = m = (l+r)*0.5
55 __ASM_EMIT("vaddps %%zmm5, %%zmm1, %%zmm1")
56 __ASM_EMIT("vmovups %%zmm0, 0x00(%[mid], %[off])")
57 __ASM_EMIT("vmovups %%zmm1, 0x40(%[mid], %[off])")
58 __ASM_EMIT("vmovups %%zmm2, 0x00(%[side], %[off])")
59 __ASM_EMIT("vmovups %%zmm3, 0x40(%[side], %[off])")
60 __ASM_EMIT("add $0x80, %[off]")
61 __ASM_EMIT32("subl $32, %[count]")
62 __ASM_EMIT64("sub $32, %[count]")
63 __ASM_EMIT("jae 1b")
64 __ASM_EMIT("2:")
65 // 16x block
66 __ASM_EMIT32("addl $16, %[count]")
67 __ASM_EMIT64("add $16, %[count]")
68 __ASM_EMIT("jl 4f")
69 __ASM_EMIT("vmulps 0x00(%[left], %[off]), %%ymm6, %%ymm0") // ymm0 = l*0.5
70 __ASM_EMIT("vmulps 0x20(%[left], %[off]), %%ymm7, %%ymm1")
71 __ASM_EMIT("vmulps 0x00(%[right], %[off]), %%ymm6, %%ymm4") // ymm4 = r*0.5
72 __ASM_EMIT("vmulps 0x20(%[right], %[off]), %%ymm7, %%ymm5")
73 __ASM_EMIT("vsubps %%ymm4, %%ymm0, %%ymm2") // ymm2 = s = (l-r)*0.5
74 __ASM_EMIT("vsubps %%ymm5, %%ymm1, %%ymm3")
75 __ASM_EMIT("vaddps %%ymm4, %%ymm0, %%ymm0") // ymm0 = m = (l+r)*0.5
76 __ASM_EMIT("vaddps %%ymm5, %%ymm1, %%ymm1")
77 __ASM_EMIT("vmovups %%ymm0, 0x00(%[mid], %[off])")
78 __ASM_EMIT("vmovups %%ymm1, 0x20(%[mid], %[off])")
79 __ASM_EMIT("vmovups %%ymm2, 0x00(%[side], %[off])")
80 __ASM_EMIT("vmovups %%ymm3, 0x20(%[side], %[off])")
81 __ASM_EMIT32("subl $16, %[count]")
82 __ASM_EMIT64("sub $16, %[count]")
83 __ASM_EMIT("add $0x40, %[off]")
84 __ASM_EMIT("4:")
85 // 8x block
86 __ASM_EMIT32("addl $8, %[count]")
87 __ASM_EMIT64("add $8, %[count]")
88 __ASM_EMIT("jl 6f")
89 __ASM_EMIT("vmulps 0x00(%[left], %[off]), %%ymm6, %%ymm0") // ymm0 = l*0.5
90 __ASM_EMIT("vmulps 0x00(%[right], %[off]), %%ymm7, %%ymm4") // ymm4 = r*0.5
91 __ASM_EMIT("vsubps %%ymm4, %%ymm0, %%ymm2") // ymm2 = s = (l-r)*0.5
92 __ASM_EMIT("vaddps %%ymm4, %%ymm0, %%ymm0") // ymm0 = m = (l+r)*0.5
93 __ASM_EMIT("vmovups %%ymm0, 0x00(%[mid], %[off])")
94 __ASM_EMIT("vmovups %%ymm2, 0x00(%[side], %[off])")
95 __ASM_EMIT("add $0x20, %[off]")
96 __ASM_EMIT32("subl $8, %[count]")
97 __ASM_EMIT64("sub $8, %[count]")
98 __ASM_EMIT("6:")
99 // 4x block
100 __ASM_EMIT32("addl $4, %[count]")
101 __ASM_EMIT64("add $4, %[count]")
102 __ASM_EMIT("jl 8f")
103 __ASM_EMIT("vmulps 0x00(%[left], %[off]), %%xmm6, %%xmm0") // xmm0 = l*0.5
104 __ASM_EMIT("vmulps 0x00(%[right], %[off]), %%xmm7, %%xmm4") // xmm4 = r*0.5
105 __ASM_EMIT("vsubps %%xmm4, %%xmm0, %%xmm2") // xmm2 = s = (l-r)*0.5
106 __ASM_EMIT("vaddps %%xmm4, %%xmm0, %%xmm0") // xmm0 = m = (l+r)*0.5
107 __ASM_EMIT("vmovups %%xmm0, 0x00(%[mid], %[off])")
108 __ASM_EMIT("vmovups %%xmm2, 0x00(%[side], %[off])")
109 __ASM_EMIT("add $0x10, %[off]")
110 __ASM_EMIT32("subl $4, %[count]")
111 __ASM_EMIT64("sub $4, %[count]")
112 __ASM_EMIT("8:")
113 // 1x blocks
114 __ASM_EMIT32("addl $3, %[count]")
115 __ASM_EMIT64("add $3, %[count]")
116 __ASM_EMIT("jl 10f")
117 __ASM_EMIT("9:")
118 __ASM_EMIT("vmulss 0x00(%[left], %[off]), %%xmm6, %%xmm0") // xmm0 = l*0.5
119 __ASM_EMIT("vmulss 0x00(%[right], %[off]), %%xmm7, %%xmm4") // xmm4 = r*0.5
120 __ASM_EMIT("vsubss %%xmm4, %%xmm0, %%xmm2") // xmm2 = s = (l-r)*0.5
121 __ASM_EMIT("vaddss %%xmm4, %%xmm0, %%xmm0") // xmm0 = m = (l+r)*0.5
122 __ASM_EMIT("vmovss %%xmm0, 0x00(%[mid], %[off])")
123 __ASM_EMIT("vmovss %%xmm2, 0x00(%[side], %[off])")
124 __ASM_EMIT("add $0x04, %[off]")
125 __ASM_EMIT32("decl %[count]")
126 __ASM_EMIT64("dec %[count]")
127 __ASM_EMIT("jge 9b")
128 __ASM_EMIT("10:")
129
130 : [off] "=&r" (off),
131 [count] X86_PGREG (count)
132 : [left] "r"(l), [right] "r" (r),
133 [mid] "r" (m), [side] "r" (s),
134 [X_HALF] "m" (msmatrix_const)
135 : "cc", "memory",
136 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
137 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
138 );
139 }
140
141 #define LR_TO_PART(P, L, R, OP) \
142 __ASM_EMIT("vmovaps %[X_HALF], %%zmm7") \
143 __ASM_EMIT("xor %[off], %[off]") \
144 __ASM_EMIT("vmovaps %%zmm7, %%zmm6") \
145 /* 64x blocks */ \
146 __ASM_EMIT("sub $64, %[count]") \
147 __ASM_EMIT("jb 2f") \
148 __ASM_EMIT("1:") \
149 __ASM_EMIT("vmovups 0x00(%[" L "], %[off]), %%zmm0") /* zmm0 = l */ \
150 __ASM_EMIT("vmovups 0x40(%[" L "], %[off]), %%zmm1") \
151 __ASM_EMIT("vmovups 0x80(%[" L "], %[off]), %%zmm2") \
152 __ASM_EMIT("vmovups 0xc0(%[" L "], %[off]), %%zmm3") \
153 __ASM_EMIT(OP "ps 0x00(%[" R "], %[off]), %%zmm0, %%zmm0") /* zmm0 = l op r */ \
154 __ASM_EMIT(OP "ps 0x40(%[" R "], %[off]), %%zmm1, %%zmm1") \
155 __ASM_EMIT(OP "ps 0x80(%[" R "], %[off]), %%zmm2, %%zmm2") \
156 __ASM_EMIT(OP "ps 0xc0(%[" R "], %[off]), %%zmm3, %%zmm3") \
157 __ASM_EMIT("vmulps %%zmm6, %%zmm0, %%zmm0") /* zmm0 = (l op r) * 0.5 */ \
158 __ASM_EMIT("vmulps %%zmm7, %%zmm1, %%zmm1") \
159 __ASM_EMIT("vmulps %%zmm6, %%zmm2, %%zmm2") \
160 __ASM_EMIT("vmulps %%zmm7, %%zmm3, %%zmm3") \
161 __ASM_EMIT("vmovups %%zmm0, 0x00(%[" P "], %[off])") \
162 __ASM_EMIT("vmovups %%zmm1, 0x40(%[" P "], %[off])") \
163 __ASM_EMIT("vmovups %%zmm2, 0x80(%[" P "], %[off])") \
164 __ASM_EMIT("vmovups %%zmm3, 0xc0(%[" P "], %[off])") \
165 __ASM_EMIT("add $0x100, %[off]") \
166 __ASM_EMIT("sub $64, %[count]") \
167 __ASM_EMIT("jae 1b") \
168 __ASM_EMIT("2:") \
169 /* 32x block */ \
170 __ASM_EMIT("add $32, %[count]") \
171 __ASM_EMIT("jl 4f") \
172 __ASM_EMIT("vmovups 0x00(%[" L "], %[off]), %%ymm0") /* ymm0 = l */ \
173 __ASM_EMIT("vmovups 0x20(%[" L "], %[off]), %%ymm1") \
174 __ASM_EMIT("vmovups 0x40(%[" L "], %[off]), %%ymm2") \
175 __ASM_EMIT("vmovups 0x60(%[" L "], %[off]), %%ymm3") \
176 __ASM_EMIT(OP "ps 0x00(%[" R "], %[off]), %%ymm0, %%ymm0") /* ymm0 = l op r */ \
177 __ASM_EMIT(OP "ps 0x20(%[" R "], %[off]), %%ymm1, %%ymm1") \
178 __ASM_EMIT(OP "ps 0x40(%[" R "], %[off]), %%ymm2, %%ymm2") \
179 __ASM_EMIT(OP "ps 0x60(%[" R "], %[off]), %%ymm3, %%ymm3") \
180 __ASM_EMIT("vmulps %%ymm6, %%ymm0, %%ymm0") /* ymm0 = (l op r) * 0.5 */ \
181 __ASM_EMIT("vmulps %%ymm7, %%ymm1, %%ymm1") \
182 __ASM_EMIT("vmulps %%ymm6, %%ymm2, %%ymm2") \
183 __ASM_EMIT("vmulps %%ymm7, %%ymm3, %%ymm3") \
184 __ASM_EMIT("vmovups %%ymm0, 0x00(%[" P "], %[off])") \
185 __ASM_EMIT("vmovups %%ymm1, 0x20(%[" P "], %[off])") \
186 __ASM_EMIT("vmovups %%ymm2, 0x40(%[" P "], %[off])") \
187 __ASM_EMIT("vmovups %%ymm3, 0x60(%[" P "], %[off])") \
188 __ASM_EMIT("sub $32, %[count]") \
189 __ASM_EMIT("add $0x80, %[off]") \
190 __ASM_EMIT("4:") \
191 /* 16x block */ \
192 __ASM_EMIT("add $16, %[count]") \
193 __ASM_EMIT("jl 6f") \
194 __ASM_EMIT("vmovups 0x00(%[" L "], %[off]), %%ymm0") /* ymm0 = l */ \
195 __ASM_EMIT("vmovups 0x20(%[" L "], %[off]), %%ymm1") \
196 __ASM_EMIT(OP "ps 0x00(%[" R "], %[off]), %%ymm0, %%ymm0") /* ymm0 = l op r */ \
197 __ASM_EMIT(OP "ps 0x20(%[" R "], %[off]), %%ymm1, %%ymm1") \
198 __ASM_EMIT("vmulps %%ymm6, %%ymm0, %%ymm0") /* ymm0 = (l op r) * 0.5 */ \
199 __ASM_EMIT("vmulps %%ymm7, %%ymm1, %%ymm1") \
200 __ASM_EMIT("vmovups %%ymm0, 0x00(%[" P "], %[off])") \
201 __ASM_EMIT("vmovups %%ymm1, 0x20(%[" P "], %[off])") \
202 __ASM_EMIT("sub $16, %[count]") \
203 __ASM_EMIT("add $0x40, %[off]") \
204 __ASM_EMIT("6:") \
205 /* 8x block */ \
206 __ASM_EMIT("add $8, %[count]") \
207 __ASM_EMIT("jl 8f") \
208 __ASM_EMIT("vmovups 0x00(%[" L "], %[off]), %%ymm0") /* ymm0 = l */ \
209 __ASM_EMIT(OP "ps 0x00(%[" R "], %[off]), %%ymm0, %%ymm0") /* ymm0 = l op r */ \
210 __ASM_EMIT("vmulps %%ymm6, %%ymm0, %%ymm0") /* ymm0 = (l op r) * 0.5 */ \
211 __ASM_EMIT("vmovups %%ymm0, 0x00(%[" P "], %[off])") \
212 __ASM_EMIT("sub $8, %[count]") \
213 __ASM_EMIT("add $0x20, %[off]") \
214 __ASM_EMIT("8:") \
215 /* 4x block */ \
216 __ASM_EMIT("add $4, %[count]") \
217 __ASM_EMIT("jl 10f") \
218 __ASM_EMIT("vmovups 0x00(%[" L "], %[off]), %%xmm0") /* xmm0 = l */ \
219 __ASM_EMIT(OP "ps 0x00(%[" R "], %[off]), %%xmm0, %%xmm0") /* xmm0 = l op r */ \
220 __ASM_EMIT("vmulps %%xmm6, %%xmm0, %%xmm0") /* xmm0 = (l op r) * 0.5 */ \
221 __ASM_EMIT("vmovups %%xmm0, 0x00(%[" P "], %[off])") \
222 __ASM_EMIT("sub $4, %[count]") \
223 __ASM_EMIT("add $0x10, %[off]") \
224 __ASM_EMIT("10:") \
225 /* 1x blocks */ \
226 __ASM_EMIT("add $3, %[count]") \
227 __ASM_EMIT("jl 12f") \
228 __ASM_EMIT("11:") \
229 __ASM_EMIT("vmovss 0x00(%[" L "], %[off]), %%xmm0") /* xmm0 = l */ \
230 __ASM_EMIT(OP "ss 0x00(%[" R "], %[off]), %%xmm0, %%xmm0") /* xmm0 = l op r */ \
231 __ASM_EMIT("vmulss %%xmm6, %%xmm0, %%xmm0") /* xmm0 = (l op r) * 0.5 */ \
232 __ASM_EMIT("vmovss %%xmm0, 0x00(%[" P "], %[off])") \
233 __ASM_EMIT("add $0x04, %[off]") \
234 __ASM_EMIT("dec %[count]") \
235 __ASM_EMIT("jge 11b") \
236 __ASM_EMIT("12:")
237
238 void lr_to_mid(float *m, const float *l, const float *r, size_t count)
239 {
240 IF_ARCH_X86(size_t off);
241 ARCH_X86_ASM (
242 LR_TO_PART("mid", "left", "right", "vadd")
243 : [off] "=&r" (off), [count] "+r" (count)
244 : [left] "r" (l), [right] "r" (r),
245 [mid] "r" (m),
246 [X_HALF] "m" (msmatrix_const)
247 : "cc", "memory",
248 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
249 "%xmm6", "%xmm7"
250 );
251 }
252
253 void lr_to_side(float *s, const float *l, const float *r, size_t count)
254 {
255 IF_ARCH_X86(size_t off);
256 ARCH_X86_ASM (
257 LR_TO_PART("side", "left", "right", "vsub")
258 : [off] "=&r" (off), [count] "+r" (count)
259 : [left] "r" (l), [right] "r" (r),
260 [side] "r" (s),
261 [X_HALF] "m" (msmatrix_const)
262 : "cc", "memory",
263 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
264 "%xmm6", "%xmm7"
265 );
266 }
267
268 #undef LR_TO_PART
269
270 void ms_to_lr(float *l, float *r, const float *m, const float *s, size_t count)
271 {
272 IF_ARCH_X86(size_t off);
273 ARCH_X86_ASM(
274 __ASM_EMIT("xor %[off], %[off]")
275 // 64x blocks
276 __ASM_EMIT32("subl $64, %[count]")
277 __ASM_EMIT64("sub $64, %[count]")
278 __ASM_EMIT("jb 2f")
279 __ASM_EMIT("1:")
280 __ASM_EMIT("vmovups 0x00(%[mid], %[off]), %%zmm0") // zmm0 = m
281 __ASM_EMIT("vmovups 0x40(%[mid], %[off]), %%zmm1")
282 __ASM_EMIT("vmovups 0x80(%[mid], %[off]), %%zmm2")
283 __ASM_EMIT("vmovups 0xc0(%[mid], %[off]), %%zmm3")
284 __ASM_EMIT("vsubps 0x00(%[side], %[off]), %%zmm0, %%zmm4") // zmm4 = m - s
285 __ASM_EMIT("vsubps 0x40(%[side], %[off]), %%zmm1, %%zmm5")
286 __ASM_EMIT("vsubps 0x80(%[side], %[off]), %%zmm2, %%zmm6")
287 __ASM_EMIT("vsubps 0xc0(%[side], %[off]), %%zmm3, %%zmm7")
288 __ASM_EMIT("vaddps 0x00(%[side], %[off]), %%zmm0, %%zmm0") // zmm0 = m + s
289 __ASM_EMIT("vaddps 0x40(%[side], %[off]), %%zmm1, %%zmm1")
290 __ASM_EMIT("vaddps 0x80(%[side], %[off]), %%zmm2, %%zmm2")
291 __ASM_EMIT("vaddps 0xc0(%[side], %[off]), %%zmm3, %%zmm3")
292 __ASM_EMIT("vmovups %%zmm0, 0x00(%[left], %[off])")
293 __ASM_EMIT("vmovups %%zmm1, 0x40(%[left], %[off])")
294 __ASM_EMIT("vmovups %%zmm2, 0x80(%[left], %[off])")
295 __ASM_EMIT("vmovups %%zmm3, 0xc0(%[left], %[off])")
296 __ASM_EMIT("vmovups %%zmm4, 0x00(%[right], %[off])")
297 __ASM_EMIT("vmovups %%zmm5, 0x40(%[right], %[off])")
298 __ASM_EMIT("vmovups %%zmm6, 0x80(%[right], %[off])")
299 __ASM_EMIT("vmovups %%zmm7, 0xc0(%[right], %[off])")
300 __ASM_EMIT("add $0x100, %[off]")
301 __ASM_EMIT32("subl $64, %[count]")
302 __ASM_EMIT64("sub $64, %[count]")
303 __ASM_EMIT("jae 1b")
304 __ASM_EMIT("2:")
305 // 32x blocks
306 __ASM_EMIT32("addl $32, %[count]")
307 __ASM_EMIT64("add $32, %[count]")
308 __ASM_EMIT("jl 4f")
309 __ASM_EMIT("vmovups 0x00(%[mid], %[off]), %%ymm0") // ymm0 = m
310 __ASM_EMIT("vmovups 0x20(%[mid], %[off]), %%ymm1")
311 __ASM_EMIT("vmovups 0x40(%[mid], %[off]), %%ymm2")
312 __ASM_EMIT("vmovups 0x60(%[mid], %[off]), %%ymm3")
313 __ASM_EMIT("vsubps 0x00(%[side], %[off]), %%ymm0, %%ymm4") // ymm4 = m - s
314 __ASM_EMIT("vsubps 0x20(%[side], %[off]), %%ymm1, %%ymm5")
315 __ASM_EMIT("vsubps 0x40(%[side], %[off]), %%ymm2, %%ymm6")
316 __ASM_EMIT("vsubps 0x60(%[side], %[off]), %%ymm3, %%ymm7")
317 __ASM_EMIT("vaddps 0x00(%[side], %[off]), %%ymm0, %%ymm0") // ymm0 = m + s
318 __ASM_EMIT("vaddps 0x20(%[side], %[off]), %%ymm1, %%ymm1")
319 __ASM_EMIT("vaddps 0x40(%[side], %[off]), %%ymm2, %%ymm2")
320 __ASM_EMIT("vaddps 0x60(%[side], %[off]), %%ymm3, %%ymm3")
321 __ASM_EMIT("vmovups %%ymm0, 0x00(%[left], %[off])")
322 __ASM_EMIT("vmovups %%ymm1, 0x20(%[left], %[off])")
323 __ASM_EMIT("vmovups %%ymm2, 0x40(%[left], %[off])")
324 __ASM_EMIT("vmovups %%ymm3, 0x60(%[left], %[off])")
325 __ASM_EMIT("vmovups %%ymm4, 0x00(%[right], %[off])")
326 __ASM_EMIT("vmovups %%ymm5, 0x20(%[right], %[off])")
327 __ASM_EMIT("vmovups %%ymm6, 0x40(%[right], %[off])")
328 __ASM_EMIT("vmovups %%ymm7, 0x60(%[right], %[off])")
329 __ASM_EMIT32("subl $32, %[count]")
330 __ASM_EMIT64("sub $32, %[count]")
331 __ASM_EMIT("add $0x80, %[off]")
332 __ASM_EMIT("4:")
333 // 16x block
334 __ASM_EMIT32("addl $16, %[count]")
335 __ASM_EMIT64("add $16, %[count]")
336 __ASM_EMIT("jl 6f")
337 __ASM_EMIT("vmovups 0x00(%[mid], %[off]), %%ymm0") // ymm0 = m
338 __ASM_EMIT("vmovups 0x20(%[mid], %[off]), %%ymm1")
339 __ASM_EMIT("vsubps 0x00(%[side], %[off]), %%ymm0, %%ymm4") // ymm4 = m - s
340 __ASM_EMIT("vsubps 0x20(%[side], %[off]), %%ymm1, %%ymm5")
341 __ASM_EMIT("vaddps 0x00(%[side], %[off]), %%ymm0, %%ymm0") // ymm0 = m + s
342 __ASM_EMIT("vaddps 0x20(%[side], %[off]), %%ymm1, %%ymm1")
343 __ASM_EMIT("vmovups %%ymm0, 0x00(%[left], %[off])")
344 __ASM_EMIT("vmovups %%ymm1, 0x20(%[left], %[off])")
345 __ASM_EMIT("vmovups %%ymm4, 0x00(%[right], %[off])")
346 __ASM_EMIT("vmovups %%ymm5, 0x20(%[right], %[off])")
347 __ASM_EMIT32("subl $16, %[count]")
348 __ASM_EMIT64("sub $16, %[count]")
349 __ASM_EMIT("add $0x40, %[off]")
350 __ASM_EMIT("6:")
351 // 8x block
352 __ASM_EMIT32("addl $8, %[count]")
353 __ASM_EMIT64("add $8, %[count]")
354 __ASM_EMIT("jl 8f")
355 __ASM_EMIT("vmovups 0x00(%[mid], %[off]), %%ymm0") // ymm0 = m
356 __ASM_EMIT("vsubps 0x00(%[side], %[off]), %%ymm0, %%ymm4") // ymm4 = m - s
357 __ASM_EMIT("vaddps 0x00(%[side], %[off]), %%ymm0, %%ymm0") // ymm0 = m + s
358 __ASM_EMIT("vmovups %%ymm0, 0x00(%[left], %[off])")
359 __ASM_EMIT("vmovups %%ymm4, 0x00(%[right], %[off])")
360 __ASM_EMIT32("subl $8, %[count]")
361 __ASM_EMIT64("sub $8, %[count]")
362 __ASM_EMIT("add $0x20, %[off]")
363 __ASM_EMIT("8:")
364 // 4x block
365 __ASM_EMIT32("addl $4, %[count]")
366 __ASM_EMIT64("add $4, %[count]")
367 __ASM_EMIT("jl 10f")
368 __ASM_EMIT("vmovups 0x00(%[mid], %[off]), %%xmm0") // xmm0 = m
369 __ASM_EMIT("vsubps 0x00(%[side], %[off]), %%xmm0, %%xmm4") // xmm4 = m - s
370 __ASM_EMIT("vaddps 0x00(%[side], %[off]), %%xmm0, %%xmm0") // xmm0 = m + s
371 __ASM_EMIT("vmovups %%xmm0, 0x00(%[left], %[off])")
372 __ASM_EMIT("vmovups %%xmm4, 0x00(%[right], %[off])")
373 __ASM_EMIT32("subl $4, %[count]")
374 __ASM_EMIT64("sub $4, %[count]")
375 __ASM_EMIT("add $0x10, %[off]")
376 __ASM_EMIT("10:")
377 // 1x blocks
378 __ASM_EMIT32("addl $3, %[count]")
379 __ASM_EMIT64("add $3, %[count]")
380 __ASM_EMIT("jl 12f")
381 __ASM_EMIT("11:")
382 __ASM_EMIT("vmovss 0x00(%[mid], %[off]), %%xmm0") // xmm0 = m
383 __ASM_EMIT("vsubss 0x00(%[side], %[off]), %%xmm0, %%xmm4") // xmm4 = m - s
384 __ASM_EMIT("vaddss 0x00(%[side], %[off]), %%xmm0, %%xmm0") // xmm0 = m + s
385 __ASM_EMIT("vmovss %%xmm0, 0x00(%[left], %[off])")
386 __ASM_EMIT("vmovss %%xmm4, 0x00(%[right], %[off])")
387 __ASM_EMIT("add $0x04, %[off]")
388 __ASM_EMIT32("decl %[count]")
389 __ASM_EMIT64("dec %[count]")
390 __ASM_EMIT("jge 11b")
391 __ASM_EMIT("12:")
392
393 : [off] "=&r" (off),
394 [count] X86_PGREG (count)
395 : [left] "r"(l), [right] "r" (r),
396 [mid] "r" (m), [side] "r" (s)
397 : "cc", "memory",
398 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
399 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
400 );
401 }
402
403 #define MS_TO_PART(P, M, S, OP) \
404 __ASM_EMIT("xor %[off], %[off]") \
405 /* 64x blocks */ \
406 __ASM_EMIT("sub $64, %[count]") \
407 __ASM_EMIT("jb 2f") \
408 __ASM_EMIT("1:") \
409 __ASM_EMIT("vmovups 0x00(%[" M "], %[off]), %%zmm0") /* zmm0 = m */ \
410 __ASM_EMIT("vmovups 0x40(%[" M "], %[off]), %%zmm1") \
411 __ASM_EMIT("vmovups 0x80(%[" M "], %[off]), %%zmm2") \
412 __ASM_EMIT("vmovups 0xc0(%[" M "], %[off]), %%zmm3") \
413 __ASM_EMIT(OP "ps 0x00(%[" S "], %[off]), %%zmm0, %%zmm0") /* zmm4 = m - s */ \
414 __ASM_EMIT(OP "ps 0x40(%[" S "], %[off]), %%zmm1, %%zmm1") \
415 __ASM_EMIT(OP "ps 0x80(%[" S "], %[off]), %%zmm2, %%zmm2") \
416 __ASM_EMIT(OP "ps 0xc0(%[" S "], %[off]), %%zmm3, %%zmm3") \
417 __ASM_EMIT("vmovups %%zmm0, 0x00(%[" P "], %[off])") \
418 __ASM_EMIT("vmovups %%zmm1, 0x40(%[" P "], %[off])") \
419 __ASM_EMIT("vmovups %%zmm2, 0x80(%[" P "], %[off])") \
420 __ASM_EMIT("vmovups %%zmm3, 0xc0(%[" P "], %[off])") \
421 __ASM_EMIT("add $0x100, %[off]") \
422 __ASM_EMIT("sub $64, %[count]") \
423 __ASM_EMIT("jae 1b") \
424 __ASM_EMIT("2:") \
425 /* 32x blocks */ \
426 __ASM_EMIT("add $32, %[count]") \
427 __ASM_EMIT("jl 4f") \
428 __ASM_EMIT("vmovups 0x00(%[" M "], %[off]), %%ymm0") /* ymm0 = m */ \
429 __ASM_EMIT("vmovups 0x20(%[" M "], %[off]), %%ymm1") \
430 __ASM_EMIT("vmovups 0x40(%[" M "], %[off]), %%ymm2") \
431 __ASM_EMIT("vmovups 0x60(%[" M "], %[off]), %%ymm3") \
432 __ASM_EMIT(OP "ps 0x00(%[" S "], %[off]), %%ymm0, %%ymm0") /* ymm4 = m - s */ \
433 __ASM_EMIT(OP "ps 0x20(%[" S "], %[off]), %%ymm1, %%ymm1") \
434 __ASM_EMIT(OP "ps 0x40(%[" S "], %[off]), %%ymm2, %%ymm2") \
435 __ASM_EMIT(OP "ps 0x60(%[" S "], %[off]), %%ymm3, %%ymm3") \
436 __ASM_EMIT("vmovups %%ymm0, 0x00(%[" P "], %[off])") \
437 __ASM_EMIT("vmovups %%ymm1, 0x20(%[" P "], %[off])") \
438 __ASM_EMIT("vmovups %%ymm2, 0x40(%[" P "], %[off])") \
439 __ASM_EMIT("vmovups %%ymm3, 0x60(%[" P "], %[off])") \
440 __ASM_EMIT("sub $32, %[count]") \
441 __ASM_EMIT("add $0x80, %[off]") \
442 __ASM_EMIT("4:") \
443 /* 16x block */ \
444 __ASM_EMIT("add $16, %[count]") \
445 __ASM_EMIT("jl 6f") \
446 __ASM_EMIT("vmovups 0x00(%[" M "], %[off]), %%ymm0") /* ymm0 = m */ \
447 __ASM_EMIT("vmovups 0x20(%[" M "], %[off]), %%ymm1") \
448 __ASM_EMIT(OP "ps 0x00(%[" S "], %[off]), %%ymm0, %%ymm0") /* ymm4 = m - s */ \
449 __ASM_EMIT(OP "ps 0x20(%[" S "], %[off]), %%ymm1, %%ymm1") \
450 __ASM_EMIT("vmovups %%ymm0, 0x00(%[" P "], %[off])") \
451 __ASM_EMIT("vmovups %%ymm1, 0x20(%[" P "], %[off])") \
452 __ASM_EMIT("sub $16, %[count]") \
453 __ASM_EMIT("add $0x40, %[off]") \
454 __ASM_EMIT("6:") \
455 /* 8x block */ \
456 __ASM_EMIT("add $8, %[count]") \
457 __ASM_EMIT("jl 8f") \
458 __ASM_EMIT("vmovups 0x00(%[" M "], %[off]), %%ymm0") /* ymm0 = m */ \
459 __ASM_EMIT(OP "ps 0x00(%[" S "], %[off]), %%ymm0, %%ymm0") /* ymm4 = m - s */ \
460 __ASM_EMIT("vmovups %%ymm0, 0x00(%[" P "], %[off])") \
461 __ASM_EMIT("sub $8, %[count]") \
462 __ASM_EMIT("add $0x20, %[off]") \
463 __ASM_EMIT("8:") \
464 /* 4x block */ \
465 __ASM_EMIT("add $4, %[count]") \
466 __ASM_EMIT("jl 10f") \
467 __ASM_EMIT("vmovups 0x00(%[" M "], %[off]), %%xmm0") /* xmm0 = m */ \
468 __ASM_EMIT(OP "ps 0x00(%[" S "], %[off]), %%xmm0, %%xmm0") /* xmm4 = m - s */ \
469 __ASM_EMIT("vmovups %%xmm0, 0x00(%[" P "], %[off])") \
470 __ASM_EMIT("sub $4, %[count]") \
471 __ASM_EMIT("add $0x10, %[off]") \
472 __ASM_EMIT("10:") \
473 /* 1x blocks */ \
474 __ASM_EMIT("add $3, %[count]") \
475 __ASM_EMIT("jl 12f") \
476 __ASM_EMIT("11:") \
477 __ASM_EMIT("vmovss 0x00(%[" M "], %[off]), %%xmm0") /* xmm0 = m */ \
478 __ASM_EMIT(OP "ss 0x00(%[" S "], %[off]), %%xmm0, %%xmm0") /* xmm4 = m - s */ \
479 __ASM_EMIT("vmovss %%xmm0, 0x00(%[" P "], %[off])") \
480 __ASM_EMIT("add $0x04, %[off]") \
481 __ASM_EMIT("dec %[count]") \
482 __ASM_EMIT("jge 11b") \
483 __ASM_EMIT("12:")
484
485 void ms_to_left(float *l, const float *m, const float *s, size_t count)
486 {
487 size_t off;
488
489 ARCH_X86_ASM (
490 MS_TO_PART("left", "mid", "side", "vadd")
491 : [off] "=&r" (off), [count] "+r" (count)
492 : [left] "r" (l),
493 [mid] "r" (m), [side] "r" (s)
494 : "cc", "memory",
495 "%xmm0", "%xmm1", "%xmm2", "%xmm3"
496 );
497 }
498
499 void ms_to_right(float *r, const float *m, const float *s, size_t count)
500 {
501 size_t off;
502
503 ARCH_X86_ASM (
504 MS_TO_PART("right", "mid", "side", "vsub")
505 : [off] "=&r" (off), [count] "+r" (count)
506 : [right] "r" (r),
507 [mid] "r" (m), [side] "r" (s)
508 : "cc", "memory",
509 "%xmm0", "%xmm1", "%xmm2", "%xmm3"
510 );
511 }
512
513 #undef MS_TO_PART
514
515 } /* namespace avx512 */
516 } /* namespace lsp */
517
518
519
520
521 #endif /* PRIVATE_DSP_ARCH_X86_AVX512_MSMATRIX_H_ */