Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/x86/avx512/pmath/lramp.h

raw
1 /*
2 * Copyright (C) 2023 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2023 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 26 окт. 2023 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_X86_AVX512_PMATH_LRAMP_H_
23 #define PRIVATE_DSP_ARCH_X86_AVX512_PMATH_LRAMP_H_
24
25 #ifndef PRIVATE_DSP_ARCH_X86_AVX512_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_X86_AVX512_IMPL */
28
29 namespace lsp
30 {
31 namespace avx512
32 {
33 IF_ARCH_X86(
34 static const float lramp_const[] __lsp_aligned64 =
35 {
36 0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, /* Initial values 0..7 */
37 8.0f, 9.0f, 10.0f, 11.0f, 12.0f, 13.0f, 14.0f, 15.0f, /* Initial values 8..15 */
38 16.0f, 17.0f, 18.0f, 19.0f, 20.0f, 21.0f, 22.0f, 23.0f, /* Initial values 16..23 */
39 24.0f, 25.0f, 26.0f, 27.0f, 28.0f, 29.0f, 30.0f, 31.0f, /* Initial values 8..15 */
40 LSP_DSP_VEC16(32.0f) /* Step */
41 };
42 )
43
44 void lramp_set1(float *dst, float v1, float v2, size_t count)
45 {
46 float delta = v2 - v1;
47 if (delta == 0.0f)
48 {
49 dsp::fill(dst, v1, count);
50 return;
51 }
52 if (count == 0)
53 return;
54
55 delta /= count;
56 IF_ARCH_X86( size_t off );
57 ARCH_X86_ASM(
58 __ASM_EMIT("xor %[off], %[off]")
59 __ASM_EMIT("vbroadcastss %[v1], %%zmm6") /* zmm6 = v1 */
60 __ASM_EMIT("vbroadcastss %[delta], %%zmm7") /* zmm7 = delta */
61 __ASM_EMIT("vmovaps 0x00(%[CC]), %%zmm3") /* zmm3 = i[0] */
62 __ASM_EMIT("vmovaps 0x40(%[CC]), %%zmm4") /* zmm4 = i[1] */
63 __ASM_EMIT("vmovaps 0x80(%[CC]), %%zmm5") /* zmm5 = step */
64 /* 32x blocks */
65 __ASM_EMIT("sub $32, %[count]")
66 __ASM_EMIT("jb 2f")
67 __ASM_EMIT("1:")
68 __ASM_EMIT("vmulps %%zmm7, %%zmm3, %%zmm0") /* zmm0 = i[0]*delta */
69 __ASM_EMIT("vmulps %%zmm7, %%zmm4, %%zmm1") /* zmm1 = i[1]*delta */
70 __ASM_EMIT("vaddps %%zmm5, %%zmm3, %%zmm3") /* zmm3 = x[0]' = x[0] + step */
71 __ASM_EMIT("vaddps %%zmm5, %%zmm4, %%zmm4") /* zmm4 = x[1]' = x[1] + step */
72 __ASM_EMIT("vaddps %%zmm6, %%zmm0, %%zmm0") /* zmm0 = v1 + i[0]*delta */
73 __ASM_EMIT("vaddps %%zmm6, %%zmm1, %%zmm1") /* zmm1 = v1 + i[1]*delta */
74 __ASM_EMIT("vmovups %%zmm0, 0x00(%[dst], %[off])")
75 __ASM_EMIT("vmovups %%zmm1, 0x40(%[dst], %[off])")
76 __ASM_EMIT("add $0x80, %[off]")
77 __ASM_EMIT("sub $32, %[count]")
78 __ASM_EMIT("jae 1b")
79 __ASM_EMIT("2:")
80 /* 16x block */
81 __ASM_EMIT("add $16, %[count]")
82 __ASM_EMIT("jl 4f")
83 __ASM_EMIT("vmulps %%zmm7, %%zmm3, %%zmm0") /* zmm0 = i[0]*delta */
84 __ASM_EMIT("vmovaps %%zmm4, %%zmm3") /* zmm3 = x[0]' = x[0] + step */
85 __ASM_EMIT("vaddps %%zmm6, %%zmm0, %%zmm0") /* zmm0 = v1 + i[0]*delta */
86 __ASM_EMIT("vmovups %%zmm0, 0x00(%[dst], %[off])")
87 __ASM_EMIT("sub $16, %[count]")
88 __ASM_EMIT("add $0x40, %[off]")
89 __ASM_EMIT("4:")
90 /* 8x blocks */
91 __ASM_EMIT("add $8, %[count]")
92 __ASM_EMIT("jl 4f")
93 __ASM_EMIT("vmulps %%ymm7, %%ymm3, %%ymm0") /* ymm0 = i[0]*delta */
94 __ASM_EMIT("vextractf32x8 $1, %%zmm3, %%ymm3") /* ymm3 = x[0]' = x[0] + step */
95 __ASM_EMIT("vaddps %%ymm6, %%ymm0, %%ymm0") /* ymm0 = v1 + i[0]*delta */
96 __ASM_EMIT("vmovups %%ymm0, 0x00(%[dst], %[off])")
97 __ASM_EMIT("sub $8, %[count]")
98 __ASM_EMIT("add $0x20, %[off]")
99 __ASM_EMIT("4:")
100 /* 4x block */
101 __ASM_EMIT("add $4, %[count]")
102 __ASM_EMIT("jl 6f")
103 __ASM_EMIT("vmulps %%xmm7, %%xmm3, %%xmm0") /* xmm0 = i[0]*delta */
104 __ASM_EMIT("vextractf128 $1, %%ymm3, %%xmm3") /* xmm3 = x[0]' = x[0] + step */
105 __ASM_EMIT("vaddps %%xmm6, %%xmm0, %%xmm0") /* xmm0 = v1 + i[0]*delta */
106 __ASM_EMIT("vmovups %%xmm0, 0x00(%[dst], %[off])")
107 __ASM_EMIT("sub $4, %[count]")
108 __ASM_EMIT("add $0x10, %[off]")
109 __ASM_EMIT("6:")
110 /* 1x blocks */
111 __ASM_EMIT("add $3, %[count]")
112 __ASM_EMIT("jl 8f")
113 __ASM_EMIT("vmulps %%xmm7, %%xmm3, %%xmm0") /* xmm0 = x[0]*delta */
114 __ASM_EMIT("vaddps %%xmm6, %%xmm0, %%xmm0") /* xmm0 = v1 + i[0]*delta */
115 __ASM_EMIT("7:")
116 __ASM_EMIT("vmovss %%xmm0, 0x00(%[dst], %[off])")
117 __ASM_EMIT("vshufps $0x39, %%xmm0, %%xmm0, %%xmm0") /* shift xmm0 */
118 __ASM_EMIT("add $0x04, %[off]")
119 __ASM_EMIT("dec %[count]")
120 __ASM_EMIT("jge 7b")
121 __ASM_EMIT("8:")
122
123 : [count] "+r" (count), [off] "=&r" (off)
124 : [dst] "r" (dst),
125 [CC] "r" (lramp_const),
126 [delta] "m" (delta),
127 [v1] "m" (v1)
128 : "cc", "memory",
129 "xmm0", "xmm1", "xmm2", "xmm3",
130 "xmm4", "xmm5", "xmm6", "xmm7"
131 );
132 }
133
134 #define LRAMP_CORE(DST, SRC) \
135 __ASM_EMIT("xor %[off], %[off]") \
136 __ASM_EMIT("vbroadcastss %[v1], %%zmm6") /* zmm6 = v1 */ \
137 __ASM_EMIT("vbroadcastss %[delta], %%zmm7") /* zmm7 = delta */ \
138 __ASM_EMIT("vmovaps 0x00(%[CC]), %%zmm3") /* zmm3 = i[0] */ \
139 __ASM_EMIT("vmovaps 0x40(%[CC]), %%zmm4") /* zmm4 = i[1] */ \
140 __ASM_EMIT("vmovaps 0x80(%[CC]), %%zmm5") /* zmm5 = step */ \
141 /* 32x blocks */ \
142 __ASM_EMIT("sub $32, %[count]") \
143 __ASM_EMIT("jb 2f") \
144 __ASM_EMIT("1:") \
145 __ASM_EMIT("vmulps %%zmm7, %%zmm3, %%zmm0") /* zmm0 = i[0]*delta */ \
146 __ASM_EMIT("vmulps %%zmm7, %%zmm4, %%zmm1") /* zmm1 = i[1]*delta */ \
147 __ASM_EMIT("vaddps %%zmm5, %%zmm3, %%zmm3") /* zmm3 = x[0]' = x[0] + step */ \
148 __ASM_EMIT("vaddps %%zmm5, %%zmm4, %%zmm4") /* zmm4 = x[1]' = x[1] + step */ \
149 __ASM_EMIT("vaddps %%zmm6, %%zmm0, %%zmm0") /* zmm0 = v1 + i[0]*delta */ \
150 __ASM_EMIT("vaddps %%zmm6, %%zmm1, %%zmm1") /* zmm1 = v1 + i[1]*delta */ \
151 __ASM_EMIT("vmulps 0x00(%[" SRC "], %[off]), %%zmm0, %%zmm0") /* zmm0 = src[0] * (v1 + i[0]*delta) */ \
152 __ASM_EMIT("vmulps 0x40(%[" SRC "], %[off]), %%zmm1, %%zmm1") /* zmm1 = src[1] * (v1 + i[1]*delta) */ \
153 __ASM_EMIT("vmovups %%zmm0, 0x00(%[" DST "], %[off])") \
154 __ASM_EMIT("vmovups %%zmm1, 0x40(%[" DST "], %[off])") \
155 __ASM_EMIT("add $0x80, %[off]") \
156 __ASM_EMIT("sub $32, %[count]") \
157 __ASM_EMIT("jae 1b") \
158 __ASM_EMIT("2:") \
159 /* 16x blocks */ \
160 __ASM_EMIT("add $16, %[count]") \
161 __ASM_EMIT("jl 4f") \
162 __ASM_EMIT("vmulps %%zmm7, %%zmm3, %%zmm0") /* zmm0 = i[0]*delta */ \
163 __ASM_EMIT("vmovaps %%zmm4, %%zmm3") /* zmm3 = x[0]' = x[0] + step */ \
164 __ASM_EMIT("vaddps %%zmm6, %%zmm0, %%zmm0") /* zmm0 = v1 + i[0]*delta */ \
165 __ASM_EMIT("vmulps 0x00(%[" SRC "], %[off]), %%zmm0, %%zmm0") /* zmm0 = src[0] * (v1 + i[0]*delta) */ \
166 __ASM_EMIT("vmovups %%zmm0, 0x00(%[" DST "], %[off])") \
167 __ASM_EMIT("sub $16, %[count]") \
168 __ASM_EMIT("add $0x40, %[off]") \
169 __ASM_EMIT("4:") \
170 /* 8x blocks */ \
171 __ASM_EMIT("add $8, %[count]") \
172 __ASM_EMIT("jl 6f") \
173 __ASM_EMIT("vmulps %%ymm7, %%ymm3, %%ymm0") /* ymm0 = i[0]*delta */ \
174 __ASM_EMIT("vextractf32x8 $1, %%zmm3, %%ymm3") /* ymm3 = x[0]' = x[0] + step */ \
175 __ASM_EMIT("vaddps %%ymm6, %%ymm0, %%ymm0") /* ymm0 = v1 + i[0]*delta */ \
176 __ASM_EMIT("vmulps 0x00(%[" SRC "], %[off]), %%ymm0, %%ymm0") /* ymm0 = src[0] * (v1 + i[0]*delta) */ \
177 __ASM_EMIT("vmovups %%ymm0, 0x00(%[" DST "], %[off])") \
178 __ASM_EMIT("sub $8, %[count]") \
179 __ASM_EMIT("add $0x20, %[off]") \
180 __ASM_EMIT("6:") \
181 /* 4x block */ \
182 __ASM_EMIT("add $4, %[count]") \
183 __ASM_EMIT("jl 8f") \
184 __ASM_EMIT("vmulps %%xmm7, %%xmm3, %%xmm0") /* xmm0 = i[0]*delta */ \
185 __ASM_EMIT("vextractf128 $1, %%ymm3, %%xmm3") /* xmm3 = x[0]' = x[0] + step */ \
186 __ASM_EMIT("vaddps %%xmm6, %%xmm0, %%xmm0") /* xmm0 = v1 + i[0]*delta */ \
187 __ASM_EMIT("vmulps 0x00(%[" SRC "], %[off]), %%xmm0, %%xmm0") /* ymm0 = src[0] * (v1 + i[0]*delta) */ \
188 __ASM_EMIT("vmovups %%xmm0, 0x00(%[" DST "], %[off])") \
189 __ASM_EMIT("sub $4, %[count]") \
190 __ASM_EMIT("add $0x10, %[off]") \
191 __ASM_EMIT("8:") \
192 /* 1x blocks */ \
193 __ASM_EMIT("add $3, %[count]") \
194 __ASM_EMIT("jl 10f") \
195 __ASM_EMIT("9:") \
196 __ASM_EMIT("vmulss %%xmm7, %%xmm3, %%xmm0") /* xmm0 = x[0]*delta */ \
197 __ASM_EMIT("vshufps $0x39, %%xmm3, %%xmm3, %%xmm3") /* shift xmm3 */ \
198 __ASM_EMIT("vaddss %%xmm6, %%xmm0, %%xmm0") /* xmm0 = v1 + i[0]*delta */ \
199 __ASM_EMIT("vmulss 0x00(%[" SRC "], %[off]), %%xmm0, %%xmm0") /* ymm0 = src[0] * (v1 + i[0]*delta) */ \
200 __ASM_EMIT("vmovss %%xmm0, 0x00(%[" DST "], %[off])") \
201 __ASM_EMIT("add $0x04, %[off]") \
202 __ASM_EMIT("dec %[count]") \
203 __ASM_EMIT("jge 9b") \
204 __ASM_EMIT("10:")
205
206
207 void lramp1(float *dst, float v1, float v2, size_t count)
208 {
209 float delta = v2 - v1;
210 if (delta == 0.0f)
211 {
212 dsp::mul_k2(dst, v1, count);
213 return;
214 }
215 if (count == 0)
216 return;
217
218 delta /= count;
219 IF_ARCH_X86( size_t off );
220 ARCH_X86_ASM(
221 LRAMP_CORE("dst", "dst")
222 : [count] "+r" (count), [off] "=&r" (off)
223 : [dst] "r" (dst),
224 [CC] "r" (lramp_const),
225 [delta] "m" (delta),
226 [v1] "m" (v1)
227 : "cc", "memory",
228 "xmm0", "xmm1", "xmm2", "xmm3",
229 "xmm4", "xmm5", "xmm6", "xmm7"
230 );
231 }
232
233 void lramp2(float *dst, const float *src, float v1, float v2, size_t count)
234 {
235 float delta = v2 - v1;
236 if (delta == 0.0f)
237 {
238 dsp::mul_k3(dst, src, v1, count);
239 return;
240 }
241 if (count == 0)
242 return;
243
244 delta /= count;
245 IF_ARCH_X86( size_t off );
246 ARCH_X86_ASM(
247 LRAMP_CORE("dst", "src")
248 : [count] "+r" (count), [off] "=&r" (off)
249 : [dst] "r" (dst), [src] "r" (src),
250 [CC] "r" (lramp_const),
251 [delta] "m" (delta),
252 [v1] "m" (v1)
253 : "cc", "memory",
254 "xmm0", "xmm1", "xmm2", "xmm3",
255 "xmm4", "xmm5", "xmm6", "xmm7"
256 );
257 }
258
259 #undef LRAMP_CORE
260
261 #define OP_DSEL(a, b) a
262 #define OP_RSEL(a, b) b
263
264 #define LRAMP_OP3_CORE(DST, A, B, OP, SEL) \
265 __ASM_EMIT("xor %[off], %[off]") \
266 __ASM_EMIT("vbroadcastss %[v1], %%zmm6") /* zmm6 = v1 */ \
267 __ASM_EMIT("vbroadcastss %[delta], %%zmm7") /* zmm7 = delta */ \
268 __ASM_EMIT("vmovaps 0x00(%[CC]), %%zmm4") /* zmm4 = i[0] */ \
269 __ASM_EMIT("vmovaps 0x40(%[CC]), %%zmm5") /* zmm5 = i[1] */ \
270 /* 32x blocks */ \
271 __ASM_EMIT("sub $32, %[count]") \
272 __ASM_EMIT("jb 2f") \
273 __ASM_EMIT("1:") \
274 __ASM_EMIT("vmulps %%zmm7, %%zmm4, %%zmm2") /* zmm2 = i[0]*delta */ \
275 __ASM_EMIT("vmulps %%zmm7, %%zmm5, %%zmm3") /* zmm3 = i[1]*delta */ \
276 __ASM_EMIT("vmovups 0x00(%[" A "], %[off]), %%zmm0") /* zmm0 = a[0] */ \
277 __ASM_EMIT("vmovups 0x40(%[" A "], %[off]), %%zmm1") /* zmm1 = a[1] */ \
278 __ASM_EMIT("vaddps 0x80(%[CC]), %%zmm4, %%zmm4") /* zmm4 = x[0]' = x[0] + step */ \
279 __ASM_EMIT("vaddps 0x80(%[CC]), %%zmm5, %%zmm5") /* zmm5 = x[1]' = x[1] + step */ \
280 __ASM_EMIT("vaddps %%zmm6, %%zmm2, %%zmm2") /* zmm2 = v1 + i[0]*delta */ \
281 __ASM_EMIT("vaddps %%zmm6, %%zmm3, %%zmm3") /* zmm3 = v1 + i[1]*delta */ \
282 __ASM_EMIT("vmulps 0x00(%[" B "], %[off]), %%zmm2, %%zmm2") /* zmm0 = b[0] * (v1 + i[0]*delta) */ \
283 __ASM_EMIT("vmulps 0x40(%[" B "], %[off]), %%zmm3, %%zmm3") /* zmm1 = b[1] * (v1 + i[1]*delta) */ \
284 __ASM_EMIT(OP "ps " SEL("%%zmm2, %%zmm0, %%zmm0", "%%zmm0, %%zmm2, %%zmm0")) /* zmm0 = a[0] OP b[0] * (v1 + i[0]*delta) */ \
285 __ASM_EMIT(OP "ps " SEL("%%zmm3, %%zmm1, %%zmm1", "%%zmm1, %%zmm3, %%zmm1")) /* zmm1 = a[1] OP b[1] * (v1 + i[1]*delta) */ \
286 __ASM_EMIT("vmovups %%zmm0, 0x00(%[" DST "], %[off])") \
287 __ASM_EMIT("vmovups %%zmm1, 0x40(%[" DST "], %[off])") \
288 __ASM_EMIT("add $0x80, %[off]") \
289 __ASM_EMIT("sub $32, %[count]") \
290 __ASM_EMIT("jae 1b") \
291 __ASM_EMIT("2:") \
292 /* 16x blocks */ \
293 __ASM_EMIT("add $16, %[count]") \
294 __ASM_EMIT("jl 4f") \
295 __ASM_EMIT("1:") \
296 __ASM_EMIT("vmulps %%zmm7, %%zmm4, %%zmm2") /* zmm2 = i[0]*delta */ \
297 __ASM_EMIT("vmovups 0x00(%[" A "], %[off]), %%zmm0") /* zmm0 = a[0] */ \
298 __ASM_EMIT("vmovaps %%zmm5, %%zmm4") /* zmm4 = x[0]' = x[0] + step */ \
299 __ASM_EMIT("vaddps %%zmm6, %%zmm2, %%zmm2") /* zmm2 = v1 + i[0]*delta */ \
300 __ASM_EMIT("vmulps 0x00(%[" B "], %[off]), %%zmm2, %%zmm2") /* zmm0 = b[0] * (v1 + i[0]*delta) */ \
301 __ASM_EMIT(OP "ps " SEL("%%zmm2, %%zmm0, %%zmm0", "%%zmm0, %%zmm2, %%zmm0")) /* zmm0 = a[0] OP b[0] * (v1 + i[0]*delta) */ \
302 __ASM_EMIT("vmovups %%zmm0, 0x00(%[" DST "], %[off])") \
303 __ASM_EMIT("add $0x40, %[off]") \
304 __ASM_EMIT("sub $16, %[count]") \
305 __ASM_EMIT("4:") \
306 /* 8x blocks */ \
307 __ASM_EMIT("add $8, %[count]") \
308 __ASM_EMIT("jl 6f") \
309 __ASM_EMIT("vmulps %%ymm7, %%ymm4, %%ymm2") /* ymm2 = i[0]*delta */ \
310 __ASM_EMIT("vmovups 0x00(%[" A "], %[off]), %%ymm0") /* ymm0 = a[0] */ \
311 __ASM_EMIT("vextractf32x8 $1, %%zmm4, %%ymm4") /* ymm4 = x[0]' = x[0] + step */ \
312 __ASM_EMIT("vaddps %%ymm6, %%ymm2, %%ymm2") /* zmm2 = v1 + i[0]*delta */ \
313 __ASM_EMIT("vmulps 0x00(%[" B "], %[off]), %%ymm2, %%ymm2") /* ymm0 = b[0] * (v1 + i[0]*delta) */ \
314 __ASM_EMIT(OP "ps " SEL("%%ymm2, %%ymm0, %%ymm0", "%%ymm0, %%ymm2, %%ymm0")) /* ymm0 = a[0] OP b[0] * (v1 + i[0]*delta) */ \
315 __ASM_EMIT("vmovups %%ymm0, 0x00(%[" DST "], %[off])") \
316 __ASM_EMIT("add $0x20, %[off]") \
317 __ASM_EMIT("sub $8, %[count]") \
318 __ASM_EMIT("6:") \
319 /* 4x block */ \
320 __ASM_EMIT("add $4, %[count]") \
321 __ASM_EMIT("jl 8f") \
322 __ASM_EMIT("vmulps %%xmm7, %%xmm4, %%xmm2") /* xmm2 = i[0]*delta */ \
323 __ASM_EMIT("vmovups 0x00(%[" A "], %[off]), %%xmm0") /* xmm0 = a[0] */ \
324 __ASM_EMIT("vextractf128 $1, %%ymm4, %%xmm4") /* xmm4 = x[0]' = x[0] + step */ \
325 __ASM_EMIT("vaddps %%xmm6, %%xmm2, %%xmm2") /* xmm2 = v1 + i[0]*delta */ \
326 __ASM_EMIT("vmulps 0x00(%[" B "], %[off]), %%xmm2, %%xmm2") /* xmm0 = b[0] * (v1 + i[0]*delta) */ \
327 __ASM_EMIT(OP "ps " SEL("%%xmm2, %%xmm0, %%xmm0", "%%xmm0, %%xmm2, %%xmm0")) /* xmm0 = a[0] OP b[0] * (v1 + i[0]*delta) */ \
328 __ASM_EMIT("vmovups %%xmm0, 0x00(%[" DST "], %[off])") \
329 __ASM_EMIT("add $0x10, %[off]") \
330 __ASM_EMIT("sub $4, %[count]") \
331 __ASM_EMIT("8:") \
332 /* 1x blocks */ \
333 __ASM_EMIT("add $3, %[count]") \
334 __ASM_EMIT("jl 10f") \
335 __ASM_EMIT("9:") \
336 __ASM_EMIT("vmulss %%xmm7, %%xmm4, %%xmm2") /* xmm2 = i[0]*delta */ \
337 __ASM_EMIT("vmovss 0x00(%[" A "], %[off]), %%xmm0") /* xmm0 = a[0] */ \
338 __ASM_EMIT("vshufps $0x39, %%xmm4, %%xmm4, %%xmm4") /* shift xmm4 */ \
339 __ASM_EMIT("vaddss %%xmm6, %%xmm2, %%xmm2") /* xmm2 = v1 + i[0]*delta */ \
340 __ASM_EMIT("vmulss 0x00(%[" B "], %[off]), %%xmm2, %%xmm2") /* xmm0 = b[0] * (v1 + i[0]*delta) */ \
341 __ASM_EMIT(OP "ss " SEL("%%xmm2, %%xmm0, %%xmm0", "%%xmm0, %%xmm2, %%xmm0")) /* xmm0 = a[0] OP b[0] * (v1 + i[0]*delta) */ \
342 __ASM_EMIT("vmovss %%xmm0, 0x00(%[" DST "], %[off])") \
343 __ASM_EMIT("add $0x04, %[off]") \
344 __ASM_EMIT("dec %[count]") \
345 __ASM_EMIT("jge 9b") \
346 __ASM_EMIT("10:")
347
348 void lramp_add2(float *dst, const float *src, float v1, float v2, size_t count)
349 {
350 float delta = v2 - v1;
351 if (delta == 0.0f)
352 {
353 dsp::fmadd_k3(dst, src, v1, count);
354 return;
355 }
356 if (count == 0)
357 return;
358
359 delta /= count;
360 IF_ARCH_X86( size_t off );
361 ARCH_X86_ASM(
362 LRAMP_OP3_CORE("dst", "dst", "src", "vadd", OP_DSEL)
363 : [count] "+r" (count), [off] "=&r" (off)
364 : [dst] "r" (dst), [src] "r" (src),
365 [CC] "r" (lramp_const),
366 [delta] "m" (delta),
367 [v1] "m" (v1)
368 : "cc", "memory",
369 "xmm0", "xmm1", "xmm2", "xmm3",
370 "xmm4", "xmm5", "xmm6", "xmm7"
371 );
372 }
373
374 void lramp_sub2(float *dst, const float *src, float v1, float v2, size_t count)
375 {
376 float delta = v2 - v1;
377 if (delta == 0.0f)
378 {
379 dsp::fmsub_k3(dst, src, v1, count);
380 return;
381 }
382 if (count == 0)
383 return;
384
385 delta /= count;
386 IF_ARCH_X86( size_t off );
387 ARCH_X86_ASM(
388 LRAMP_OP3_CORE("dst", "dst", "src", "vsub", OP_DSEL)
389 : [count] "+r" (count), [off] "=&r" (off)
390 : [dst] "r" (dst), [src] "r" (src),
391 [CC] "r" (lramp_const),
392 [delta] "m" (delta),
393 [v1] "m" (v1)
394 : "cc", "memory",
395 "xmm0", "xmm1", "xmm2", "xmm3",
396 "xmm4", "xmm5", "xmm6", "xmm7"
397 );
398 }
399
400 void lramp_rsub2(float *dst, const float *src, float v1, float v2, size_t count)
401 {
402 float delta = v2 - v1;
403 if (delta == 0.0f)
404 {
405 dsp::fmrsub_k3(dst, src, v1, count);
406 return;
407 }
408 if (count == 0)
409 return;
410
411 delta /= count;
412 IF_ARCH_X86( size_t off );
413 ARCH_X86_ASM(
414 LRAMP_OP3_CORE("dst", "dst", "src", "vsub", OP_RSEL)
415 : [count] "+r" (count), [off] "=&r" (off)
416 : [dst] "r" (dst), [src] "r" (src),
417 [CC] "r" (lramp_const),
418 [delta] "m" (delta),
419 [v1] "m" (v1)
420 : "cc", "memory",
421 "xmm0", "xmm1", "xmm2", "xmm3",
422 "xmm4", "xmm5", "xmm6", "xmm7"
423 );
424 }
425
426 void lramp_mul2(float *dst, const float *src, float v1, float v2, size_t count)
427 {
428 float delta = v2 - v1;
429 if (delta == 0.0f)
430 {
431 dsp::fmmul_k3(dst, src, v1, count);
432 return;
433 }
434 if (count == 0)
435 return;
436
437 delta /= count;
438 IF_ARCH_X86( size_t off );
439 ARCH_X86_ASM(
440 LRAMP_OP3_CORE("dst", "dst", "src", "vmul", OP_DSEL)
441 : [count] "+r" (count), [off] "=&r" (off)
442 : [dst] "r" (dst), [src] "r" (src),
443 [CC] "r" (lramp_const),
444 [delta] "m" (delta),
445 [v1] "m" (v1)
446 : "cc", "memory",
447 "xmm0", "xmm1", "xmm2", "xmm3",
448 "xmm4", "xmm5", "xmm6", "xmm7"
449 );
450 }
451
452 void lramp_div2(float *dst, const float *src, float v1, float v2, size_t count)
453 {
454 float delta = v2 - v1;
455 if (delta == 0.0f)
456 {
457 dsp::fmdiv_k3(dst, src, v1, count);
458 return;
459 }
460 if (count == 0)
461 return;
462
463 delta /= count;
464 IF_ARCH_X86( size_t off );
465 ARCH_X86_ASM(
466 LRAMP_OP3_CORE("dst", "dst", "src", "vdiv", OP_DSEL)
467 : [count] "+r" (count), [off] "=&r" (off)
468 : [dst] "r" (dst), [src] "r" (src),
469 [CC] "r" (lramp_const),
470 [delta] "m" (delta),
471 [v1] "m" (v1)
472 : "cc", "memory",
473 "xmm0", "xmm1", "xmm2", "xmm3",
474 "xmm4", "xmm5", "xmm6", "xmm7"
475 );
476 }
477
478 void lramp_rdiv2(float *dst, const float *src, float v1, float v2, size_t count)
479 {
480 float delta = v2 - v1;
481 if (delta == 0.0f)
482 {
483 dsp::fmrdiv_k3(dst, src, v1, count);
484 return;
485 }
486 if (count == 0)
487 return;
488
489 delta /= count;
490 IF_ARCH_X86( size_t off );
491 ARCH_X86_ASM(
492 LRAMP_OP3_CORE("dst", "dst", "src", "vdiv", OP_RSEL)
493 : [count] "+r" (count), [off] "=&r" (off)
494 : [dst] "r" (dst), [src] "r" (src),
495 [CC] "r" (lramp_const),
496 [delta] "m" (delta),
497 [v1] "m" (v1)
498 : "cc", "memory",
499 "xmm0", "xmm1", "xmm2", "xmm3",
500 "xmm4", "xmm5", "xmm6", "xmm7"
501 );
502 }
503
504 void lramp_add3(float *dst, const float *a, const float *b, float v1, float v2, size_t count)
505 {
506 float delta = v2 - v1;
507 if (delta == 0.0f)
508 {
509 dsp::fmadd_k4(dst, a, b, v1, count);
510 return;
511 }
512 if (count == 0)
513 return;
514
515 delta /= count;
516 IF_ARCH_X86( size_t off );
517 ARCH_X86_ASM(
518 LRAMP_OP3_CORE("dst", "a", "b", "vadd", OP_DSEL)
519 : [off] "=&r" (off),
520 __IF_32([count] "+g" (count))
521 __IF_64([count] "+r" (count))
522 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b),
523 [CC] "r" (lramp_const),
524 [delta] "m" (delta),
525 [v1] "m" (v1)
526 : "cc", "memory",
527 "xmm0", "xmm1", "xmm2", "xmm3",
528 "xmm4", "xmm5", "xmm6", "xmm7"
529 );
530 }
531
532 void lramp_sub3(float *dst, const float *a, const float *b, float v1, float v2, size_t count)
533 {
534 float delta = v2 - v1;
535 if (delta == 0.0f)
536 {
537 dsp::fmsub_k4(dst, a, b, v1, count);
538 return;
539 }
540 if (count == 0)
541 return;
542
543 delta /= count;
544 IF_ARCH_X86( size_t off );
545 ARCH_X86_ASM(
546 LRAMP_OP3_CORE("dst", "a", "b", "vsub", OP_DSEL)
547 : [off] "=&r" (off),
548 __IF_32([count] "+g" (count))
549 __IF_64([count] "+r" (count))
550 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b),
551 [CC] "r" (lramp_const),
552 [delta] "m" (delta),
553 [v1] "m" (v1)
554 : "cc", "memory",
555 "xmm0", "xmm1", "xmm2", "xmm3",
556 "xmm4", "xmm5", "xmm6", "xmm7"
557 );
558 }
559
560 void lramp_rsub3(float *dst, const float *a, const float *b, float v1, float v2, size_t count)
561 {
562 float delta = v2 - v1;
563 if (delta == 0.0f)
564 {
565 dsp::fmrsub_k4(dst, a, b, v1, count);
566 return;
567 }
568 if (count == 0)
569 return;
570
571 delta /= count;
572 IF_ARCH_X86( size_t off );
573 ARCH_X86_ASM(
574 LRAMP_OP3_CORE("dst", "a", "b", "vsub", OP_RSEL)
575 : [off] "=&r" (off),
576 __IF_32([count] "+g" (count))
577 __IF_64([count] "+r" (count))
578 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b),
579 [CC] "r" (lramp_const),
580 [delta] "m" (delta),
581 [v1] "m" (v1)
582 : "cc", "memory",
583 "xmm0", "xmm1", "xmm2", "xmm3",
584 "xmm4", "xmm5", "xmm6", "xmm7"
585 );
586 }
587
588 void lramp_mul3(float *dst, const float *a, const float *b, float v1, float v2, size_t count)
589 {
590 float delta = v2 - v1;
591 if (delta == 0.0f)
592 {
593 dsp::fmmul_k4(dst, a, b, v1, count);
594 return;
595 }
596 if (count == 0)
597 return;
598
599 delta /= count;
600 IF_ARCH_X86( size_t off );
601 ARCH_X86_ASM(
602 LRAMP_OP3_CORE("dst", "a", "b", "vmul", OP_DSEL)
603 : [off] "=&r" (off),
604 __IF_32([count] "+g" (count))
605 __IF_64([count] "+r" (count))
606 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b),
607 [CC] "r" (lramp_const),
608 [delta] "m" (delta),
609 [v1] "m" (v1)
610 : "cc", "memory",
611 "xmm0", "xmm1", "xmm2", "xmm3",
612 "xmm4", "xmm5", "xmm6", "xmm7"
613 );
614 }
615
616 void lramp_div3(float *dst, const float *a, const float *b, float v1, float v2, size_t count)
617 {
618 float delta = v2 - v1;
619 if (delta == 0.0f)
620 {
621 dsp::fmdiv_k4(dst, a, b, v1, count);
622 return;
623 }
624 if (count == 0)
625 return;
626
627 delta /= count;
628 IF_ARCH_X86( size_t off );
629 ARCH_X86_ASM(
630 LRAMP_OP3_CORE("dst", "a", "b", "vdiv", OP_DSEL)
631 : [off] "=&r" (off),
632 __IF_32([count] "+g" (count))
633 __IF_64([count] "+r" (count))
634 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b),
635 [CC] "r" (lramp_const),
636 [delta] "m" (delta),
637 [v1] "m" (v1)
638 : "cc", "memory",
639 "xmm0", "xmm1", "xmm2", "xmm3",
640 "xmm4", "xmm5", "xmm6", "xmm7"
641 );
642 }
643
644 void lramp_rdiv3(float *dst, const float *a, const float *b, float v1, float v2, size_t count)
645 {
646 float delta = v2 - v1;
647 if (delta == 0.0f)
648 {
649 dsp::fmrdiv_k4(dst, a, b, v1, count);
650 return;
651 }
652 if (count == 0)
653 return;
654
655 delta /= count;
656 IF_ARCH_X86( size_t off );
657 ARCH_X86_ASM(
658 LRAMP_OP3_CORE("dst", "a", "b", "vdiv", OP_RSEL)
659 : [off] "=&r" (off),
660 __IF_32([count] "+g" (count))
661 __IF_64([count] "+r" (count))
662 : [dst] "r" (dst), [a] "r" (a), [b] "r" (b),
663 [CC] "r" (lramp_const),
664 [delta] "m" (delta),
665 [v1] "m" (v1)
666 : "cc", "memory",
667 "xmm0", "xmm1", "xmm2", "xmm3",
668 "xmm4", "xmm5", "xmm6", "xmm7"
669 );
670 }
671
672 #undef LRAMP_OP3_CORE
673 #undef OP_DSEL
674 #undef OP_RSEL
675
676 } /* namespace avx512 */
677 } /* namespace lsp */
678
679
680
681
682 #endif /* PRIVATE_DSP_ARCH_X86_AVX512_IMPL */