Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/x86/avx2/pmath/lanczos.h

raw
1 /*
2 * Copyright (C) 2026 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2026 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 17 апр. 2025 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_X86_AVX2_PMATH_LANCZOS_H_
23 #define PRIVATE_DSP_ARCH_X86_AVX2_PMATH_LANCZOS_H_
24
25 #ifndef PRIVATE_DSP_ARCH_X86_AVX2_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_X86_AVX2_IMPL */
28
29 #include <private/dsp/arch/x86/avx2/pmath/sin.h>
30
31 namespace lsp
32 {
33 namespace avx2
34 {
35 IF_ARCH_X86(
36 static const uint32_t lanczos_const[] __lsp_aligned32 =
37 {
38 LSP_DSP_VEC8(0x7fffffff), // +0x00: Mask for fabsf
39 LSP_DSP_VEC8(0x38d1b717), // +0x20: Sinc threshold = 1e-4
40 LSP_DSP_VEC8(0x3f800000), // +0x40: 1.0
41 };
42 )
43
44 typedef struct lanczos_gen_t
45 {
46 float k[8]; // +0x00: Multiplier
47 float p[8]; // +0x20: Initial phase
48 float a[8]; // +0x40: Number of lobes
49 float i[8]; // +0x60: Indices
50 float x1[8]; // +0x80: Computed X1
51 float d[8]; // +0xa0: Denominator
52 float t[8]; // +0xc0: Time range
53 } lanczos_gen_t;
54
55 #define LANCZOS_GEN_FUNC_X8 \
56 /* ymm0 = x1 */ \
57 __ASM_EMIT("vmulps 0x40(%[state]), %%ymm0, %%ymm4") /* ymm4 = x2 = x1*a */ \
58 __ASM_EMIT("vmovaps %%ymm0, 0x80(%[state])") /* *x1 = x1 */ \
59 __ASM_EMIT("vmulps %%ymm0, %%ymm4, %%ymm2") /* ymm2 = d = x1 * x2 */ \
60 __ASM_EMIT("vaddps 0x00 + %[S2C], %%ymm0, %%ymm0") /* ymm0 = x1 + PI/2 */ \
61 __ASM_EMIT("vaddps 0x00 + %[S2C], %%ymm4, %%ymm4") /* ymm4 = x2 + PI/2 */ \
62 __ASM_EMIT("vmovaps %%ymm2, 0xa0(%[state])") /* *d = d */ \
63 SINF_X_PLUS_PI_2_CORE_X16 /* ymm0 = sinf(x1), ymm4 = sinf(x2) */ \
64 __ASM_EMIT("vmovaps 0x80(%[state]), %%ymm1") /* ymm1 = x1 */ \
65 __ASM_EMIT("vmulps %%ymm4, %%ymm0, %%ymm0") /* ymm0 = sinf(x1)*sinf(x2) */ \
66 __ASM_EMIT("vandps 0x00 + %[LC], %%ymm1, %%ymm1") /* ymm1 = fabsf(x) */ \
67 __ASM_EMIT("vdivps 0xa0(%[state]), %%ymm0, %%ymm0") /* ymm0 = F = (sinf(x1)*sinf(x2)) / (x1 * x2) */ \
68 __ASM_EMIT("vcmpps $1, 0x20 + %[LC], %%ymm1, %%ymm2") /* ymm2 = [ fabsf(x) < 1e-4 ] */ \
69 __ASM_EMIT("vcmpps $1, 0xc0(%[state]), %%ymm1, %%ymm1") /* ymm1 = [ fabsf(x) < t ] */ \
70 __ASM_EMIT("vblendvps %%ymm2, 0x40 + %[LC], %%ymm0, %%ymm0") /* ymm0 = [ fabsf(x) >= 1e-4 ] ? f : 1.0 */ \
71 __ASM_EMIT("vandps %%ymm1, %%ymm0, %%ymm0") /* ymm0 = [ fabsf(x) < t ] ? ([ fabsf(x) >= 1e-4 ] ? f : 1.0) : 0.0 */
72
73 #define LANCZOS_GEN_FUNC_X4 \
74 /* xmm0 = x1 */ \
75 __ASM_EMIT("vmulps 0x40(%[state]), %%xmm0, %%xmm4" ) /* xmm4 = x2 = x1*a */ \
76 __ASM_EMIT("vmovaps %%xmm0, 0x80(%[state])") /* *x1 = x1 */ \
77 __ASM_EMIT("vmulps %%xmm0, %%xmm4, %%xmm2") /* xmm2 = d = x1 * x2 */ \
78 __ASM_EMIT("vinsertf128 $1, %%xmm4, %%ymm0, %%ymm0") /* ymm0 = x1 x1 x1 x1 x2 x2 x2 x2 */ \
79 __ASM_EMIT("vmovaps %%xmm2, 0xa0(%[state])") /* *d = d */ \
80 __ASM_EMIT("vaddps 0x00 + %[S2C], %%ymm0, %%ymm0") /* xmm0 = x1 + PI/2 ... x2 + PI/2 ... */ \
81 SINF_X_PLUS_PI_2_CORE_X8 /* ymm0 = sinf(x1) ... sinf(x2) ... */ \
82 __ASM_EMIT("vextractf128 $1, %%ymm0, %%xmm4") /* xmm4 = sif(x2) */ \
83 __ASM_EMIT("vmovaps 0x80(%[state]), %%xmm1") /* xmm1 = x1 */ \
84 __ASM_EMIT("vmulps %%xmm4, %%xmm0, %%xmm0") /* xmm0 = sinf(x1)*sinf(x2) */ \
85 __ASM_EMIT("vandps 0x00 + %[LC], %%xmm1, %%xmm1") /* xmm1 = fabsf(x) */ \
86 __ASM_EMIT("vdivps 0xa0(%[state]), %%xmm0, %%xmm0") /* xmm0 = F = (sinf(x1)*sinf(x2)) / (x1 * x2) */ \
87 __ASM_EMIT("vcmpps $1, 0x20 + %[LC], %%xmm1, %%xmm2") /* xmm2 = [ fabsf(x) < 1e-4 ] */ \
88 __ASM_EMIT("vcmpps $1, 0xc0(%[state]), %%xmm1, %%xmm1") /* xmm1 = [ fabsf(x) < t ] */ \
89 __ASM_EMIT("vblendvps %%xmm2, 0x40 + %[LC], %%xmm0, %%xmm0") /* xmm0 = [ fabsf(x) >= 1e-4 ] ? f : 1.0 */ \
90 __ASM_EMIT("vandps %%xmm1, %%xmm0, %%xmm0") /* xmm0 = [ fabsf(x) < t ] ? ([ fabsf(x) >= 1e-4 ] ? f : 1.0) : 0.0 */
91
92
93 void lanczos1(float *dst, float k, float p, float t, float a, size_t count)
94 {
95 IF_ARCH_X86(
96 lanczos_gen_t state __lsp_aligned32;
97 float stub[8] __lsp_aligned16;
98 );
99
100 ARCH_X86_ASM(
101 // Prepare
102 __ASM_EMIT("vbroadcastss %[k], %%ymm4") // ymm4 = k
103 __ASM_EMIT("vbroadcastss %[p], %%ymm5") // ymm5 = p
104 __ASM_EMIT("vbroadcastss %[a], %%ymm6") // ymm6 = a
105 __ASM_EMIT("vbroadcastss %[t], %%ymm7") // ymm7 = t
106 __ASM_EMIT("vmovaps %%ymm4, 0x00(%[state])")
107 __ASM_EMIT("vmovaps %%ymm5, 0x20(%[state])")
108 __ASM_EMIT("vmovaps %%ymm6, 0x40(%[state])")
109 __ASM_EMIT("vmovaps %%ymm7, 0xc0(%[state])")
110 __ASM_EMIT("vmovaps 0x00 + %[LGEN], %%ymm1") // ymm1 = i = 0 1 2 3 4 5 6 7
111 // x8 blocks
112 __ASM_EMIT("sub $8, %[count]")
113 __ASM_EMIT("jb 2f")
114 __ASM_EMIT("1:")
115 __ASM_EMIT("vmulps 0x00(%[state]), %%ymm1, %%ymm0") // ymm0 = k*i
116 __ASM_EMIT("vaddps 0x60 + %[LGEN], %%ymm1, %%ymm1") // ymm1 = i' = i + step
117 __ASM_EMIT("vsubps 0x20(%[state]), %%ymm0, %%ymm0") // ymm0 = x1 = k*i - p
118 __ASM_EMIT("vmovaps %%ymm1, 0x60(%[state])") // *i = i'
119 LANCZOS_GEN_FUNC_X8
120 __ASM_EMIT("vmovaps 0x60(%[state]), %%ymm1") // ymm1 = i'
121 __ASM_EMIT("vmovups %%ymm0, 0x00(%[dst])")
122 __ASM_EMIT("add $0x20, %[dst]")
123 __ASM_EMIT("sub $8, %[count]")
124 __ASM_EMIT("jae 1b")
125 __ASM_EMIT("2:")
126 // x4 block
127 __ASM_EMIT("add $4, %[count]")
128 __ASM_EMIT("jl 4f")
129 __ASM_EMIT("vmulps 0x00(%[state]), %%xmm1, %%xmm0") // xmm0 = k*i
130 __ASM_EMIT("vaddps 0x80 + %[LGEN], %%xmm1, %%xmm1") // xmm1 = i' = i + step
131 __ASM_EMIT("vsubps 0x20(%[state]), %%xmm0, %%xmm0") // xmm0 = x1 = k*i - p
132 __ASM_EMIT("vmovaps %%xmm1, 0x60(%[state])") // *i = i'
133 LANCZOS_GEN_FUNC_X4
134 __ASM_EMIT("vmovaps 0x60(%[state]), %%xmm1") // xmm1 = i'
135 __ASM_EMIT("vmovups %%xmm0, 0x00(%[dst])")
136 __ASM_EMIT("add $0x10, %[dst]")
137 __ASM_EMIT("sub $4, %[count]")
138 __ASM_EMIT("4:")
139 // Tail: 1x-3x block
140 __ASM_EMIT("add $4, %[count]")
141 __ASM_EMIT("jle 8f")
142 __ASM_EMIT("vmulps 0x00(%[state]), %%xmm1, %%xmm0") // xmm0 = k*i
143 __ASM_EMIT("vsubps 0x20(%[state]), %%xmm0, %%xmm0") // xmm0 = x1 = k*i - p
144 LANCZOS_GEN_FUNC_X4
145 __ASM_EMIT("test $1, %[count]")
146 __ASM_EMIT("jz 6f")
147 __ASM_EMIT("vmovss %%xmm0, 0x00(%[dst])")
148 __ASM_EMIT("vshufps $0x39, %%xmm0, %%xmm0, %%xmm0")
149 __ASM_EMIT("add $4, %[dst]")
150 __ASM_EMIT("6:")
151 __ASM_EMIT("test $2, %[count]")
152 __ASM_EMIT("jz 8f")
153 __ASM_EMIT("vmovlps %%xmm0, 0x00(%[dst])")
154 // End
155 __ASM_EMIT("8:")
156
157 : [dst] "+r" (dst), [count] "+r" (count)
158 : [state] "r" (&state),
159 [S2C] "o" (sinf_const),
160 [LGEN] "o" (kp_gen_const),
161 [LC] "o" (lanczos_const),
162 [stub] "o" (stub),
163 [k] "m" (k),
164 [p] "m" (p),
165 [a] "m" (a),
166 [t] "m" (t)
167 : "cc", "memory",
168 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
169 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
170 );
171 }
172
173 #define LANCZOS_GEN_X64_FUNC_X8 \
174 /* ymm8 = k */ \
175 /* ymm9 = p */ \
176 /* ymm10 = t */ \
177 /* ymm11 = a */ \
178 /* ymm12 = x1 */ \
179 /* ymm13 = d */ \
180 /* ymm14 = i */ \
181 /* ymm15 = step */ \
182 __ASM_EMIT("vmulps %%ymm11, %%ymm12, %%ymm4") /* ymm12 = x2 = x1*a */ \
183 __ASM_EMIT("vmulps %%ymm12, %%ymm4, %%ymm13") /* ymm13 = d = x1 * x2 */ \
184 __ASM_EMIT("vaddps 0x00 + %[S2C], %%ymm12, %%ymm0") /* ymm0 = x1 + PI/2 */ \
185 __ASM_EMIT("vaddps 0x00 + %[S2C], %%ymm4, %%ymm4") /* ymm4 = x2 + PI/2 */ \
186 SINF_X_PLUS_PI_2_CORE_X16 /* ymm0 = sinf(x1), ymm4 = sinf(x2) */ \
187 __ASM_EMIT("vmulps %%ymm4, %%ymm0, %%ymm0") /* ymm0 = sinf(x1)*sinf(x2) */ \
188 __ASM_EMIT("vandps 0x00 + %[LC], %%ymm12, %%ymm12") /* ymm12 = fabsf(x1) */ \
189 __ASM_EMIT("vdivps %%ymm13, %%ymm0, %%ymm0") /* ymm0 = F = (sinf(x1)*sinf(x2)) / (x1 * x2) */ \
190 __ASM_EMIT("vcmpps $1, 0x20 + %[LC], %%ymm12, %%ymm2") /* ymm2 = [ fabsf(x1) < 1e-4 ] */ \
191 __ASM_EMIT("vcmpps $1, %%ymm10, %%ymm12, %%ymm1") /* ymm1 = [ fabsf(x1) < t ] */ \
192 __ASM_EMIT("vblendvps %%ymm2, 0x40 + %[LC], %%ymm0, %%ymm0") /* ymm0 = [ fabsf(x1) >= 1e-4 ] ? f : 1.0 */ \
193 __ASM_EMIT("vandps %%ymm1, %%ymm0, %%ymm0") /* ymm0 = [ fabsf(x1) < t ] ? ([ fabsf(x1) >= 1e-4 ] ? f : 1.0) : 0.0 */
194
195 #define LANCZOS_GEN_X64_FUNC_X4 \
196 /* xmm0 = x1 */ \
197 __ASM_EMIT("vmulps %%xmm11, %%xmm12, %%xmm4") /* xmm12 = x2 = x1*a */ \
198 __ASM_EMIT("vmulps %%xmm12, %%xmm4, %%xmm13") /* xmm13 = d = x1 * x2 */ \
199 __ASM_EMIT("vinsertf128 $1, %%xmm4, %%ymm12, %%ymm0") /* ymm0 = x1 x1 x1 x1 x2 x2 x2 x2 */ \
200 __ASM_EMIT("vaddps 0x00 + %[S2C], %%ymm0, %%ymm0") /* ymm0 = x1 + PI/2 ... x2 + PI/2 */ \
201 SINF_X_PLUS_PI_2_CORE_X8 /* ymm0 = sinf(x1) ... sinf(x2) */ \
202 __ASM_EMIT("vextractf128 $1, %%ymm0, %%xmm4") /* xmm4 = sif(x2) */ \
203 __ASM_EMIT("vmulps %%ymm4, %%ymm0, %%ymm0") /* ymm0 = sinf(x1)*sinf(x2) */ \
204 __ASM_EMIT("vandps 0x00 + %[LC], %%ymm12, %%ymm12") /* ymm12 = fabsf(x1) */ \
205 __ASM_EMIT("vdivps %%ymm13, %%ymm0, %%ymm0") /* ymm0 = F = (sinf(x1)*sinf(x2)) / (x1 * x2) */ \
206 __ASM_EMIT("vcmpps $1, 0x20 + %[LC], %%ymm12, %%ymm2") /* ymm2 = [ fabsf(x1) < 1e-4 ] */ \
207 __ASM_EMIT("vcmpps $1, %%ymm10, %%ymm12, %%ymm1") /* ymm1 = [ fabsf(x1) < t ] */ \
208 __ASM_EMIT("vblendvps %%ymm2, 0x40 + %[LC], %%ymm0, %%ymm0") /* ymm0 = [ fabsf(x1) >= 1e-4 ] ? f : 1.0 */ \
209 __ASM_EMIT("vandps %%ymm1, %%ymm0, %%ymm0") /* ymm0 = [ fabsf(x1) < t ] ? ([ fabsf(x1) >= 1e-4 ] ? f : 1.0) : 0.0 */
210
211 void x64_lanczos1(float *dst, float k, float p, float t, float a, size_t count)
212 {
213 ARCH_X86_64_ASM(
214 // Prepare
215 __ASM_EMIT("vbroadcastss %[k], %%ymm8") // ymm8 = k
216 __ASM_EMIT("vbroadcastss %[p], %%ymm9") // ymm9 = p
217 __ASM_EMIT("vbroadcastss %[t], %%ymm10") // ymm10 = t
218 __ASM_EMIT("vbroadcastss %[a], %%ymm11") // ymm11 = a
219 __ASM_EMIT("vmovaps 0x00 + %[LGEN], %%ymm14") // ymm14 = i = 0 1 2 3 4 5 6 7
220 // x8 blocks
221 __ASM_EMIT("sub $8, %[count]")
222 __ASM_EMIT("jb 2f")
223 __ASM_EMIT("vmovaps 0x60 + %[LGEN], %%ymm15") // ymm15 = step
224 __ASM_EMIT("1:")
225 __ASM_EMIT("vmulps %%ymm8, %%ymm14, %%ymm12") // ymm12 = k*i
226 __ASM_EMIT("vaddps %%ymm15, %%ymm14, %%ymm14") // ymm14 = i' = i + step
227 __ASM_EMIT("vsubps %%ymm9, %%ymm12, %%ymm12") // ymm12 = x1 = k*i - p
228 LANCZOS_GEN_X64_FUNC_X8
229 __ASM_EMIT("vmovups %%ymm0, 0x00(%[dst])")
230 __ASM_EMIT("add $0x20, %[dst]")
231 __ASM_EMIT("sub $8, %[count]")
232 __ASM_EMIT("jae 1b")
233 __ASM_EMIT("2:")
234 // x4 block
235 __ASM_EMIT("add $4, %[count]")
236 __ASM_EMIT("jl 4f")
237 __ASM_EMIT("vmulps %%xmm8, %%xmm14, %%xmm12") // xmm12 = k*i
238 __ASM_EMIT("vaddps 0x80 + %[LGEN], %%xmm14, %%xmm14") // xmm14 = i' = i + step
239 __ASM_EMIT("vsubps %%xmm9, %%xmm12, %%xmm12") // xmm12 = x1 = k*i - p
240 LANCZOS_GEN_X64_FUNC_X4
241 __ASM_EMIT("vmovups %%xmm0, 0x00(%[dst])")
242 __ASM_EMIT("add $0x10, %[dst]")
243 __ASM_EMIT("sub $4, %[count]")
244 __ASM_EMIT("4:")
245 // Tail: 1x-3x block
246 __ASM_EMIT("add $4, %[count]")
247 __ASM_EMIT("jle 8f")
248 __ASM_EMIT("vmulps %%xmm8, %%xmm14, %%xmm12") // xmm12 = k*i
249 __ASM_EMIT("vsubps %%xmm9, %%xmm12, %%xmm12") // xmm12 = x1 = k*i - p
250 LANCZOS_GEN_X64_FUNC_X4
251 __ASM_EMIT("test $1, %[count]")
252 __ASM_EMIT("jz 6f")
253 __ASM_EMIT("vmovss %%xmm0, 0x00(%[dst])")
254 __ASM_EMIT("vshufps $0x39, %%xmm0, %%xmm0, %%xmm0")
255 __ASM_EMIT("add $4, %[dst]")
256 __ASM_EMIT("6:")
257 __ASM_EMIT("test $2, %[count]")
258 __ASM_EMIT("jz 8f")
259 __ASM_EMIT("vmovlps %%xmm0, 0x00(%[dst])")
260 // End
261 __ASM_EMIT("8:")
262
263 : [dst] "+r" (dst), [count] "+r" (count)
264 : [S2C] "o" (sinf_const),
265 [LGEN] "o" (kp_gen_const),
266 [LC] "o" (lanczos_const),
267 [k] "m" (k),
268 [p] "m" (p),
269 [a] "m" (a),
270 [t] "m" (t)
271 : "cc", "memory",
272 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
273 "%xmm4", "%xmm5", "%xmm6", "%xmm7",
274 "%xmm8", "%xmm9", "%xmm10", "%xmm11",
275 "%xmm12", "%xmm13", "%xmm14", "%xmm15"
276 );
277 }
278
279
280 #define LANCZOS_GEN_FUNC_X8_FMA3 \
281 /* ymm0 = x1 */ \
282 __ASM_EMIT("vmulps 0x40(%[state]), %%ymm0, %%ymm4") /* ymm4 = x2 = x1*a */ \
283 __ASM_EMIT("vmovaps %%ymm0, 0x80(%[state])") /* *x1 = x1 */ \
284 __ASM_EMIT("vmulps %%ymm0, %%ymm4, %%ymm2") /* ymm2 = d = x1 * x2 */ \
285 __ASM_EMIT("vaddps 0x00 + %[S2C], %%ymm0, %%ymm0") /* ymm0 = x1 + PI/2 */ \
286 __ASM_EMIT("vaddps 0x00 + %[S2C], %%ymm4, %%ymm4") /* ymm4 = x2 + PI/2 */ \
287 __ASM_EMIT("vmovaps %%ymm2, 0xa0(%[state])") /* *d = d */ \
288 SINF_X_PLUS_PI_2_CORE_X16_FMA3 /* ymm0 = sinf(x1), ymm4 = sinf(x2) */ \
289 __ASM_EMIT("vmovaps 0x80(%[state]), %%ymm1") /* ymm1 = x1 */ \
290 __ASM_EMIT("vmulps %%ymm4, %%ymm0, %%ymm0") /* ymm0 = sinf(x1)*sinf(x2) */ \
291 __ASM_EMIT("vandps 0x00 + %[LC], %%ymm1, %%ymm1") /* ymm1 = fabsf(x) */ \
292 __ASM_EMIT("vdivps 0xa0(%[state]), %%ymm0, %%ymm0") /* ymm0 = F = (sinf(x1)*sinf(x2)) / (x1 * x2) */ \
293 __ASM_EMIT("vcmpps $1, 0x20 + %[LC], %%ymm1, %%ymm2") /* ymm2 = [ fabsf(x) < 1e-4 ] */ \
294 __ASM_EMIT("vcmpps $1, 0xc0(%[state]), %%ymm1, %%ymm1") /* ymm1 = [ fabsf(x) < t ] */ \
295 __ASM_EMIT("vblendvps %%ymm2, 0x40 + %[LC], %%ymm0, %%ymm0") /* ymm0 = [ fabsf(x) >= 1e-4 ] ? f : 1.0 */ \
296 __ASM_EMIT("vandps %%ymm1, %%ymm0, %%ymm0") /* ymm0 = [ fabsf(x) < t ] ? ([ fabsf(x) >= 1e-4 ] ? f : 1.0) : 0.0 */
297
298 #define LANCZOS_GEN_FUNC_X4_FMA3 \
299 /* xmm0 = x1 */ \
300 __ASM_EMIT("vmulps 0x40(%[state]), %%xmm0, %%xmm4") /* xmm4 = x2 = x1*a */ \
301 __ASM_EMIT("vmovaps %%xmm0, 0x80(%[state])") /* *x1 = x1 */ \
302 __ASM_EMIT("vmulps %%xmm0, %%xmm4, %%xmm2") /* xmm2 = d = x1 * x2 */ \
303 __ASM_EMIT("vinsertf128 $1, %%xmm4, %%ymm0, %%ymm0") /* ymm0 = x1 x1 x1 x1 x2 x2 x2 x2 */ \
304 __ASM_EMIT("vmovaps %%xmm2, 0xa0(%[state])") /* *d = d */ \
305 __ASM_EMIT("vaddps 0x00 + %[S2C], %%ymm0, %%ymm0") /* xmm0 = x1 + PI/2 ... x2 + PI/2 ... */ \
306 SINF_X_PLUS_PI_2_CORE_X8_FMA3 /* ymm0 = sinf(x1) ... sinf(x2) */ \
307 __ASM_EMIT("vextractf128 $1, %%ymm0, %%xmm4") /* xmm4 = sif(x2) */ \
308 __ASM_EMIT("vmovaps 0x80(%[state]), %%xmm1") /* xmm1 = x1 */ \
309 __ASM_EMIT("vmulps %%xmm4, %%xmm0, %%xmm0") /* xmm0 = sinf(x1)*sinf(x2) */ \
310 __ASM_EMIT("vandps 0x00 + %[LC], %%xmm1, %%xmm1") /* xmm1 = fabsf(x) */ \
311 __ASM_EMIT("vdivps 0xa0(%[state]), %%xmm0, %%xmm0") /* xmm0 = F = (sinf(x1)*sinf(x2)) / (x1 * x2) */ \
312 __ASM_EMIT("vcmpps $1, 0x20 + %[LC], %%xmm1, %%xmm2") /* xmm2 = [ fabsf(x) < 1e-4 ] */ \
313 __ASM_EMIT("vcmpps $1,0xc0(%[state]), %%xmm1, %%xmm1") /* xmm1 = [ fabsf(x) < t ] */ \
314 __ASM_EMIT("vblendvps %%xmm2, 0x40 + %[LC], %%xmm0, %%xmm0") /* xmm0 = [ fabsf(x) >= 1e-4 ] ? f : 1.0 */ \
315 __ASM_EMIT("vandps %%xmm1, %%xmm0, %%xmm0") /* xmm0 = [ fabsf(x) < t ] ? ([ fabsf(x) >= 1e-4 ] ? f : 1.0) : 0.0 */
316
317 void lanczos1_fma3(float *dst, float k, float p, float t, float a, size_t count)
318 {
319 IF_ARCH_X86(
320 lanczos_gen_t state __lsp_aligned32;
321 float stub[8] __lsp_aligned16;
322 );
323
324 ARCH_X86_ASM(
325 // Prepare
326 __ASM_EMIT("vbroadcastss %[k], %%ymm4") // ymm4 = k
327 __ASM_EMIT("vbroadcastss %[p], %%ymm5") // ymm5 = p
328 __ASM_EMIT("vbroadcastss %[a], %%ymm6") // ymm6 = a
329 __ASM_EMIT("vbroadcastss %[t], %%ymm7") // ymm7 = t
330 __ASM_EMIT("vmovaps %%ymm4, 0x00(%[state])")
331 __ASM_EMIT("vmovaps %%ymm5, 0x20(%[state])")
332 __ASM_EMIT("vmovaps %%ymm6, 0x40(%[state])")
333 __ASM_EMIT("vmovaps %%ymm7, 0xc0(%[state])")
334 __ASM_EMIT("vmovaps 0x00 + %[LGEN], %%ymm1") // ymm1 = i = 0 1 2 3 4 5 6 7
335 // x8 blocks
336 __ASM_EMIT("sub $8, %[count]")
337 __ASM_EMIT("jb 2f")
338 __ASM_EMIT("1:")
339 __ASM_EMIT("vmulps 0x00(%[state]), %%ymm1, %%ymm0") // ymm0 = k*i
340 __ASM_EMIT("vaddps 0x60 + %[LGEN], %%ymm1, %%ymm1") // ymm1 = i' = i + step
341 __ASM_EMIT("vsubps 0x20(%[state]), %%ymm0, %%ymm0") // ymm0 = x1 = k*i - p
342 __ASM_EMIT("vmovaps %%ymm1, 0x60(%[state])") // *i = i'
343 LANCZOS_GEN_FUNC_X8_FMA3
344 __ASM_EMIT("vmovaps 0x60(%[state]), %%ymm1") // ymm1 = i'
345 __ASM_EMIT("vmovups %%ymm0, 0x00(%[dst])")
346 __ASM_EMIT("add $0x20, %[dst]")
347 __ASM_EMIT("sub $8, %[count]")
348 __ASM_EMIT("jae 1b")
349 __ASM_EMIT("2:")
350 // x4 block
351 __ASM_EMIT("add $4, %[count]")
352 __ASM_EMIT("jl 4f")
353 __ASM_EMIT("vmulps 0x00(%[state]), %%xmm1, %%xmm0") // xmm0 = k*i
354 __ASM_EMIT("vaddps 0x80 + %[LGEN], %%xmm1, %%xmm1") // xmm1 = i' = i + step
355 __ASM_EMIT("vsubps 0x20(%[state]), %%xmm0, %%xmm0") // xmm0 = x1 = k*i - p
356 __ASM_EMIT("vmovaps %%xmm1, 0x60(%[state])") // *i = i'
357 LANCZOS_GEN_FUNC_X4_FMA3
358 __ASM_EMIT("vmovaps 0x60(%[state]), %%xmm1") // xmm1 = i'
359 __ASM_EMIT("vmovups %%xmm0, 0x00(%[dst])")
360 __ASM_EMIT("add $0x10, %[dst]")
361 __ASM_EMIT("sub $4, %[count]")
362 __ASM_EMIT("4:")
363 // Tail: 1x-3x block
364 __ASM_EMIT("add $4, %[count]")
365 __ASM_EMIT("jle 8f")
366 __ASM_EMIT("vmulps 0x00(%[state]), %%xmm1, %%xmm0") // xmm0 = k*i
367 __ASM_EMIT("vsubps 0x20(%[state]), %%xmm0, %%xmm0") // xmm0 = x1 = k*i - p
368 LANCZOS_GEN_FUNC_X4_FMA3
369 __ASM_EMIT("test $1, %[count]")
370 __ASM_EMIT("jz 6f")
371 __ASM_EMIT("vmovss %%xmm0, 0x00(%[dst])")
372 __ASM_EMIT("vshufps $0x39, %%xmm0, %%xmm0, %%xmm0")
373 __ASM_EMIT("add $4, %[dst]")
374 __ASM_EMIT("6:")
375 __ASM_EMIT("test $2, %[count]")
376 __ASM_EMIT("jz 8f")
377 __ASM_EMIT("vmovlps %%xmm0, 0x00(%[dst])")
378 // End
379 __ASM_EMIT("8:")
380
381 : [dst] "+r" (dst), [count] "+r" (count)
382 : [state] "r" (&state),
383 [S2C] "o" (sinf_const),
384 [LGEN] "o" (kp_gen_const),
385 [LC] "o" (lanczos_const),
386 [stub] "o" (stub),
387 [k] "m" (k),
388 [p] "m" (p),
389 [a] "m" (a),
390 [t] "m" (t)
391 : "cc", "memory",
392 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
393 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
394 );
395 }
396
397 #define LANCZOS_GEN_X64_FUNC_X8_FMA3 \
398 /* ymm8 = k */ \
399 /* ymm9 = p */ \
400 /* ymm10 = t */ \
401 /* ymm11 = a */ \
402 /* ymm12 = x1 */ \
403 /* ymm13 = d */ \
404 /* ymm14 = i */ \
405 /* ymm15 = step */ \
406 __ASM_EMIT("vmulps %%ymm11, %%ymm12, %%ymm4") /* ymm12 = x2 = x1*a */ \
407 __ASM_EMIT("vmulps %%ymm12, %%ymm4, %%ymm13") /* ymm13 = d = x1 * x2 */ \
408 __ASM_EMIT("vaddps 0x00 + %[S2C], %%ymm12, %%ymm0") /* ymm0 = x1 + PI/2 */ \
409 __ASM_EMIT("vaddps 0x00 + %[S2C], %%ymm4, %%ymm4") /* ymm4 = x2 + PI/2 */ \
410 SINF_X_PLUS_PI_2_CORE_X16_FMA3 /* ymm0 = sinf(x1), ymm4 = sinf(x2) */ \
411 __ASM_EMIT("vmulps %%ymm4, %%ymm0, %%ymm0") /* ymm0 = sinf(x1)*sinf(x2) */ \
412 __ASM_EMIT("vandps 0x00 + %[LC], %%ymm12, %%ymm12") /* ymm12 = fabsf(x1) */ \
413 __ASM_EMIT("vdivps %%ymm13, %%ymm0, %%ymm0") /* ymm0 = F = (sinf(x1)*sinf(x2)) / (x1 * x2) */ \
414 __ASM_EMIT("vcmpps $1, 0x20 + %[LC], %%ymm12, %%ymm2") /* ymm2 = [ fabsf(x1) < 1e-4 ] */ \
415 __ASM_EMIT("vcmpps $1, %%ymm10, %%ymm12, %%ymm1") /* ymm1 = [ fabsf(x1) < t ] */ \
416 __ASM_EMIT("vblendvps %%ymm2, 0x40 + %[LC], %%ymm0, %%ymm0") /* ymm0 = [ fabsf(x1) >= 1e-4 ] ? f : 1.0 */ \
417 __ASM_EMIT("vandps %%ymm1, %%ymm0, %%ymm0") /* ymm0 = [ fabsf(x1) < t ] ? ([ fabsf(x1) >= 1e-4 ] ? f : 1.0) : 0.0 */
418
419 #define LANCZOS_GEN_X64_FUNC_X4_FMA3 \
420 /* xmm0 = x1 */ \
421 __ASM_EMIT("vmulps %%xmm11, %%xmm12, %%xmm4") /* xmm12 = x2 = x1*a */ \
422 __ASM_EMIT("vmulps %%xmm12, %%xmm4, %%xmm13") /* xmm13 = d = x1 * x2 */ \
423 __ASM_EMIT("vinsertf128 $1, %%xmm4, %%ymm12, %%ymm0") /* ymm0 = x1 x1 x1 x1 x2 x2 x2 x2 */ \
424 __ASM_EMIT("vaddps 0x00 + %[S2C], %%ymm0, %%ymm0") /* ymm0 = x1 + PI/2 ... x2 + PI/2 */ \
425 SINF_X_PLUS_PI_2_CORE_X8_FMA3 /* ymm0 = sinf(x1) ... sinf(x2) */ \
426 __ASM_EMIT("vextractf128 $1, %%ymm0, %%xmm4") /* xmm4 = sif(x2) */ \
427 __ASM_EMIT("vmulps %%ymm4, %%ymm0, %%ymm0") /* ymm0 = sinf(x1)*sinf(x2) */ \
428 __ASM_EMIT("vandps 0x00 + %[LC], %%ymm12, %%ymm12") /* ymm12 = fabsf(x1) */ \
429 __ASM_EMIT("vdivps %%ymm13, %%ymm0, %%ymm0") /* ymm0 = F = (sinf(x1)*sinf(x2)) / (x1 * x2) */ \
430 __ASM_EMIT("vcmpps $1, 0x20 + %[LC], %%ymm12, %%ymm2") /* ymm2 = [ fabsf(x1) < 1e-4 ] */ \
431 __ASM_EMIT("vcmpps $1, %%ymm10, %%ymm12, %%ymm1") /* ymm1 = [ fabsf(x1) < t ] */ \
432 __ASM_EMIT("vblendvps %%ymm2, 0x40 + %[LC], %%ymm0, %%ymm0") /* ymm0 = [ fabsf(x1) >= 1e-4 ] ? f : 1.0 */ \
433 __ASM_EMIT("vandps %%ymm1, %%ymm0, %%ymm0") /* ymm0 = [ fabsf(x1) < t ] ? ([ fabsf(x1) >= 1e-4 ] ? f : 1.0) : 0.0 */
434
435 void x64_lanczos1_fma3(float *dst, float k, float p, float t, float a, size_t count)
436 {
437 ARCH_X86_64_ASM(
438 // Prepare
439 __ASM_EMIT("vbroadcastss %[k], %%ymm8") // ymm8 = k
440 __ASM_EMIT("vbroadcastss %[p], %%ymm9") // ymm9 = p
441 __ASM_EMIT("vbroadcastss %[t], %%ymm10") // ymm10 = t
442 __ASM_EMIT("vbroadcastss %[a], %%ymm11") // ymm11 = a
443 __ASM_EMIT("vmovaps 0x00 + %[LGEN], %%ymm14") // ymm14 = i = 0 1 2 3 4 5 6 7
444 // x8 blocks
445 __ASM_EMIT("sub $8, %[count]")
446 __ASM_EMIT("jb 2f")
447 __ASM_EMIT("vmovaps 0x60 + %[LGEN], %%ymm15") // ymm15 = step
448 __ASM_EMIT("1:")
449 __ASM_EMIT("vmulps %%ymm8, %%ymm14, %%ymm12") // ymm12 = k*i
450 __ASM_EMIT("vaddps %%ymm15, %%ymm14, %%ymm14") // ymm14 = i' = i + step
451 __ASM_EMIT("vsubps %%ymm9, %%ymm12, %%ymm12") // ymm12 = x1 = k*i - p
452 LANCZOS_GEN_X64_FUNC_X8_FMA3
453 __ASM_EMIT("vmovups %%ymm0, 0x00(%[dst])")
454 __ASM_EMIT("add $0x20, %[dst]")
455 __ASM_EMIT("sub $8, %[count]")
456 __ASM_EMIT("jae 1b")
457 __ASM_EMIT("2:")
458 // x4 block
459 __ASM_EMIT("add $4, %[count]")
460 __ASM_EMIT("jl 4f")
461 __ASM_EMIT("vmulps %%xmm8, %%xmm14, %%xmm12") // xmm12 = k*i
462 __ASM_EMIT("vaddps 0x80 + %[LGEN], %%xmm14, %%xmm14") // xmm14 = i' = i + step
463 __ASM_EMIT("vsubps %%xmm9, %%xmm12, %%xmm12") // xmm12 = x1 = k*i - p
464 LANCZOS_GEN_X64_FUNC_X4_FMA3
465 __ASM_EMIT("vmovups %%xmm0, 0x00(%[dst])")
466 __ASM_EMIT("add $0x10, %[dst]")
467 __ASM_EMIT("sub $4, %[count]")
468 __ASM_EMIT("4:")
469 // Tail: 1x-3x block
470 __ASM_EMIT("add $4, %[count]")
471 __ASM_EMIT("jle 8f")
472 __ASM_EMIT("vmulps %%xmm8, %%xmm14, %%xmm12") // xmm12 = k*i
473 __ASM_EMIT("vsubps %%xmm9, %%xmm12, %%xmm12") // xmm12 = x1 = k*i - p
474 LANCZOS_GEN_X64_FUNC_X4_FMA3
475 __ASM_EMIT("test $1, %[count]")
476 __ASM_EMIT("jz 6f")
477 __ASM_EMIT("vmovss %%xmm0, 0x00(%[dst])")
478 __ASM_EMIT("vshufps $0x39, %%xmm0, %%xmm0, %%xmm0")
479 __ASM_EMIT("add $4, %[dst]")
480 __ASM_EMIT("6:")
481 __ASM_EMIT("test $2, %[count]")
482 __ASM_EMIT("jz 8f")
483 __ASM_EMIT("vmovlps %%xmm0, 0x00(%[dst])")
484 // End
485 __ASM_EMIT("8:")
486
487 : [dst] "+r" (dst), [count] "+r" (count)
488 : [S2C] "o" (sinf_const),
489 [LGEN] "o" (kp_gen_const),
490 [LC] "o" (lanczos_const),
491 [k] "m" (k),
492 [p] "m" (p),
493 [a] "m" (a),
494 [t] "m" (t)
495 : "cc", "memory",
496 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
497 "%xmm4", "%xmm5", "%xmm6", "%xmm7",
498 "%xmm8", "%xmm9", "%xmm10", "%xmm11",
499 "%xmm12", "%xmm13", "%xmm14", "%xmm15"
500 );
501 }
502
503 } /* namespace avx2 */
504 } /* namespace lsp */
505
506
507 #endif /* PRIVATE_DSP_ARCH_X86_AVX2_PMATH_LANCZOS_H_ */