Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/x86/avx512/pmath/lanczos.h

raw
1 /*
2 * Copyright (C) 2026 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2026 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 18 апр. 2025 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_X86_AVX512_PMATH_LANCZOS_H_
23 #define PRIVATE_DSP_ARCH_X86_AVX512_PMATH_LANCZOS_H_
24
25 #ifndef PRIVATE_DSP_ARCH_X86_AVX512_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_X86_AVX512_IMPL */
28
29 #include <private/dsp/arch/x86/avx512/pmath/sin.h>
30
31 namespace lsp
32 {
33 namespace avx512
34 {
35 IF_ARCH_X86(
36 static const uint32_t lanczos_const[] __lsp_aligned64 =
37 {
38 LSP_DSP_VEC16(0x7fffffff), // +0x00: Mask for fabsf
39 LSP_DSP_VEC16(0x38d1b717), // +0x40: Sinc threshold = 1e-4
40 LSP_DSP_VEC16(0x3f800000), // +0x80: 1.0
41 };
42 )
43
44 typedef struct lanczos_gen_t
45 {
46 float k[16]; // +0x000: Multiplier
47 float p[16]; // +0x040: Initial phase
48 float a[16]; // +0x080: Number of lobes
49 float i[16]; // +0x0c0: Indices
50 float x1[16]; // +0x100: Computed X1
51 float d[16]; // +0x140: Denominator
52 float t[16]; // +0x180: Time range
53 } lanczos_gen_t;
54
55 #define LANCZOS_GEN_FUNC_X16 \
56 /* zmm0 = x1 */ \
57 __ASM_EMIT("vmulps 0x080(%[state]), %%zmm0, %%zmm4") /* zmm4 = x2 = x1*a */ \
58 __ASM_EMIT("vmovaps %%zmm0, 0x100(%[state])") /* *x1 = x1 */ \
59 __ASM_EMIT("vmulps %%zmm0, %%zmm4, %%zmm2") /* zmm2 = d = x1 * x2 */ \
60 __ASM_EMIT("vaddps 0x00 + %[S2C], %%zmm0, %%zmm0") /* zmm0 = x1 + PI/2 */ \
61 __ASM_EMIT("vaddps 0x00 + %[S2C], %%zmm4, %%zmm4") /* zmm4 = x2 + PI/2 */ \
62 __ASM_EMIT("vmovaps %%zmm2, 0x140(%[state])") /* *d = d */ \
63 SINF_X_PLUS_PI_2_CORE_X32 /* zmm0 = sinf(x1), zmm4 = sinf(x2) */ \
64 __ASM_EMIT("vmovaps 0x100(%[state]), %%zmm1") /* zmm1 = x1 */ \
65 __ASM_EMIT("vmulps %%zmm4, %%zmm0, %%zmm0") /* zmm0 = sinf(x1)*sinf(x2) */ \
66 __ASM_EMIT("vandps 0x00 + %[LC], %%zmm1, %%zmm1") /* zmm1 = fabsf(x) */ \
67 __ASM_EMIT("vdivps 0x140(%[state]), %%zmm0, %%zmm0") /* zmm0 = F = (sinf(x1)*sinf(x2)) / (x1 * x2) */ \
68 __ASM_EMIT("vcmpps $1, 0x40 + %[LC], %%zmm1, %%k1") /* k1 = [ fabsf(x) < 1e-4 ] */ \
69 __ASM_EMIT("vcmpps $5, 0x180(%[state]), %%zmm1, %%k2") /* k2 = [ fabsf(x) >= t ] */ \
70 __ASM_EMIT("vmovaps 0x80 + %[LC], %%zmm0 %{%%k1%}") /* zmm0 = [ fabsf(x) >= 1e-4 ] ? f : 1.0 */ \
71 __ASM_EMIT("vxorps %%zmm0, %%zmm0, %%zmm0 %{%%k2%}") /* zmm0 = [ fabsf(x) < t ] ? ([ fabsf(x) >= 1e-4 ] ? f : 1.0) : 0.0 */
72
73 #define LANCZOS_GEN_FUNC_X8 \
74 /* ymm0 = x1 */ \
75 __ASM_EMIT("vmulps 0x080(%[state]), %%ymm0, %%ymm4") /* ymm4 = x2 = x1*a */ \
76 __ASM_EMIT("vmovaps %%ymm0, 0x100(%[state])") /* *x1 = x1 */ \
77 __ASM_EMIT("vmulps %%ymm0, %%ymm4, %%ymm2") /* ymm2 = d = x1 * x2 */ \
78 __ASM_EMIT("vinsertf64x4 $1, %%ymm4, %%zmm0, %%zmm0") /* zmm0 = x1 ... x2 ... */ \
79 __ASM_EMIT("vmovaps %%ymm2, 0x140(%[state])") /* *d = d */ \
80 __ASM_EMIT("vaddps 0x00 + %[S2C], %%zmm0, %%zmm0") /* zmm0 = x1 + PI/2 ... x2 + PI/2 */ \
81 SINF_X_PLUS_PI_2_CORE_X16 /* ymm0 = sinf(x1), ymm4 = sinf(x2) */ \
82 __ASM_EMIT("vextractf64x4 $1, %%zmm0, %%ymm4") /* ymm4 = sinf(x2) */ \
83 __ASM_EMIT("vmovaps 0x100(%[state]), %%ymm1") /* ymm1 = x1 */ \
84 __ASM_EMIT("vmulps %%ymm4, %%ymm0, %%ymm0") /* ymm0 = sinf(x1)*sinf(x2) */ \
85 __ASM_EMIT("vandps 0x00 + %[LC], %%ymm1, %%ymm1") /* ymm1 = fabsf(x) */ \
86 __ASM_EMIT("vdivps 0x140(%[state]), %%ymm0, %%ymm0") /* ymm0 = F = (sinf(x1)*sinf(x2)) / (x1 * x2) */ \
87 __ASM_EMIT("vcmpps $1, 0x40 + %[LC], %%ymm1, %%ymm2") /* ymm2 = [ fabsf(x) < 1e-4 ] */ \
88 __ASM_EMIT("vcmpps $1, 0x180(%[state]), %%ymm1, %%ymm1") /* ymm1 = [ fabsf(x) < t ] */ \
89 __ASM_EMIT("vblendvps %%ymm2, 0x80 + %[LC], %%ymm0, %%ymm0") /* ymm0 = [ fabsf(x) >= 1e-4 ] ? f : 1.0 */ \
90 __ASM_EMIT("vandps %%ymm1, %%ymm0, %%ymm0") /* ymm0 = [ fabsf(x) < t ] ? ([ fabsf(x) >= 1e-4 ] ? f : 1.0) : 0.0 */
91
92 #define LANCZOS_GEN_FUNC_X4 \
93 /* xmm0 = x1 */ \
94 __ASM_EMIT("vmulps 0x080(%[state]), %%xmm0, %%xmm4") /* xmm4 = x2 = x1*a */ \
95 __ASM_EMIT("vmovaps %%xmm0, 0x100(%[state])") /* *x1 = x1 */ \
96 __ASM_EMIT("vmulps %%xmm0, %%xmm4, %%xmm2") /* xmm2 = d = x1 * x2 */ \
97 __ASM_EMIT("vinsertf128 $1, %%xmm4, %%ymm0, %%ymm0") /* ymm0 = x1 x1 x1 x1 x2 x2 x2 x2 */ \
98 __ASM_EMIT("vmovaps %%xmm2, 0x140(%[state])") /* *d = d */ \
99 __ASM_EMIT("vaddps 0x00 + %[S2C], %%ymm0, %%ymm0") /* xmm0 = x1 + PI/2 ... x2 + PI/2 ... */ \
100 SINF_X_PLUS_PI_2_CORE_X8 /* ymm0 = sinf(x1) ... sinf(x2) ... */ \
101 __ASM_EMIT("vextractf128 $1, %%ymm0, %%xmm4") /* xmm4 = sinf(x2) */ \
102 __ASM_EMIT("vmovaps 0x100(%[state]), %%xmm1") /* xmm1 = x1 */ \
103 __ASM_EMIT("vmulps %%xmm4, %%xmm0, %%xmm0") /* xmm0 = sinf(x1)*sinf(x2) */ \
104 __ASM_EMIT("vandps 0x00 + %[LC], %%xmm1, %%xmm1") /* xmm1 = fabsf(x) */ \
105 __ASM_EMIT("vdivps 0x140(%[state]), %%xmm0, %%xmm0") /* xmm0 = F = (sinf(x1)*sinf(x2)) / (x1 * x2) */ \
106 __ASM_EMIT("vcmpps $1, 0x40 + %[LC], %%xmm1, %%xmm2") /* xmm2 = [ fabsf(x) < 1e-4 ] */ \
107 __ASM_EMIT("vcmpps $1, 0x180(%[state]), %%xmm1, %%xmm1") /* xmm1 = [ fabsf(x) < t ] */ \
108 __ASM_EMIT("vblendvps %%xmm2, 0x80 + %[LC], %%xmm0, %%xmm0") /* xmm0 = [ fabsf(x) >= 1e-4 ] ? f : 1.0 */ \
109 __ASM_EMIT("vandps %%xmm1, %%xmm0, %%xmm0") /* xmm0 = [ fabsf(x) < t ] ? ([ fabsf(x) >= 1e-4 ] ? f : 1.0) : 0.0 */
110
111 void lanczos1(float *dst, float k, float p, float t, float a, size_t count)
112 {
113 IF_ARCH_X86(
114 lanczos_gen_t state __lsp_aligned64;
115 float stub[16] __lsp_aligned64;
116 );
117
118 ARCH_X86_ASM(
119 // Prepare
120 __ASM_EMIT("vbroadcastss %[k], %%zmm4") // zmm4 = k
121 __ASM_EMIT("vbroadcastss %[p], %%zmm5") // zmm5 = p
122 __ASM_EMIT("vbroadcastss %[a], %%zmm6") // zmm6 = a
123 __ASM_EMIT("vbroadcastss %[t], %%zmm7") // zmm7 = t
124 __ASM_EMIT("vmovaps %%zmm4, 0x000(%[state])")
125 __ASM_EMIT("vmovaps %%zmm5, 0x040(%[state])")
126 __ASM_EMIT("vmovaps %%zmm6, 0x080(%[state])")
127 __ASM_EMIT("vmovaps %%zmm7, 0x180(%[state])")
128 __ASM_EMIT("vmovaps 0x00 + %[LGEN], %%zmm1") // zmm1 = i = 0 1 2 3 4 5 6 7 ...
129 // x16 blocks
130 __ASM_EMIT("sub $16, %[count]")
131 __ASM_EMIT("jb 2f")
132 __ASM_EMIT("1:")
133 __ASM_EMIT("vmulps 0x000(%[state]), %%zmm1, %%zmm0") // zmm0 = k*i
134 __ASM_EMIT("vaddps 0x0c0 + %[LGEN], %%zmm1, %%zmm1") // zmm1 = i' = i + step
135 __ASM_EMIT("vsubps 0x040(%[state]), %%zmm0, %%zmm0") // zmm0 = x1 = k*i - p
136 __ASM_EMIT("vmovaps %%zmm1, 0x0c0(%[state])") // *i = i'
137 LANCZOS_GEN_FUNC_X16
138 __ASM_EMIT("vmovaps 0x0c0(%[state]), %%zmm1") // zmm1 = i'
139 __ASM_EMIT("vmovups %%zmm0, 0x00(%[dst])")
140 __ASM_EMIT("add $0x40, %[dst]")
141 __ASM_EMIT("sub $16, %[count]")
142 __ASM_EMIT("jae 1b")
143 __ASM_EMIT("2:")
144 // x8 blocks
145 __ASM_EMIT("add $8, %[count]")
146 __ASM_EMIT("jl 4f")
147 __ASM_EMIT("vmulps 0x000(%[state]), %%ymm1, %%ymm0") // ymm0 = k*i
148 __ASM_EMIT("vaddps 0x100 + %[LGEN], %%ymm1, %%ymm1") // ymm1 = i' = i + step
149 __ASM_EMIT("vsubps 0x040(%[state]), %%ymm0, %%ymm0") // ymm0 = x1 = k*i - p
150 __ASM_EMIT("vmovaps %%ymm1, 0x0c0(%[state])") // *i = i'
151 LANCZOS_GEN_FUNC_X8
152 __ASM_EMIT("vmovaps 0x0c0(%[state]), %%ymm1") // ymm1 = i'
153 __ASM_EMIT("vmovups %%ymm0, 0x00(%[dst])")
154 __ASM_EMIT("add $0x20, %[dst]")
155 __ASM_EMIT("sub $8, %[count]")
156 __ASM_EMIT("4:")
157 // x4 block
158 __ASM_EMIT("add $4, %[count]")
159 __ASM_EMIT("jl 6f")
160 __ASM_EMIT("vmulps 0x000(%[state]), %%xmm1, %%xmm0") // xmm0 = k*i
161 __ASM_EMIT("vaddps 0x120 + %[LGEN], %%xmm1, %%xmm1") // xmm1 = i' = i + step
162 __ASM_EMIT("vsubps 0x040(%[state]), %%xmm0, %%xmm0") // xmm0 = x1 = k*i - p
163 __ASM_EMIT("vmovaps %%xmm1, 0x0c0(%[state])") // *i = i'
164 LANCZOS_GEN_FUNC_X4
165 __ASM_EMIT("vmovaps 0x0c0(%[state]), %%xmm1") // xmm1 = i'
166 __ASM_EMIT("vmovups %%xmm0, 0x00(%[dst])")
167 __ASM_EMIT("add $0x10, %[dst]")
168 __ASM_EMIT("sub $4, %[count]")
169 __ASM_EMIT("6:")
170 // Tail: 1x-3x block
171 __ASM_EMIT("add $4, %[count]")
172 __ASM_EMIT("jle 10f")
173 __ASM_EMIT("vmulps 0x000(%[state]), %%xmm1, %%xmm0") // xmm0 = k*i
174 __ASM_EMIT("vsubps 0x040(%[state]), %%xmm0, %%xmm0") // xmm0 = x1 = k*i - p
175 LANCZOS_GEN_FUNC_X4
176 __ASM_EMIT("test $1, %[count]")
177 __ASM_EMIT("jz 8f")
178 __ASM_EMIT("vmovss %%xmm0, 0x00(%[dst])")
179 __ASM_EMIT("vshufps $0x39, %%xmm0, %%xmm0, %%xmm0")
180 __ASM_EMIT("add $4, %[dst]")
181 __ASM_EMIT("8:")
182 __ASM_EMIT("test $2, %[count]")
183 __ASM_EMIT("jz 10f")
184 __ASM_EMIT("vmovlps %%xmm0, 0x00(%[dst])")
185 // End
186 __ASM_EMIT("10:")
187
188 : [dst] "+r" (dst), [count] "+r" (count)
189 : [state] "r" (&state),
190 [S2C] "o" (sinf_const),
191 [LGEN] "o" (kp_gen_const),
192 [LC] "o" (lanczos_const),
193 [stub] "o" (stub),
194 [k] "m" (k),
195 [p] "m" (p),
196 [a] "m" (a),
197 [t] "m" (t)
198 : "cc", "memory",
199 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
200 "%xmm4", "%xmm5", "%xmm6", "%xmm7",
201 "%k1", "%k2"
202 );
203 }
204
205 #define LANCZOS_GEN_X64_FUNC_X16 \
206 /* zmm8 = k */ \
207 /* zmm9 = p */ \
208 /* zmm10 = t */ \
209 /* zmm11 = a */ \
210 /* zmm12 = x1 */ \
211 /* zmm13 = d */ \
212 /* zmm14 = i */ \
213 /* zmm15 = step */ \
214 __ASM_EMIT("vmulps %%zmm11, %%zmm12, %%zmm4") /* zmm12 = x2 = x1*a */ \
215 __ASM_EMIT("vmulps %%zmm12, %%zmm4, %%zmm13") /* zmm13 = d = x1 * x2 */ \
216 __ASM_EMIT("vaddps 0x00 + %[S2C], %%zmm12, %%zmm0") /* zmm0 = x1 + PI/2 */ \
217 __ASM_EMIT("vaddps 0x00 + %[S2C], %%zmm4, %%zmm4") /* zmm4 = x2 + PI/2 */ \
218 SINF_X_PLUS_PI_2_CORE_X32 /* zmm0 = sinf(x1), zmm4 = sinf(x2) */ \
219 __ASM_EMIT("vmulps %%zmm4, %%zmm0, %%zmm0") /* zmm0 = sinf(x1)*sinf(x2) */ \
220 __ASM_EMIT("vandps 0x00 + %[LC], %%zmm12, %%zmm12") /* zmm12 = fabsf(x1) */ \
221 __ASM_EMIT("vdivps %%zmm13, %%zmm0, %%zmm0") /* zmm0 = F = (sinf(x1)*sinf(x2)) / (x1 * x2) */ \
222 __ASM_EMIT("vcmpps $1, 0x40 + %[LC], %%zmm12, %%k1") /* k1 = [ fabsf(x1) < 1e-4 ] */ \
223 __ASM_EMIT("vcmpps $5, %%zmm10, %%zmm1, %%k2") /* k2 = [ fabsf(x1) >= t ] */ \
224 __ASM_EMIT("vmovaps 0x80 + %[LC], %%zmm0 %{%%k1%}") /* zmm0 = [ fabsf(x1) >= 1e-4 ] ? f : 1.0 */ \
225 __ASM_EMIT("vxorps %%zmm0, %%zmm0, %%zmm0 %{%%k2%}") /* zmm0 = [ fabsf(x1) < t ] ? ([ fabsf(x1) >= 1e-4 ] ? f : 1.0) : 0.0 */
226
227 #define LANCZOS_GEN_X64_FUNC_X8 \
228 /* ymm8 = k */ \
229 /* ymm9 = p */ \
230 /* ymm10 = t */ \
231 /* ymm11 = a */ \
232 /* ymm12 = x1 */ \
233 /* ymm13 = d */ \
234 /* ymm14 = i */ \
235 /* ymm15 = step */ \
236 __ASM_EMIT("vmulps %%ymm11, %%ymm12, %%ymm4") /* ymm12 = x2 = x1*a */ \
237 __ASM_EMIT("vmulps %%ymm12, %%ymm4, %%ymm13") /* ymm13 = d = x1 * x2 */ \
238 __ASM_EMIT("vinsertf64x4 $1, %%ymm4, %%zmm12, %%zmm0") /* zmm0 = x1 ... x2 ... */ \
239 __ASM_EMIT("vaddps 0x00 + %[S2C], %%zmm0, %%zmm0") /* zmm0 = x1 + PI/2 ... x2 + PI/2 */ \
240 SINF_X_PLUS_PI_2_CORE_X16 /* ymm0 = sinf(x1), ymm4 = sinf(x2) */ \
241 __ASM_EMIT("vextractf64x4 $1, %%zmm0, %%ymm4") /* ymm4 = sinf(x2) */ \
242 __ASM_EMIT("vmulps %%ymm4, %%ymm0, %%ymm0") /* ymm0 = sinf(x1)*sinf(x2) */ \
243 __ASM_EMIT("vandps 0x00 + %[LC], %%ymm12, %%ymm12") /* ymm12 = fabsf(x1) */ \
244 __ASM_EMIT("vdivps %%ymm13, %%ymm0, %%ymm0") /* ymm0 = F = (sinf(x1)*sinf(x2)) / (x1 * x2) */ \
245 __ASM_EMIT("vcmpps $1, 0x40 + %[LC], %%ymm12, %%ymm2") /* ymm2 = [ fabsf(x1) < 1e-4 ] */ \
246 __ASM_EMIT("vcmpps $1, %%ymm10, %%ymm12, %%ymm1") /* ymm1 = [ fabsf(x1) < t ] */ \
247 __ASM_EMIT("vblendvps %%ymm2, 0x80 + %[LC], %%ymm0, %%ymm0") /* ymm0 = [ fabsf(x1) >= 1e-4 ] ? f : 1.0 */ \
248 __ASM_EMIT("vandps %%ymm1, %%ymm0, %%ymm0") /* ymm0 = [ fabsf(x1) < t ] ? ([ fabsf(x1) >= 1e-4 ] ? f : 1.0) : 0.0 */
249
250 #define LANCZOS_GEN_X64_FUNC_X4 \
251 /* xmm0 = x1 */ \
252 __ASM_EMIT("vmulps %%xmm11, %%xmm12, %%xmm4") /* xmm12 = x2 = x1*a */ \
253 __ASM_EMIT("vmulps %%xmm12, %%xmm4, %%xmm13") /* xmm13 = d = x1 * x2 */ \
254 __ASM_EMIT("vinsertf128 $1, %%xmm4, %%ymm12, %%ymm0") /* ymm0 = x1 x1 x1 x1 x2 x2 x2 x2 */ \
255 __ASM_EMIT("vaddps 0x00 + %[S2C], %%ymm0, %%ymm0") /* ymm0 = x1 + PI/2 ... x2 + PI/2 */ \
256 SINF_X_PLUS_PI_2_CORE_X8 /* ymm0 = sinf(x1) ... sinf(x2) */ \
257 __ASM_EMIT("vextractf128 $1, %%ymm0, %%xmm4") /* xmm4 = sif(x2) */ \
258 __ASM_EMIT("vmulps %%ymm4, %%ymm0, %%ymm0") /* ymm0 = sinf(x1)*sinf(x2) */ \
259 __ASM_EMIT("vandps 0x00 + %[LC], %%ymm12, %%ymm12") /* ymm12 = fabsf(x1) */ \
260 __ASM_EMIT("vdivps %%ymm13, %%ymm0, %%ymm0") /* ymm0 = F = (sinf(x1)*sinf(x2)) / (x1 * x2) */ \
261 __ASM_EMIT("vcmpps $1, 0x40 + %[LC], %%ymm12, %%ymm2") /* ymm2 = [ fabsf(x1) < 1e-4 ] */ \
262 __ASM_EMIT("vcmpps $1, %%ymm10, %%ymm12, %%ymm1") /* ymm1 = [ fabsf(x1) < t ] */ \
263 __ASM_EMIT("vblendvps %%ymm2, 0x80 + %[LC], %%ymm0, %%ymm0") /* ymm0 = [ fabsf(x1) >= 1e-4 ] ? f : 1.0 */ \
264 __ASM_EMIT("vandps %%ymm1, %%ymm0, %%ymm0") /* ymm0 = [ fabsf(x1) < t ] ? ([ fabsf(x1) >= 1e-4 ] ? f : 1.0) : 0.0 */
265
266 void x64_lanczos1(float *dst, float k, float p, float t, float a, size_t count)
267 {
268 ARCH_X86_64_ASM(
269 // Prepare
270 __ASM_EMIT("vbroadcastss %[k], %%zmm8") // zmm8 = k
271 __ASM_EMIT("vbroadcastss %[p], %%zmm9") // zmm9 = p
272 __ASM_EMIT("vbroadcastss %[t], %%zmm10") // zmm10 = t
273 __ASM_EMIT("vbroadcastss %[a], %%zmm11") // zmm11 = a
274 __ASM_EMIT("vmovaps 0x00 + %[LGEN], %%zmm14") // zmm14 = i = 0 1 2 3 4 5 6 7 ...
275 // x16 blocks
276 __ASM_EMIT("sub $16, %[count]")
277 __ASM_EMIT("jb 2f")
278 __ASM_EMIT("vmovaps 0xc0 + %[LGEN], %%zmm15") // zmm15 = step
279 __ASM_EMIT("1:")
280 __ASM_EMIT("vmulps %%zmm8, %%zmm14, %%zmm12") // zmm12 = k*i
281 __ASM_EMIT("vaddps %%zmm15, %%zmm14, %%zmm14") // zmm14 = i' = i + step
282 __ASM_EMIT("vsubps %%zmm9, %%zmm12, %%zmm12") // zmm12 = x1 = k*i - p
283 LANCZOS_GEN_X64_FUNC_X16
284 __ASM_EMIT("vmovups %%zmm0, 0x00(%[dst])")
285 __ASM_EMIT("add $0x40, %[dst]")
286 __ASM_EMIT("sub $16, %[count]")
287 __ASM_EMIT("jae 1b")
288 __ASM_EMIT("2:")
289 // x8 blocks
290 __ASM_EMIT("add $8, %[count]")
291 __ASM_EMIT("jl 4f")
292 __ASM_EMIT("vmulps %%ymm8, %%ymm14, %%ymm12") // ymm12 = k*i
293 __ASM_EMIT("vaddps 0x100 + %[LGEN], %%ymm14, %%ymm14") // ymm14 = i' = i + step
294 __ASM_EMIT("vsubps %%ymm9, %%ymm12, %%ymm12") // ymm12 = x1 = k*i - p
295 LANCZOS_GEN_X64_FUNC_X8
296 __ASM_EMIT("vmovups %%ymm0, 0x00(%[dst])")
297 __ASM_EMIT("add $0x20, %[dst]")
298 __ASM_EMIT("sub $8, %[count]")
299 __ASM_EMIT("4:")
300 // x4 block
301 __ASM_EMIT("add $4, %[count]")
302 __ASM_EMIT("jl 6f")
303 __ASM_EMIT("vmulps %%xmm8, %%xmm14, %%xmm12") // xmm12 = k*i
304 __ASM_EMIT("vaddps 0x120 + %[LGEN], %%xmm14, %%xmm14") // xmm14 = i' = i + step
305 __ASM_EMIT("vsubps %%xmm9, %%xmm12, %%xmm12") // xmm12 = x1 = k*i - p
306 LANCZOS_GEN_X64_FUNC_X4
307 __ASM_EMIT("vmovups %%xmm0, 0x00(%[dst])")
308 __ASM_EMIT("add $0x10, %[dst]")
309 __ASM_EMIT("sub $4, %[count]")
310 __ASM_EMIT("6:")
311 // Tail: 1x-3x block
312 __ASM_EMIT("add $4, %[count]")
313 __ASM_EMIT("jle 10f")
314 __ASM_EMIT("vmulps %%xmm8, %%xmm14, %%xmm12") // xmm12 = k*i
315 __ASM_EMIT("vsubps %%xmm9, %%xmm12, %%xmm12") // xmm12 = x1 = k*i - p
316 LANCZOS_GEN_X64_FUNC_X4
317 __ASM_EMIT("test $1, %[count]")
318 __ASM_EMIT("jz 8f")
319 __ASM_EMIT("vmovss %%xmm0, 0x00(%[dst])")
320 __ASM_EMIT("vshufps $0x39, %%xmm0, %%xmm0, %%xmm0")
321 __ASM_EMIT("add $4, %[dst]")
322 __ASM_EMIT("8:")
323 __ASM_EMIT("test $2, %[count]")
324 __ASM_EMIT("jz 10f")
325 __ASM_EMIT("vmovlps %%xmm0, 0x00(%[dst])")
326 // End
327 __ASM_EMIT("10:")
328
329 : [dst] "+r" (dst), [count] "+r" (count)
330 : [S2C] "o" (sinf_const),
331 [LGEN] "o" (kp_gen_const),
332 [LC] "o" (lanczos_const),
333 [k] "m" (k),
334 [p] "m" (p),
335 [a] "m" (a),
336 [t] "m" (t)
337 : "cc", "memory",
338 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
339 "%xmm4", "%xmm5", "%xmm6", "%xmm7",
340 "%xmm8", "%xmm9", "%xmm10", "%xmm11",
341 "%xmm12", "%xmm13", "%xmm14", "%xmm15"
342 );
343 }
344
345 } /* namespace avx512 */
346 } /* namespace lsp */
347
348
349
350
351 #endif /* PRIVATE_DSP_ARCH_X86_AVX512_PMATH_LANCZOS_H_ */