Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/x86/avx512/complex.h

raw
1 /*
2 * Copyright (C) 2023 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2023 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 9 сент. 2023 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_X86_AVX512_COMPLEX_H_
23 #define PRIVATE_DSP_ARCH_X86_AVX512_COMPLEX_H_
24
25 #ifndef PRIVATE_DSP_ARCH_X86_AVX512_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_X86_AVX512_IMPL */
28
29 namespace lsp
30 {
31 namespace avx512
32 {
33 #define COMPLEX_MUL3_CORE(DST, SRC1, SRC2) \
34 __ASM_EMIT ("xor %[off], %[off]") \
35 /* x16 blocks */ \
36 __ASM_EMIT32("subl $16, %[count]") \
37 __ASM_EMIT64("sub $16, %[count]") \
38 __ASM_EMIT ("jb 2f") \
39 __ASM_EMIT ("1:") \
40 __ASM_EMIT32("mov %[" SRC1 "_re], %[ptr_re]") \
41 __ASM_EMIT32("mov %[" SRC1 "_im], %[ptr_im]") \
42 __ASM_EMIT32("vmovups 0x00(%[ptr_re], %[off]), %%zmm4") /* zmm4 = ar */ \
43 __ASM_EMIT32("vmovups 0x00(%[ptr_im], %[off]), %%zmm5") /* zmm5 = ai */ \
44 __ASM_EMIT64("vmovups 0x00(%[" SRC1 "_re], %[off]), %%zmm4") /* zmm4 = ar */ \
45 __ASM_EMIT64("vmovups 0x00(%[" SRC1 "_im], %[off]), %%zmm5") /* zmm5 = ai */ \
46 __ASM_EMIT ("vmovups 0x00(%[" SRC2 "_re], %[off]), %%zmm0") /* zmm0 = br */ \
47 __ASM_EMIT ("vmovups 0x00(%[" SRC2 "_im], %[off]), %%zmm1") /* zmm1 = bi */ \
48 __ASM_EMIT ("vmulps %%zmm0, %%zmm5, %%zmm2") /* zmm2 = ai*br */ \
49 __ASM_EMIT ("vmulps %%zmm1, %%zmm5, %%zmm3") /* zmm3 = ai*bi */ \
50 __ASM_EMIT32("mov %[" DST "_re], %[ptr_re]") \
51 __ASM_EMIT32("mov %[" DST "_im], %[ptr_im]") \
52 __ASM_EMIT ("vfmsub132ps %%zmm4, %%zmm3, %%zmm0") /* ymm0 = ar*br - ai*bi = r */ \
53 __ASM_EMIT ("vfmadd132ps %%zmm4, %%zmm2, %%zmm1") /* ymm1 = ar*bi + ai*br = i */ \
54 __ASM_EMIT32("vmovups %%zmm0, 0x00(%[ptr_re], %[off])") \
55 __ASM_EMIT32("vmovups %%zmm1, 0x00(%[ptr_im], %[off])") \
56 __ASM_EMIT64("vmovups %%zmm0, 0x00(%[" DST "_re], %[off])") \
57 __ASM_EMIT64("vmovups %%zmm1, 0x00(%[" DST "_im], %[off])") \
58 __ASM_EMIT ("add $0x40, %[off]") \
59 __ASM_EMIT32("subl $16, %[count]") \
60 __ASM_EMIT64("sub $16, %[count]") \
61 __ASM_EMIT ("jae 1b") \
62 __ASM_EMIT ("2:") \
63 /* x8 blocks */ \
64 __ASM_EMIT32("addl $8, %[count]") \
65 __ASM_EMIT64("add $8, %[count]") \
66 __ASM_EMIT ("jl 4f") \
67 __ASM_EMIT32("mov %[" SRC1 "_re], %[ptr_re]") \
68 __ASM_EMIT32("mov %[" SRC1 "_im], %[ptr_im]") \
69 __ASM_EMIT32("vmovups 0x00(%[ptr_re], %[off]), %%ymm4") /* ymm4 = ar */ \
70 __ASM_EMIT32("vmovups 0x00(%[ptr_im], %[off]), %%ymm5") /* ymm5 = ai */ \
71 __ASM_EMIT64("vmovups 0x00(%[" SRC1 "_re], %[off]), %%ymm4") /* ymm4 = ar */ \
72 __ASM_EMIT64("vmovups 0x00(%[" SRC1 "_im], %[off]), %%ymm5") /* ymm5 = ai */ \
73 __ASM_EMIT ("vmovups 0x00(%[" SRC2 "_re], %[off]), %%ymm0") /* ymm0 = br */ \
74 __ASM_EMIT ("vmovups 0x00(%[" SRC2 "_im], %[off]), %%ymm1") /* ymm1 = bi */ \
75 __ASM_EMIT ("vmulps %%ymm0, %%ymm5, %%ymm2") /* ymm2 = ai*br */ \
76 __ASM_EMIT ("vmulps %%ymm1, %%ymm5, %%ymm3") /* ymm3 = ai*bi */ \
77 __ASM_EMIT32("mov %[" DST "_re], %[ptr_re]") \
78 __ASM_EMIT32("mov %[" DST "_im], %[ptr_im]") \
79 __ASM_EMIT ("vfmsub132ps %%ymm4, %%ymm3, %%ymm0") /* ymm0 = ar*br - ai*bi = r */ \
80 __ASM_EMIT ("vfmadd132ps %%ymm4, %%ymm2, %%ymm1") /* ymm1 = ar*bi + ai*br = i */ \
81 __ASM_EMIT32("vmovups %%ymm0, 0x00(%[ptr_re], %[off])") \
82 __ASM_EMIT32("vmovups %%ymm1, 0x00(%[ptr_im], %[off])") \
83 __ASM_EMIT64("vmovups %%ymm0, 0x00(%[" DST "_re], %[off])") \
84 __ASM_EMIT64("vmovups %%ymm1, 0x00(%[" DST "_im], %[off])") \
85 __ASM_EMIT ("add $0x20, %[off]") \
86 __ASM_EMIT32("subl $8, %[count]") \
87 __ASM_EMIT64("sub $8, %[count]") \
88 __ASM_EMIT ("jae 1b") \
89 __ASM_EMIT ("4:") \
90 /* 4x block */ \
91 __ASM_EMIT32("addl $4, %[count]") \
92 __ASM_EMIT64("add $4, %[count]") \
93 __ASM_EMIT ("jl 6f") \
94 __ASM_EMIT32("mov %[" SRC1 "_re], %[ptr_re]") \
95 __ASM_EMIT32("mov %[" SRC1 "_im], %[ptr_im]") \
96 __ASM_EMIT32("vmovups 0x00(%[ptr_re], %[off]), %%xmm4") /* xmm4 = ar */ \
97 __ASM_EMIT32("vmovups 0x00(%[ptr_im], %[off]), %%xmm5") /* xmm5 = ai */ \
98 __ASM_EMIT64("vmovups 0x00(%[" SRC1 "_re], %[off]), %%xmm4") /* xmm4 = ar */ \
99 __ASM_EMIT64("vmovups 0x00(%[" SRC1 "_im], %[off]), %%xmm5") /* xmm5 = ai */ \
100 __ASM_EMIT ("vmovups 0x00(%[" SRC2 "_re], %[off]), %%xmm0") /* xmm0 = br */ \
101 __ASM_EMIT ("vmovups 0x00(%[" SRC2 "_im], %[off]), %%xmm1") /* xmm1 = bi */ \
102 __ASM_EMIT ("vmulps %%xmm0, %%xmm5, %%xmm2") /* xmm2 = ai*br */ \
103 __ASM_EMIT ("vmulps %%xmm1, %%xmm5, %%xmm3") /* xmm3 = ai*bi */ \
104 __ASM_EMIT32("mov %[" DST "_re], %[ptr_re]") \
105 __ASM_EMIT32("mov %[" DST "_im], %[ptr_im]") \
106 __ASM_EMIT ("vfmsub132ps %%xmm4, %%xmm3, %%xmm0") /* xmm0 = ar*br - ai*bi = r */ \
107 __ASM_EMIT ("vfmadd132ps %%xmm4, %%xmm2, %%xmm1") /* xmm1 = ar*bi + ai*br = i */ \
108 __ASM_EMIT32("vmovups %%xmm0, 0x00(%[ptr_re], %[off])") \
109 __ASM_EMIT32("vmovups %%xmm1, 0x00(%[ptr_im], %[off])") \
110 __ASM_EMIT64("vmovups %%xmm0, 0x00(%[" DST "_re], %[off])") \
111 __ASM_EMIT64("vmovups %%xmm1, 0x00(%[" DST "_im], %[off])") \
112 __ASM_EMIT ("add $0x10, %[off]") \
113 __ASM_EMIT32("subl $4, %[count]") \
114 __ASM_EMIT64("sub $4, %[count]") \
115 __ASM_EMIT ("6:") \
116 /* 1x blocks */ \
117 __ASM_EMIT32("addl $3, %[count]") \
118 __ASM_EMIT64("add $3, %[count]") \
119 __ASM_EMIT ("jl 8f") \
120 __ASM_EMIT ("7:") \
121 __ASM_EMIT32("mov %[" SRC1 "_re], %[ptr_re]") \
122 __ASM_EMIT32("mov %[" SRC1 "_im], %[ptr_im]") \
123 __ASM_EMIT32("vmovss 0x00(%[ptr_re], %[off]), %%xmm4") /* xmm4 = ar */ \
124 __ASM_EMIT32("vmovss 0x00(%[ptr_im], %[off]), %%xmm5") /* xmm5 = ai */ \
125 __ASM_EMIT64("vmovss 0x00(%[" SRC1 "_re], %[off]), %%xmm4") /* xmm4 = ar */ \
126 __ASM_EMIT64("vmovss 0x00(%[" SRC1 "_im], %[off]), %%xmm5") /* xmm5 = ai */ \
127 __ASM_EMIT ("vmovss 0x00(%[" SRC2 "_re], %[off]), %%xmm0") /* xmm0 = br */ \
128 __ASM_EMIT ("vmovss 0x00(%[" SRC2 "_im], %[off]), %%xmm1") /* xmm1 = bi */ \
129 __ASM_EMIT ("vmulps %%xmm0, %%xmm5, %%xmm2") /* xmm2 = ai*br */ \
130 __ASM_EMIT ("vmulps %%xmm1, %%xmm5, %%xmm3") /* xmm3 = ai*bi */ \
131 __ASM_EMIT32("mov %[" DST "_re], %[ptr_re]") \
132 __ASM_EMIT32("mov %[" DST "_im], %[ptr_im]") \
133 __ASM_EMIT ("vfmsub132ss %%xmm4, %%xmm3, %%xmm0") /* xmm0 = ar*br - ai*bi = r */ \
134 __ASM_EMIT ("vfmadd132ss %%xmm4, %%xmm2, %%xmm1") /* xmm1 = ar*bi + ai*br = i */ \
135 __ASM_EMIT32("vmovss %%xmm0, 0x00(%[ptr_re], %[off])") \
136 __ASM_EMIT32("vmovss %%xmm1, 0x00(%[ptr_im], %[off])") \
137 __ASM_EMIT64("vmovss %%xmm0, 0x00(%[" DST "_re], %[off])") \
138 __ASM_EMIT64("vmovss %%xmm1, 0x00(%[" DST "_im], %[off])") \
139 __ASM_EMIT ("add $0x04, %[off]") \
140 __ASM_EMIT32("decl %[count]") \
141 __ASM_EMIT64("dec %[count]") \
142 __ASM_EMIT ("jge 7b") \
143 __ASM_EMIT ("8:")
144
145 void complex_mul3(float *dst_re, float *dst_im, const float *src1_re, const float *src1_im, const float *src2_re, const float *src2_im, size_t count)
146 {
147 IF_ARCH_I386(float *ptr_re, *ptr_im);
148 IF_ARCH_X86(size_t off);
149 ARCH_X86_ASM
150 (
151 COMPLEX_MUL3_CORE("dst", "src1", "src2")
152 : __IF_32(
153 [ptr_re] "=&r" (ptr_re), [ptr_im] "=&r" (ptr_im),
154 )
155 [count] X86_PGREG (count),
156 [off] "=&r" (off)
157 : [dst_re] X86_GREG (dst_re), [dst_im] X86_GREG (dst_im),
158 [src1_re] X86_GREG (src1_re), [src1_im] X86_GREG (src1_im),
159 [src2_re] "r" (src2_re), [src2_im] "r" (src2_im)
160 : "cc", "memory",
161 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
162 "%xmm4", "%xmm5"
163 );
164 }
165
166 #define COMPLEX_MUL2_CORE(DST, SRC, SEL) \
167 __ASM_EMIT ("xor %[off], %[off]") \
168 /* x16 blocks */ \
169 __ASM_EMIT32("subl $16, %[count]") \
170 __ASM_EMIT64("sub $16, %[count]") \
171 __ASM_EMIT ("jb 2f") \
172 __ASM_EMIT ("1:") \
173 __ASM_EMIT ("vmovups 0x00(%[" DST "_re], %[off]), %%zmm4") /* zmm4 = ar */ \
174 __ASM_EMIT ("vmovups 0x00(%[" DST "_im], %[off]), %%zmm5") /* zmm5 = ai */ \
175 __ASM_EMIT ("vmovups 0x00(%[" SRC "_re], %[off]), %%zmm0") /* zmm0 = br */ \
176 __ASM_EMIT ("vmovups 0x00(%[" SRC "_im], %[off]), %%zmm1") /* zmm1 = bi */ \
177 __ASM_EMIT ("vmulps %%zmm0, %%zmm5, %%zmm2") /* ymm2 = ai*br */ \
178 __ASM_EMIT ("vmulps %%zmm1, %%zmm5, %%zmm3") /* ymm3 = ai*bi */ \
179 __ASM_EMIT ("vfmsub132ps %%zmm4, %%zmm3, %%zmm0") /* ymm0 = ar*br - ai*bi = r */ \
180 __ASM_EMIT ("vfmadd132ps %%zmm4, %%zmm2, %%zmm1") /* ymm1 = ar*bi + ai*br = i */ \
181 __ASM_EMIT ("vmovups %%zmm0, 0x00(%[" DST "_re], %[off])") \
182 __ASM_EMIT ("vmovups %%zmm1, 0x00(%[" DST "_im], %[off])") \
183 __ASM_EMIT ("add $0x40, %[off]") \
184 __ASM_EMIT32("subl $16, %[count]") \
185 __ASM_EMIT64("sub $16, %[count]") \
186 __ASM_EMIT ("jae 1b") \
187 __ASM_EMIT ("2:") \
188 /* x8 blocks */ \
189 __ASM_EMIT32("addl $8, %[count]") \
190 __ASM_EMIT64("add $8, %[count]") \
191 __ASM_EMIT ("jl 4f") \
192 __ASM_EMIT ("vmovups 0x00(%[" DST "_re], %[off]), %%ymm4") /* ymm4 = ar */ \
193 __ASM_EMIT ("vmovups 0x00(%[" DST "_im], %[off]), %%ymm5") /* ymm5 = ai */ \
194 __ASM_EMIT ("vmovups 0x00(%[" SRC "_re], %[off]), %%ymm0") /* ymm0 = br */ \
195 __ASM_EMIT ("vmovups 0x00(%[" SRC "_im], %[off]), %%ymm1") /* ymm1 = bi */ \
196 __ASM_EMIT ("vmulps %%ymm0, %%ymm5, %%ymm2") /* ymm2 = ai*br */ \
197 __ASM_EMIT ("vmulps %%ymm1, %%ymm5, %%ymm3") /* ymm3 = ai*bi */ \
198 __ASM_EMIT ("vfmsub132ps %%ymm4, %%ymm3, %%ymm0") /* ymm0 = ar*br - ai*bi = r */ \
199 __ASM_EMIT ("vfmadd132ps %%ymm4, %%ymm2, %%ymm1") /* ymm1 = ar*bi + ai*br = i */ \
200 __ASM_EMIT ("vmovups %%ymm0, 0x00(%[" DST "_re], %[off])") \
201 __ASM_EMIT ("vmovups %%ymm1, 0x00(%[" DST "_im], %[off])") \
202 __ASM_EMIT ("add $0x20, %[off]") \
203 __ASM_EMIT32("subl $8, %[count]") \
204 __ASM_EMIT64("sub $8, %[count]") \
205 __ASM_EMIT ("jae 1b") \
206 __ASM_EMIT ("4:") \
207 /* 4x block */ \
208 __ASM_EMIT32("addl $4, %[count]") \
209 __ASM_EMIT64("add $4, %[count]") \
210 __ASM_EMIT ("jl 6f") \
211 __ASM_EMIT ("vmovups 0x00(%[" DST "_re], %[off]), %%xmm4") /* xmm4 = ar */ \
212 __ASM_EMIT ("vmovups 0x00(%[" DST "_im], %[off]), %%xmm5") /* xmm5 = ai */ \
213 __ASM_EMIT ("vmovups 0x00(%[" SRC "_re], %[off]), %%xmm0") /* xmm0 = br */ \
214 __ASM_EMIT ("vmovups 0x00(%[" SRC "_im], %[off]), %%xmm1") /* xmm1 = bi */ \
215 __ASM_EMIT ("vmulps %%xmm0, %%xmm5, %%xmm2") /* xmm2 = ai*br */ \
216 __ASM_EMIT ("vmulps %%xmm1, %%xmm5, %%xmm3") /* xmm3 = ai*bi */ \
217 __ASM_EMIT ("vfmsub132ps %%xmm4, %%xmm3, %%xmm0") /* xmm0 = ar*br - ai*bi = r */ \
218 __ASM_EMIT ("vfmadd132ps %%xmm4, %%xmm2, %%xmm1") /* xmm1 = ar*bi + ai*br = i */ \
219 __ASM_EMIT ("vmovups %%xmm0, 0x00(%[" DST "_re], %[off])") \
220 __ASM_EMIT ("vmovups %%xmm1, 0x00(%[" DST "_im], %[off])") \
221 __ASM_EMIT ("add $0x10, %[off]") \
222 __ASM_EMIT32("subl $4, %[count]") \
223 __ASM_EMIT64("sub $4, %[count]") \
224 __ASM_EMIT ("6:") \
225 /* 1x blocks */ \
226 __ASM_EMIT32("addl $3, %[count]") \
227 __ASM_EMIT64("add $3, %[count]") \
228 __ASM_EMIT ("jl 8f") \
229 __ASM_EMIT ("7:") \
230 __ASM_EMIT ("vmovss 0x00(%[" DST "_re], %[off]), %%xmm4") /* xmm4 = ar */ \
231 __ASM_EMIT ("vmovss 0x00(%[" DST "_im], %[off]), %%xmm5") /* xmm5 = ai */ \
232 __ASM_EMIT ("vmovss 0x00(%[" SRC "_re], %[off]), %%xmm0") /* xmm0 = br */ \
233 __ASM_EMIT ("vmovss 0x00(%[" SRC "_im], %[off]), %%xmm1") /* xmm1 = bi */ \
234 __ASM_EMIT ("vmulps %%xmm0, %%xmm5, %%xmm2") /* xmm2 = ai*br */ \
235 __ASM_EMIT ("vmulps %%xmm1, %%xmm5, %%xmm3") /* xmm3 = ai*bi */ \
236 __ASM_EMIT ("vfmsub132ss %%xmm4, %%xmm3, %%xmm0") /* xmm0 = ar*br - ai*bi = r */ \
237 __ASM_EMIT ("vfmadd132ss %%xmm4, %%xmm2, %%xmm1") /* xmm1 = ar*bi + ai*br = i */ \
238 __ASM_EMIT ("vmovss %%xmm0, 0x00(%[" DST "_re], %[off])") \
239 __ASM_EMIT ("vmovss %%xmm1, 0x00(%[" DST "_im], %[off])") \
240 __ASM_EMIT ("add $0x04, %[off]") \
241 __ASM_EMIT32("decl %[count]") \
242 __ASM_EMIT64("dec %[count]") \
243 __ASM_EMIT ("jge 7b") \
244 __ASM_EMIT ("8:")
245
246 void complex_mul2(float *dst_re, float *dst_im, const float *src_re, const float *src_im, size_t count)
247 {
248 IF_ARCH_X86(size_t off);
249 ARCH_X86_ASM
250 (
251 COMPLEX_MUL2_CORE("dst", "src", FMA_OFF)
252 : [count] X86_PGREG (count),
253 [off] "=&r" (off)
254 : [dst_re] "r" (dst_re), [dst_im] "r" (dst_im),
255 [src_re] "r" (src_re), [src_im] "r" (src_im)
256 : "cc", "memory",
257 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
258 "%xmm4", "%xmm5"
259 );
260 }
261
262 #define COMPLEX_MOD_CORE(DST, SRC) \
263 __ASM_EMIT ("xor %[off], %[off]") \
264 /* x64 blocks */ \
265 __ASM_EMIT ("sub $64, %[count]") \
266 __ASM_EMIT ("jb 2f") \
267 __ASM_EMIT ("1:") \
268 __ASM_EMIT ("vmovups 0x00(%[" SRC "_re], %[off]), %%zmm0") /* zmm0 = r */ \
269 __ASM_EMIT ("vmovups 0x40(%[" SRC "_re], %[off]), %%zmm1") \
270 __ASM_EMIT ("vmovups 0x80(%[" SRC "_re], %[off]), %%zmm2") \
271 __ASM_EMIT ("vmovups 0xc0(%[" SRC "_re], %[off]), %%zmm3") \
272 __ASM_EMIT ("vmovups 0x00(%[" SRC "_im], %[off]), %%zmm4") /* zmm4 = i */ \
273 __ASM_EMIT ("vmovups 0x40(%[" SRC "_im], %[off]), %%zmm5") \
274 __ASM_EMIT ("vmovups 0x80(%[" SRC "_im], %[off]), %%zmm6") \
275 __ASM_EMIT ("vmovups 0xc0(%[" SRC "_im], %[off]), %%zmm7") \
276 __ASM_EMIT ("vmulps %%zmm0, %%zmm0, %%zmm0") /* zmm0 = r*r */ \
277 __ASM_EMIT ("vmulps %%zmm1, %%zmm1, %%zmm1") \
278 __ASM_EMIT ("vmulps %%zmm2, %%zmm2, %%zmm2") \
279 __ASM_EMIT ("vmulps %%zmm3, %%zmm3, %%zmm3") \
280 __ASM_EMIT ("vfmadd231ps %%zmm4, %%zmm4, %%zmm0") /* zmm0 = r*r + i*i */ \
281 __ASM_EMIT ("vfmadd231ps %%zmm5, %%zmm5, %%zmm1") \
282 __ASM_EMIT ("vfmadd231ps %%zmm6, %%zmm6, %%zmm2") \
283 __ASM_EMIT ("vfmadd231ps %%zmm7, %%zmm7, %%zmm3") \
284 __ASM_EMIT ("vsqrtps %%zmm0, %%zmm0") /* zmm0 = sqrt(r*r + i*i) */ \
285 __ASM_EMIT ("vsqrtps %%zmm1, %%zmm1") \
286 __ASM_EMIT ("vsqrtps %%zmm2, %%zmm2") \
287 __ASM_EMIT ("vsqrtps %%zmm3, %%zmm3") \
288 __ASM_EMIT ("vmovups %%zmm0, 0x00(%[" DST "], %[off])") \
289 __ASM_EMIT ("vmovups %%zmm1, 0x40(%[" DST "], %[off])") \
290 __ASM_EMIT ("vmovups %%zmm2, 0x80(%[" DST "], %[off])") \
291 __ASM_EMIT ("vmovups %%zmm3, 0xc0(%[" DST "], %[off])") \
292 __ASM_EMIT ("add $0x100, %[off]") \
293 __ASM_EMIT ("sub $64, %[count]") \
294 __ASM_EMIT ("jae 1b") \
295 __ASM_EMIT ("2:") \
296 /* x32 block */ \
297 __ASM_EMIT ("add $32, %[count]") \
298 __ASM_EMIT ("jl 4f") \
299 __ASM_EMIT ("vmovups 0x00(%[" SRC "_re], %[off]), %%zmm0") /* zmm0 = r */ \
300 __ASM_EMIT ("vmovups 0x40(%[" SRC "_re], %[off]), %%zmm1") \
301 __ASM_EMIT ("vmovups 0x00(%[" SRC "_im], %[off]), %%zmm4") /* zmm4 = i */ \
302 __ASM_EMIT ("vmovups 0x40(%[" SRC "_im], %[off]), %%zmm5") \
303 __ASM_EMIT ("vmulps %%zmm0, %%zmm0, %%zmm0") /* zmm0 = r*r */ \
304 __ASM_EMIT ("vmulps %%zmm1, %%zmm1, %%zmm1") \
305 __ASM_EMIT ("vfmadd231ps %%zmm4, %%zmm4, %%zmm0") /* zmm0 = r*r + i*i */ \
306 __ASM_EMIT ("vfmadd231ps %%zmm5, %%zmm5, %%zmm1") \
307 __ASM_EMIT ("vsqrtps %%zmm0, %%zmm0") /* zmm0 = sqrt(r*r + i*i) */ \
308 __ASM_EMIT ("vsqrtps %%zmm1, %%zmm1") \
309 __ASM_EMIT ("vmovups %%zmm0, 0x00(%[" DST "], %[off])") \
310 __ASM_EMIT ("vmovups %%zmm1, 0x40(%[" DST "], %[off])") \
311 __ASM_EMIT ("sub $32, %[count]") \
312 __ASM_EMIT ("add $0x80, %[off]") \
313 __ASM_EMIT ("4:") \
314 /* x16 blocks */ \
315 __ASM_EMIT ("add $16, %[count]") \
316 __ASM_EMIT ("jl 6f") \
317 __ASM_EMIT ("vmovups 0x00(%[" SRC "_re], %[off]), %%ymm0") /* ymm0 = r */ \
318 __ASM_EMIT ("vmovups 0x20(%[" SRC "_re], %[off]), %%ymm1") \
319 __ASM_EMIT ("vmovups 0x00(%[" SRC "_im], %[off]), %%ymm2") /* ymm2 = i */ \
320 __ASM_EMIT ("vmovups 0x20(%[" SRC "_im], %[off]), %%ymm3") \
321 __ASM_EMIT ("vmulps %%ymm0, %%ymm0, %%ymm0") /* ymm0 = r*r */ \
322 __ASM_EMIT ("vmulps %%ymm1, %%ymm1, %%ymm1") \
323 __ASM_EMIT ("vfmadd231ps %%ymm2, %%ymm2, %%ymm0") /* ymm0 = r*r + i*i */ \
324 __ASM_EMIT ("vfmadd231ps %%ymm3, %%ymm3, %%ymm1") \
325 __ASM_EMIT ("vsqrtps %%ymm0, %%ymm0") /* ymm0 = sqrt(r*r + i*i) */ \
326 __ASM_EMIT ("vsqrtps %%ymm1, %%ymm1") \
327 __ASM_EMIT ("vmovups %%ymm0, 0x00(%[" DST "], %[off])") \
328 __ASM_EMIT ("vmovups %%ymm1, 0x20(%[" DST "], %[off])") \
329 __ASM_EMIT ("sub $16, %[count]") \
330 __ASM_EMIT ("add $0x40, %[off]") \
331 __ASM_EMIT ("6:") \
332 /* x8 block */ \
333 __ASM_EMIT ("add $8, %[count]") \
334 __ASM_EMIT ("jl 8f") \
335 __ASM_EMIT ("vmovups 0x00(%[" SRC "_re], %[off]), %%ymm0") /* ymm0 = r */ \
336 __ASM_EMIT ("vmovups 0x00(%[" SRC "_im], %[off]), %%ymm2") /* ymm2 = i */ \
337 __ASM_EMIT ("vmulps %%ymm0, %%ymm0, %%ymm0") /* ymm0 = r*r */ \
338 __ASM_EMIT ("vfmadd231ps %%ymm2, %%ymm2, %%ymm0") /* ymm0 = r*r + i*i */ \
339 __ASM_EMIT ("vsqrtps %%ymm0, %%ymm0") /* ymm0 = sqrt(r*r + i*i) */ \
340 __ASM_EMIT ("vmovups %%ymm0, 0x00(%[" DST "], %[off])") \
341 __ASM_EMIT ("add $0x20, %[off]") \
342 __ASM_EMIT ("sub $8, %[count]") \
343 __ASM_EMIT ("8:") \
344 /* x4 block */ \
345 __ASM_EMIT ("add $4, %[count]") \
346 __ASM_EMIT ("jl 10f") \
347 __ASM_EMIT ("vmovups 0x00(%[" SRC "_re], %[off]), %%xmm0") /* xmm0 = r */ \
348 __ASM_EMIT ("vmovups 0x00(%[" SRC "_im], %[off]), %%xmm2") /* xmm2 = i */ \
349 __ASM_EMIT ("vmulps %%xmm0, %%xmm0, %%xmm0") /* xmm0 = r*r */ \
350 __ASM_EMIT ("vfmadd231ps %%xmm2, %%xmm2, %%xmm0") /* xmm0 = r*r + i*i */ \
351 __ASM_EMIT ("vsqrtps %%xmm0, %%xmm0") /* xmm0 = sqrt(r*r + i*i) */ \
352 __ASM_EMIT ("vmovups %%xmm0, 0x00(%[" DST "], %[off])") \
353 __ASM_EMIT ("add $0x10, %[off]") \
354 __ASM_EMIT ("sub $4, %[count]") \
355 __ASM_EMIT ("10:") \
356 /* x1 block */ \
357 __ASM_EMIT ("add $3, %[count]") \
358 __ASM_EMIT ("jl 12f") \
359 __ASM_EMIT ("11:") \
360 __ASM_EMIT ("vmovss 0x00(%[" SRC "_re], %[off]), %%xmm0") /* xmm0 = r */ \
361 __ASM_EMIT ("vmovss 0x00(%[" SRC "_im], %[off]), %%xmm2") /* xmm2 = i */ \
362 __ASM_EMIT ("vmulss %%xmm0, %%xmm0, %%xmm0") /* xmm0 = r*r */ \
363 __ASM_EMIT ("vfmadd231ps %%xmm2, %%xmm2, %%xmm0") /* xmm0 = r*r + i*i */ \
364 __ASM_EMIT ("vsqrtss %%xmm0, %%xmm0, %%xmm0") /* xmm0 = sqrt(r*r + i*i) */ \
365 __ASM_EMIT ("vmovss %%xmm0, 0x00(%[" DST "], %[off])") \
366 __ASM_EMIT ("add $0x04, %[off]") \
367 __ASM_EMIT ("dec %[count]") \
368 __ASM_EMIT ("jge 11b") \
369 __ASM_EMIT ("12:") \
370
371 void complex_mod(float *dst, const float *src_re, const float *src_im, size_t count)
372 {
373 IF_ARCH_X86(size_t off);
374 ARCH_X86_ASM
375 (
376 COMPLEX_MOD_CORE("dst", "src")
377 : [count] "+r" (count), [off] "=&r" (off)
378 : [dst] "r" (dst), [src_re] "r" (src_re), [src_im] "r" (src_im)
379 : "cc", "memory",
380 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
381 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
382 );
383 }
384
385 IF_ARCH_X86(
386 static uint32_t complex_div_const[] __lsp_aligned64 =
387 {
388 LSP_DSP_VEC16(0x3f800000), // 1.0
389 LSP_DSP_VEC16(0x80000000) // signbit
390 };
391 )
392
393 #define COMPLEX_DIV3_CORE(DSTR, DSTI, TR, TI, BR, BI) \
394 __ASM_EMIT ("xor %[off], %[off]") \
395 __ASM_EMIT ("vmovaps 0x00 + %[CC], %%zmm6") /* zmm6 = 1 */ \
396 /* x16 blocks */ \
397 __ASM_EMIT32("subl $16, %[count]") \
398 __ASM_EMIT64("sub $16, %[count]") \
399 __ASM_EMIT ("jb 2f") \
400 __ASM_EMIT ("1:") \
401 __ASM_EMIT ("vmovups 0x00(%[" BR "], %[off]), %%zmm2") /* zmm2 = br */ \
402 __ASM_EMIT ("vmovups 0x00(%[" BI "], %[off]), %%zmm3") /* zmm3 = bi */ \
403 __ASM_EMIT32("vmovups 0x00(%[ptr_re], %[off]), %%zmm0") /* zmm0 = tr */ \
404 __ASM_EMIT32("vmovups 0x00(%[ptr_re], %[off]), %%zmm1") /* zmm1 = ti */ \
405 __ASM_EMIT64("vmovups 0x00(%[" TR "], %[off]), %%zmm0") /* zmm0 = tr */ \
406 __ASM_EMIT64("vmovups 0x00(%[" TI "], %[off]), %%zmm1") /* zmm1 = ti */ \
407 __ASM_EMIT32("mov %[" DSTR "], %[ptr_re]") \
408 __ASM_EMIT32("mov %[" DSTI "], %[ptr_im]") \
409 __ASM_EMIT ("vmulps %%zmm2, %%zmm2, %%zmm7") /* zmm7 = br*br */ \
410 __ASM_EMIT ("vmulps %%zmm3, %%zmm1, %%zmm4") /* zmm4 = ti*bi */ \
411 __ASM_EMIT ("vmulps %%zmm3, %%zmm0, %%zmm5") /* zmm5 = tr*bi */ \
412 __ASM_EMIT ("vfmadd231ps %%zmm3, %%zmm3, %%zmm7") /* zmm7 = M = br*br + bi*bi */ \
413 __ASM_EMIT ("vmovaps 0x00 + %[CC], %%zmm6") /* zmm6 = 1 */ \
414 __ASM_EMIT ("vdivps %%zmm7, %%zmm6, %%zmm7") /* zmm7 = R = 1/M */ \
415 __ASM_EMIT ("vfmadd132ps %%zmm2, %%zmm4, %%zmm0") /* zmm0 = r = tr*br + ti*bi */ \
416 __ASM_EMIT ("vfmsub132ps %%zmm2, %%zmm5, %%zmm1") /* zmm1 = i = tr*br - tr*bi */ \
417 __ASM_EMIT ("vmulps %%zmm7, %%zmm0, %%zmm0") /* zmm0 = r/R */ \
418 __ASM_EMIT ("vmulps %%zmm7, %%zmm1, %%zmm1") /* zmm1 = i/R */ \
419 __ASM_EMIT32("vmovups %%zmm0, 0x00(%[ptr_re], %[off])") \
420 __ASM_EMIT32("vmovups %%zmm1, 0x00(%[ptr_re], %[off])") \
421 __ASM_EMIT64("vmovups %%zmm0, 0x00(%[" DSTR "], %[off])") \
422 __ASM_EMIT64("vmovups %%zmm1, 0x00(%[" DSTI "], %[off])") \
423 __ASM_EMIT ("add $0x40, %[off]") \
424 __ASM_EMIT32("subl $16, %[count]") \
425 __ASM_EMIT64("sub $16, %[count]") \
426 __ASM_EMIT ("jae 1b") \
427 __ASM_EMIT ("2:") \
428 /* x8 blocks */ \
429 __ASM_EMIT32("addl $8, %[count]") \
430 __ASM_EMIT64("add $8, %[count]") \
431 __ASM_EMIT ("jl 4f") \
432 __ASM_EMIT32("mov %[" TR "], %[ptr_re]") \
433 __ASM_EMIT32("mov %[" TI "], %[ptr_im]") \
434 __ASM_EMIT ("vmovups 0x00(%[" BR "], %[off]), %%ymm2") /* ymm2 = br */ \
435 __ASM_EMIT ("vmovups 0x00(%[" BI "], %[off]), %%ymm3") /* ymm3 = bi */ \
436 __ASM_EMIT32("vmovups 0x00(%[ptr_re], %[off]), %%ymm0") /* ymm0 = tr */ \
437 __ASM_EMIT32("vmovups 0x00(%[ptr_re], %[off]), %%ymm1") /* ymm1 = ti */ \
438 __ASM_EMIT64("vmovups 0x00(%[" TR "], %[off]), %%ymm0") /* ymm0 = tr */ \
439 __ASM_EMIT64("vmovups 0x00(%[" TI "], %[off]), %%ymm1") /* ymm1 = ti */ \
440 __ASM_EMIT32("mov %[" DSTR "], %[ptr_re]") \
441 __ASM_EMIT32("mov %[" DSTI "], %[ptr_im]") \
442 __ASM_EMIT ("vmulps %%ymm2, %%ymm2, %%ymm7") /* ymm7 = br*br */ \
443 __ASM_EMIT ("vmulps %%ymm3, %%ymm1, %%ymm4") /* ymm4 = ti*bi */ \
444 __ASM_EMIT ("vmulps %%ymm3, %%ymm0, %%ymm5") /* ymm5 = tr*bi */ \
445 __ASM_EMIT ("vfmadd231ps %%ymm3, %%ymm3, %%ymm7") /* ymm7 = M = br*br + bi*bi */ \
446 __ASM_EMIT ("vmovaps 0x00 + %[CC], %%ymm6") /* ymm6 = 1 */ \
447 __ASM_EMIT ("vdivps %%ymm7, %%ymm6, %%ymm7") /* ymm7 = R = 1/M */ \
448 __ASM_EMIT ("vfmadd132ps %%ymm2, %%ymm4, %%ymm0") /* ymm0 = r = tr*br + ti*bi */ \
449 __ASM_EMIT ("vfmsub132ps %%ymm2, %%ymm5, %%ymm1") /* ymm1 = i = tr*br - tr*bi */ \
450 __ASM_EMIT ("vmulps %%ymm7, %%ymm0, %%ymm0") /* ymm0 = r/R */ \
451 __ASM_EMIT ("vmulps %%ymm7, %%ymm1, %%ymm1") /* ymm1 = i/R */ \
452 __ASM_EMIT32("vmovups %%ymm0, 0x00(%[ptr_re], %[off])") \
453 __ASM_EMIT32("vmovups %%ymm1, 0x00(%[ptr_re], %[off])") \
454 __ASM_EMIT64("vmovups %%ymm0, 0x00(%[" DSTR "], %[off])") \
455 __ASM_EMIT64("vmovups %%ymm1, 0x00(%[" DSTI "], %[off])") \
456 __ASM_EMIT32("subl $8, %[count]") \
457 __ASM_EMIT64("sub $8, %[count]") \
458 __ASM_EMIT ("add $0x20, %[off]") \
459 __ASM_EMIT ("4:") \
460 /* 4x block */ \
461 __ASM_EMIT32("addl $4, %[count]") \
462 __ASM_EMIT64("add $4, %[count]") \
463 __ASM_EMIT ("jl 6f") \
464 __ASM_EMIT32("mov %[" TR "], %[ptr_re]") \
465 __ASM_EMIT32("mov %[" TI "], %[ptr_im]") \
466 __ASM_EMIT ("vmovups 0x00(%[" BR "], %[off]), %%xmm2") /* xmm2 = br */ \
467 __ASM_EMIT ("vmovups 0x00(%[" BI "], %[off]), %%xmm3") /* xmm3 = bi */ \
468 __ASM_EMIT32("vmovups 0x00(%[ptr_re], %[off]), %%xmm0") /* xmm0 = tr */ \
469 __ASM_EMIT32("vmovups 0x00(%[ptr_re], %[off]), %%xmm1") /* xmm1 = ti */ \
470 __ASM_EMIT64("vmovups 0x00(%[" TR "], %[off]), %%xmm0") /* xmm0 = tr */ \
471 __ASM_EMIT64("vmovups 0x00(%[" TI "], %[off]), %%xmm1") /* xmm1 = ti */ \
472 __ASM_EMIT32("mov %[" DSTR "], %[ptr_re]") \
473 __ASM_EMIT32("mov %[" DSTI "], %[ptr_im]") \
474 __ASM_EMIT ("vmulps %%xmm2, %%xmm2, %%xmm7") /* xmm7 = br*br */ \
475 __ASM_EMIT ("vmulps %%xmm3, %%xmm1, %%xmm4") /* xmm4 = ti*bi */ \
476 __ASM_EMIT ("vmulps %%xmm3, %%xmm0, %%xmm5") /* xmm5 = tr*bi */ \
477 __ASM_EMIT ("vfmadd231ps %%xmm3, %%xmm3, %%xmm7") /* xmm7 = M = br*br + bi*bi */ \
478 __ASM_EMIT ("vmovaps 0x00 + %[CC], %%xmm6") /* xmm6 = 1 */ \
479 __ASM_EMIT ("vdivps %%xmm7, %%xmm6, %%xmm7") /* xmm7 = R = 1/M */ \
480 __ASM_EMIT ("vfmadd132ps %%xmm2, %%xmm4, %%xmm0") /* xmm0 = r = tr*br + ti*bi */ \
481 __ASM_EMIT ("vfmsub132ps %%xmm2, %%xmm5, %%xmm1") /* xmm1 = i = tr*br - tr*bi */ \
482 __ASM_EMIT ("vmulps %%xmm7, %%xmm0, %%xmm0") /* xmm0 = r/R */ \
483 __ASM_EMIT ("vmulps %%xmm7, %%xmm1, %%xmm1") /* xmm1 = i/R */ \
484 __ASM_EMIT32("vmovups %%xmm0, 0x00(%[ptr_re], %[off])") \
485 __ASM_EMIT32("vmovups %%xmm1, 0x00(%[ptr_re], %[off])") \
486 __ASM_EMIT64("vmovups %%xmm0, 0x00(%[" DSTR "], %[off])") \
487 __ASM_EMIT64("vmovups %%xmm1, 0x00(%[" DSTI "], %[off])") \
488 __ASM_EMIT32("subl $4, %[count]") \
489 __ASM_EMIT64("sub $4, %[count]") \
490 __ASM_EMIT ("add $0x10, %[off]") \
491 __ASM_EMIT ("6:") \
492 /* 1x blocks */ \
493 __ASM_EMIT32("addl $3, %[count]") \
494 __ASM_EMIT64("add $3, %[count]") \
495 __ASM_EMIT ("jl 8f") \
496 __ASM_EMIT ("7:") \
497 __ASM_EMIT32("mov %[" TR "], %[ptr_re]") \
498 __ASM_EMIT32("mov %[" TI "], %[ptr_im]") \
499 __ASM_EMIT ("vmovss 0x00(%[" BR "], %[off]), %%xmm2") /* xmm2 = br */ \
500 __ASM_EMIT ("vmovss 0x00(%[" BI "], %[off]), %%xmm3") /* xmm3 = bi */ \
501 __ASM_EMIT32("vmovss 0x00(%[ptr_re], %[off]), %%xmm0") /* xmm0 = tr */ \
502 __ASM_EMIT32("vmovss 0x00(%[ptr_re], %[off]), %%xmm1") /* xmm1 = ti */ \
503 __ASM_EMIT64("vmovss 0x00(%[" TR "], %[off]), %%xmm0") /* xmm0 = tr */ \
504 __ASM_EMIT64("vmovss 0x00(%[" TI "], %[off]), %%xmm1") /* xmm1 = ti */ \
505 __ASM_EMIT32("mov %[" DSTR "], %[ptr_re]") \
506 __ASM_EMIT32("mov %[" DSTI "], %[ptr_im]") \
507 __ASM_EMIT ("vmulss %%xmm2, %%xmm2, %%xmm7") /* xmm7 = br*br */ \
508 __ASM_EMIT ("vmulss %%xmm3, %%xmm1, %%xmm4") /* xmm4 = ti*bi */ \
509 __ASM_EMIT ("vmulss %%xmm3, %%xmm0, %%xmm5") /* xmm5 = tr*bi */ \
510 __ASM_EMIT ("vfmadd231ss %%xmm3, %%xmm3, %%xmm7") /* xmm7 = M = br*br + bi*bi */ \
511 __ASM_EMIT ("vmovaps 0x00 + %[CC], %%xmm6") /* xmm6 = 1 */ \
512 __ASM_EMIT ("vdivss %%xmm7, %%xmm6, %%xmm7") /* xmm7 = R = 1/M */ \
513 __ASM_EMIT ("vfmadd132ss %%xmm2, %%xmm4, %%xmm0") /* xmm0 = r = tr*br + ti*bi */ \
514 __ASM_EMIT ("vfmsub132ss %%xmm2, %%xmm5, %%xmm1") /* xmm1 = i = tr*br - tr*bi */ \
515 __ASM_EMIT ("vmulss %%xmm7, %%xmm0, %%xmm0") /* xmm0 = r/R */ \
516 __ASM_EMIT ("vmulss %%xmm7, %%xmm1, %%xmm1") /* xmm1 = i/R */ \
517 __ASM_EMIT32("vmovss %%xmm0, 0x00(%[ptr_re], %[off])") \
518 __ASM_EMIT32("vmovss %%xmm1, 0x00(%[ptr_re], %[off])") \
519 __ASM_EMIT64("vmovss %%xmm0, 0x00(%[" DSTR "], %[off])") \
520 __ASM_EMIT64("vmovss %%xmm1, 0x00(%[" DSTI "], %[off])") \
521 __ASM_EMIT ("add $0x04, %[off]") \
522 __ASM_EMIT32("decl %[count]") \
523 __ASM_EMIT64("dec %[count]") \
524 __ASM_EMIT ("jge 7b") \
525 __ASM_EMIT ("8:")
526
527 void complex_div3(float *dst_re, float *dst_im,
528 const float *src1_re, const float *src1_im,
529 const float *src2_re, const float *src2_im,
530 size_t count)
531 {
532 IF_ARCH_I386(float *ptr_re, *ptr_im);
533 IF_ARCH_X86(size_t off);
534 ARCH_X86_ASM
535 (
536 COMPLEX_DIV3_CORE("dst_re", "dst_im", "src1_re", "src1_im", "src2_re", "src2_im")
537 : __IF_32(
538 [ptr_re] "=&r" (ptr_re), [ptr_im] "=&r" (ptr_im),
539 )
540 [count] X86_PGREG (count),
541 [off] "=&r" (off)
542 : [dst_re] X86_GREG (dst_re), [dst_im] X86_GREG (dst_im),
543 [src1_re] X86_GREG (src1_re), [src1_im] X86_GREG (src1_im),
544 [src2_re] "r" (src2_re), [src2_im] "r" (src2_im),
545 [CC] "o" (complex_div_const)
546 : "cc", "memory",
547 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
548 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
549 );
550 }
551
552 #undef COMPLEX_DIV3_CORE
553
554 #define COMPLEX_DIV2_CORE(DSTR, DSTI, TR, TI, BR, BI) \
555 __ASM_EMIT ("xor %[off], %[off]") \
556 __ASM_EMIT ("vmovaps 0x00 + %[CC], %%zmm6") /* zmm6 = 1 */ \
557 /* x16 blocks */ \
558 __ASM_EMIT32("subl $16, %[count]") \
559 __ASM_EMIT64("sub $16, %[count]") \
560 __ASM_EMIT ("jb 2f") \
561 __ASM_EMIT ("1:") \
562 __ASM_EMIT ("vmovups 0x00(%[" BR "], %[off]), %%zmm2") /* zmm2 = br */ \
563 __ASM_EMIT ("vmovups 0x00(%[" BI "], %[off]), %%zmm3") /* zmm3 = bi */ \
564 __ASM_EMIT ("vmovups 0x00(%[" TR "], %[off]), %%zmm0") /* zmm0 = tr */ \
565 __ASM_EMIT ("vmovups 0x00(%[" TI "], %[off]), %%zmm1") /* zmm1 = ti */ \
566 __ASM_EMIT ("vmulps %%zmm2, %%zmm2, %%zmm7") /* zmm7 = br*br */ \
567 __ASM_EMIT ("vmulps %%zmm3, %%zmm1, %%zmm4") /* zmm4 = ti*bi */ \
568 __ASM_EMIT ("vmulps %%zmm3, %%zmm0, %%zmm5") /* zmm5 = tr*bi */ \
569 __ASM_EMIT ("vfmadd231ps %%zmm3, %%zmm3, %%zmm7") /* zmm7 = M = br*br + bi*bi */ \
570 __ASM_EMIT ("vdivps %%zmm7, %%zmm6, %%zmm7") /* zmm7 = R = 1/M */ \
571 __ASM_EMIT ("vfmadd132ps %%zmm2, %%zmm4, %%zmm0") /* zmm0 = r = tr*br + ti*bi */ \
572 __ASM_EMIT ("vfmsub132ps %%zmm2, %%zmm5, %%zmm1") /* zmm1 = i = tr*br - tr*bi */ \
573 __ASM_EMIT ("vmulps %%zmm7, %%zmm0, %%zmm0") /* zmm0 = r/R */ \
574 __ASM_EMIT ("vmulps %%zmm7, %%zmm1, %%zmm1") /* zmm1 = i/R */ \
575 __ASM_EMIT ("vmovups %%zmm0, 0x00(%[" DSTR "], %[off])") \
576 __ASM_EMIT ("vmovups %%zmm1, 0x00(%[" DSTI "], %[off])") \
577 __ASM_EMIT ("add $0x40, %[off]") \
578 __ASM_EMIT32("subl $16, %[count]") \
579 __ASM_EMIT64("sub $16, %[count]") \
580 __ASM_EMIT ("jae 1b") \
581 __ASM_EMIT ("2:") \
582 /* x8 block */ \
583 __ASM_EMIT32("addl $8, %[count]") \
584 __ASM_EMIT64("add $8, %[count]") \
585 __ASM_EMIT ("jl 4f") \
586 __ASM_EMIT ("vmovups 0x00(%[" BR "], %[off]), %%ymm2") /* ymm2 = br */ \
587 __ASM_EMIT ("vmovups 0x00(%[" BI "], %[off]), %%ymm3") /* ymm3 = bi */ \
588 __ASM_EMIT ("vmovups 0x00(%[" TR "], %[off]), %%ymm0") /* ymm0 = tr */ \
589 __ASM_EMIT ("vmovups 0x00(%[" TI "], %[off]), %%ymm1") /* ymm1 = ti */ \
590 __ASM_EMIT ("vmulps %%ymm2, %%ymm2, %%ymm7") /* ymm7 = br*br */ \
591 __ASM_EMIT ("vmulps %%ymm3, %%ymm1, %%ymm4") /* ymm4 = ti*bi */ \
592 __ASM_EMIT ("vmulps %%ymm3, %%ymm0, %%ymm5") /* ymm5 = tr*bi */ \
593 __ASM_EMIT ("vfmadd231ps %%ymm3, %%ymm3, %%ymm7") /* ymm7 = M = br*br + bi*bi */ \
594 __ASM_EMIT ("vdivps %%ymm7, %%ymm6, %%ymm7") /* ymm7 = R = 1/M */ \
595 __ASM_EMIT ("vfmadd132ps %%ymm2, %%ymm4, %%ymm0") /* ymm0 = r = tr*br + ti*bi */ \
596 __ASM_EMIT ("vfmsub132ps %%ymm2, %%ymm5, %%ymm1") /* ymm1 = i = tr*br - tr*bi */ \
597 __ASM_EMIT ("vmulps %%ymm7, %%ymm0, %%ymm0") /* ymm0 = r/R */ \
598 __ASM_EMIT ("vmulps %%ymm7, %%ymm1, %%ymm1") /* ymm1 = i/R */ \
599 __ASM_EMIT ("vmovups %%ymm0, 0x00(%[" DSTR "], %[off])") \
600 __ASM_EMIT ("vmovups %%ymm1, 0x00(%[" DSTI "], %[off])") \
601 __ASM_EMIT32("subl $8, %[count]") \
602 __ASM_EMIT64("sub $8, %[count]") \
603 __ASM_EMIT ("add $0x20, %[off]") \
604 __ASM_EMIT ("4:") \
605 /* 4x block */ \
606 __ASM_EMIT32("addl $4, %[count]") \
607 __ASM_EMIT64("add $4, %[count]") \
608 __ASM_EMIT ("jl 6f") \
609 __ASM_EMIT ("vmovups 0x00(%[" BR "], %[off]), %%xmm2") /* xmm2 = br */ \
610 __ASM_EMIT ("vmovups 0x00(%[" BI "], %[off]), %%xmm3") /* xmm3 = bi */ \
611 __ASM_EMIT ("vmovups 0x00(%[" TR "], %[off]), %%xmm0") /* xmm0 = tr */ \
612 __ASM_EMIT ("vmovups 0x00(%[" TI "], %[off]), %%xmm1") /* xmm1 = ti */ \
613 __ASM_EMIT ("vmulps %%xmm2, %%xmm2, %%xmm7") /* xmm7 = br*br */ \
614 __ASM_EMIT ("vmulps %%xmm3, %%xmm1, %%xmm4") /* xmm4 = ti*bi */ \
615 __ASM_EMIT ("vmulps %%xmm3, %%xmm0, %%xmm5") /* xmm5 = tr*bi */ \
616 __ASM_EMIT ("vfmadd231ps %%xmm3, %%xmm3, %%xmm7") /* xmm7 = M = br*br + bi*bi */ \
617 __ASM_EMIT ("vdivps %%xmm7, %%xmm6, %%xmm7") /* xmm7 = R = 1/M */ \
618 __ASM_EMIT ("vfmadd132ps %%xmm2, %%xmm4, %%xmm0") /* xmm0 = r = tr*br + ti*bi */ \
619 __ASM_EMIT ("vfmsub132ps %%xmm2, %%xmm5, %%xmm1") /* xmm1 = i = tr*br - tr*bi */ \
620 __ASM_EMIT ("vmulps %%xmm7, %%xmm0, %%xmm0") /* xmm0 = r/R */ \
621 __ASM_EMIT ("vmulps %%xmm7, %%xmm1, %%xmm1") /* xmm1 = i/R */ \
622 __ASM_EMIT ("vmovups %%xmm0, 0x00(%[" DSTR "], %[off])") \
623 __ASM_EMIT ("vmovups %%xmm1, 0x00(%[" DSTI "], %[off])") \
624 __ASM_EMIT32("subl $4, %[count]") \
625 __ASM_EMIT64("sub $4, %[count]") \
626 __ASM_EMIT ("add $0x10, %[off]") \
627 __ASM_EMIT ("6:") \
628 /* 1x blocks */ \
629 __ASM_EMIT32("addl $3, %[count]") \
630 __ASM_EMIT64("add $3, %[count]") \
631 __ASM_EMIT ("jl 8f") \
632 __ASM_EMIT ("7:") \
633 __ASM_EMIT ("vmovss 0x00(%[" BR "], %[off]), %%xmm2") /* xmm2 = br */ \
634 __ASM_EMIT ("vmovss 0x00(%[" BI "], %[off]), %%xmm3") /* xmm3 = bi */ \
635 __ASM_EMIT ("vmovss 0x00(%[" TR "], %[off]), %%xmm0") /* xmm0 = tr */ \
636 __ASM_EMIT ("vmovss 0x00(%[" TI "], %[off]), %%xmm1") /* xmm1 = ti */ \
637 __ASM_EMIT ("vmulss %%xmm2, %%xmm2, %%xmm7") /* xmm7 = br*br */ \
638 __ASM_EMIT ("vmulss %%xmm3, %%xmm1, %%xmm4") /* xmm4 = ti*bi */ \
639 __ASM_EMIT ("vmulss %%xmm3, %%xmm0, %%xmm5") /* xmm5 = tr*bi */ \
640 __ASM_EMIT ("vfmadd231ss %%xmm3, %%xmm3, %%xmm7") /* xmm7 = M = br*br + bi*bi */ \
641 __ASM_EMIT ("vmovaps 0x00 + %[CC], %%xmm6") /* xmm6 = 1 */ \
642 __ASM_EMIT ("vdivss %%xmm7, %%xmm6, %%xmm7") /* xmm7 = R = 1/M */ \
643 __ASM_EMIT ("vfmadd132ss %%xmm2, %%xmm4, %%xmm0") /* xmm0 = r = tr*br + ti*bi */ \
644 __ASM_EMIT ("vfmsub132ss %%xmm2, %%xmm5, %%xmm1") /* xmm1 = i = tr*br - tr*bi */ \
645 __ASM_EMIT ("vmulss %%xmm7, %%xmm0, %%xmm0") /* xmm0 = r/R */ \
646 __ASM_EMIT ("vmulss %%xmm7, %%xmm1, %%xmm1") /* xmm1 = i/R */ \
647 __ASM_EMIT ("vmovss %%xmm0, 0x00(%[" DSTR "], %[off])") \
648 __ASM_EMIT ("vmovss %%xmm1, 0x00(%[" DSTI "], %[off])") \
649 __ASM_EMIT ("add $0x04, %[off]") \
650 __ASM_EMIT32("decl %[count]") \
651 __ASM_EMIT64("dec %[count]") \
652 __ASM_EMIT ("jge 7b") \
653 __ASM_EMIT ("8:")
654
655 void complex_div2(float *dst_re, float *dst_im, const float *src_re, const float *src_im, size_t count)
656 {
657 IF_ARCH_X86(size_t off);
658 ARCH_X86_ASM
659 (
660 COMPLEX_DIV2_CORE("dst_re", "dst_im", "dst_re", "dst_im", "src_re", "src_im")
661 : [count] X86_PGREG (count),
662 [off] "=&r" (off)
663 : [dst_re] "r" (dst_re), [dst_im] "r" (dst_im),
664 [src_re] "r" (src_re), [src_im] "r" (src_im),
665 [CC] "o" (complex_div_const)
666 : "cc", "memory",
667 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
668 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
669 );
670 }
671
672 void complex_rdiv2(float *dst_re, float *dst_im, const float *src_re, const float *src_im, size_t count)
673 {
674 IF_ARCH_X86(size_t off);
675 ARCH_X86_ASM
676 (
677 COMPLEX_DIV2_CORE("dst_re", "dst_im", "src_re", "src_im", "dst_re", "dst_im")
678 : [count] X86_PGREG (count),
679 [off] "=&r" (off)
680 : [dst_re] "r" (dst_re), [dst_im] "r" (dst_im),
681 [src_re] "r" (src_re), [src_im] "r" (src_im),
682 [CC] "o" (complex_div_const)
683 : "cc", "memory",
684 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
685 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
686 );
687 }
688
689 #undef COMPLEX_DIV2_CORE
690
691 #define COMPLEX_RCP_CORE(DST, SRC, SEL) \
692 __ASM_EMIT ("xor %[off], %[off]") \
693 __ASM_EMIT ("vmovaps 0x00 + %[CC], %%zmm6") /* zmm6 = 1 */ \
694 /* x32 blocks */ \
695 __ASM_EMIT32("subl $32, %[count]") \
696 __ASM_EMIT64("sub $32, %[count]") \
697 __ASM_EMIT ("jb 2f") \
698 __ASM_EMIT ("1:") \
699 __ASM_EMIT ("vmovups 0x00(%[" SRC "_re], %[off]), %%zmm0") /* zmm0 = ar */ \
700 __ASM_EMIT ("vmovups 0x40(%[" SRC "_re], %[off]), %%zmm1") \
701 __ASM_EMIT ("vmovups 0x00(%[" SRC "_im], %[off]), %%zmm2") /* zmm2 = ai */ \
702 __ASM_EMIT ("vmovups 0x40(%[" SRC "_im], %[off]), %%zmm3") \
703 __ASM_EMIT ("vmulps %%zmm0, %%zmm0, %%zmm4") /* zmm4 = ar*ar */ \
704 __ASM_EMIT ("vmulps %%zmm1, %%zmm1, %%zmm5") \
705 __ASM_EMIT ("vfmadd231ps %%zmm2, %%zmm2, %%zmm4") /* zmm4 = R = ar*ar+ai*ai */ \
706 __ASM_EMIT ("vfmadd231ps %%zmm3, %%zmm3, %%zmm5") \
707 __ASM_EMIT ("vdivps %%zmm4, %%zmm6, %%zmm4") /* zmm4 = 1/R */ \
708 __ASM_EMIT ("vdivps %%zmm5, %%zmm6, %%zmm5") \
709 __ASM_EMIT ("vpxord 0x40 + %[CC], %%zmm2, %%zmm2") /* zmm2 = -ai */ \
710 __ASM_EMIT ("vpxord 0x40 + %[CC], %%zmm3, %%zmm3") \
711 __ASM_EMIT ("vmulps %%zmm4, %%zmm0, %%zmm0") /* zmm0 = ar/R */ \
712 __ASM_EMIT ("vmulps %%zmm5, %%zmm1, %%zmm1") \
713 __ASM_EMIT ("vmulps %%zmm4, %%zmm2, %%zmm2") /* zmm1 = -ai/R */ \
714 __ASM_EMIT ("vmulps %%zmm5, %%zmm3, %%zmm3") \
715 __ASM_EMIT ("vmovups %%zmm0, 0x00(%[" DST "_re], %[off])") \
716 __ASM_EMIT ("vmovups %%zmm1, 0x40(%[" DST "_re], %[off])") \
717 __ASM_EMIT ("vmovups %%zmm2, 0x00(%[" DST "_im], %[off])") \
718 __ASM_EMIT ("vmovups %%zmm3, 0x40(%[" DST "_im], %[off])") \
719 __ASM_EMIT ("add $0x80, %[off]") \
720 __ASM_EMIT32("subl $32, %[count]") \
721 __ASM_EMIT64("sub $32, %[count]") \
722 __ASM_EMIT ("jae 1b") \
723 __ASM_EMIT ("2:") \
724 /* x16 blocks */ \
725 __ASM_EMIT32("addl $16, %[count]") \
726 __ASM_EMIT64("add $16, %[count]") \
727 __ASM_EMIT ("jl 4f") \
728 __ASM_EMIT ("vmovups 0x00(%[" SRC "_re], %[off]), %%ymm0") /* ymm0 = ar */ \
729 __ASM_EMIT ("vmovups 0x20(%[" SRC "_re], %[off]), %%ymm1") \
730 __ASM_EMIT ("vmovups 0x00(%[" SRC "_im], %[off]), %%ymm2") /* ymm2 = ai */ \
731 __ASM_EMIT ("vmovups 0x20(%[" SRC "_im], %[off]), %%ymm3") \
732 __ASM_EMIT ("vmulps %%ymm0, %%ymm0, %%ymm4") /* ymm4 = ar*ar */ \
733 __ASM_EMIT ("vmulps %%ymm1, %%ymm1, %%ymm5") \
734 __ASM_EMIT ("vfmadd231ps %%ymm2, %%ymm2, %%ymm4") /* ymm4 = R = ar*ar+ai*ai */ \
735 __ASM_EMIT ("vfmadd231ps %%ymm3, %%ymm3, %%ymm5") \
736 __ASM_EMIT ("vdivps %%ymm4, %%ymm6, %%ymm4") /* ymm4 = 1/R */ \
737 __ASM_EMIT ("vdivps %%ymm5, %%ymm6, %%ymm5") \
738 __ASM_EMIT ("vpxord 0x40 + %[CC], %%ymm2, %%ymm2") /* ymm2 = -ai */ \
739 __ASM_EMIT ("vpxord 0x40 + %[CC], %%ymm3, %%ymm3") \
740 __ASM_EMIT ("vmulps %%ymm4, %%ymm0, %%ymm0") /* ymm0 = ar/R */ \
741 __ASM_EMIT ("vmulps %%ymm5, %%ymm1, %%ymm1") \
742 __ASM_EMIT ("vmulps %%ymm4, %%ymm2, %%ymm2") /* ymm1 = -ai/R */ \
743 __ASM_EMIT ("vmulps %%ymm5, %%ymm3, %%ymm3") \
744 __ASM_EMIT ("vmovups %%ymm0, 0x00(%[" DST "_re], %[off])") \
745 __ASM_EMIT ("vmovups %%ymm1, 0x20(%[" DST "_re], %[off])") \
746 __ASM_EMIT ("vmovups %%ymm2, 0x00(%[" DST "_im], %[off])") \
747 __ASM_EMIT ("vmovups %%ymm3, 0x20(%[" DST "_im], %[off])") \
748 __ASM_EMIT ("add $0x40, %[off]") \
749 __ASM_EMIT32("subl $16, %[count]") \
750 __ASM_EMIT64("sub $16, %[count]") \
751 __ASM_EMIT ("jae 1b") \
752 __ASM_EMIT ("4:") \
753 /* 8x block */ \
754 __ASM_EMIT32("addl $8, %[count]") \
755 __ASM_EMIT64("add $8, %[count]") \
756 __ASM_EMIT ("jl 6f") \
757 __ASM_EMIT ("vmovups 0x00(%[" SRC "_re], %[off]), %%xmm0") /* xmm0 = ar */ \
758 __ASM_EMIT ("vmovups 0x10(%[" SRC "_re], %[off]), %%xmm1") \
759 __ASM_EMIT ("vmovups 0x00(%[" SRC "_im], %[off]), %%xmm2") /* xmm2 = ai */ \
760 __ASM_EMIT ("vmovups 0x10(%[" SRC "_im], %[off]), %%xmm3") \
761 __ASM_EMIT ("vmulps %%xmm0, %%xmm0, %%xmm4") /* xmm4 = ar*ar */ \
762 __ASM_EMIT ("vmulps %%xmm1, %%xmm1, %%xmm5") \
763 __ASM_EMIT ("vfmadd231ps %%xmm2, %%xmm2, %%xmm4") /* xmm4 = R = ar*ar+ai*ai */ \
764 __ASM_EMIT ("vfmadd231ps %%xmm3, %%xmm3, %%xmm5") \
765 __ASM_EMIT ("vdivps %%xmm4, %%xmm6, %%xmm4") /* xmm4 = 1/R */ \
766 __ASM_EMIT ("vdivps %%xmm5, %%xmm6, %%xmm5") \
767 __ASM_EMIT ("vpxord 0x40 + %[CC], %%xmm2, %%xmm2") /* xmm2 = -ai */ \
768 __ASM_EMIT ("vpxord 0x40 + %[CC], %%xmm3, %%xmm3") \
769 __ASM_EMIT ("vmulps %%xmm4, %%xmm0, %%xmm0") /* xmm0 = ar/R */ \
770 __ASM_EMIT ("vmulps %%xmm5, %%xmm1, %%xmm1") \
771 __ASM_EMIT ("vmulps %%xmm4, %%xmm2, %%xmm2") /* xmm1 = -ai/R */ \
772 __ASM_EMIT ("vmulps %%xmm5, %%xmm3, %%xmm3") \
773 __ASM_EMIT ("vmovups %%xmm0, 0x00(%[" DST "_re], %[off])") \
774 __ASM_EMIT ("vmovups %%xmm1, 0x10(%[" DST "_re], %[off])") \
775 __ASM_EMIT ("vmovups %%xmm2, 0x00(%[" DST "_im], %[off])") \
776 __ASM_EMIT ("vmovups %%xmm3, 0x10(%[" DST "_im], %[off])") \
777 __ASM_EMIT32("subl $8, %[count]") \
778 __ASM_EMIT64("sub $8, %[count]") \
779 __ASM_EMIT ("add $0x20, %[off]") \
780 __ASM_EMIT ("6:") \
781 /* 4x block */ \
782 __ASM_EMIT32("addl $4, %[count]") \
783 __ASM_EMIT64("add $4, %[count]") \
784 __ASM_EMIT ("jl 8f") \
785 __ASM_EMIT ("vmovups 0x00(%[" SRC "_re], %[off]), %%xmm0") /* xmm0 = ar */ \
786 __ASM_EMIT ("vmovups 0x00(%[" SRC "_im], %[off]), %%xmm2") /* xmm2 = ai */ \
787 __ASM_EMIT ("vmulps %%xmm0, %%xmm0, %%xmm4") /* xmm4 = ar*ar */ \
788 __ASM_EMIT ("vfmadd231ps %%xmm2, %%xmm2, %%xmm4") /* xmm4 = R = ar*ar+ai*ai */ \
789 __ASM_EMIT ("vdivps %%xmm4, %%xmm6, %%xmm4") /* xmm4 = 1/R */ \
790 __ASM_EMIT ("vpxord 0x40 + %[CC], %%xmm2, %%xmm2") /* xmm2 = -ai */ \
791 __ASM_EMIT ("vmulps %%xmm4, %%xmm0, %%xmm0") /* xmm0 = ar/R */ \
792 __ASM_EMIT ("vmulps %%xmm4, %%xmm2, %%xmm2") /* xmm1 = -ai/R */ \
793 __ASM_EMIT ("vmovups %%xmm0, 0x00(%[" DST "_re], %[off])") \
794 __ASM_EMIT ("vmovups %%xmm2, 0x00(%[" DST "_im], %[off])") \
795 __ASM_EMIT32("subl $4, %[count]") \
796 __ASM_EMIT64("sub $4, %[count]") \
797 __ASM_EMIT ("add $0x10, %[off]") \
798 __ASM_EMIT ("8:") \
799 /* 1x blocks */ \
800 __ASM_EMIT32("addl $3, %[count]") \
801 __ASM_EMIT64("add $3, %[count]") \
802 __ASM_EMIT ("jl 10f") \
803 __ASM_EMIT ("9:") \
804 __ASM_EMIT ("vmovss 0x00(%[" SRC "_re], %[off]), %%xmm0") /* xmm0 = ar */ \
805 __ASM_EMIT ("vmovss 0x00(%[" SRC "_im], %[off]), %%xmm2") /* xmm2 = ai */ \
806 __ASM_EMIT ("vmulss %%xmm0, %%xmm0, %%xmm4") /* xmm4 = ar*ar */ \
807 __ASM_EMIT ("vfmadd231ss %%xmm2, %%xmm2, %%xmm4") /* xmm4 = R = ar*ar+ai*ai */ \
808 __ASM_EMIT ("vdivss %%xmm4, %%xmm6, %%xmm4") /* xmm4 = 1/R */ \
809 __ASM_EMIT ("vpxord 0x40 + %[CC], %%xmm2, %%xmm2") /* xmm2 = -ai */ \
810 __ASM_EMIT ("vmulss %%xmm4, %%xmm0, %%xmm0") /* xmm0 = ar/R */ \
811 __ASM_EMIT ("vmulss %%xmm4, %%xmm2, %%xmm2") /* xmm1 = -ai/R */ \
812 __ASM_EMIT ("vmovss %%xmm0, 0x00(%[" DST "_re], %[off])") \
813 __ASM_EMIT ("vmovss %%xmm2, 0x00(%[" DST "_im], %[off])") \
814 __ASM_EMIT ("add $0x04, %[off]") \
815 __ASM_EMIT32("decl %[count]") \
816 __ASM_EMIT64("dec %[count]") \
817 __ASM_EMIT ("jge 9b") \
818 __ASM_EMIT ("10:")
819
820 void complex_rcp1(float *dst_re, float *dst_im, size_t count)
821 {
822 IF_ARCH_X86(size_t off);
823 ARCH_X86_ASM
824 (
825 COMPLEX_RCP_CORE("dst", "dst", FMA_OFF)
826 : [count] "+r" (count), [off] "=&r" (off)
827 : [dst_re] "r" (dst_re), [dst_im] "r" (dst_im),
828 [CC] "o" (complex_div_const)
829 : "cc", "memory",
830 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
831 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
832 );
833 }
834
835 void complex_rcp2(float *dst_re, float *dst_im, const float *src_re, const float *src_im, size_t count)
836 {
837 IF_ARCH_X86(size_t off);
838 ARCH_X86_ASM
839 (
840 COMPLEX_RCP_CORE("dst", "src", FMA_OFF)
841 : [count] X86_PGREG (count),
842 [off] "=&r" (off)
843 : [dst_re] "r" (dst_re), [dst_im] "r" (dst_im),
844 [src_re] "r" (src_re), [src_im] "r" (src_im),
845 [CC] "o" (complex_div_const)
846 : "cc", "memory",
847 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
848 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
849 );
850 }
851
852 #undef COMPLEX_RCP_CORE
853
854
855 } /* namespace avx512 */
856 } /* namespace lsp */
857
858
859
860 #endif /* PRIVATE_DSP_ARCH_X86_AVX512_COMPLEX_H_ */