Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/x86/avx/complex.h

raw
1 /*
2 * Copyright (C) 2023 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2023 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 31 мар. 2020 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_X86_AVX_COMPLEX_H_
23 #define PRIVATE_DSP_ARCH_X86_AVX_COMPLEX_H_
24
25 #ifndef PRIVATE_DSP_ARCH_X86_AVX_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_X86_AVX_IMPL */
28
29 namespace lsp
30 {
31 namespace avx
32 {
33 #define FMA_OFF(a, b) a
34 #define FMA_ON(a, b) b
35 #define FMA_PASS(x) __ASM_EMIT(x)
36 #define FMA_SKIP(x)
37
38 IF_ARCH_X86(
39 static uint32_t complex_div_const[] __lsp_aligned32 =
40 {
41 LSP_DSP_VEC8(0x3f800000), // 1.0
42 LSP_DSP_VEC8(0x80000000) // signbit
43 };
44 )
45
46 #define COMPLEX_MUL3_CORE(DST, SRC1, SRC2, SEL) \
47 __ASM_EMIT ("xor %[off], %[off]") \
48 /* x8 blocks */ \
49 __ASM_EMIT32("subl $8, %[count]") \
50 __ASM_EMIT64("sub $8, %[count]") \
51 __ASM_EMIT ("jb 2f") \
52 __ASM_EMIT ("1:") \
53 __ASM_EMIT32("mov %[" SRC1 "_re], %[ptr_re]") \
54 __ASM_EMIT32("mov %[" SRC1 "_im], %[ptr_im]") \
55 __ASM_EMIT64("vmovups 0x00(%[" SRC1 "_re], %[off]), %%ymm4") /* ymm4 = ar */ \
56 __ASM_EMIT64("vmovups 0x00(%[" SRC1 "_im], %[off]), %%ymm5") /* ymm5 = ai */ \
57 __ASM_EMIT32("vmovups 0x00(%[ptr_re], %[off]), %%ymm4") /* ymm4 = ar */ \
58 __ASM_EMIT32("vmovups 0x00(%[ptr_im], %[off]), %%ymm5") /* ymm5 = ai */ \
59 __ASM_EMIT ("vmovups 0x00(%[" SRC2 "_re], %[off]), %%ymm0") /* ymm0 = br */ \
60 __ASM_EMIT ("vmovups 0x00(%[" SRC2 "_im], %[off]), %%ymm1") /* ymm1 = bi */ \
61 __ASM_EMIT ("vmulps %%ymm0, %%ymm5, %%ymm2") /* ymm2 = ai*br */ \
62 __ASM_EMIT ("vmulps %%ymm1, %%ymm5, %%ymm3") /* ymm3 = ai*bi */ \
63 __ASM_EMIT (SEL("vmulps %%ymm0, %%ymm4, %%ymm0", "")) /* ymm0 = ar*br */ \
64 __ASM_EMIT (SEL("vmulps %%ymm1, %%ymm4, %%ymm1", "")) /* ymm1 = ar*bi */ \
65 __ASM_EMIT32("mov %[" DST "_re], %[ptr_re]") \
66 __ASM_EMIT32("mov %[" DST "_im], %[ptr_im]") \
67 __ASM_EMIT (SEL("vsubps %%ymm3, %%ymm0, %%ymm0", "vfmsub132ps %%ymm4, %%ymm3, %%ymm0")) /* ymm0 = ar*br - ai*bi = r */ \
68 __ASM_EMIT (SEL("vaddps %%ymm2, %%ymm1, %%ymm1", "vfmadd132ps %%ymm4, %%ymm2, %%ymm1")) /* ymm1 = ar*bi + ai*br = i */ \
69 __ASM_EMIT32("vmovups %%ymm0, 0x00(%[ptr_re], %[off])") \
70 __ASM_EMIT32("vmovups %%ymm1, 0x00(%[ptr_im], %[off])") \
71 __ASM_EMIT64("vmovups %%ymm0, 0x00(%[" DST "_re], %[off])") \
72 __ASM_EMIT64("vmovups %%ymm1, 0x00(%[" DST "_im], %[off])") \
73 __ASM_EMIT ("add $0x20, %[off]") \
74 __ASM_EMIT32("subl $8, %[count]") \
75 __ASM_EMIT64("sub $8, %[count]") \
76 __ASM_EMIT ("jae 1b") \
77 __ASM_EMIT ("2:") \
78 /* 4x block */ \
79 __ASM_EMIT32("addl $4, %[count]") \
80 __ASM_EMIT64("add $4, %[count]") \
81 __ASM_EMIT ("jl 4f") \
82 __ASM_EMIT32("mov %[" SRC1 "_re], %[ptr_re]") \
83 __ASM_EMIT32("mov %[" SRC1 "_im], %[ptr_im]") \
84 __ASM_EMIT64("vmovups 0x00(%[" SRC1 "_re], %[off]), %%xmm4") /* xmm4 = ar */ \
85 __ASM_EMIT64("vmovups 0x00(%[" SRC1 "_im], %[off]), %%xmm5") /* xmm5 = ai */ \
86 __ASM_EMIT32("vmovups 0x00(%[ptr_re], %[off]), %%xmm4") /* xmm4 = ar */ \
87 __ASM_EMIT32("vmovups 0x00(%[ptr_im], %[off]), %%xmm5") /* xmm5 = ai */ \
88 __ASM_EMIT ("vmovups 0x00(%[" SRC2 "_re], %[off]), %%xmm0") /* xmm0 = br */ \
89 __ASM_EMIT ("vmovups 0x00(%[" SRC2 "_im], %[off]), %%xmm1") /* xmm1 = bi */ \
90 __ASM_EMIT ("vmulps %%xmm0, %%xmm5, %%xmm2") /* xmm2 = ai*br */ \
91 __ASM_EMIT ("vmulps %%xmm1, %%xmm5, %%xmm3") /* xmm3 = ai*bi */ \
92 __ASM_EMIT (SEL("vmulps %%xmm0, %%xmm4, %%xmm0", "")) /* xmm0 = ar*br */ \
93 __ASM_EMIT (SEL("vmulps %%xmm1, %%xmm4, %%xmm1", "")) /* xmm1 = ar*bi */ \
94 __ASM_EMIT32("mov %[" DST "_re], %[ptr_re]") \
95 __ASM_EMIT32("mov %[" DST "_im], %[ptr_im]") \
96 __ASM_EMIT (SEL("vsubps %%xmm3, %%xmm0, %%xmm0", "vfmsub132ps %%xmm4, %%xmm3, %%xmm0")) /* xmm0 = ar*br - ai*bi = r */ \
97 __ASM_EMIT (SEL("vaddps %%xmm2, %%xmm1, %%xmm1", "vfmadd132ps %%xmm4, %%xmm2, %%xmm1")) /* xmm1 = ar*bi + ai*br = i */ \
98 __ASM_EMIT32("vmovups %%xmm0, 0x00(%[ptr_re], %[off])") \
99 __ASM_EMIT32("vmovups %%xmm1, 0x00(%[ptr_im], %[off])") \
100 __ASM_EMIT64("vmovups %%xmm0, 0x00(%[" DST "_re], %[off])") \
101 __ASM_EMIT64("vmovups %%xmm1, 0x00(%[" DST "_im], %[off])") \
102 __ASM_EMIT ("add $0x10, %[off]") \
103 __ASM_EMIT32("subl $4, %[count]") \
104 __ASM_EMIT64("sub $4, %[count]") \
105 __ASM_EMIT ("4:") \
106 /* 1x blocks */ \
107 __ASM_EMIT32("addl $3, %[count]") \
108 __ASM_EMIT64("add $3, %[count]") \
109 __ASM_EMIT ("jl 6f") \
110 __ASM_EMIT ("5:") \
111 __ASM_EMIT32("mov %[" SRC1 "_re], %[ptr_re]") \
112 __ASM_EMIT32("mov %[" SRC1 "_im], %[ptr_im]") \
113 __ASM_EMIT64("vmovss 0x00(%[" SRC1 "_re], %[off]), %%xmm4") /* xmm4 = ar */ \
114 __ASM_EMIT64("vmovss 0x00(%[" SRC1 "_im], %[off]), %%xmm5") /* xmm5 = ai */ \
115 __ASM_EMIT32("vmovss 0x00(%[ptr_re], %[off]), %%xmm4") /* xmm4 = ar */ \
116 __ASM_EMIT32("vmovss 0x00(%[ptr_im], %[off]), %%xmm5") /* xmm5 = ai */ \
117 __ASM_EMIT ("vmovss 0x00(%[" SRC2 "_re], %[off]), %%xmm0") /* xmm0 = br */ \
118 __ASM_EMIT ("vmovss 0x00(%[" SRC2 "_im], %[off]), %%xmm1") /* xmm1 = bi */ \
119 __ASM_EMIT ("vmulps %%xmm0, %%xmm5, %%xmm2") /* xmm2 = ai*br */ \
120 __ASM_EMIT ("vmulps %%xmm1, %%xmm5, %%xmm3") /* xmm3 = ai*bi */ \
121 __ASM_EMIT (SEL("vmulss %%xmm0, %%xmm4, %%xmm0", "")) /* xmm0 = ar*br */ \
122 __ASM_EMIT (SEL("vmulss %%xmm1, %%xmm4, %%xmm1", "")) /* xmm1 = ar*bi */ \
123 __ASM_EMIT32("mov %[" DST "_re], %[ptr_re]") \
124 __ASM_EMIT32("mov %[" DST "_im], %[ptr_im]") \
125 __ASM_EMIT (SEL("vsubss %%xmm3, %%xmm0, %%xmm0", "vfmsub132ss %%xmm4, %%xmm3, %%xmm0")) /* xmm0 = ar*br - ai*bi = r */ \
126 __ASM_EMIT (SEL("vaddss %%xmm2, %%xmm1, %%xmm1", "vfmadd132ss %%xmm4, %%xmm2, %%xmm1")) /* xmm1 = ar*bi + ai*br = i */ \
127 __ASM_EMIT32("vmovss %%xmm0, 0x00(%[ptr_re], %[off])") \
128 __ASM_EMIT32("vmovss %%xmm1, 0x00(%[ptr_im], %[off])") \
129 __ASM_EMIT64("vmovss %%xmm0, 0x00(%[" DST "_re], %[off])") \
130 __ASM_EMIT64("vmovss %%xmm1, 0x00(%[" DST "_im], %[off])") \
131 __ASM_EMIT ("add $0x04, %[off]") \
132 __ASM_EMIT32("decl %[count]") \
133 __ASM_EMIT64("dec %[count]") \
134 __ASM_EMIT ("jge 5b") \
135 __ASM_EMIT ("6:")
136
137 void complex_mul3(float *dst_re, float *dst_im, const float *src1_re, const float *src1_im, const float *src2_re, const float *src2_im, size_t count)
138 {
139 IF_ARCH_I386(float *ptr_re, *ptr_im);
140 IF_ARCH_X86(size_t off);
141 ARCH_X86_ASM
142 (
143 COMPLEX_MUL3_CORE("dst", "src1", "src2", FMA_OFF)
144 : __IF_32(
145 [ptr_re] "=&r" (ptr_re), [ptr_im] "=&r" (ptr_im),
146 )
147 [count] X86_PGREG (count),
148 [off] "=&r" (off)
149 : [dst_re] X86_GREG (dst_re), [dst_im] X86_GREG (dst_im),
150 [src1_re] X86_GREG (src1_re), [src1_im] X86_GREG (src1_im),
151 [src2_re] "r" (src2_re), [src2_im] "r" (src2_im)
152 : "cc", "memory",
153 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
154 "%xmm4", "%xmm5"
155 );
156 }
157
158 void complex_mul3_fma3(float *dst_re, float *dst_im, const float *src1_re, const float *src1_im, const float *src2_re, const float *src2_im, size_t count)
159 {
160 IF_ARCH_I386(float *ptr_re, *ptr_im);
161 IF_ARCH_X86(size_t off);
162 ARCH_X86_ASM
163 (
164 COMPLEX_MUL3_CORE("dst", "src1", "src2", FMA_ON)
165 : __IF_32(
166 [ptr_re] "=&r" (ptr_re), [ptr_im] "=&r" (ptr_im),
167 )
168 [count] X86_PGREG (count),
169 [off] "=&r" (off)
170 : [dst_re] X86_GREG (dst_re), [dst_im] X86_GREG (dst_im),
171 [src1_re] X86_GREG (src1_re), [src1_im] X86_GREG (src1_im),
172 [src2_re] "r" (src2_re), [src2_im] "r" (src2_im)
173 : "cc", "memory",
174 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
175 "%xmm4", "%xmm5"
176 );
177 }
178
179 #undef COMPLEX_MUL3_CORE
180
181 #define COMPLEX_MUL2_CORE(DST, SRC, SEL) \
182 __ASM_EMIT ("xor %[off], %[off]") \
183 /* x8 blocks */ \
184 __ASM_EMIT32("subl $8, %[count]") \
185 __ASM_EMIT64("sub $8, %[count]") \
186 __ASM_EMIT ("jb 2f") \
187 __ASM_EMIT ("1:") \
188 __ASM_EMIT ("vmovups 0x00(%[" DST "_re], %[off]), %%ymm4") /* ymm4 = ar */ \
189 __ASM_EMIT ("vmovups 0x00(%[" DST "_im], %[off]), %%ymm5") /* ymm5 = ai */ \
190 __ASM_EMIT ("vmovups 0x00(%[" SRC "_re], %[off]), %%ymm0") /* ymm0 = br */ \
191 __ASM_EMIT ("vmovups 0x00(%[" SRC "_im], %[off]), %%ymm1") /* ymm1 = bi */ \
192 __ASM_EMIT ("vmulps %%ymm0, %%ymm5, %%ymm2") /* ymm2 = ai*br */ \
193 __ASM_EMIT ("vmulps %%ymm1, %%ymm5, %%ymm3") /* ymm3 = ai*bi */ \
194 __ASM_EMIT (SEL("vmulps %%ymm0, %%ymm4, %%ymm0", "")) /* ymm0 = ar*br */ \
195 __ASM_EMIT (SEL("vmulps %%ymm1, %%ymm4, %%ymm1", "")) /* ymm1 = ar*bi */ \
196 __ASM_EMIT (SEL("vsubps %%ymm3, %%ymm0, %%ymm0", "vfmsub132ps %%ymm4, %%ymm3, %%ymm0")) /* ymm0 = ar*br - ai*bi = r */ \
197 __ASM_EMIT (SEL("vaddps %%ymm2, %%ymm1, %%ymm1", "vfmadd132ps %%ymm4, %%ymm2, %%ymm1")) /* ymm1 = ar*bi + ai*br = i */ \
198 __ASM_EMIT ("vmovups %%ymm0, 0x00(%[" DST "_re], %[off])") \
199 __ASM_EMIT ("vmovups %%ymm1, 0x00(%[" DST "_im], %[off])") \
200 __ASM_EMIT ("add $0x20, %[off]") \
201 __ASM_EMIT32("subl $8, %[count]") \
202 __ASM_EMIT64("sub $8, %[count]") \
203 __ASM_EMIT ("jae 1b") \
204 __ASM_EMIT ("2:") \
205 /* 4x block */ \
206 __ASM_EMIT32("addl $4, %[count]") \
207 __ASM_EMIT64("add $4, %[count]") \
208 __ASM_EMIT ("jl 4f") \
209 __ASM_EMIT ("vmovups 0x00(%[" DST "_re], %[off]), %%xmm4") /* xmm4 = ar */ \
210 __ASM_EMIT ("vmovups 0x00(%[" DST "_im], %[off]), %%xmm5") /* xmm5 = ai */ \
211 __ASM_EMIT ("vmovups 0x00(%[" SRC "_re], %[off]), %%xmm0") /* xmm0 = br */ \
212 __ASM_EMIT ("vmovups 0x00(%[" SRC "_im], %[off]), %%xmm1") /* xmm1 = bi */ \
213 __ASM_EMIT ("vmulps %%xmm0, %%xmm5, %%xmm2") /* xmm2 = ai*br */ \
214 __ASM_EMIT ("vmulps %%xmm1, %%xmm5, %%xmm3") /* xmm3 = ai*bi */ \
215 __ASM_EMIT (SEL("vmulps %%xmm0, %%xmm4, %%xmm0", "")) /* xmm0 = ar*br */ \
216 __ASM_EMIT (SEL("vmulps %%xmm1, %%xmm4, %%xmm1", "")) /* xmm1 = ar*bi */ \
217 __ASM_EMIT (SEL("vsubps %%xmm3, %%xmm0, %%xmm0", "vfmsub132ps %%xmm4, %%xmm3, %%xmm0")) /* xmm0 = ar*br - ai*bi = r */ \
218 __ASM_EMIT (SEL("vaddps %%xmm2, %%xmm1, %%xmm1", "vfmadd132ps %%xmm4, %%xmm2, %%xmm1")) /* xmm1 = ar*bi + ai*br = i */ \
219 __ASM_EMIT ("vmovups %%xmm0, 0x00(%[" DST "_re], %[off])") \
220 __ASM_EMIT ("vmovups %%xmm1, 0x00(%[" DST "_im], %[off])") \
221 __ASM_EMIT ("add $0x10, %[off]") \
222 __ASM_EMIT32("subl $4, %[count]") \
223 __ASM_EMIT64("sub $4, %[count]") \
224 __ASM_EMIT ("4:") \
225 /* 1x blocks */ \
226 __ASM_EMIT32("addl $3, %[count]") \
227 __ASM_EMIT64("add $3, %[count]") \
228 __ASM_EMIT ("jl 6f") \
229 __ASM_EMIT ("5:") \
230 __ASM_EMIT ("vmovss 0x00(%[" DST "_re], %[off]), %%xmm4") /* xmm4 = ar */ \
231 __ASM_EMIT ("vmovss 0x00(%[" DST "_im], %[off]), %%xmm5") /* xmm5 = ai */ \
232 __ASM_EMIT ("vmovss 0x00(%[" SRC "_re], %[off]), %%xmm0") /* xmm0 = br */ \
233 __ASM_EMIT ("vmovss 0x00(%[" SRC "_im], %[off]), %%xmm1") /* xmm1 = bi */ \
234 __ASM_EMIT ("vmulps %%xmm0, %%xmm5, %%xmm2") /* xmm2 = ai*br */ \
235 __ASM_EMIT ("vmulps %%xmm1, %%xmm5, %%xmm3") /* xmm3 = ai*bi */ \
236 __ASM_EMIT (SEL("vmulss %%xmm0, %%xmm4, %%xmm0", "")) /* xmm0 = ar*br */ \
237 __ASM_EMIT (SEL("vmulss %%xmm1, %%xmm4, %%xmm1", "")) /* xmm1 = ar*bi */ \
238 __ASM_EMIT (SEL("vsubss %%xmm3, %%xmm0, %%xmm0", "vfmsub132ss %%xmm4, %%xmm3, %%xmm0")) /* xmm0 = ar*br - ai*bi = r */ \
239 __ASM_EMIT (SEL("vaddss %%xmm2, %%xmm1, %%xmm1", "vfmadd132ss %%xmm4, %%xmm2, %%xmm1")) /* xmm1 = ar*bi + ai*br = i */ \
240 __ASM_EMIT ("vmovss %%xmm0, 0x00(%[" DST "_re], %[off])") \
241 __ASM_EMIT ("vmovss %%xmm1, 0x00(%[" DST "_im], %[off])") \
242 __ASM_EMIT ("add $0x04, %[off]") \
243 __ASM_EMIT32("decl %[count]") \
244 __ASM_EMIT64("dec %[count]") \
245 __ASM_EMIT ("jge 5b") \
246 __ASM_EMIT ("6:")
247
248 void complex_mul2(float *dst_re, float *dst_im, const float *src_re, const float *src_im, size_t count)
249 {
250 IF_ARCH_X86(size_t off);
251 ARCH_X86_ASM
252 (
253 COMPLEX_MUL2_CORE("dst", "src", FMA_OFF)
254 : [count] X86_PGREG (count),
255 [off] "=&r" (off)
256 : [dst_re] "r" (dst_re), [dst_im] "r" (dst_im),
257 [src_re] "r" (src_re), [src_im] "r" (src_im)
258 : "cc", "memory",
259 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
260 "%xmm4", "%xmm5"
261 );
262 }
263
264 void complex_mul2_fma3(float *dst_re, float *dst_im, const float *src_re, const float *src_im, size_t count)
265 {
266 IF_ARCH_X86(size_t off);
267 ARCH_X86_ASM
268 (
269 COMPLEX_MUL2_CORE("dst", "src", FMA_ON)
270 : [count] X86_PGREG (count),
271 [off] "=&r" (off)
272 : [dst_re] "r" (dst_re), [dst_im] "r" (dst_im),
273 [src_re] "r" (src_re), [src_im] "r" (src_im)
274 : "cc", "memory",
275 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
276 "%xmm4", "%xmm5"
277 );
278 }
279
280 #undef COMPLEX_MUL2_CORE
281
282 #define COMPLEX_MOD_CORE(DST, SRC, SEL) \
283 __ASM_EMIT ("xor %[off], %[off]") \
284 /* x16 blocks */ \
285 __ASM_EMIT ("sub $16, %[count]") \
286 __ASM_EMIT ("jb 2f") \
287 __ASM_EMIT ("1:") \
288 __ASM_EMIT ("vmovups 0x00(%[" SRC "_re], %[off]), %%ymm0") /* ymm0 = r */ \
289 __ASM_EMIT ("vmovups 0x20(%[" SRC "_re], %[off]), %%ymm1") \
290 __ASM_EMIT ("vmovups 0x00(%[" SRC "_im], %[off]), %%ymm2") /* ymm2 = i */ \
291 __ASM_EMIT ("vmovups 0x20(%[" SRC "_im], %[off]), %%ymm3") \
292 __ASM_EMIT ("vmulps %%ymm0, %%ymm0, %%ymm0") /* ymm0 = r*r */ \
293 __ASM_EMIT ("vmulps %%ymm1, %%ymm1, %%ymm1") \
294 __ASM_EMIT (SEL("vmulps %%ymm2, %%ymm2, %%ymm2", "")) /* ymm2 = i*i */ \
295 __ASM_EMIT (SEL("vmulps %%ymm3, %%ymm3, %%ymm3", "")) \
296 __ASM_EMIT (SEL("vaddps %%ymm2, %%ymm0, %%ymm0", "vfmadd231ps %%ymm2, %%ymm2, %%ymm0")) /* ymm0 = r*r + i*i */ \
297 __ASM_EMIT (SEL("vaddps %%ymm3, %%ymm1, %%ymm1", "vfmadd231ps %%ymm3, %%ymm3, %%ymm1")) \
298 __ASM_EMIT ("vsqrtps %%ymm0, %%ymm0") /* ymm0 = sqrt(r*r + i*i) */ \
299 __ASM_EMIT ("vsqrtps %%ymm1, %%ymm1") \
300 __ASM_EMIT ("vmovups %%ymm0, 0x00(%[" DST "], %[off])") \
301 __ASM_EMIT ("vmovups %%ymm1, 0x20(%[" DST "], %[off])") \
302 __ASM_EMIT ("add $0x40, %[off]") \
303 __ASM_EMIT ("sub $16, %[count]") \
304 __ASM_EMIT ("jae 1b") \
305 __ASM_EMIT ("2:") \
306 /* x8 block */ \
307 __ASM_EMIT ("add $8, %[count]") \
308 __ASM_EMIT ("jl 4f") \
309 __ASM_EMIT ("vmovups 0x00(%[" SRC "_re], %[off]), %%ymm0") /* ymm0 = r */ \
310 __ASM_EMIT ("vmovups 0x00(%[" SRC "_im], %[off]), %%ymm2") /* ymm2 = i */ \
311 __ASM_EMIT ("vmulps %%ymm0, %%ymm0, %%ymm0") /* ymm0 = r*r */ \
312 __ASM_EMIT (SEL("vmulps %%ymm2, %%ymm2, %%ymm2", "")) /* ymm2 = i*i */ \
313 __ASM_EMIT (SEL("vaddps %%ymm2, %%ymm0, %%ymm0", "vfmadd231ps %%ymm2, %%ymm2, %%ymm0")) /* ymm0 = r*r + i*i */ \
314 __ASM_EMIT ("vsqrtps %%ymm0, %%ymm0") /* ymm0 = sqrt(r*r + i*i) */ \
315 __ASM_EMIT ("vmovups %%ymm0, 0x00(%[" DST "], %[off])") \
316 __ASM_EMIT ("add $0x20, %[off]") \
317 __ASM_EMIT ("sub $8, %[count]") \
318 __ASM_EMIT ("4:") \
319 /* x4 block */ \
320 __ASM_EMIT ("add $4, %[count]") \
321 __ASM_EMIT ("jl 6f") \
322 __ASM_EMIT ("vmovups 0x00(%[" SRC "_re], %[off]), %%xmm0") /* xmm0 = r */ \
323 __ASM_EMIT ("vmovups 0x00(%[" SRC "_im], %[off]), %%xmm2") /* xmm2 = i */ \
324 __ASM_EMIT ("vmulps %%xmm0, %%xmm0, %%xmm0") /* xmm0 = r*r */ \
325 __ASM_EMIT (SEL("vmulps %%xmm2, %%xmm2, %%xmm2", "")) /* xmm2 = i*i */ \
326 __ASM_EMIT (SEL("vaddps %%xmm2, %%xmm0, %%xmm0", "vfmadd231ps %%xmm2, %%xmm2, %%xmm0")) /* xmm0 = r*r + i*i */ \
327 __ASM_EMIT ("vsqrtps %%xmm0, %%xmm0") /* xmm0 = sqrt(r*r + i*i) */ \
328 __ASM_EMIT ("vmovups %%xmm0, 0x00(%[" DST "], %[off])") \
329 __ASM_EMIT ("add $0x10, %[off]") \
330 __ASM_EMIT ("sub $4, %[count]") \
331 __ASM_EMIT ("6:") \
332 /* x1 block */ \
333 __ASM_EMIT ("add $3, %[count]") \
334 __ASM_EMIT ("jl 8f") \
335 __ASM_EMIT ("7:") \
336 __ASM_EMIT ("vmovss 0x00(%[" SRC "_re], %[off]), %%xmm0") /* xmm0 = r */ \
337 __ASM_EMIT ("vmovss 0x00(%[" SRC "_im], %[off]), %%xmm2") /* xmm2 = i */ \
338 __ASM_EMIT ("vmulss %%xmm0, %%xmm0, %%xmm0") /* xmm0 = r*r */ \
339 __ASM_EMIT (SEL("vmulss %%xmm2, %%xmm2, %%xmm2", "")) /* xmm2 = i*i */ \
340 __ASM_EMIT (SEL("vaddss %%xmm2, %%xmm0, %%xmm0", "vfmadd231ps %%xmm2, %%xmm2, %%xmm0")) /* xmm0 = r*r + i*i */ \
341 __ASM_EMIT ("vsqrtss %%xmm0, %%xmm0, %%xmm0") /* xmm0 = sqrt(r*r + i*i) */ \
342 __ASM_EMIT ("vmovss %%xmm0, 0x00(%[" DST "], %[off])") \
343 __ASM_EMIT ("add $0x04, %[off]") \
344 __ASM_EMIT ("dec %[count]") \
345 __ASM_EMIT ("jge 7b") \
346 __ASM_EMIT ("8:") \
347
348 void complex_mod(float *dst, const float *src_re, const float *src_im, size_t count)
349 {
350 IF_ARCH_X86(size_t off);
351 ARCH_X86_ASM
352 (
353 COMPLEX_MOD_CORE("dst", "src", FMA_OFF)
354 : [count] "+r" (count), [off] "=&r" (off)
355 : [dst] "r" (dst), [src_re] "r" (src_re), [src_im] "r" (src_im)
356 : "cc", "memory",
357 "%xmm0", "%xmm1", "%xmm2", "%xmm3"
358 );
359 }
360
361 void complex_mod_fma3(float *dst, const float *src_re, const float *src_im, size_t count)
362 {
363 IF_ARCH_X86(size_t off);
364 ARCH_X86_ASM
365 (
366 COMPLEX_MOD_CORE("dst", "src", FMA_ON)
367 : [count] "+r" (count), [off] "=&r" (off)
368 : [dst] "r" (dst), [src_re] "r" (src_re), [src_im] "r" (src_im)
369 : "cc", "memory",
370 "%xmm0", "%xmm1", "%xmm2", "%xmm3"
371 );
372 }
373
374 #undef COMPLEX_MOD_CORE
375
376 #define COMPLEX_DIV3_CORE(DSTR, DSTI, TR, TI, BR, BI, __ASM_NOFMA, __ASM_FMA) \
377 __ASM_EMIT ("xor %[off], %[off]") \
378 /* x8 blocks */ \
379 __ASM_EMIT32("subl $8, %[count]") \
380 __ASM_EMIT64("sub $8, %[count]") \
381 __ASM_EMIT ("jb 2f") \
382 __ASM_EMIT ("1:") \
383 __ASM_EMIT32("mov %[" TR "], %[ptr_re]") \
384 __ASM_EMIT32("mov %[" TI "], %[ptr_im]") \
385 __ASM_EMIT ("vmovups 0x00(%[" BR "], %[off]), %%ymm2") /* ymm2 = br */ \
386 __ASM_EMIT ("vmovups 0x00(%[" BI "], %[off]), %%ymm3") /* ymm3 = bi */ \
387 __ASM_EMIT32("vmovups 0x00(%[ptr_re], %[off]), %%ymm0") /* ymm0 = tr */ \
388 __ASM_EMIT32("vmovups 0x00(%[ptr_im], %[off]), %%ymm1") /* ymm1 = ti */ \
389 __ASM_EMIT64("vmovups 0x00(%[" TR "], %[off]), %%ymm0") /* ymm0 = tr */ \
390 __ASM_EMIT64("vmovups 0x00(%[" TI "], %[off]), %%ymm1") /* ymm1 = ti */ \
391 __ASM_EMIT32("mov %[" DSTR "], %[ptr_re]") \
392 __ASM_EMIT32("mov %[" DSTI "], %[ptr_im]") \
393 __ASM_EMIT ("vmulps %%ymm2, %%ymm2, %%ymm7") /* ymm7 = br*br */ \
394 __ASM_NOFMA ("vmulps %%ymm3, %%ymm3, %%ymm6") /* ymm6 = bi*bi */ \
395 __ASM_EMIT ("vmulps %%ymm3, %%ymm1, %%ymm4") /* ymm4 = ti*bi */ \
396 __ASM_EMIT ("vmulps %%ymm3, %%ymm0, %%ymm5") /* ymm5 = tr*bi */ \
397 __ASM_NOFMA ("vaddps %%ymm6, %%ymm7, %%ymm7") /* ymm7 = M = br*br + bi*bi */ \
398 __ASM_FMA ("vfmadd231ps %%ymm3, %%ymm3, %%ymm7") /* ymm7 = M = br*br + bi*bi */ \
399 __ASM_EMIT ("vmovaps 0x00 + %[CC], %%ymm6") /* ymm6 = 1 */ \
400 __ASM_NOFMA ("vmulps %%ymm2, %%ymm0, %%ymm0") /* ymm0 = tr*br */ \
401 __ASM_NOFMA ("vmulps %%ymm2, %%ymm1, %%ymm1") /* ymm1 = ti*br */ \
402 __ASM_EMIT ("vdivps %%ymm7, %%ymm6, %%ymm7") /* ymm7 = R = 1/M */ \
403 __ASM_NOFMA ("vaddps %%ymm4, %%ymm0, %%ymm0") /* ymm0 = r = tr*br + ti*bi */ \
404 __ASM_NOFMA ("vsubps %%ymm5, %%ymm1, %%ymm1") /* ymm1 = i = ti*br - tr*bi */ \
405 __ASM_FMA ("vfmadd132ps %%ymm2, %%ymm4, %%ymm0") /* ymm0 = r = tr*br + ti*bi */ \
406 __ASM_FMA ("vfmsub132ps %%ymm2, %%ymm5, %%ymm1") /* ymm1 = i = tr*br - tr*bi */ \
407 __ASM_EMIT ("vmulps %%ymm7, %%ymm0, %%ymm0") /* ymm0 = r/R */ \
408 __ASM_EMIT ("vmulps %%ymm7, %%ymm1, %%ymm1") /* ymm1 = i/R */ \
409 __ASM_EMIT32("vmovups %%ymm0, 0x00(%[ptr_re], %[off])") \
410 __ASM_EMIT32("vmovups %%ymm1, 0x00(%[ptr_im], %[off])") \
411 __ASM_EMIT64("vmovups %%ymm0, 0x00(%[" DSTR "], %[off])") \
412 __ASM_EMIT64("vmovups %%ymm1, 0x00(%[" DSTI "], %[off])") \
413 __ASM_EMIT ("add $0x20, %[off]") \
414 __ASM_EMIT32("subl $8, %[count]") \
415 __ASM_EMIT64("sub $8, %[count]") \
416 __ASM_EMIT ("jae 1b") \
417 __ASM_EMIT ("2:") \
418 /* 4x block */ \
419 __ASM_EMIT32("addl $4, %[count]") \
420 __ASM_EMIT64("add $4, %[count]") \
421 __ASM_EMIT ("jl 4f") \
422 __ASM_EMIT32("mov %[" TR "], %[ptr_re]") \
423 __ASM_EMIT32("mov %[" TI "], %[ptr_im]") \
424 __ASM_EMIT ("vmovups 0x00(%[" BR "], %[off]), %%xmm2") /* xmm2 = br */ \
425 __ASM_EMIT ("vmovups 0x00(%[" BI "], %[off]), %%xmm3") /* xmm3 = bi */ \
426 __ASM_EMIT32("vmovups 0x00(%[ptr_re], %[off]), %%xmm0") /* xmm0 = tr */ \
427 __ASM_EMIT32("vmovups 0x00(%[ptr_im], %[off]), %%xmm1") /* xmm1 = ti */ \
428 __ASM_EMIT64("vmovups 0x00(%[" TR "], %[off]), %%xmm0") /* xmm0 = tr */ \
429 __ASM_EMIT64("vmovups 0x00(%[" TI "], %[off]), %%xmm1") /* xmm1 = ti */ \
430 __ASM_EMIT32("mov %[" DSTR "], %[ptr_re]") \
431 __ASM_EMIT32("mov %[" DSTI "], %[ptr_im]") \
432 __ASM_EMIT ("vmulps %%xmm2, %%xmm2, %%xmm7") /* xmm7 = br*br */ \
433 __ASM_NOFMA ("vmulps %%xmm3, %%xmm3, %%xmm6") /* xmm6 = bi*bi */ \
434 __ASM_EMIT ("vmulps %%xmm3, %%xmm1, %%xmm4") /* xmm4 = ti*bi */ \
435 __ASM_EMIT ("vmulps %%xmm3, %%xmm0, %%xmm5") /* xmm5 = tr*bi */ \
436 __ASM_NOFMA ("vaddps %%xmm6, %%xmm7, %%xmm7") /* xmm7 = M = br*br + bi*bi */ \
437 __ASM_FMA ("vfmadd231ps %%xmm3, %%xmm3, %%xmm7") /* xmm7 = M = br*br + bi*bi */ \
438 __ASM_EMIT ("vmovaps 0x00 + %[CC], %%xmm6") /* xmm6 = 1 */ \
439 __ASM_NOFMA ("vmulps %%xmm2, %%xmm0, %%xmm0") /* xmm0 = tr*br */ \
440 __ASM_NOFMA ("vmulps %%xmm2, %%xmm1, %%xmm1") /* xmm1 = ti*br */ \
441 __ASM_EMIT ("vdivps %%xmm7, %%xmm6, %%xmm7") /* xmm7 = R = 1/M */ \
442 __ASM_NOFMA ("vaddps %%xmm4, %%xmm0, %%xmm0") /* xmm0 = r = tr*br + ti*bi */ \
443 __ASM_NOFMA ("vsubps %%xmm5, %%xmm1, %%xmm1") /* xmm1 = i = ti*br - tr*bi */ \
444 __ASM_FMA ("vfmadd132ps %%xmm2, %%xmm4, %%xmm0") /* xmm0 = r = tr*br + ti*bi */ \
445 __ASM_FMA ("vfmsub132ps %%xmm2, %%xmm5, %%xmm1") /* xmm1 = i = tr*br - tr*bi */ \
446 __ASM_EMIT ("vmulps %%xmm7, %%xmm0, %%xmm0") /* xmm0 = r/R */ \
447 __ASM_EMIT ("vmulps %%xmm7, %%xmm1, %%xmm1") /* xmm1 = i/R */ \
448 __ASM_EMIT32("vmovups %%xmm0, 0x00(%[ptr_re], %[off])") \
449 __ASM_EMIT32("vmovups %%xmm1, 0x00(%[ptr_im], %[off])") \
450 __ASM_EMIT64("vmovups %%xmm0, 0x00(%[" DSTR "], %[off])") \
451 __ASM_EMIT64("vmovups %%xmm1, 0x00(%[" DSTI "], %[off])") \
452 __ASM_EMIT32("subl $4, %[count]") \
453 __ASM_EMIT64("sub $4, %[count]") \
454 __ASM_EMIT ("add $0x10, %[off]") \
455 __ASM_EMIT ("4:") \
456 /* 1x blocks */ \
457 __ASM_EMIT32("addl $3, %[count]") \
458 __ASM_EMIT64("add $3, %[count]") \
459 __ASM_EMIT ("jl 6f") \
460 __ASM_EMIT ("5:") \
461 __ASM_EMIT32("mov %[" TR "], %[ptr_re]") \
462 __ASM_EMIT32("mov %[" TI "], %[ptr_im]") \
463 __ASM_EMIT ("vmovss 0x00(%[" BR "], %[off]), %%xmm2") /* xmm2 = br */ \
464 __ASM_EMIT ("vmovss 0x00(%[" BI "], %[off]), %%xmm3") /* xmm3 = bi */ \
465 __ASM_EMIT32("vmovss 0x00(%[ptr_re], %[off]), %%xmm0") /* xmm0 = tr */ \
466 __ASM_EMIT32("vmovss 0x00(%[ptr_im], %[off]), %%xmm1") /* xmm1 = ti */ \
467 __ASM_EMIT64("vmovss 0x00(%[" TR "], %[off]), %%xmm0") /* xmm0 = tr */ \
468 __ASM_EMIT64("vmovss 0x00(%[" TI "], %[off]), %%xmm1") /* xmm1 = ti */ \
469 __ASM_EMIT32("mov %[" DSTR "], %[ptr_re]") \
470 __ASM_EMIT32("mov %[" DSTI "], %[ptr_im]") \
471 __ASM_EMIT ("vmulss %%xmm2, %%xmm2, %%xmm7") /* xmm7 = br*br */ \
472 __ASM_NOFMA ("vmulss %%xmm3, %%xmm3, %%xmm6") /* xmm6 = bi*bi */ \
473 __ASM_EMIT ("vmulss %%xmm3, %%xmm1, %%xmm4") /* xmm4 = ti*bi */ \
474 __ASM_EMIT ("vmulss %%xmm3, %%xmm0, %%xmm5") /* xmm5 = tr*bi */ \
475 __ASM_NOFMA ("vaddss %%xmm6, %%xmm7, %%xmm7") /* xmm7 = M = br*br + bi*bi */ \
476 __ASM_FMA ("vfmadd231ss %%xmm3, %%xmm3, %%xmm7") /* xmm7 = M = br*br + bi*bi */ \
477 __ASM_EMIT ("vmovaps 0x00 + %[CC], %%xmm6") /* xmm6 = 1 */ \
478 __ASM_NOFMA ("vmulss %%xmm2, %%xmm0, %%xmm0") /* xmm0 = tr*br */ \
479 __ASM_NOFMA ("vmulss %%xmm2, %%xmm1, %%xmm1") /* xmm1 = ti*br */ \
480 __ASM_EMIT ("vdivss %%xmm7, %%xmm6, %%xmm7") /* xmm7 = R = 1/M */ \
481 __ASM_NOFMA ("vaddss %%xmm4, %%xmm0, %%xmm0") /* xmm0 = r = tr*br + ti*bi */ \
482 __ASM_NOFMA ("vsubss %%xmm5, %%xmm1, %%xmm1") /* xmm1 = i = ti*br - tr*bi */ \
483 __ASM_FMA ("vfmadd132ss %%xmm2, %%xmm4, %%xmm0") /* xmm0 = r = tr*br + ti*bi */ \
484 __ASM_FMA ("vfmsub132ss %%xmm2, %%xmm5, %%xmm1") /* xmm1 = i = tr*br - tr*bi */ \
485 __ASM_EMIT ("vmulss %%xmm7, %%xmm0, %%xmm0") /* xmm0 = r/R */ \
486 __ASM_EMIT ("vmulss %%xmm7, %%xmm1, %%xmm1") /* xmm1 = i/R */ \
487 __ASM_EMIT32("vmovss %%xmm0, 0x00(%[ptr_re], %[off])") \
488 __ASM_EMIT32("vmovss %%xmm1, 0x00(%[ptr_im], %[off])") \
489 __ASM_EMIT64("vmovss %%xmm0, 0x00(%[" DSTR "], %[off])") \
490 __ASM_EMIT64("vmovss %%xmm1, 0x00(%[" DSTI "], %[off])") \
491 __ASM_EMIT ("add $0x04, %[off]") \
492 __ASM_EMIT32("decl %[count]") \
493 __ASM_EMIT64("dec %[count]") \
494 __ASM_EMIT ("jge 5b") \
495 __ASM_EMIT ("6:")
496
497 void complex_div3(float *dst_re, float *dst_im,
498 const float *src1_re, const float *src1_im,
499 const float *src2_re, const float *src2_im,
500 size_t count)
501 {
502 IF_ARCH_I386(float *ptr_re, *ptr_im);
503 IF_ARCH_X86(size_t off);
504 ARCH_X86_ASM
505 (
506 COMPLEX_DIV3_CORE("dst_re", "dst_im", "src1_re", "src1_im", "src2_re", "src2_im", FMA_PASS, FMA_SKIP)
507 : __IF_32(
508 [ptr_re] "=&r" (ptr_re), [ptr_im] "=&r" (ptr_im),
509 )
510 [count] X86_PGREG (count),
511 [off] "=&r" (off)
512 : [dst_re] X86_GREG (dst_re), [dst_im] X86_GREG (dst_im),
513 [src1_re] X86_GREG (src1_re), [src1_im] X86_GREG (src1_im),
514 [src2_re] "r" (src2_re), [src2_im] "r" (src2_im),
515 [CC] "o" (complex_div_const)
516 : "cc", "memory",
517 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
518 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
519 );
520 }
521
522 void complex_div3_fma3(float *dst_re, float *dst_im,
523 const float *src1_re, const float *src1_im,
524 const float *src2_re, const float *src2_im,
525 size_t count)
526 {
527 IF_ARCH_I386(float *ptr_re, *ptr_im);
528 IF_ARCH_X86(size_t off);
529 ARCH_X86_ASM
530 (
531 COMPLEX_DIV3_CORE("dst_re", "dst_im", "src1_re", "src1_im", "src2_re", "src2_im", FMA_SKIP, FMA_PASS)
532 : __IF_32(
533 [ptr_re] "=&r" (ptr_re), [ptr_im] "=&r" (ptr_im),
534 )
535 [count] X86_PGREG (count),
536 [off] "=&r" (off)
537 : [dst_re] X86_GREG (dst_re), [dst_im] X86_GREG (dst_im),
538 [src1_re] X86_GREG (src1_re), [src1_im] X86_GREG (src1_im),
539 [src2_re] "r" (src2_re), [src2_im] "r" (src2_im),
540 [CC] "o" (complex_div_const)
541 : "cc", "memory",
542 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
543 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
544 );
545 }
546
547 #undef COMPLEX_DIV3_CORE
548
549 #define COMPLEX_DIV2_CORE(DSTR, DSTI, TR, TI, BR, BI, __ASM_NOFMA, __ASM_FMA) \
550 __ASM_EMIT ("xor %[off], %[off]") \
551 /* x8 blocks */ \
552 __ASM_EMIT32("subl $8, %[count]") \
553 __ASM_EMIT64("sub $8, %[count]") \
554 __ASM_EMIT ("jb 2f") \
555 __ASM_EMIT ("1:") \
556 __ASM_EMIT ("vmovups 0x00(%[" BR "], %[off]), %%ymm2") /* ymm2 = br */ \
557 __ASM_EMIT ("vmovups 0x00(%[" BI "], %[off]), %%ymm3") /* ymm3 = bi */ \
558 __ASM_EMIT ("vmovups 0x00(%[" TR "], %[off]), %%ymm0") /* ymm0 = tr */ \
559 __ASM_EMIT ("vmovups 0x00(%[" TI "], %[off]), %%ymm1") /* ymm1 = ti */ \
560 __ASM_EMIT ("vmulps %%ymm2, %%ymm2, %%ymm7") /* ymm7 = br*br */ \
561 __ASM_NOFMA ("vmulps %%ymm3, %%ymm3, %%ymm6") /* ymm6 = bi*bi */ \
562 __ASM_EMIT ("vmulps %%ymm3, %%ymm1, %%ymm4") /* ymm4 = ti*bi */ \
563 __ASM_EMIT ("vmulps %%ymm3, %%ymm0, %%ymm5") /* ymm5 = tr*bi */ \
564 __ASM_NOFMA ("vaddps %%ymm6, %%ymm7, %%ymm7") /* ymm7 = M = br*br + bi*bi */ \
565 __ASM_FMA ("vfmadd231ps %%ymm3, %%ymm3, %%ymm7") /* ymm7 = M = br*br + bi*bi */ \
566 __ASM_EMIT ("vmovaps 0x00 + %[CC], %%ymm6") /* ymm6 = 1 */ \
567 __ASM_NOFMA ("vmulps %%ymm2, %%ymm0, %%ymm0") /* ymm0 = tr*br */ \
568 __ASM_NOFMA ("vmulps %%ymm2, %%ymm1, %%ymm1") /* ymm1 = ti*br */ \
569 __ASM_EMIT ("vdivps %%ymm7, %%ymm6, %%ymm7") /* ymm7 = R = 1/M */ \
570 __ASM_NOFMA ("vaddps %%ymm4, %%ymm0, %%ymm0") /* ymm0 = r = tr*br + ti*bi */ \
571 __ASM_NOFMA ("vsubps %%ymm5, %%ymm1, %%ymm1") /* ymm1 = i = ti*br - tr*bi */ \
572 __ASM_FMA ("vfmadd132ps %%ymm2, %%ymm4, %%ymm0") /* ymm0 = r = tr*br + ti*bi */ \
573 __ASM_FMA ("vfmsub132ps %%ymm2, %%ymm5, %%ymm1") /* ymm1 = i = tr*br - tr*bi */ \
574 __ASM_EMIT ("vmulps %%ymm7, %%ymm0, %%ymm0") /* ymm0 = r/R */ \
575 __ASM_EMIT ("vmulps %%ymm7, %%ymm1, %%ymm1") /* ymm1 = i/R */ \
576 __ASM_EMIT ("vmovups %%ymm0, 0x00(%[" DSTR "], %[off])") \
577 __ASM_EMIT ("vmovups %%ymm1, 0x00(%[" DSTI "], %[off])") \
578 __ASM_EMIT ("add $0x20, %[off]") \
579 __ASM_EMIT32("subl $8, %[count]") \
580 __ASM_EMIT64("sub $8, %[count]") \
581 __ASM_EMIT ("jae 1b") \
582 __ASM_EMIT ("2:") \
583 /* 4x block */ \
584 __ASM_EMIT32("addl $4, %[count]") \
585 __ASM_EMIT64("add $4, %[count]") \
586 __ASM_EMIT ("jl 4f") \
587 __ASM_EMIT ("vmovups 0x00(%[" BR "], %[off]), %%xmm2") /* xmm2 = br */ \
588 __ASM_EMIT ("vmovups 0x00(%[" BI "], %[off]), %%xmm3") /* xmm3 = bi */ \
589 __ASM_EMIT ("vmovups 0x00(%[" TR "], %[off]), %%xmm0") /* xmm0 = tr */ \
590 __ASM_EMIT ("vmovups 0x00(%[" TI "], %[off]), %%xmm1") /* xmm1 = ti */ \
591 __ASM_EMIT ("vmulps %%xmm2, %%xmm2, %%xmm7") /* xmm7 = br*br */ \
592 __ASM_NOFMA ("vmulps %%xmm3, %%xmm3, %%xmm6") /* xmm6 = bi*bi */ \
593 __ASM_EMIT ("vmulps %%xmm3, %%xmm1, %%xmm4") /* xmm4 = ti*bi */ \
594 __ASM_EMIT ("vmulps %%xmm3, %%xmm0, %%xmm5") /* xmm5 = tr*bi */ \
595 __ASM_NOFMA ("vaddps %%xmm6, %%xmm7, %%xmm7") /* xmm7 = M = br*br + bi*bi */ \
596 __ASM_FMA ("vfmadd231ps %%xmm3, %%xmm3, %%xmm7") /* xmm7 = M = br*br + bi*bi */ \
597 __ASM_EMIT ("vmovaps 0x00 + %[CC], %%xmm6") /* xmm6 = 1 */ \
598 __ASM_NOFMA ("vmulps %%xmm2, %%xmm0, %%xmm0") /* xmm0 = tr*br */ \
599 __ASM_NOFMA ("vmulps %%xmm2, %%xmm1, %%xmm1") /* xmm1 = ti*br */ \
600 __ASM_EMIT ("vdivps %%xmm7, %%xmm6, %%xmm7") /* xmm7 = R = 1/M */ \
601 __ASM_NOFMA ("vaddps %%xmm4, %%xmm0, %%xmm0") /* xmm0 = r = tr*br + ti*bi */ \
602 __ASM_NOFMA ("vsubps %%xmm5, %%xmm1, %%xmm1") /* xmm1 = i = ti*br - tr*bi */ \
603 __ASM_FMA ("vfmadd132ps %%xmm2, %%xmm4, %%xmm0") /* xmm0 = r = tr*br + ti*bi */ \
604 __ASM_FMA ("vfmsub132ps %%xmm2, %%xmm5, %%xmm1") /* xmm1 = i = tr*br - tr*bi */ \
605 __ASM_EMIT ("vmulps %%xmm7, %%xmm0, %%xmm0") /* xmm0 = r/R */ \
606 __ASM_EMIT ("vmulps %%xmm7, %%xmm1, %%xmm1") /* xmm1 = i/R */ \
607 __ASM_EMIT ("vmovups %%xmm0, 0x00(%[" DSTR "], %[off])") \
608 __ASM_EMIT ("vmovups %%xmm1, 0x00(%[" DSTI "], %[off])") \
609 __ASM_EMIT32("subl $4, %[count]") \
610 __ASM_EMIT64("sub $4, %[count]") \
611 __ASM_EMIT ("add $0x10, %[off]") \
612 __ASM_EMIT ("4:") \
613 /* 1x blocks */ \
614 __ASM_EMIT32("addl $3, %[count]") \
615 __ASM_EMIT64("add $3, %[count]") \
616 __ASM_EMIT ("jl 6f") \
617 __ASM_EMIT ("5:") \
618 __ASM_EMIT ("vmovss 0x00(%[" BR "], %[off]), %%xmm2") /* xmm2 = br */ \
619 __ASM_EMIT ("vmovss 0x00(%[" BI "], %[off]), %%xmm3") /* xmm3 = bi */ \
620 __ASM_EMIT ("vmovss 0x00(%[" TR "], %[off]), %%xmm0") /* xmm0 = tr */ \
621 __ASM_EMIT ("vmovss 0x00(%[" TI "], %[off]), %%xmm1") /* xmm1 = ti */ \
622 __ASM_EMIT ("vmulss %%xmm2, %%xmm2, %%xmm7") /* xmm7 = br*br */ \
623 __ASM_NOFMA ("vmulss %%xmm3, %%xmm3, %%xmm6") /* xmm6 = bi*bi */ \
624 __ASM_EMIT ("vmulss %%xmm3, %%xmm1, %%xmm4") /* xmm4 = ti*bi */ \
625 __ASM_EMIT ("vmulss %%xmm3, %%xmm0, %%xmm5") /* xmm5 = tr*bi */ \
626 __ASM_NOFMA ("vaddss %%xmm6, %%xmm7, %%xmm7") /* xmm7 = M = br*br + bi*bi */ \
627 __ASM_FMA ("vfmadd231ss %%xmm3, %%xmm3, %%xmm7") /* xmm7 = M = br*br + bi*bi */ \
628 __ASM_EMIT ("vmovaps 0x00 + %[CC], %%xmm6") /* xmm6 = 1 */ \
629 __ASM_NOFMA ("vmulss %%xmm2, %%xmm0, %%xmm0") /* xmm0 = tr*br */ \
630 __ASM_NOFMA ("vmulss %%xmm2, %%xmm1, %%xmm1") /* xmm1 = ti*br */ \
631 __ASM_EMIT ("vdivss %%xmm7, %%xmm6, %%xmm7") /* xmm7 = R = 1/M */ \
632 __ASM_NOFMA ("vaddss %%xmm4, %%xmm0, %%xmm0") /* xmm0 = r = tr*br + ti*bi */ \
633 __ASM_NOFMA ("vsubss %%xmm5, %%xmm1, %%xmm1") /* xmm1 = i = ti*br - tr*bi */ \
634 __ASM_FMA ("vfmadd132ss %%xmm2, %%xmm4, %%xmm0") /* xmm0 = r = tr*br + ti*bi */ \
635 __ASM_FMA ("vfmsub132ss %%xmm2, %%xmm5, %%xmm1") /* xmm1 = i = tr*br - tr*bi */ \
636 __ASM_EMIT ("vmulss %%xmm7, %%xmm0, %%xmm0") /* xmm0 = r/R */ \
637 __ASM_EMIT ("vmulss %%xmm7, %%xmm1, %%xmm1") /* xmm1 = i/R */ \
638 __ASM_EMIT ("vmovss %%xmm0, 0x00(%[" DSTR "], %[off])") \
639 __ASM_EMIT ("vmovss %%xmm1, 0x00(%[" DSTI "], %[off])") \
640 __ASM_EMIT ("add $0x04, %[off]") \
641 __ASM_EMIT32("decl %[count]") \
642 __ASM_EMIT64("dec %[count]") \
643 __ASM_EMIT ("jge 5b") \
644 __ASM_EMIT ("6:")
645
646 void complex_div2(float *dst_re, float *dst_im, const float *src_re, const float *src_im, size_t count)
647 {
648 IF_ARCH_X86(size_t off);
649 ARCH_X86_ASM
650 (
651 COMPLEX_DIV2_CORE("dst_re", "dst_im", "dst_re", "dst_im", "src_re", "src_im", FMA_PASS, FMA_SKIP)
652 : [count] X86_PGREG (count),
653 [off] "=&r" (off)
654 : [dst_re] "r" (dst_re), [dst_im] "r" (dst_im),
655 [src_re] "r" (src_re), [src_im] "r" (src_im),
656 [CC] "m" (complex_div_const)
657 : "cc", "memory",
658 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
659 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
660 );
661 }
662
663 void complex_div2_fma3(float *dst_re, float *dst_im, const float *src_re, const float *src_im, size_t count)
664 {
665 IF_ARCH_X86(size_t off);
666 ARCH_X86_ASM
667 (
668 COMPLEX_DIV2_CORE("dst_re", "dst_im", "dst_re", "dst_im", "src_re", "src_im", FMA_SKIP, FMA_PASS)
669 : [count] X86_PGREG (count),
670 [off] "=&r" (off)
671 : [dst_re] "r" (dst_re), [dst_im] "r" (dst_im),
672 [src_re] "r" (src_re), [src_im] "r" (src_im),
673 [CC] "m" (complex_div_const)
674 : "cc", "memory",
675 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
676 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
677 );
678 }
679
680 void complex_rdiv2(float *dst_re, float *dst_im, const float *src_re, const float *src_im, size_t count)
681 {
682 IF_ARCH_X86(size_t off);
683 ARCH_X86_ASM
684 (
685 COMPLEX_DIV2_CORE("dst_re", "dst_im", "src_re", "src_im", "dst_re", "dst_im", FMA_PASS, FMA_SKIP)
686 : [count] X86_PGREG (count),
687 [off] "=&r" (off)
688 : [dst_re] "r" (dst_re), [dst_im] "r" (dst_im),
689 [src_re] "r" (src_re), [src_im] "r" (src_im),
690 [CC] "m" (complex_div_const)
691 : "cc", "memory",
692 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
693 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
694 );
695 }
696
697 void complex_rdiv2_fma3(float *dst_re, float *dst_im, const float *src_re, const float *src_im, size_t count)
698 {
699 IF_ARCH_X86(size_t off);
700 ARCH_X86_ASM
701 (
702 COMPLEX_DIV2_CORE("dst_re", "dst_im", "src_re", "src_im", "dst_re", "dst_im", FMA_SKIP, FMA_PASS)
703 : [count] X86_PGREG (count),
704 [off] "=&r" (off)
705 : [dst_re] "r" (dst_re), [dst_im] "r" (dst_im),
706 [src_re] "r" (src_re), [src_im] "r" (src_im),
707 [CC] "m" (complex_div_const)
708 : "cc", "memory",
709 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
710 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
711 );
712 }
713
714 #undef COMPLEX_DIV2_CORE
715
716 #define COMPLEX_RCP_CORE(DST, SRC, SEL) \
717 __ASM_EMIT ("xor %[off], %[off]") \
718 /* x16 blocks */ \
719 __ASM_EMIT32("subl $16, %[count]") \
720 __ASM_EMIT64("sub $16, %[count]") \
721 __ASM_EMIT ("jb 2f") \
722 __ASM_EMIT ("1:") \
723 __ASM_EMIT ("vmovups 0x00(%[" SRC "_re], %[off]), %%ymm0") /* ymm0 = ar */ \
724 __ASM_EMIT ("vmovups 0x20(%[" SRC "_re], %[off]), %%ymm1") \
725 __ASM_EMIT ("vmovups 0x00(%[" SRC "_im], %[off]), %%ymm2") /* ymm2 = ai */ \
726 __ASM_EMIT ("vmovups 0x20(%[" SRC "_im], %[off]), %%ymm3") \
727 __ASM_EMIT ("vmulps %%ymm0, %%ymm0, %%ymm4") /* ymm4 = ar*ar */ \
728 __ASM_EMIT ("vmulps %%ymm1, %%ymm1, %%ymm5") \
729 __ASM_EMIT (SEL("vmulps %%ymm2, %%ymm2, %%ymm6", "")) /* ymm6 = ai*ai */ \
730 __ASM_EMIT (SEL("vmulps %%ymm3, %%ymm3, %%ymm7", "")) \
731 __ASM_EMIT (SEL("vaddps %%ymm6, %%ymm4, %%ymm4", "vfmadd231ps %%ymm2, %%ymm2, %%ymm4")) /* ymm4 = R = ar*ar+ai*ai */ \
732 __ASM_EMIT (SEL("vaddps %%ymm7, %%ymm5, %%ymm5", "vfmadd231ps %%ymm3, %%ymm3, %%ymm5")) \
733 __ASM_EMIT ("vmovaps 0x00 + %[CC], %%ymm6") /* ymm6 = 1 */ \
734 __ASM_EMIT ("vxorps 0x20 + %[CC], %%ymm2, %%ymm2") /* ymm1 = -ai */ \
735 __ASM_EMIT ("vxorps 0x20 + %[CC], %%ymm3, %%ymm3") \
736 __ASM_EMIT ("vdivps %%ymm4, %%ymm6, %%ymm4") /* ymm4 = 1/R */ \
737 __ASM_EMIT ("vdivps %%ymm5, %%ymm6, %%ymm5") \
738 __ASM_EMIT ("vmulps %%ymm4, %%ymm0, %%ymm0") /* ymm0 = ar/R */ \
739 __ASM_EMIT ("vmulps %%ymm5, %%ymm1, %%ymm1") \
740 __ASM_EMIT ("vmulps %%ymm4, %%ymm2, %%ymm2") /* ymm1 = -ai/R */ \
741 __ASM_EMIT ("vmulps %%ymm5, %%ymm3, %%ymm3") \
742 __ASM_EMIT ("vmovups %%ymm0, 0x00(%[" DST "_re], %[off])") \
743 __ASM_EMIT ("vmovups %%ymm1, 0x20(%[" DST "_re], %[off])") \
744 __ASM_EMIT ("vmovups %%ymm2, 0x00(%[" DST "_im], %[off])") \
745 __ASM_EMIT ("vmovups %%ymm3, 0x20(%[" DST "_im], %[off])") \
746 __ASM_EMIT ("add $0x40, %[off]") \
747 __ASM_EMIT32("subl $16, %[count]") \
748 __ASM_EMIT64("sub $16, %[count]") \
749 __ASM_EMIT ("jae 1b") \
750 __ASM_EMIT ("2:") \
751 /* 8x block */ \
752 __ASM_EMIT32("addl $8, %[count]") \
753 __ASM_EMIT64("add $8, %[count]") \
754 __ASM_EMIT ("jl 4f") \
755 __ASM_EMIT ("vmovups 0x00(%[" SRC "_re], %[off]), %%ymm0") /* ymm0 = ar */ \
756 __ASM_EMIT ("vmovups 0x00(%[" SRC "_im], %[off]), %%ymm2") /* ymm2 = ai */ \
757 __ASM_EMIT ("vmulps %%ymm0, %%ymm0, %%ymm4") /* ymm4 = ar*ar */ \
758 __ASM_EMIT (SEL("vmulps %%ymm2, %%ymm2, %%ymm6", "")) /* ymm6 = ai*ai */ \
759 __ASM_EMIT (SEL("vaddps %%ymm6, %%ymm4, %%ymm4", "vfmadd231ps %%ymm2, %%ymm2, %%ymm4")) /* ymm4 = R = ar*ar+ai*ai */ \
760 __ASM_EMIT ("vmovaps 0x00 + %[CC], %%ymm6") /* ymm6 = 1 */ \
761 __ASM_EMIT ("vxorps 0x20 + %[CC], %%ymm2, %%ymm2") /* ymm1 = -ai */ \
762 __ASM_EMIT ("vdivps %%ymm4, %%ymm6, %%ymm4") /* ymm4 = 1/R */ \
763 __ASM_EMIT ("vmulps %%ymm4, %%ymm0, %%ymm0") /* ymm0 = ar/R */ \
764 __ASM_EMIT ("vmulps %%ymm4, %%ymm2, %%ymm2") /* ymm1 = -ai/R */ \
765 __ASM_EMIT ("vmovups %%ymm0, 0x00(%[" DST "_re], %[off])") \
766 __ASM_EMIT ("vmovups %%ymm2, 0x00(%[" DST "_im], %[off])") \
767 __ASM_EMIT ("add $0x20, %[off]") \
768 __ASM_EMIT32("subl $8, %[count]") \
769 __ASM_EMIT64("sub $8, %[count]") \
770 __ASM_EMIT ("4:") \
771 /* 4x block */ \
772 __ASM_EMIT32("addl $4, %[count]") \
773 __ASM_EMIT64("add $4, %[count]") \
774 __ASM_EMIT ("jl 6f") \
775 __ASM_EMIT ("vmovups 0x00(%[" SRC "_re], %[off]), %%xmm0") /* xmm0 = ar */ \
776 __ASM_EMIT ("vmovups 0x00(%[" SRC "_im], %[off]), %%xmm2") /* xmm2 = ai */ \
777 __ASM_EMIT ("vmulps %%xmm0, %%xmm0, %%xmm4") /* xmm4 = ar*ar */ \
778 __ASM_EMIT (SEL("vmulps %%xmm2, %%xmm2, %%xmm6", "")) /* xmm6 = ai*ai */ \
779 __ASM_EMIT (SEL("vaddps %%xmm6, %%xmm4, %%xmm4", "vfmadd231ps %%xmm2, %%xmm2, %%xmm4")) /* xmm4 = R = ar*ar+ai*ai */ \
780 __ASM_EMIT ("vmovaps 0x00 + %[CC], %%xmm6") /* xmm6 = 1 */ \
781 __ASM_EMIT ("vxorps 0x20 + %[CC], %%xmm2, %%xmm2") /* xmm1 = -ai */ \
782 __ASM_EMIT ("vdivps %%xmm4, %%xmm6, %%xmm4") /* xmm4 = 1/R */ \
783 __ASM_EMIT ("vmulps %%xmm4, %%xmm0, %%xmm0") /* xmm0 = ar/R */ \
784 __ASM_EMIT ("vmulps %%xmm4, %%xmm2, %%xmm2") /* xmm1 = -ai/R */ \
785 __ASM_EMIT ("vmovups %%xmm0, 0x00(%[" DST "_re], %[off])") \
786 __ASM_EMIT ("vmovups %%xmm2, 0x00(%[" DST "_im], %[off])") \
787 __ASM_EMIT ("add $0x10, %[off]") \
788 __ASM_EMIT32("subl $4, %[count]") \
789 __ASM_EMIT64("sub $4, %[count]") \
790 __ASM_EMIT ("6:") \
791 /* 1x blocks */ \
792 __ASM_EMIT32("addl $3, %[count]") \
793 __ASM_EMIT64("add $3, %[count]") \
794 __ASM_EMIT ("jl 8f") \
795 __ASM_EMIT ("7:") \
796 __ASM_EMIT ("vmovss 0x00(%[" SRC "_re], %[off]), %%xmm0") /* xmm0 = ar */ \
797 __ASM_EMIT ("vmovss 0x00(%[" SRC "_im], %[off]), %%xmm2") /* xmm2 = ai */ \
798 __ASM_EMIT ("vmulss %%xmm0, %%xmm0, %%xmm4") /* xmm4 = ar*ar */ \
799 __ASM_EMIT (SEL("vmulss %%xmm2, %%xmm2, %%xmm6", "")) /* xmm6 = ai*ai */ \
800 __ASM_EMIT (SEL("vaddss %%xmm6, %%xmm4, %%xmm4", "vfmadd231ss %%xmm2, %%xmm2, %%xmm4")) /* xmm4 = R = ar*ar+ai*ai */ \
801 __ASM_EMIT ("vmovaps 0x00 + %[CC], %%xmm6") /* xmm6 = 1 */ \
802 __ASM_EMIT ("vxorps 0x20 + %[CC], %%xmm2, %%xmm2") /* xmm1 = -ai */ \
803 __ASM_EMIT ("vdivss %%xmm4, %%xmm6, %%xmm4") /* xmm4 = 1/R */ \
804 __ASM_EMIT ("vmulss %%xmm4, %%xmm0, %%xmm0") /* xmm0 = ar/R */ \
805 __ASM_EMIT ("vmulss %%xmm4, %%xmm2, %%xmm2") /* xmm1 = -ai/R */ \
806 __ASM_EMIT ("vmovss %%xmm0, 0x00(%[" DST "_re], %[off])") \
807 __ASM_EMIT ("vmovss %%xmm2, 0x00(%[" DST "_im], %[off])") \
808 __ASM_EMIT ("add $0x04, %[off]") \
809 __ASM_EMIT32("decl %[count]") \
810 __ASM_EMIT64("dec %[count]") \
811 __ASM_EMIT ("jge 7b") \
812 __ASM_EMIT ("8:")
813
814 void complex_rcp1(float *dst_re, float *dst_im, size_t count)
815 {
816 IF_ARCH_X86(size_t off);
817 ARCH_X86_ASM
818 (
819 COMPLEX_RCP_CORE("dst", "dst", FMA_OFF)
820 : [count] "+r" (count), [off] "=&r" (off)
821 : [dst_re] "r" (dst_re), [dst_im] "r" (dst_im),
822 [CC] "m" (complex_div_const)
823 : "cc", "memory",
824 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
825 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
826 );
827 }
828
829 void complex_rcp1_fma3(float *dst_re, float *dst_im, size_t count)
830 {
831 IF_ARCH_X86(size_t off);
832 ARCH_X86_ASM
833 (
834 COMPLEX_RCP_CORE("dst", "dst", FMA_ON)
835 : [count] "+r" (count), [off] "=&r" (off)
836 : [dst_re] "r" (dst_re), [dst_im] "r" (dst_im),
837 [CC] "m" (complex_div_const)
838 : "cc", "memory",
839 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
840 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
841 );
842 }
843
844 void complex_rcp2(float *dst_re, float *dst_im, const float *src_re, const float *src_im, size_t count)
845 {
846 IF_ARCH_X86(size_t off);
847 ARCH_X86_ASM
848 (
849 COMPLEX_RCP_CORE("dst", "src", FMA_OFF)
850 : [count] X86_PGREG (count),
851 [off] "=&r" (off)
852 : [dst_re] "r" (dst_re), [dst_im] "r" (dst_im),
853 [src_re] "r" (src_re), [src_im] "r" (src_im),
854 [CC] "m" (complex_div_const)
855 : "cc", "memory",
856 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
857 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
858 );
859 }
860
861 void complex_rcp2_fma3(float *dst_re, float *dst_im, const float *src_re, const float *src_im, size_t count)
862 {
863 IF_ARCH_X86(size_t off);
864 ARCH_X86_ASM
865 (
866 COMPLEX_RCP_CORE("dst", "src", FMA_ON)
867 : [count] X86_PGREG (count),
868 [off] "=&r" (off)
869 : [dst_re] "r" (dst_re), [dst_im] "r" (dst_im),
870 [src_re] "r" (src_re), [src_im] "r" (src_im),
871 [CC] "m" (complex_div_const)
872 : "cc", "memory",
873 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
874 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
875 );
876 }
877
878 #undef COMPLEX_RCP_CORE
879
880 #undef FMA_OFF
881 #undef FMA_ON
882 #undef FMA_PASS
883 #undef FMA_SKIP
884 } /* namespace avx */
885 } /* namespace lsp */
886
887 #endif /* PRIVATE_DSP_ARCH_X86_AVX_COMPLEX_H_ */