Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/x86/avx/fft/p_repack.h

raw
1 /*
2 * Copyright (C) 2020 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2020 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 31 мар. 2020 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_X86_AVX_FFT_P_REPACK_H_
23 #define PRIVATE_DSP_ARCH_X86_AVX_FFT_P_REPACK_H_
24
25 #ifndef PRIVATE_DSP_ARCH_X86_AVX_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_X86_AVX_IMPL */
28
29 namespace lsp
30 {
31 namespace avx
32 {
33 static inline void packed_fft_repack(float *dst, size_t rank)
34 {
35 size_t blocks = 1 << rank;
36
37 // Perform 4-element butterflies
38 ARCH_X86_ASM
39 (
40 // 16x blocks
41 __ASM_EMIT("sub $16, %[blocks]")
42 __ASM_EMIT("jb 2f")
43 __ASM_EMIT("1:")
44 __ASM_EMIT("vmovups 0x00(%[dst]), %%ymm0") /* ymm0 = r0 r1 r2 r3 r4 r5 r6 r7 */
45 __ASM_EMIT("vmovups 0x20(%[dst]), %%ymm1") /* ymm1 = i0 i1 i2 i3 i4 i5 i6 i7 */
46 __ASM_EMIT("vmovups 0x40(%[dst]), %%ymm2") /* ymm2 = r8 r9 r10 r11 r12 r13 r14 r15 */
47 __ASM_EMIT("vmovups 0x60(%[dst]), %%ymm3") /* ymm3 = i8 i9 i10 i11 i12 i13 i14 i15 */
48 __ASM_EMIT("vunpcklps %%ymm1, %%ymm0, %%ymm4") /* ymm4 = r0 i0 r1 i1 r4 i4 r5 i5 */
49 __ASM_EMIT("vunpckhps %%ymm1, %%ymm0, %%ymm5") /* ymm5 = r2 i2 r3 i3 r6 i6 r7 i7 */
50 __ASM_EMIT("vunpcklps %%ymm3, %%ymm2, %%ymm0") /* ymm0 = r8 i8 r9 i9 r12 i12 r13 i13 */
51 __ASM_EMIT("vunpckhps %%ymm3, %%ymm2, %%ymm1") /* ymm1 = r10 i10 r12 i12 r14 i14 r15 i15 */
52 __ASM_EMIT("vmovups %%xmm4, 0x00(%[dst])")
53 __ASM_EMIT("vmovups %%xmm5, 0x10(%[dst])")
54 __ASM_EMIT("vextractf128 $1, %%ymm4, 0x20(%[dst])")
55 __ASM_EMIT("vextractf128 $1, %%ymm5, 0x30(%[dst])")
56 __ASM_EMIT("vmovups %%xmm0, 0x40(%[dst])")
57 __ASM_EMIT("vmovups %%xmm1, 0x50(%[dst])")
58 __ASM_EMIT("vextractf128 $1, %%ymm0, 0x60(%[dst])")
59 __ASM_EMIT("vextractf128 $1, %%ymm1, 0x70(%[dst])")
60 __ASM_EMIT("add $0x80, %[dst]")
61 __ASM_EMIT("sub $16, %[blocks]")
62 __ASM_EMIT("jae 1b")
63 __ASM_EMIT("2:")
64 // 8x block
65 __ASM_EMIT("add $8, %[blocks]")
66 __ASM_EMIT("jl 4f")
67 __ASM_EMIT("vmovups 0x00(%[dst]), %%ymm0") /* ymm0 = r0 r1 r2 r3 r4 r5 r6 r7 */
68 __ASM_EMIT("vmovups 0x20(%[dst]), %%ymm1") /* ymm1 = i0 i1 i2 i3 i4 i5 i6 i7 */
69 __ASM_EMIT("vunpcklps %%ymm1, %%ymm0, %%ymm4") /* ymm4 = r0 i0 r1 i1 r4 i4 r5 i5 */
70 __ASM_EMIT("vunpckhps %%ymm1, %%ymm0, %%ymm5") /* ymm5 = r2 i2 r3 i3 r6 i6 r7 i7 */
71 __ASM_EMIT("vmovups %%xmm4, 0x00(%[dst])")
72 __ASM_EMIT("vmovups %%xmm5, 0x10(%[dst])")
73 __ASM_EMIT("vextractf128 $1, %%ymm4, 0x20(%[dst])")
74 __ASM_EMIT("vextractf128 $1, %%ymm5, 0x30(%[dst])")
75 __ASM_EMIT("4:")
76
77 : [dst] "+r"(dst), [blocks] "+r" (blocks)
78 :
79 : "cc", "memory",
80 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
81 "%xmm4", "%xmm5"
82 );
83 }
84
85 static inline void packed_fft_repack_normalize(float *dst, size_t rank)
86 {
87 size_t blocks = 1 << rank;
88 float norm = 1.0f / float(blocks);
89
90 // Perform 4-element butterflies
91 ARCH_X86_ASM
92 (
93 __ASM_EMIT("vbroadcastss %[norm], %%ymm6")
94 __ASM_EMIT("vmovaps %%ymm6, %%ymm7")
95 // 16x blocks
96 __ASM_EMIT("sub $16, %[blocks]")
97 __ASM_EMIT("jb 2f")
98 __ASM_EMIT("1:")
99 __ASM_EMIT("vmulps 0x00(%[dst]), %%ymm6, %%ymm0") /* ymm0 = r0 r1 r2 r3 r4 r5 r6 r7 */
100 __ASM_EMIT("vmulps 0x20(%[dst]), %%ymm7, %%ymm1") /* ymm1 = i0 i1 i2 i3 i4 i5 i6 i7 */
101 __ASM_EMIT("vmulps 0x40(%[dst]), %%ymm6, %%ymm2") /* ymm2 = r8 r9 r10 r11 r12 r13 r14 r15 */
102 __ASM_EMIT("vmulps 0x60(%[dst]), %%ymm7, %%ymm3") /* ymm3 = i8 i9 i10 i11 i12 i13 i14 i15 */
103 __ASM_EMIT("vunpcklps %%ymm1, %%ymm0, %%ymm4") /* ymm4 = r0 i0 r1 i1 r4 i4 r5 i5 */
104 __ASM_EMIT("vunpckhps %%ymm1, %%ymm0, %%ymm5") /* ymm5 = r2 i2 r3 i3 r6 i6 r7 i7 */
105 __ASM_EMIT("vunpcklps %%ymm3, %%ymm2, %%ymm0") /* ymm0 = r8 i8 r9 i9 r12 i12 r13 i13 */
106 __ASM_EMIT("vunpckhps %%ymm3, %%ymm2, %%ymm1") /* ymm1 = r10 i10 r12 i12 r14 i14 r15 i15 */
107 __ASM_EMIT("vmovups %%xmm4, 0x00(%[dst])")
108 __ASM_EMIT("vmovups %%xmm5, 0x10(%[dst])")
109 __ASM_EMIT("vextractf128 $1, %%ymm4, 0x20(%[dst])")
110 __ASM_EMIT("vextractf128 $1, %%ymm5, 0x30(%[dst])")
111 __ASM_EMIT("vmovups %%xmm0, 0x40(%[dst])")
112 __ASM_EMIT("vmovups %%xmm1, 0x50(%[dst])")
113 __ASM_EMIT("vextractf128 $1, %%ymm0, 0x60(%[dst])")
114 __ASM_EMIT("vextractf128 $1, %%ymm1, 0x70(%[dst])")
115 __ASM_EMIT("add $0x80, %[dst]")
116 __ASM_EMIT("sub $16, %[blocks]")
117 __ASM_EMIT("jae 1b")
118 __ASM_EMIT("2:")
119 // 8x block
120 __ASM_EMIT("add $8, %[blocks]")
121 __ASM_EMIT("jl 4f")
122 __ASM_EMIT("vmulps 0x00(%[dst]), %%ymm6, %%ymm0") /* ymm0 = r0 r1 r2 r3 r4 r5 r6 r7 */
123 __ASM_EMIT("vmulps 0x20(%[dst]), %%ymm7, %%ymm1") /* ymm1 = i0 i1 i2 i3 i4 i5 i6 i7 */
124 __ASM_EMIT("vunpcklps %%ymm1, %%ymm0, %%ymm4") /* ymm4 = r0 i0 r1 i1 r4 i4 r5 i5 */
125 __ASM_EMIT("vunpckhps %%ymm1, %%ymm0, %%ymm5") /* ymm5 = r2 i2 r3 i3 r6 i6 r7 i7 */
126 __ASM_EMIT("vmovups %%xmm4, 0x00(%[dst])")
127 __ASM_EMIT("vmovups %%xmm5, 0x10(%[dst])")
128 __ASM_EMIT("vextractf128 $1, %%ymm4, 0x20(%[dst])")
129 __ASM_EMIT("vextractf128 $1, %%ymm5, 0x30(%[dst])")
130 __ASM_EMIT("4:")
131
132 : [dst] "+r"(dst), [blocks] "+r" (blocks)
133 : [norm] "m" (norm)
134 : "cc", "memory",
135 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
136 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
137 );
138 }
139 }
140 }
141
142 #endif /* PRIVATE_DSP_ARCH_X86_AVX_FFT_P_REPACK_H_ */