Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/x86/avx512/fft/p_repack.h

raw
1 /*
2 * Copyright (C) 2026 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2026 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 18 мар. 2026 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_X86_AVX512_FFT_P_REPACK_H_
23 #define PRIVATE_DSP_ARCH_X86_AVX512_FFT_P_REPACK_H_
24
25 #ifndef PRIVATE_DSP_ARCH_X86_AVX512_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_X86_AVX512_IMPL */
28
29 namespace lsp
30 {
31 namespace avx512
32 {
33 static inline void packed_fft_repack(float *dst, size_t rank)
34 {
35 size_t blocks = 1 << rank;
36
37 ARCH_X86_ASM
38 (
39 __ASM_EMIT("kmovw 0x00 + %[MASK], %%k4")
40 __ASM_EMIT("kmovw 0x02 + %[MASK], %%k5")
41 // 32x blocks
42 __ASM_EMIT("sub $32, %[blocks]")
43 __ASM_EMIT("jb 2f")
44 __ASM_EMIT("1:")
45 __ASM_EMIT("vexpandps 0x000(%[dst]), %%zmm0 %{%%k4%}%{z%}") /* zmm0 = r0 0 r1 0 r2 0 r3 0 r4 0 r5 0 r6 0 r7 0 */
46 __ASM_EMIT("vexpandps 0x020(%[dst]), %%zmm1 %{%%k4%}%{z%}") /* zmm1 = r8 0 r9 0 r10 0 r11 0 r12 0 r13 0 r14 0 r15 0 */
47 __ASM_EMIT("vexpandps 0x040(%[dst]), %%zmm0 %{%%k5%}") /* zmm0 = r0 i0 r1 i1 r2 i2 r3 i3 r4 i4 r5 i5 r6 i6 r7 i7 */
48 __ASM_EMIT("vexpandps 0x060(%[dst]), %%zmm1 %{%%k5%}") /* zmm1 = r8 i8 r9 i9 r10 i10 r11 i11 r12 i12 r13 i13 r14 i14 r15 i15 */
49 __ASM_EMIT("vexpandps 0x080(%[dst]), %%zmm2 %{%%k4%}%{z%}")
50 __ASM_EMIT("vexpandps 0x0a0(%[dst]), %%zmm3 %{%%k4%}%{z%}")
51 __ASM_EMIT("vexpandps 0x0c0(%[dst]), %%zmm2 %{%%k5%}")
52 __ASM_EMIT("vmovups 0x0e0(%[dst]), %%ymm4")
53 __ASM_EMIT("vexpandps %%zmm4, %%zmm3 %{%%k5%}")
54 __ASM_EMIT("vmovups %%zmm0, 0x000(%[dst])")
55 __ASM_EMIT("vmovups %%zmm1, 0x040(%[dst])")
56 __ASM_EMIT("vmovups %%zmm2, 0x080(%[dst])")
57 __ASM_EMIT("vmovups %%zmm3, 0x0c0(%[dst])")
58 __ASM_EMIT("add $0x100, %[dst]")
59 __ASM_EMIT("sub $32, %[blocks]")
60 __ASM_EMIT("jae 1b")
61 // 16x block
62 __ASM_EMIT("2:")
63 __ASM_EMIT("add $16, %[blocks]")
64 __ASM_EMIT("jl 4f")
65 __ASM_EMIT("vexpandps 0x000(%[dst]), %%zmm0 %{%%k4%}%{z%}") /* zmm0 = r0 0 r1 0 r2 0 r3 0 r4 0 r5 0 r6 0 r7 0 */
66 __ASM_EMIT("vexpandps 0x020(%[dst]), %%zmm1 %{%%k4%}%{z%}") /* zmm1 = r8 0 r9 0 r10 0 r11 0 r12 0 r13 0 r14 0 r15 0 */
67 __ASM_EMIT("vexpandps 0x040(%[dst]), %%zmm0 %{%%k5%}") /* zmm0 = r0 i0 r1 i1 r2 i2 r3 i3 r4 i4 r5 i5 r6 i6 r7 i7 */
68 __ASM_EMIT("vmovups 0x060(%[dst]), %%ymm4")
69 __ASM_EMIT("vexpandps %%zmm4, %%zmm1 %{%%k5%}") /* zmm1 = r8 i8 r9 i9 r10 i10 r11 i11 r12 i12 r13 i13 r14 i14 r15 i15 */
70 __ASM_EMIT("vmovups %%zmm0, 0x000(%[dst])")
71 __ASM_EMIT("vmovups %%zmm1, 0x040(%[dst])")
72 __ASM_EMIT("sub $16, %[blocks]")
73 __ASM_EMIT("add $0x100, %[dst]")
74 // 8x block
75 __ASM_EMIT("4:")
76 __ASM_EMIT("add $8, %[blocks]")
77 __ASM_EMIT("jl 6f")
78 __ASM_EMIT("vmovups 0x00(%[dst]), %%ymm0") /* ymm0 = r0 r1 r2 r3 r4 r5 r6 r7 */
79 __ASM_EMIT("vmovups 0x20(%[dst]), %%ymm1") /* ymm1 = i0 i1 i2 i3 i4 i5 i6 i7 */
80 __ASM_EMIT("vunpcklps %%ymm1, %%ymm0, %%ymm4") /* ymm4 = r0 i0 r1 i1 r4 i4 r5 i5 */
81 __ASM_EMIT("vunpckhps %%ymm1, %%ymm0, %%ymm5") /* ymm5 = r2 i2 r3 i3 r6 i6 r7 i7 */
82 __ASM_EMIT("vmovups %%xmm4, 0x00(%[dst])")
83 __ASM_EMIT("vmovups %%xmm5, 0x10(%[dst])")
84 __ASM_EMIT("vextractf128 $1, %%ymm4, 0x20(%[dst])")
85 __ASM_EMIT("vextractf128 $1, %%ymm5, 0x30(%[dst])")
86 __ASM_EMIT("6:")
87
88 : [dst] "+r"(dst), [blocks] "+r" (blocks)
89 : [MASK] "o" (FFT_REPACK_MASKS)
90 : "cc", "memory",
91 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
92 "%xmm4",
93 "%k4", "%k5"
94 );
95 }
96
97 static inline void packed_fft_repack_normalize(float *dst, size_t rank)
98 {
99 size_t blocks = 1 << rank;
100 const float norm = 1.0f / float(blocks);
101
102 // Perform 4-element butterflies
103 ARCH_X86_ASM
104 (
105 __ASM_EMIT("vbroadcastss %[norm], %%zmm6")
106 __ASM_EMIT("kmovw 0x00 + %[MASK], %%k4")
107 __ASM_EMIT("kmovw 0x02 + %[MASK], %%k5")
108 __ASM_EMIT("vmovaps %%zmm6, %%zmm7")
109 // 32x blocks
110 __ASM_EMIT("sub $32, %[blocks]")
111 __ASM_EMIT("jb 2f")
112 __ASM_EMIT("1:")
113 __ASM_EMIT("vexpandps 0x000(%[dst]), %%zmm0 %{%%k4%}%{z%}") /* zmm0 = r0 0 r1 0 r2 0 r3 0 r4 0 r5 0 r6 0 r7 0 */
114 __ASM_EMIT("vexpandps 0x020(%[dst]), %%zmm1 %{%%k4%}%{z%}") /* zmm1 = r8 0 r9 0 r10 0 r11 0 r12 0 r13 0 r14 0 r15 0 */
115 __ASM_EMIT("vexpandps 0x040(%[dst]), %%zmm0 %{%%k5%}") /* zmm0 = r0 i0 r1 i1 r2 i2 r3 i3 r4 i4 r5 i5 r6 i6 r7 i7 */
116 __ASM_EMIT("vexpandps 0x060(%[dst]), %%zmm1 %{%%k5%}") /* zmm1 = r8 i8 r9 i9 r10 i10 r11 i11 r12 i12 r13 i13 r14 i14 r15 i15 */
117 __ASM_EMIT("vexpandps 0x080(%[dst]), %%zmm2 %{%%k4%}%{z%}")
118 __ASM_EMIT("vexpandps 0x0a0(%[dst]), %%zmm3 %{%%k4%}%{z%}")
119 __ASM_EMIT("vexpandps 0x0c0(%[dst]), %%zmm2 %{%%k5%}")
120 __ASM_EMIT("vmovups 0x0e0(%[dst]), %%ymm4")
121 __ASM_EMIT("vexpandps %%zmm4, %%zmm3 %{%%k5%}")
122 __ASM_EMIT("vmulps %%zmm6, %%zmm0, %%zmm0")
123 __ASM_EMIT("vmulps %%zmm7, %%zmm1, %%zmm1")
124 __ASM_EMIT("vmulps %%zmm6, %%zmm2, %%zmm2")
125 __ASM_EMIT("vmulps %%zmm7, %%zmm3, %%zmm3")
126 __ASM_EMIT("vmovups %%zmm0, 0x000(%[dst])")
127 __ASM_EMIT("vmovups %%zmm1, 0x040(%[dst])")
128 __ASM_EMIT("vmovups %%zmm2, 0x080(%[dst])")
129 __ASM_EMIT("vmovups %%zmm3, 0x0c0(%[dst])")
130 __ASM_EMIT("add $0x100, %[dst]")
131 __ASM_EMIT("sub $32, %[blocks]")
132 __ASM_EMIT("jae 1b")
133 // 16x block
134 __ASM_EMIT("2:")
135 __ASM_EMIT("add $16, %[blocks]")
136 __ASM_EMIT("jl 4f")
137 __ASM_EMIT("vexpandps 0x000(%[dst]), %%zmm0 %{%%k4%}%{z%}") /* zmm0 = r0 0 r1 0 r2 0 r3 0 r4 0 r5 0 r6 0 r7 0 */
138 __ASM_EMIT("vexpandps 0x020(%[dst]), %%zmm1 %{%%k4%}%{z%}") /* zmm1 = r8 0 r9 0 r10 0 r11 0 r12 0 r13 0 r14 0 r15 0 */
139 __ASM_EMIT("vexpandps 0x040(%[dst]), %%zmm0 %{%%k5%}") /* zmm0 = r0 i0 r1 i1 r2 i2 r3 i3 r4 i4 r5 i5 r6 i6 r7 i7 */
140 __ASM_EMIT("vmovups 0x060(%[dst]), %%ymm4")
141 __ASM_EMIT("vexpandps %%zmm4, %%zmm1 %{%%k5%}") /* zmm1 = r8 i8 r9 i9 r10 i10 r11 i11 r12 i12 r13 i13 r14 i14 r15 i15 */
142 __ASM_EMIT("vmulps %%zmm6, %%zmm0, %%zmm0")
143 __ASM_EMIT("vmulps %%zmm7, %%zmm1, %%zmm1")
144 __ASM_EMIT("vmovups %%zmm0, 0x000(%[dst])")
145 __ASM_EMIT("vmovups %%zmm1, 0x040(%[dst])")
146 __ASM_EMIT("sub $16, %[blocks]")
147 __ASM_EMIT("add $0x100, %[dst]")
148 // 8x block
149 __ASM_EMIT("4:")
150 __ASM_EMIT("add $8, %[blocks]")
151 __ASM_EMIT("jl 6f")
152 __ASM_EMIT("vmulps 0x00(%[dst]), %%ymm6, %%ymm0") /* ymm0 = r0 r1 r2 r3 r4 r5 r6 r7 */
153 __ASM_EMIT("vmulps 0x20(%[dst]), %%ymm7, %%ymm1") /* ymm1 = i0 i1 i2 i3 i4 i5 i6 i7 */
154 __ASM_EMIT("vunpcklps %%ymm1, %%ymm0, %%ymm4") /* ymm4 = r0 i0 r1 i1 r4 i4 r5 i5 */
155 __ASM_EMIT("vunpckhps %%ymm1, %%ymm0, %%ymm5") /* ymm5 = r2 i2 r3 i3 r6 i6 r7 i7 */
156 __ASM_EMIT("vmovups %%xmm4, 0x00(%[dst])")
157 __ASM_EMIT("vmovups %%xmm5, 0x10(%[dst])")
158 __ASM_EMIT("vextractf128 $1, %%ymm4, 0x20(%[dst])")
159 __ASM_EMIT("vextractf128 $1, %%ymm5, 0x30(%[dst])")
160 __ASM_EMIT("6:")
161
162 : [dst] "+r"(dst), [blocks] "+r" (blocks)
163 : [norm] "m" (norm),
164 [MASK] "o" (FFT_REPACK_MASKS)
165 : "cc", "memory",
166 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
167 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
168 );
169 }
170 } /* namespace avx512 */
171 } /* namespace lsp */
172
173
174 #endif /* PRIVATE_DSP_ARCH_X86_AVX512_FFT_P_REPACK_H_ */