Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/x86/sse3/copy.h

raw
1 /*
2 * Copyright (C) 2020 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2020 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 31 мар. 2020 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_X86_SSE3_COPY_H_
23 #define PRIVATE_DSP_ARCH_X86_SSE3_COPY_H_
24
25 #ifndef PRIVATE_DSP_ARCH_X86_SSE3_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_X86_SSE3_IMPL */
28
29 namespace lsp
30 {
31 namespace sse3
32 {
33 void copy(float *dst, const float *src, size_t count)
34 {
35 IF_ARCH_X86(size_t off);
36
37 ARCH_X86_ASM
38 (
39 __ASM_EMIT("cmp %[dst], %[src]")
40 __ASM_EMIT("je 10f")
41
42 /* Destination is aligned */
43 __ASM_EMIT("xor %[off], %[off]")
44 __ASM_EMIT("sub $0x20, %[count]")
45 __ASM_EMIT("jb 2f")
46
47 /* Block of 32 items */
48 __ASM_EMIT("1:")
49 __ASM_EMIT("lddqu 0x00(%[src], %[off]), %%xmm0")
50 __ASM_EMIT("lddqu 0x10(%[src], %[off]), %%xmm1")
51 __ASM_EMIT("lddqu 0x20(%[src], %[off]), %%xmm2")
52 __ASM_EMIT("lddqu 0x30(%[src], %[off]), %%xmm3")
53 __ASM_EMIT("lddqu 0x40(%[src], %[off]), %%xmm4")
54 __ASM_EMIT("lddqu 0x50(%[src], %[off]), %%xmm5")
55 __ASM_EMIT("lddqu 0x60(%[src], %[off]), %%xmm6")
56 __ASM_EMIT("lddqu 0x70(%[src], %[off]), %%xmm7")
57 __ASM_EMIT("movdqu %%xmm0, 0x00(%[dst], %[off])")
58 __ASM_EMIT("movdqu %%xmm1, 0x10(%[dst], %[off])")
59 __ASM_EMIT("movdqu %%xmm2, 0x20(%[dst], %[off])")
60 __ASM_EMIT("movdqu %%xmm3, 0x30(%[dst], %[off])")
61 __ASM_EMIT("movdqu %%xmm4, 0x40(%[dst], %[off])")
62 __ASM_EMIT("movdqu %%xmm5, 0x50(%[dst], %[off])")
63 __ASM_EMIT("movdqu %%xmm6, 0x60(%[dst], %[off])")
64 __ASM_EMIT("movdqu %%xmm7, 0x70(%[dst], %[off])")
65 __ASM_EMIT("add $0x80, %[off]")
66 __ASM_EMIT("sub $0x20, %[count]")
67 __ASM_EMIT("jae 1b")
68 /* Block of 16 items */
69 __ASM_EMIT("2:")
70 __ASM_EMIT("add $0x10, %[count]")
71 __ASM_EMIT("jl 4f")
72 __ASM_EMIT("lddqu 0x00(%[src], %[off]), %%xmm0")
73 __ASM_EMIT("lddqu 0x10(%[src], %[off]), %%xmm1")
74 __ASM_EMIT("lddqu 0x20(%[src], %[off]), %%xmm2")
75 __ASM_EMIT("lddqu 0x30(%[src], %[off]), %%xmm3")
76 __ASM_EMIT("movdqu %%xmm0, 0x00(%[dst], %[off])")
77 __ASM_EMIT("movdqu %%xmm1, 0x10(%[dst], %[off])")
78 __ASM_EMIT("movdqu %%xmm2, 0x20(%[dst], %[off])")
79 __ASM_EMIT("movdqu %%xmm3, 0x30(%[dst], %[off])")
80 __ASM_EMIT("add $0x40, %[off]")
81 __ASM_EMIT("sub $0x10, %[count]")
82 /* Block of 8 items */
83 __ASM_EMIT("4:")
84 __ASM_EMIT("add $0x08, %[count]")
85 __ASM_EMIT("jl 6f")
86 __ASM_EMIT("lddqu 0x00(%[src], %[off]), %%xmm0")
87 __ASM_EMIT("lddqu 0x10(%[src], %[off]), %%xmm1")
88 __ASM_EMIT("movdqu %%xmm0, 0x00(%[dst], %[off])")
89 __ASM_EMIT("movdqu %%xmm1, 0x10(%[dst], %[off])")
90 __ASM_EMIT("add $0x20, %[off]")
91 __ASM_EMIT("sub $0x08, %[count]")
92 /* Block of 4 items */
93 __ASM_EMIT("6:")
94 __ASM_EMIT("add $0x04, %[count]")
95 __ASM_EMIT("jl 8f")
96 __ASM_EMIT("lddqu 0x00(%[src], %[off]), %%xmm0")
97 __ASM_EMIT("movdqu %%xmm0, 0x00(%[dst], %[off])")
98 __ASM_EMIT("add $0x10, %[off]")
99 __ASM_EMIT("sub $0x04, %[count]")
100 /* 1x items */
101 __ASM_EMIT("8:")
102 __ASM_EMIT("add $0x03, %[count]")
103 __ASM_EMIT("jl 10f")
104 __ASM_EMIT("9:")
105 __ASM_EMIT("movss 0x00(%[src], %[off]), %%xmm0")
106 __ASM_EMIT("movss %%xmm0, 0x00(%[dst], %[off])")
107 __ASM_EMIT("add $0x04, %[off]")
108 __ASM_EMIT("dec %[count]")
109 __ASM_EMIT("jge 9b")
110
111 __ASM_EMIT("10:")
112
113 : [src] "+r" (src), [dst] "+r"(dst),
114 [count] "+r" (count), [off] "=&r" (off)
115 :
116 : "cc", "memory",
117 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
118 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
119 );
120 }
121
122 void copy_backward(float *dst, const float *src, size_t count)
123 {
124 IF_ARCH_X86(size_t off);
125
126 ARCH_X86_ASM
127 (
128 __ASM_EMIT("cmp %[dst], %[src]")
129 __ASM_EMIT("je 10f")
130
131 __ASM_EMIT("lea (,%[count], 4), %[off]")
132 __ASM_EMIT("sub $0x20, %[count]")
133 __ASM_EMIT("jb 2f")
134
135 /* Block of 32 items */
136 __ASM_EMIT("1:")
137 __ASM_EMIT("sub $0x80, %[off]")
138 __ASM_EMIT("lddqu 0x00(%[src], %[off]), %%xmm0")
139 __ASM_EMIT("lddqu 0x10(%[src], %[off]), %%xmm1")
140 __ASM_EMIT("lddqu 0x20(%[src], %[off]), %%xmm2")
141 __ASM_EMIT("lddqu 0x30(%[src], %[off]), %%xmm3")
142 __ASM_EMIT("lddqu 0x40(%[src], %[off]), %%xmm4")
143 __ASM_EMIT("lddqu 0x50(%[src], %[off]), %%xmm5")
144 __ASM_EMIT("lddqu 0x60(%[src], %[off]), %%xmm6")
145 __ASM_EMIT("lddqu 0x70(%[src], %[off]), %%xmm7")
146 __ASM_EMIT("movdqu %%xmm0, 0x00(%[dst], %[off])")
147 __ASM_EMIT("movdqu %%xmm1, 0x10(%[dst], %[off])")
148 __ASM_EMIT("movdqu %%xmm2, 0x20(%[dst], %[off])")
149 __ASM_EMIT("movdqu %%xmm3, 0x30(%[dst], %[off])")
150 __ASM_EMIT("movdqu %%xmm4, 0x40(%[dst], %[off])")
151 __ASM_EMIT("movdqu %%xmm5, 0x50(%[dst], %[off])")
152 __ASM_EMIT("movdqu %%xmm6, 0x60(%[dst], %[off])")
153 __ASM_EMIT("movdqu %%xmm7, 0x70(%[dst], %[off])")
154 __ASM_EMIT("sub $0x20, %[count]")
155 __ASM_EMIT("jae 1b")
156 /* Block of 16 items */
157 __ASM_EMIT("2:")
158 __ASM_EMIT("add $0x10, %[count]")
159 __ASM_EMIT("jl 4f")
160 __ASM_EMIT("sub $0x40, %[off]")
161 __ASM_EMIT("lddqu 0x00(%[src], %[off]), %%xmm0")
162 __ASM_EMIT("lddqu 0x10(%[src], %[off]), %%xmm1")
163 __ASM_EMIT("lddqu 0x20(%[src], %[off]), %%xmm2")
164 __ASM_EMIT("lddqu 0x30(%[src], %[off]), %%xmm3")
165 __ASM_EMIT("movdqu %%xmm0, 0x00(%[dst], %[off])")
166 __ASM_EMIT("movdqu %%xmm1, 0x10(%[dst], %[off])")
167 __ASM_EMIT("movdqu %%xmm2, 0x20(%[dst], %[off])")
168 __ASM_EMIT("movdqu %%xmm3, 0x30(%[dst], %[off])")
169 __ASM_EMIT("sub $0x10, %[count]")
170 /* Block of 8 items */
171 __ASM_EMIT("4:")
172 __ASM_EMIT("add $0x08, %[count]")
173 __ASM_EMIT("jl 6f")
174 __ASM_EMIT("sub $0x20, %[off]")
175 __ASM_EMIT("lddqu 0x00(%[src], %[off]), %%xmm0")
176 __ASM_EMIT("lddqu 0x10(%[src], %[off]), %%xmm1")
177 __ASM_EMIT("movdqu %%xmm0, 0x00(%[dst], %[off])")
178 __ASM_EMIT("movdqu %%xmm1, 0x10(%[dst], %[off])")
179 __ASM_EMIT("sub $0x08, %[count]")
180 /* Block of 4 items */
181 __ASM_EMIT("6:")
182 __ASM_EMIT("add $0x04, %[count]")
183 __ASM_EMIT("jl 8f")
184 __ASM_EMIT("sub $0x10, %[off]")
185 __ASM_EMIT("lddqu 0x00(%[src], %[off]), %%xmm0")
186 __ASM_EMIT("movdqu %%xmm0, 0x00(%[dst], %[off])")
187 __ASM_EMIT("sub $0x04, %[count]")
188 /* 1x items */
189 __ASM_EMIT("8:")
190 __ASM_EMIT("add $0x03, %[count]")
191 __ASM_EMIT("jl 10f")
192 __ASM_EMIT("9:")
193 __ASM_EMIT("sub $0x04, %[off]")
194 __ASM_EMIT("movd 0x00(%[src], %[off]), %%xmm0")
195 __ASM_EMIT("movd %%xmm0, 0x00(%[dst], %[off])")
196 __ASM_EMIT("dec %[count]")
197 __ASM_EMIT("jge 9b")
198
199 __ASM_EMIT("10:")
200
201 : [src] "+r" (src), [dst] "+r"(dst),
202 [count] "+r" (count), [off] "=&r" (off)
203 :
204 : "cc", "memory",
205 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
206 "%xmm4", "%xmm5", "%xmm6", "%xmm7"
207 );
208
209 #undef RCOPY_CORE
210 }
211
212 void move(float *dst, const float *src, size_t count)
213 {
214 if (dst < src)
215 sse3::copy(dst, src, count);
216 else if (dst > src)
217 sse3::copy_backward(dst, src, count);
218 }
219
220 } /* namespace sse3 */
221 } /* namespace lsp */
222
223 #endif /* PRIVATE_DSP_ARCH_X86_SSE3_COPY_H_ */