Debian Codegraph @

modules/lsp-dsp-lib/include/private/dsp/arch/x86/avx512/float.h

raw
1 /*
2 * Copyright (C) 2026 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2026 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-dsp-lib
6 * Created on: 8 сент. 2023 г.
7 *
8 * lsp-dsp-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-dsp-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #ifndef PRIVATE_DSP_ARCH_X86_AVX512_FLOAT_H_
23 #define PRIVATE_DSP_ARCH_X86_AVX512_FLOAT_H_
24
25 #ifndef PRIVATE_DSP_ARCH_X86_AVX512_IMPL
26 #error "This header should not be included directly"
27 #endif /* PRIVATE_DSP_ARCH_X86_AVX512_IMPL */
28
29 namespace lsp
30 {
31 namespace avx512
32 {
33 #define LIMIT_BODY(DST, SRC) \
34 __ASM_EMIT("xor %[off], %[off]") \
35 __ASM_EMIT("vbroadcastss %[min], %%zmm6") /* zmm6 = min */ \
36 __ASM_EMIT("vbroadcastss %[max], %%zmm7") /* zmm7 = max */ \
37 /* x64 blocks */ \
38 __ASM_EMIT("sub $64, %[count]") \
39 __ASM_EMIT("jb 2f") \
40 __ASM_EMIT("1:") \
41 __ASM_EMIT("vmovups 0x00(%[" SRC "], %[off]), %%zmm0") /* zmm0 = s0 */ \
42 __ASM_EMIT("vmovups 0x40(%[" SRC "], %[off]), %%zmm1") \
43 __ASM_EMIT("vmovups 0x80(%[" SRC "], %[off]), %%zmm2") \
44 __ASM_EMIT("vmovups 0xc0(%[" SRC "], %[off]), %%zmm3") \
45 __ASM_EMIT("vcmpps $2, %%zmm0, %%zmm6, %%k4") /* k4 = [ (s >= min) & !isnan(s) ] */ \
46 __ASM_EMIT("vcmpps $2, %%zmm1, %%zmm6, %%k5") \
47 __ASM_EMIT("vcmpps $2, %%zmm2, %%zmm6, %%k6") \
48 __ASM_EMIT("vcmpps $2, %%zmm3, %%zmm6, %%k7") \
49 __ASM_EMIT("vblendmps %%zmm0, %%zmm6, %%zmm0%{%%k4%}") /* zmm0 = (s >= min) ? s : min */ \
50 __ASM_EMIT("vblendmps %%zmm1, %%zmm6, %%zmm1%{%%k5%}") \
51 __ASM_EMIT("vblendmps %%zmm2, %%zmm6, %%zmm2%{%%k6%}") \
52 __ASM_EMIT("vblendmps %%zmm3, %%zmm6, %%zmm3%{%%k7%}") \
53 __ASM_EMIT("vcmpps $5, %%zmm0, %%zmm7, %%k4") /* k4 = [ (s <= max) & !isnan(s) ] */ \
54 __ASM_EMIT("vcmpps $5, %%zmm1, %%zmm7, %%k5") \
55 __ASM_EMIT("vcmpps $5, %%zmm2, %%zmm7, %%k6") \
56 __ASM_EMIT("vcmpps $5, %%zmm3, %%zmm7, %%k7") \
57 __ASM_EMIT("vblendmps %%zmm0, %%zmm7, %%zmm0%{%%k4%}") /* zmm0 = (s <= max) ? ((s >= min) ? s : min) : max */ \
58 __ASM_EMIT("vblendmps %%zmm1, %%zmm7, %%zmm1%{%%k5%}") \
59 __ASM_EMIT("vblendmps %%zmm2, %%zmm7, %%zmm2%{%%k6%}") \
60 __ASM_EMIT("vblendmps %%zmm3, %%zmm7, %%zmm3%{%%k7%}") \
61 __ASM_EMIT("vmovups %%zmm0, 0x00(%[" DST "], %[off])") \
62 __ASM_EMIT("vmovups %%zmm1, 0x40(%[" DST "], %[off])") \
63 __ASM_EMIT("vmovups %%zmm2, 0x80(%[" DST "], %[off])") \
64 __ASM_EMIT("vmovups %%zmm3, 0xc0(%[" DST "], %[off])") \
65 __ASM_EMIT("add $0x100, %[off]") \
66 __ASM_EMIT("sub $64, %[count]") \
67 __ASM_EMIT("jae 1b") \
68 __ASM_EMIT("2:") \
69 /* x32 block */ \
70 __ASM_EMIT("add $32, %[count]") \
71 __ASM_EMIT("jl 4f") \
72 __ASM_EMIT("vmovups 0x00(%[" SRC "], %[off]), %%zmm0") /* zmm0 = s0 */ \
73 __ASM_EMIT("vmovups 0x40(%[" SRC "], %[off]), %%zmm1") \
74 __ASM_EMIT("vcmpps $2, %%zmm0, %%zmm6, %%k4") /* k4 = [ (s >= min) & !isnan(s) ] */ \
75 __ASM_EMIT("vcmpps $2, %%zmm1, %%zmm6, %%k5") \
76 __ASM_EMIT("vblendmps %%zmm0, %%zmm6, %%zmm0%{%%k4%}") /* zmm0 = (s >= min) ? s : min */ \
77 __ASM_EMIT("vblendmps %%zmm1, %%zmm6, %%zmm1%{%%k5%}") \
78 __ASM_EMIT("vcmpps $5, %%zmm0, %%zmm7, %%k4") /* k4 = [ (s <= max) & !isnan(s) ] */ \
79 __ASM_EMIT("vcmpps $5, %%zmm1, %%zmm7, %%k5") \
80 __ASM_EMIT("vblendmps %%zmm0, %%zmm7, %%zmm0%{%%k4%}") /* zmm0 = (s <= max) ? ((s >= min) ? s : min) : max */ \
81 __ASM_EMIT("vblendmps %%zmm1, %%zmm7, %%zmm1%{%%k5%}") \
82 __ASM_EMIT("vmovups %%zmm0, 0x00(%[" DST "], %[off])") \
83 __ASM_EMIT("vmovups %%zmm1, 0x40(%[" DST "], %[off])") \
84 __ASM_EMIT("add $0x80, %[off]") \
85 __ASM_EMIT("sub $32, %[count]") \
86 __ASM_EMIT("4:") \
87 /* x16 blocks */ \
88 __ASM_EMIT("add $16, %[count]") \
89 __ASM_EMIT("jl 6f") \
90 __ASM_EMIT("vmovups 0x00(%[" SRC "], %[off]), %%ymm0") /* ymm0 = s0 */ \
91 __ASM_EMIT("vmovups 0x20(%[" SRC "], %[off]), %%ymm1") \
92 __ASM_EMIT("vcmpps $2, %%ymm0, %%ymm6, %%ymm2") /* ymm2 = c = [ (s >= min) & !isnan(s) ] */ \
93 __ASM_EMIT("vcmpps $2, %%ymm1, %%ymm6, %%ymm3") \
94 __ASM_EMIT("vblendvps %%ymm2, %%ymm0, %%ymm6, %%ymm0") /* ymm0 = (s & c) | min & (!c) */ \
95 __ASM_EMIT("vblendvps %%ymm3, %%ymm1, %%ymm6, %%ymm1") \
96 __ASM_EMIT("vcmpps $5, %%ymm0, %%ymm7, %%ymm2") /* ymm2 = c = [ (s <= max) & !isnan(s) ] */ \
97 __ASM_EMIT("vcmpps $5, %%ymm1, %%ymm7, %%ymm3") \
98 __ASM_EMIT("vblendvps %%ymm2, %%ymm0, %%ymm7, %%ymm0") /* ymm0 = (s & c) | min & (!c) */ \
99 __ASM_EMIT("vblendvps %%ymm3, %%ymm1, %%ymm7, %%ymm1") \
100 __ASM_EMIT("vmovups %%ymm0, 0x00(%[" DST "], %[off])") \
101 __ASM_EMIT("vmovups %%ymm1, 0x20(%[" DST "], %[off])") \
102 __ASM_EMIT("add $0x40, %[off]") \
103 __ASM_EMIT("sub $16, %[count]") \
104 __ASM_EMIT("6:") \
105 /* x8 block */ \
106 __ASM_EMIT("add $8, %[count]") \
107 __ASM_EMIT("jl 8f") \
108 __ASM_EMIT("vmovups 0x00(%[" SRC "], %[off]), %%xmm0") /* xmm0 = s0 */ \
109 __ASM_EMIT("vmovups 0x10(%[" SRC "], %[off]), %%xmm1") \
110 __ASM_EMIT("vcmpps $2, %%xmm0, %%xmm6, %%xmm2") /* xmm2 = c = [ (s >= min) & !isnan(s) ] */ \
111 __ASM_EMIT("vcmpps $2, %%xmm1, %%xmm6, %%xmm3") \
112 __ASM_EMIT("vblendvps %%xmm2, %%xmm0, %%xmm6, %%xmm0") /* xmm0 = (s & c) | min & (!c) */ \
113 __ASM_EMIT("vblendvps %%xmm3, %%xmm1, %%xmm6, %%xmm1") \
114 __ASM_EMIT("vcmpps $5, %%xmm0, %%xmm7, %%xmm2") /* xmm2 = c = [ (s <= max) & !isnan(s) ] */ \
115 __ASM_EMIT("vcmpps $5, %%xmm1, %%xmm7, %%xmm3") \
116 __ASM_EMIT("vblendvps %%xmm2, %%xmm0, %%xmm7, %%xmm0") /* xmm0 = (s & c) | min & (!c) */ \
117 __ASM_EMIT("vblendvps %%xmm3, %%xmm1, %%xmm7, %%xmm1") \
118 __ASM_EMIT("vmovups %%xmm0, 0x00(%[" DST "], %[off])") \
119 __ASM_EMIT("vmovups %%xmm1, 0x10(%[" DST "], %[off])") \
120 __ASM_EMIT("sub $8, %[count]") \
121 __ASM_EMIT("add $0x20, %[off]") \
122 __ASM_EMIT("8:") \
123 /* x4 block */ \
124 __ASM_EMIT("add $4, %[count]") \
125 __ASM_EMIT("jl 10f") \
126 __ASM_EMIT("vmovups 0x00(%[" SRC "], %[off]), %%xmm0") /* xmm0 = s0 */ \
127 __ASM_EMIT("vcmpps $2, %%xmm0, %%xmm6, %%xmm2") /* xmm2 = c = [ (s >= min) & !isnan(s) ] */ \
128 __ASM_EMIT("vblendvps %%xmm2, %%xmm0, %%xmm6, %%xmm0") /* xmm0 = (s & c) | min & (!c) */ \
129 __ASM_EMIT("vcmpps $5, %%xmm0, %%xmm7, %%xmm2") /* xmm2 = c = [ (s <= max) & !isnan(s) ] */ \
130 __ASM_EMIT("vblendvps %%xmm2, %%xmm0, %%xmm7, %%xmm0") /* xmm0 = (s & c) | min & (!c) */ \
131 __ASM_EMIT("vmovups %%xmm0, 0x00(%[" DST "], %[off])") \
132 __ASM_EMIT("sub $4, %[count]") \
133 __ASM_EMIT("add $0x10, %[off]") \
134 __ASM_EMIT("10:") \
135 /* x1 blocks */ \
136 __ASM_EMIT("add $3, %[count]") \
137 __ASM_EMIT("jl 12f") \
138 __ASM_EMIT("11:") \
139 __ASM_EMIT("vmovss 0x00(%[" SRC "], %[off]), %%xmm0") /* xmm0 = s0 */ \
140 __ASM_EMIT("vcmpps $2, %%xmm0, %%xmm6, %%xmm2") /* xmm2 = c = [ (s >= min) & !isnan(s) ] */ \
141 __ASM_EMIT("vblendvps %%xmm2, %%xmm0, %%xmm6, %%xmm0") /* xmm0 = (s & c) | min & (!c) */ \
142 __ASM_EMIT("vcmpps $5, %%xmm0, %%xmm7, %%xmm2") /* xmm2 = c = [ (s <= max) & !isnan(s) ] */ \
143 __ASM_EMIT("vblendvps %%xmm2, %%xmm0, %%xmm7, %%xmm0") /* xmm0 = (s & c) | min & (!c) */ \
144 __ASM_EMIT("vmovss %%xmm0, 0x00(%[" DST "], %[off])") \
145 __ASM_EMIT("add $0x04, %[off]") \
146 __ASM_EMIT("dec %[count]") \
147 __ASM_EMIT("jge 11b") \
148 __ASM_EMIT("12:")
149
150
151 void limit1(float *dst, float min, float max, size_t count)
152 {
153 IF_ARCH_X86(size_t off);
154 ARCH_X86_ASM(
155 LIMIT_BODY("dst", "dst")
156 : [count] "+r" (count), [off] "=&r" (off)
157 : [dst] "r" (dst),
158 [min] "m" (min), [max] "m" (max)
159 : "cc", "memory",
160 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
161 "%xmm6", "%xmm7",
162 "%k4", "%k5", "%k6", "%k7"
163 );
164 }
165
166 void limit2(float *dst, const float *src, float min, float max, size_t count)
167 {
168 IF_ARCH_X86(size_t off);
169 ARCH_X86_ASM(
170 LIMIT_BODY("dst", "src")
171 : [count] "+r" (count), [off] "=&r" (off)
172 : [dst] "r" (dst), [src] "r" (src),
173 [min] "m" (min), [max] "m" (max)
174 : "cc", "memory",
175 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
176 "%xmm6", "%xmm7",
177 "%k4", "%k5", "%k6", "%k7"
178 );
179 }
180
181 #undef LIMIT_BODY
182
183 #define SANITIZE_BODY(DST, SRC) \
184 __ASM_EMIT("xor %[off], %[off]") \
185 /* 64x blocks */ \
186 __ASM_EMIT("sub $64, %[count]") \
187 __ASM_EMIT("jb 2f") \
188 __ASM_EMIT("1:") \
189 __ASM_EMIT("vmovups 0x00(%[" SRC "], %[off]), %%zmm0") /* xmm0 = s */ \
190 __ASM_EMIT("vmovups 0x40(%[" SRC "], %[off]), %%zmm1") \
191 __ASM_EMIT("vmovups 0x80(%[" SRC "], %[off]), %%zmm2") \
192 __ASM_EMIT("vmovups 0xc0(%[" SRC "], %[off]), %%zmm3") \
193 __ASM_EMIT("vpandd 0x00(%[CVAL]), %%zmm0, %%zmm4") /* xmm4 = abs(s) */ \
194 __ASM_EMIT("vpandd 0x00(%[CVAL]), %%zmm1, %%zmm5") \
195 __ASM_EMIT("vpandd 0x00(%[CVAL]), %%zmm2, %%zmm6") \
196 __ASM_EMIT("vpandd 0x00(%[CVAL]), %%zmm3, %%zmm7") \
197 __ASM_EMIT("vpcmpgtd 0x40(%[CVAL]), %%zmm4, %%k4") /* k4 = [abs(s) > X_MAX] */ \
198 __ASM_EMIT("vpcmpgtd 0x40(%[CVAL]), %%zmm5, %%k5") \
199 __ASM_EMIT("vpcmpgtd 0x40(%[CVAL]), %%zmm6, %%k6") \
200 __ASM_EMIT("vpcmpgtd 0x40(%[CVAL]), %%zmm7, %%k7") \
201 __ASM_EMIT("vpcmpgtd 0x80(%[CVAL]), %%zmm4, %%k0") /* k0 = [abs(s) > X_MIN] */ \
202 __ASM_EMIT("vpcmpgtd 0x80(%[CVAL]), %%zmm5, %%k1") \
203 __ASM_EMIT("vpcmpgtd 0x80(%[CVAL]), %%zmm6, %%k2") \
204 __ASM_EMIT("vpcmpgtd 0x80(%[CVAL]), %%zmm7, %%k3") \
205 __ASM_EMIT("kandnw %%k0, %%k4, %%k4") /* k4 = [(abs(s) > X_MAX) & (abs(s) <= X_MIN)] */ \
206 __ASM_EMIT("kandnw %%k1, %%k5, %%k5") \
207 __ASM_EMIT("kandnw %%k2, %%k6, %%k6") \
208 __ASM_EMIT("kandnw %%k3, %%k7, %%k7") \
209 __ASM_EMIT("vpandd 0xc0(%[CVAL]), %%zmm0, %%zmm4") /* zmm4 = sign(s) */ \
210 __ASM_EMIT("vpandd 0xc0(%[CVAL]), %%zmm1, %%zmm5") \
211 __ASM_EMIT("vpandd 0xc0(%[CVAL]), %%zmm2, %%zmm6") \
212 __ASM_EMIT("vpandd 0xc0(%[CVAL]), %%zmm3, %%zmm7") \
213 __ASM_EMIT("vblendmps %%zmm0, %%zmm4, %%zmm0%{%%k4%}") /* zmm0 = ([(abs(s) > X_MAX) & (abs(s) <= X_MIN)]) ? s : sign(s) */ \
214 __ASM_EMIT("vblendmps %%zmm1, %%zmm5, %%zmm1%{%%k5%}") \
215 __ASM_EMIT("vblendmps %%zmm2, %%zmm6, %%zmm2%{%%k6%}") \
216 __ASM_EMIT("vblendmps %%zmm3, %%zmm7, %%zmm3%{%%k7%}") \
217 __ASM_EMIT("vmovups %%zmm0, 0x00(%[" DST "], %[off])") /* zmm0 = s' */ \
218 __ASM_EMIT("vmovups %%zmm1, 0x40(%[" DST "], %[off])") \
219 __ASM_EMIT("vmovups %%zmm2, 0x80(%[" DST "], %[off])") \
220 __ASM_EMIT("vmovups %%zmm3, 0xc0(%[" DST "], %[off])") \
221 __ASM_EMIT("add $0x100, %[off]") \
222 __ASM_EMIT("sub $64, %[count]") \
223 __ASM_EMIT("jae 1b") \
224 __ASM_EMIT("2:") \
225 /* 32x block */ \
226 __ASM_EMIT("add $32, %[count]") \
227 __ASM_EMIT("jl 4f") \
228 __ASM_EMIT("vmovups 0x00(%[" SRC "], %[off]), %%zmm0") /* xmm0 = s */ \
229 __ASM_EMIT("vmovups 0x40(%[" SRC "], %[off]), %%zmm1") \
230 __ASM_EMIT("vpandd 0x00(%[CVAL]), %%zmm0, %%zmm4") /* xmm4 = abs(s) */ \
231 __ASM_EMIT("vpandd 0x00(%[CVAL]), %%zmm1, %%zmm5") \
232 __ASM_EMIT("vpcmpgtd 0x40(%[CVAL]), %%zmm4, %%k4") /* k4 = [abs(s) > X_MAX] */ \
233 __ASM_EMIT("vpcmpgtd 0x40(%[CVAL]), %%zmm5, %%k5") \
234 __ASM_EMIT("vpcmpgtd 0x80(%[CVAL]), %%zmm4, %%k0") /* k0 = [abs(s) > X_MIN] */ \
235 __ASM_EMIT("vpcmpgtd 0x80(%[CVAL]), %%zmm5, %%k1") \
236 __ASM_EMIT("kandnw %%k0, %%k4, %%k4") /* k4 = [(abs(s) > X_MAX) & (abs(s) <= X_MIN)] */ \
237 __ASM_EMIT("kandnw %%k1, %%k5, %%k5") \
238 __ASM_EMIT("vpandd 0xc0(%[CVAL]), %%zmm0, %%zmm4") /* zmm4 = sign(s) */ \
239 __ASM_EMIT("vpandd 0xc0(%[CVAL]), %%zmm1, %%zmm5") \
240 __ASM_EMIT("vblendmps %%zmm0, %%zmm4, %%zmm0%{%%k4%}") /* zmm0 = ([(abs(s) > X_MAX) & (abs(s) <= X_MIN)]) ? s : sign(s) */ \
241 __ASM_EMIT("vblendmps %%zmm1, %%zmm5, %%zmm1%{%%k5%}") \
242 __ASM_EMIT("vmovups %%zmm0, 0x00(%[" DST "], %[off])") /* zmm0 = s' */ \
243 __ASM_EMIT("vmovups %%zmm1, 0x40(%[" DST "], %[off])") \
244 __ASM_EMIT("add $0x80, %[off]") \
245 __ASM_EMIT("sub $32, %[count]") \
246 __ASM_EMIT("4:") \
247 /* 8x blocks */ \
248 __ASM_EMIT("add $24, %[count]") \
249 __ASM_EMIT("jl 6f") \
250 __ASM_EMIT("5:") \
251 __ASM_EMIT("vmovdqu 0x00(%[" SRC "], %[off]), %%xmm0") /* xmm0 = s */ \
252 __ASM_EMIT("vmovdqu 0x10(%[" SRC "], %[off]), %%xmm4") \
253 __ASM_EMIT("vpand 0x00(%[CVAL]), %%xmm0, %%xmm1") /* xmm1 = abs(s) */ \
254 __ASM_EMIT("vpand 0x00(%[CVAL]), %%xmm4, %%xmm5") \
255 __ASM_EMIT("vpand 0xc0(%[CVAL]), %%xmm0, %%xmm2") /* xmm2 = sign(s) */ \
256 __ASM_EMIT("vpand 0xc0(%[CVAL]), %%xmm4, %%xmm6") \
257 __ASM_EMIT("vpcmpgtd 0x40(%[CVAL]), %%xmm1, %%xmm3") /* xmm3 = abs(s) > X_MAX */ \
258 __ASM_EMIT("vpcmpgtd 0x40(%[CVAL]), %%xmm5, %%xmm7") \
259 __ASM_EMIT("vpcmpgtd 0x80(%[CVAL]), %%xmm1, %%xmm1") /* xmm1 = abs(s) > X_MIN */ \
260 __ASM_EMIT("vpcmpgtd 0x80(%[CVAL]), %%xmm5, %%xmm5") \
261 __ASM_EMIT("vpandn %%xmm1, %%xmm3, %%xmm1") /* xmm1 = (abs(s) > X_MIN) & (abs(s) <= X_MAX) */ \
262 __ASM_EMIT("vpandn %%xmm5, %%xmm7, %%xmm5") \
263 __ASM_EMIT("vblendvps %%xmm1, %%xmm0, %%xmm2, %%xmm0") /* xmm0 = ((abs(s) > X_MIN) & (abs(s) <= X_MAX))) ? s : sign(s) */ \
264 __ASM_EMIT("vblendvps %%xmm5, %%xmm4, %%xmm6, %%xmm4") \
265 __ASM_EMIT("vmovdqu %%xmm0, 0x00(%[" DST "], %[off])") /* xmm0 = s */ \
266 __ASM_EMIT("vmovdqu %%xmm4, 0x10(%[" DST "], %[off])") \
267 __ASM_EMIT("add $0x20, %[off]") \
268 __ASM_EMIT("sub $8, %[count]") \
269 __ASM_EMIT("jge 5b") \
270 __ASM_EMIT("6:") \
271 /* 4x block */ \
272 __ASM_EMIT("add $4, %[count]") \
273 __ASM_EMIT("jl 8f") \
274 __ASM_EMIT("vmovdqu 0x00(%[" SRC "], %[off]), %%xmm0") /* xmm0 = s */ \
275 __ASM_EMIT("vpand 0x00(%[CVAL]), %%xmm0, %%xmm1") /* xmm1 = abs(s) */ \
276 __ASM_EMIT("vpand 0xc0(%[CVAL]), %%xmm0, %%xmm2") /* xmm2 = sign(s) */ \
277 __ASM_EMIT("vpcmpgtd 0x40(%[CVAL]), %%xmm1, %%xmm3") /* xmm3 = abs(s) > X_MAX */ \
278 __ASM_EMIT("vpcmpgtd 0x80(%[CVAL]), %%xmm1, %%xmm1") /* xmm1 = abs(s) > X_MIN */ \
279 __ASM_EMIT("vpandn %%xmm1, %%xmm3, %%xmm1") /* xmm1 = (abs(s) > X_MIN) & (abs(s) <= X_MAX) */ \
280 __ASM_EMIT("vblendvps %%xmm1, %%xmm0, %%xmm2, %%xmm0") /* xmm0 = ((abs(s) > X_MIN) & (abs(s) <= X_MAX))) ? s : sign(s) */ \
281 __ASM_EMIT("vmovdqu %%xmm0, 0x00(%[" DST "], %[off])") /* xmm0 = s */ \
282 __ASM_EMIT("add $0x10, %[off]") \
283 __ASM_EMIT("sub $4, %[count]") \
284 __ASM_EMIT("8:") \
285 /* 1x blocks */ \
286 __ASM_EMIT("add $3, %[count]") \
287 __ASM_EMIT("jl 10f") \
288 __ASM_EMIT("9:") \
289 __ASM_EMIT("vmovd 0x00(%[" SRC "], %[off]), %%xmm0") /* xmm0 = s */ \
290 __ASM_EMIT("vpand 0x00(%[CVAL]), %%xmm0, %%xmm1") /* xmm1 = abs(s) */ \
291 __ASM_EMIT("vpand 0xc0(%[CVAL]), %%xmm0, %%xmm2") /* xmm2 = sign(s) */ \
292 __ASM_EMIT("vpcmpgtd 0x40(%[CVAL]), %%xmm1, %%xmm3") /* xmm3 = abs(s) > X_MAX */ \
293 __ASM_EMIT("vpcmpgtd 0x80(%[CVAL]), %%xmm1, %%xmm1") /* xmm1 = abs(s) > X_MIN */ \
294 __ASM_EMIT("vpandn %%xmm1, %%xmm3, %%xmm1") /* xmm1 = (abs(s) > X_MIN) & (abs(s) <= X_MAX) */ \
295 __ASM_EMIT("vblendvps %%xmm1, %%xmm0, %%xmm2, %%xmm0") /* xmm0 = ((abs(s) > X_MIN) & (abs(s) <= X_MAX))) ? s : sign(s) */ \
296 __ASM_EMIT("vmovd %%xmm0, 0x00(%[" DST "], %[off])") /* xmm0 = s */ \
297 __ASM_EMIT("add $0x04, %[off]") \
298 __ASM_EMIT("dec %[count]") \
299 __ASM_EMIT("jge 9b") \
300 __ASM_EMIT("10:") \
301 /* end */
302
303 IF_ARCH_X86(
304 static uint32_t SANITIZE_CVAL[] __lsp_aligned64 =
305 {
306 LSP_DSP_VEC16(0x7fffffff), // X_ABS
307 LSP_DSP_VEC16(0x7f7fffff), // X_MAX
308 LSP_DSP_VEC16(0x007fffff), // X_MIN
309 LSP_DSP_VEC16(0x80000000), // X_SIGN
310 };
311 )
312
313 void sanitize1(float *dst, size_t count)
314 {
315 IF_ARCH_X86(size_t off);
316
317 ARCH_X86_ASM
318 (
319 SANITIZE_BODY("dst", "dst")
320 : [off] "=&r" (off), [count] "+r" (count)
321 : [dst] "r" (dst),
322 [CVAL] "r" (SANITIZE_CVAL)
323 : "cc", "memory",
324 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
325 "%xmm4", "%xmm5", "%xmm6", "%xmm7",
326 "%k0", "%k1", "%k2", "%k3",
327 "%k4", "%k5", "%k6", "%k7"
328 );
329 }
330
331 void sanitize2(float *dst, const float *src, size_t count)
332 {
333 IF_ARCH_X86(size_t off);
334
335 ARCH_X86_ASM
336 (
337 SANITIZE_BODY("dst", "src")
338 : [off] "=&r" (off), [count] "+r" (count)
339 : [dst] "r" (dst), [src] "r" (src),
340 [CVAL] "r" (SANITIZE_CVAL)
341 : "cc", "memory",
342 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
343 "%xmm4", "%xmm5", "%xmm6", "%xmm7",
344 "%k0", "%k1", "%k2", "%k3",
345 "%k4", "%k5", "%k6", "%k7"
346 );
347 }
348
349 #undef SANITIZE_BODY
350
351 IF_ARCH_X86(
352 static uint32_t XC_SATURATE[] __lsp_aligned64 =
353 {
354 LSP_DSP_VEC16(0x7fffffff), // X_ABS
355 LSP_DSP_VEC16(0x80000000), // X_SIGN
356 LSP_DSP_VEC16(0x7f800000), // X_P_INF
357 LSP_DSP_VEC16(LSP_DSP_FLOAT_SAT_P_NAN_I), // SX_P_NAN
358 LSP_DSP_VEC16(LSP_DSP_FLOAT_SAT_P_INF_I) // SX_P_INF
359 };
360 )
361
362 #define SAT_BODY(DST, SRC) \
363 __ASM_EMIT("xor %[off], %[off]") \
364 /* x64 blocks */ \
365 __ASM_EMIT("sub $64, %[count]") \
366 __ASM_EMIT("jb 2f") \
367 __ASM_EMIT("1:") \
368 __ASM_EMIT("vmovups 0x00(%[" SRC "], %[off]), %%zmm0") /* zmm0 = s */ \
369 __ASM_EMIT("vmovups 0x40(%[" SRC "], %[off]), %%zmm1") \
370 __ASM_EMIT("vmovups 0x80(%[" SRC "], %[off]), %%zmm2") \
371 __ASM_EMIT("vmovups 0xc0(%[" SRC "], %[off]), %%zmm3") \
372 __ASM_EMIT("vpandd 0x00(%[CVAL]), %%zmm0, %%zmm4") /* zmm4 = abs(s) */ \
373 __ASM_EMIT("vpandd 0x00(%[CVAL]), %%zmm1, %%zmm5") \
374 __ASM_EMIT("vpandd 0x00(%[CVAL]), %%zmm2, %%zmm6") \
375 __ASM_EMIT("vpandd 0x00(%[CVAL]), %%zmm3, %%zmm7") \
376 __ASM_EMIT("vpandd 0x40(%[CVAL]), %%zmm0, %%zmm0") /* zmm0 = sign(s) */ \
377 __ASM_EMIT("vpandd 0x40(%[CVAL]), %%zmm1, %%zmm1") \
378 __ASM_EMIT("vpandd 0x40(%[CVAL]), %%zmm2, %%zmm2") \
379 __ASM_EMIT("vpandd 0x40(%[CVAL]), %%zmm3, %%zmm3") \
380 __ASM_EMIT("vpcmpgtd 0x80(%[CVAL]), %%zmm4, %%k4") /* k4 = [abs(s) > +Inf] */ \
381 __ASM_EMIT("vpcmpgtd 0x80(%[CVAL]), %%zmm5, %%k5") \
382 __ASM_EMIT("vpcmpgtd 0x80(%[CVAL]), %%zmm6, %%k6") \
383 __ASM_EMIT("vpcmpgtd 0x80(%[CVAL]), %%zmm7, %%k7") \
384 __ASM_EMIT("vmovaps 0xc0(%[CVAL]), %%zmm4%{%%k4%}") /* zmm4 = [abs(s) > +Inf] ? PNAN: s */ \
385 __ASM_EMIT("vmovaps 0xc0(%[CVAL]), %%zmm5%{%%k5%}") \
386 __ASM_EMIT("vmovaps 0xc0(%[CVAL]), %%zmm6%{%%k6%}") \
387 __ASM_EMIT("vmovaps 0xc0(%[CVAL]), %%zmm7%{%%k7%}") \
388 __ASM_EMIT("vpcmpeqd 0x80(%[CVAL]), %%zmm4, %%k4") /* k4 = [abs(s) == +Inf] */ \
389 __ASM_EMIT("vpcmpeqd 0x80(%[CVAL]), %%zmm5, %%k5") \
390 __ASM_EMIT("vpcmpeqd 0x80(%[CVAL]), %%zmm6, %%k6") \
391 __ASM_EMIT("vpcmpeqd 0x80(%[CVAL]), %%zmm7, %%k7") \
392 __ASM_EMIT("vmovaps 0x100(%[CVAL]), %%zmm4%{%%k4%}") /* zmm4 = S = [abs(s) == +Inf] ? PINF : s */ \
393 __ASM_EMIT("vmovaps 0x100(%[CVAL]), %%zmm5%{%%k5%}") \
394 __ASM_EMIT("vmovaps 0x100(%[CVAL]), %%zmm6%{%%k6%}") \
395 __ASM_EMIT("vmovaps 0x100(%[CVAL]), %%zmm7%{%%k7%}") \
396 __ASM_EMIT("vpord %%zmm4, %%zmm0, %%zmm0") /* zmm0 = sign(s) * S */ \
397 __ASM_EMIT("vpord %%zmm5, %%zmm1, %%zmm1") \
398 __ASM_EMIT("vpord %%zmm6, %%zmm2, %%zmm2") \
399 __ASM_EMIT("vpord %%zmm7, %%zmm3, %%zmm3") \
400 __ASM_EMIT("vmovups %%zmm0, 0x00(%[" DST "], %[off])") \
401 __ASM_EMIT("vmovups %%zmm1, 0x40(%[" DST "], %[off])") \
402 __ASM_EMIT("vmovups %%zmm2, 0x80(%[" DST "], %[off])") \
403 __ASM_EMIT("vmovups %%zmm3, 0xc0(%[" DST "], %[off])") \
404 __ASM_EMIT("add $0x100, %[off]") \
405 __ASM_EMIT("sub $64, %[count]") \
406 __ASM_EMIT("jae 1b") \
407 __ASM_EMIT("2:") \
408 /* x32 block */ \
409 __ASM_EMIT("add $32, %[count]") \
410 __ASM_EMIT("jl 4f") \
411 __ASM_EMIT("vmovups 0x00(%[" SRC "], %[off]), %%zmm0") /* zmm0 = s */ \
412 __ASM_EMIT("vmovups 0x40(%[" SRC "], %[off]), %%zmm1") \
413 __ASM_EMIT("vpandd 0x00(%[CVAL]), %%zmm0, %%zmm4") /* zmm4 = abs(s) */ \
414 __ASM_EMIT("vpandd 0x00(%[CVAL]), %%zmm1, %%zmm5") \
415 __ASM_EMIT("vpandd 0x40(%[CVAL]), %%zmm0, %%zmm0") /* zmm0 = sign(s) */ \
416 __ASM_EMIT("vpandd 0x40(%[CVAL]), %%zmm1, %%zmm1") \
417 __ASM_EMIT("vpcmpgtd 0x80(%[CVAL]), %%zmm4, %%k4") /* k4 = [abs(s) > +Inf] */ \
418 __ASM_EMIT("vpcmpgtd 0x80(%[CVAL]), %%zmm5, %%k5") \
419 __ASM_EMIT("vmovaps 0xc0(%[CVAL]), %%zmm4%{%%k4%}") /* zmm4 = [abs(s) > +Inf] ? PNAN: s */ \
420 __ASM_EMIT("vmovaps 0xc0(%[CVAL]), %%zmm5%{%%k5%}") \
421 __ASM_EMIT("vpcmpeqd 0x80(%[CVAL]), %%zmm4, %%k4") /* k4 = [abs(s) == +Inf] */ \
422 __ASM_EMIT("vpcmpeqd 0x80(%[CVAL]), %%zmm5, %%k5") \
423 __ASM_EMIT("vmovaps 0x100(%[CVAL]), %%zmm4%{%%k4%}") /* zmm4 = S = [abs(s) == +Inf] ? PINF : s */ \
424 __ASM_EMIT("vmovaps 0x100(%[CVAL]), %%zmm5%{%%k5%}") \
425 __ASM_EMIT("vpord %%zmm4, %%zmm0, %%zmm0") /* zmm0 = sign(s) * S */ \
426 __ASM_EMIT("vpord %%zmm5, %%zmm1, %%zmm1") \
427 __ASM_EMIT("vmovups %%zmm0, 0x00(%[" DST "], %[off])") \
428 __ASM_EMIT("vmovups %%zmm1, 0x40(%[" DST "], %[off])") \
429 __ASM_EMIT("add $0x80, %[off]") \
430 __ASM_EMIT("sub $32, %[count]") \
431 __ASM_EMIT("4:") \
432 /* x8 blocks */ \
433 __ASM_EMIT("add $24, %[count]") \
434 __ASM_EMIT("jl 6f") \
435 __ASM_EMIT("5:") \
436 __ASM_EMIT("vmovups 0x00(%[" SRC "], %[off]), %%xmm0") /* xmm0 = s */ \
437 __ASM_EMIT("vmovups 0x10(%[" SRC "], %[off]), %%xmm1") \
438 __ASM_EMIT("vandps 0x00(%[CVAL]), %%xmm0, %%xmm2") /* xmm2 = abs(s) */ \
439 __ASM_EMIT("vandps 0x00(%[CVAL]), %%xmm1, %%xmm3") \
440 __ASM_EMIT("vandps 0x40(%[CVAL]), %%xmm0, %%xmm4") /* xmm4 = sign(s) */ \
441 __ASM_EMIT("vandps 0x40(%[CVAL]), %%xmm1, %%xmm5") \
442 __ASM_EMIT("vpcmpgtd 0x80(%[CVAL]), %%xmm2, %%xmm6") /* xmm6 = abs(s) > +Inf */ \
443 __ASM_EMIT("vpcmpgtd 0x80(%[CVAL]), %%xmm3, %%xmm7") \
444 __ASM_EMIT("vpcmpeqd 0x80(%[CVAL]), %%xmm2, %%xmm2") /* xmm2 = abs(s) == +Inf */ \
445 __ASM_EMIT("vpcmpeqd 0x80(%[CVAL]), %%xmm3, %%xmm3") \
446 __ASM_EMIT("vblendvps %%xmm6, 0xc0(%[CVAL]), %%xmm0, %%xmm0") /* xmm0 = S = s & (abs(s) <= +Inf) | PNAN & (abs(s) > +Inf) */ \
447 __ASM_EMIT("vblendvps %%xmm7, 0xc0(%[CVAL]), %%xmm1, %%xmm1") \
448 __ASM_EMIT("vblendvps %%xmm2, 0x100(%[CVAL]), %%xmm0, %%xmm0")/* xmm0 = S & (abs(S) != +Inf) | PINF & (abs(s) == +Inf) */ \
449 __ASM_EMIT("vblendvps %%xmm3, 0x100(%[CVAL]), %%xmm1, %%xmm1") \
450 __ASM_EMIT("vorps %%xmm4, %%xmm0, %%xmm0") \
451 __ASM_EMIT("vorps %%xmm5, %%xmm1, %%xmm1") \
452 __ASM_EMIT("vmovups %%xmm0, 0x00(%[" DST "], %[off])") \
453 __ASM_EMIT("vmovups %%xmm1, 0x10(%[" DST "], %[off])") \
454 __ASM_EMIT("add $0x20, %[off]") \
455 __ASM_EMIT("sub $8, %[count]") \
456 __ASM_EMIT("jge 5b") \
457 __ASM_EMIT("6:") \
458 /* x4 block */ \
459 __ASM_EMIT("add $4, %[count]") \
460 __ASM_EMIT("jl 8f") \
461 __ASM_EMIT("vmovups 0x00(%[" SRC "], %[off]), %%xmm0") /* xmm0 = s */ \
462 __ASM_EMIT("vandps 0x00(%[CVAL]), %%xmm0, %%xmm2") /* xmm2 = abs(s) */ \
463 __ASM_EMIT("vandps 0x40(%[CVAL]), %%xmm0, %%xmm4") /* xmm4 = sign(s) */ \
464 __ASM_EMIT("vpcmpgtd 0x80(%[CVAL]), %%xmm2, %%xmm6") /* xmm6 = abs(s) > +Inf */ \
465 __ASM_EMIT("vpcmpeqd 0x80(%[CVAL]), %%xmm2, %%xmm2") /* xmm2 = abs(s) == +Inf */ \
466 __ASM_EMIT("vblendvps %%xmm6, 0xc0(%[CVAL]), %%xmm0, %%xmm0") /* xmm0 = S = s & (abs(s) <= +Inf) | PNAN & (abs(s) > +Inf) */ \
467 __ASM_EMIT("vblendvps %%xmm2, 0x100(%[CVAL]), %%xmm0, %%xmm0")/* xmm0 = S & (abs(S) != +Inf) | PINF & (abs(s) == +Inf) */ \
468 __ASM_EMIT("vorps %%xmm4, %%xmm0, %%xmm0") \
469 __ASM_EMIT("vmovups %%xmm0, 0x00(%[" DST "], %[off])") \
470 __ASM_EMIT("sub $4, %[count]") \
471 __ASM_EMIT("add $0x10, %[off]") \
472 __ASM_EMIT("8:") \
473 /* x1 block */ \
474 __ASM_EMIT("add $3, %[count]") \
475 __ASM_EMIT("jl 10f") \
476 __ASM_EMIT("9:") \
477 __ASM_EMIT("vmovss 0x00(%[" SRC "], %[off]), %%xmm0") /* xmm0 = s */ \
478 __ASM_EMIT("vandps 0x00(%[CVAL]), %%xmm0, %%xmm2") /* xmm2 = abs(s) */ \
479 __ASM_EMIT("vandps 0x40(%[CVAL]), %%xmm0, %%xmm4") /* xmm4 = sign(s) */ \
480 __ASM_EMIT("vpcmpgtd 0x80(%[CVAL]), %%xmm2, %%xmm6") /* xmm6 = abs(s) > +Inf */ \
481 __ASM_EMIT("vpcmpeqd 0x80(%[CVAL]), %%xmm2, %%xmm2") /* xmm2 = abs(s) == +Inf */ \
482 __ASM_EMIT("vblendvps %%xmm6, 0xc0(%[CVAL]), %%xmm0, %%xmm0") /* xmm0 = S = s & (abs(s) <= +Inf) | PNAN & (abs(s) > +Inf) */ \
483 __ASM_EMIT("vblendvps %%xmm2, 0x100(%[CVAL]), %%xmm0, %%xmm0")/* xmm0 = S & (abs(S) != +Inf) | PINF & (abs(s) == +Inf) */ \
484 __ASM_EMIT("vorps %%xmm4, %%xmm0, %%xmm0") \
485 __ASM_EMIT("vmovss %%xmm0, 0x00(%[" DST "], %[off])") \
486 __ASM_EMIT("add $0x04, %[off]") \
487 __ASM_EMIT("dec %[count]") \
488 __ASM_EMIT("jge 9b") \
489 __ASM_EMIT("10:")
490
491 void saturate(float *dst, size_t count)
492 {
493 IF_ARCH_X86(size_t off);
494 ARCH_X86_ASM(
495 SAT_BODY("dst", "dst")
496 : [off] "=&r" (off), [count] "+r" (count)
497 : [dst] "r" (dst),
498 [CVAL] "r" (XC_SATURATE)
499 : "cc", "memory",
500 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
501 "%xmm4", "%xmm5", "%xmm6", "%xmm7",
502 "%k4", "%k5", "%k6", "%k7"
503 );
504 }
505
506 void copy_saturated(float *dst, const float *src, size_t count)
507 {
508 IF_ARCH_X86(size_t off);
509 ARCH_X86_ASM(
510 SAT_BODY("dst", "src")
511 : [off] "=&r" (off), [count] "+r" (count)
512 : [dst] "r" (dst), [src] "r" (src),
513 [CVAL] "r" (XC_SATURATE)
514 : "cc", "memory",
515 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
516 "%xmm4", "%xmm5", "%xmm6", "%xmm7",
517 "%k4", "%k5", "%k6", "%k7"
518 );
519 }
520
521 #undef SAT_BODY
522
523 #define LIMIT_SAT_BODY(DST, SRC) \
524 __ASM_EMIT("xor %[off], %[off]") \
525 __ASM_EMIT("vmovaps 0x80(%[CVAL]), %%zmm4") /* xmm4 = +1 */ \
526 __ASM_EMIT("vmovaps 0x80(%[CVAL]), %%zmm5") \
527 /* x32 blocks */ \
528 __ASM_EMIT("sub $32, %[count]") \
529 __ASM_EMIT("jb 2f") \
530 __ASM_EMIT("1:") \
531 __ASM_EMIT("vmovups 0x00(%[" SRC "], %[off]), %%zmm0") /* zmm0 = s */ \
532 __ASM_EMIT("vmovups 0x40(%[" SRC "], %[off]), %%zmm1") \
533 __ASM_EMIT("vpandd 0x00(%[CVAL]), %%zmm0, %%zmm2") /* zmm2 = abs(s) */ \
534 __ASM_EMIT("vpandd 0x00(%[CVAL]), %%zmm1, %%zmm3") \
535 __ASM_EMIT("vpandd 0x40(%[CVAL]), %%zmm0, %%zmm0") /* zmm0 = sign(s) */ \
536 __ASM_EMIT("vpandd 0x40(%[CVAL]), %%zmm1, %%zmm1") \
537 __ASM_EMIT("vcmpps $2, %%zmm4, %%zmm2, %%k4") /* k4 = [ (abs(s) <= +1) & !isnan(s) ] */ \
538 __ASM_EMIT("vcmpps $2, %%zmm5, %%zmm3, %%k5") \
539 __ASM_EMIT("vpcmpgtd 0xc0(%[CVAL]), %%zmm2, %%k6") /* k6 = [abs(s) > +Inf] */ \
540 __ASM_EMIT("vpcmpgtd 0xc0(%[CVAL]), %%zmm3, %%k7") \
541 __ASM_EMIT("vblendmps %%zmm2, %%zmm4, %%zmm2%{%%k4%}") /* zmm2 = ([ (abs(s) <= +1) & !isnan(s) ]) ? abs(s) : 1 */ \
542 __ASM_EMIT("vblendmps %%zmm3, %%zmm5, %%zmm3%{%%k5%}") \
543 __ASM_EMIT("vpord %%zmm2, %%zmm0, %%zmm0") /* zmm0 = ([ (abs(s) <= +1) & !isnan(s) ]) ? s : sign(s)*1 */ \
544 __ASM_EMIT("vpord %%zmm3, %%zmm1, %%zmm1") \
545 __ASM_EMIT("vpxord %%zmm0, %%zmm0, %%zmm0%{%%k6%}") /* zmm0 = [abs(s) > +Inf] ? 0 : ([ (abs(s) <= +1) & !isnan(s) ]) ? s : sign(s)*1 */ \
546 __ASM_EMIT("vpxord %%zmm1, %%zmm1, %%zmm1%{%%k7%}") \
547 __ASM_EMIT("vmovups %%zmm0, 0x00(%[" DST "], %[off])") \
548 __ASM_EMIT("vmovups %%zmm1, 0x40(%[" DST "], %[off])") \
549 __ASM_EMIT("add $0x80, %[off]") \
550 __ASM_EMIT("sub $32, %[count]") \
551 __ASM_EMIT("jae 1b") \
552 __ASM_EMIT("2:") \
553 /* x8 blocks */ \
554 __ASM_EMIT("add $24, %[count]") \
555 __ASM_EMIT("jl 4f") \
556 __ASM_EMIT("3:") \
557 __ASM_EMIT("vmovups 0x00(%[" SRC "], %[off]), %%xmm0") /* xmm0 = s */ \
558 __ASM_EMIT("vmovups 0x10(%[" SRC "], %[off]), %%xmm1") \
559 __ASM_EMIT("vmovaps 0x80(%[CVAL]), %%xmm2") /* xmm2 = +1 */ \
560 __ASM_EMIT("vandps 0x00(%[CVAL]), %%xmm0, %%xmm6") /* xmm6 = abs(s) */ \
561 __ASM_EMIT("vmovaps %%xmm2, %%xmm3") \
562 __ASM_EMIT("vandps 0x00(%[CVAL]), %%xmm1, %%xmm7") \
563 __ASM_EMIT("vcmpps $2, %%xmm2, %%xmm6, %%xmm2") /* xmm2 = c = [ (abs(s) <= +1) & !isnan(s) ] */ \
564 __ASM_EMIT("vcmpps $2, %%xmm3, %%xmm7, %%xmm3") \
565 __ASM_EMIT("vandps 0x40(%[CVAL]), %%xmm0, %%xmm4") /* xmm4 = sign(s) */ \
566 __ASM_EMIT("vandps 0x40(%[CVAL]), %%xmm1, %%xmm5") \
567 __ASM_EMIT("vpcmpgtd 0xc0(%[CVAL]), %%xmm6, %%xmm6") /* xmm6 = [ abs(s) > +Inf ] */ \
568 __ASM_EMIT("vpcmpgtd 0xc0(%[CVAL]), %%xmm7, %%xmm7") \
569 __ASM_EMIT("vorps 0x80(%[CVAL]), %%xmm4, %%xmm4") /* xmm4 = +1 * sign(s) */ \
570 __ASM_EMIT("vorps 0x80(%[CVAL]), %%xmm5, %%xmm5") \
571 __ASM_EMIT("vandnps %%xmm4, %%xmm6, %%xmm4") /* xmm4 = r = +1 * sign(s) & [ abs(s) <= +Inf ] */ \
572 __ASM_EMIT("vandnps %%xmm5, %%xmm7, %%xmm5") \
573 __ASM_EMIT("vblendvps %%xmm2, %%xmm0, %%xmm4, %%xmm0") /* xmm0 = (s & c) | (r & !c) */ \
574 __ASM_EMIT("vblendvps %%xmm3, %%xmm1, %%xmm5, %%xmm1") \
575 __ASM_EMIT("vmovups %%xmm0, 0x00(%[" DST "], %[off])") \
576 __ASM_EMIT("vmovups %%xmm1, 0x10(%[" DST "], %[off])") \
577 __ASM_EMIT("add $0x20, %[off]") \
578 __ASM_EMIT("sub $8, %[count]") \
579 __ASM_EMIT("jge 3b") \
580 __ASM_EMIT("4:") \
581 /* x4 block */ \
582 __ASM_EMIT("add $4, %[count]") \
583 __ASM_EMIT("jl 6f") \
584 __ASM_EMIT("vmovups 0x00(%[" SRC "], %[off]), %%xmm0") /* xmm0 = s */ \
585 __ASM_EMIT("vmovaps 0x80(%[CVAL]), %%xmm2") /* xmm2 = +1 */ \
586 __ASM_EMIT("vandps 0x00(%[CVAL]), %%xmm0, %%xmm6") /* xmm6 = abs(s) */ \
587 __ASM_EMIT("vcmpps $2, %%xmm2, %%xmm6, %%xmm2") /* xmm2 = c = [ (abs(s) <= +1) & !isnan(s) ] */ \
588 __ASM_EMIT("vandps 0x40(%[CVAL]), %%xmm0, %%xmm4") /* xmm4 = sign(s) */ \
589 __ASM_EMIT("vpcmpgtd 0xc0(%[CVAL]), %%xmm6, %%xmm6") /* xmm6 = [ abs(s) > +Inf ] */ \
590 __ASM_EMIT("vorps 0x80(%[CVAL]), %%xmm4, %%xmm4") /* xmm4 = +1 * sign(s) */ \
591 __ASM_EMIT("vandnps %%xmm4, %%xmm6, %%xmm4") /* xmm4 = r = +1 * sign(s) & [ abs(s) <= +Inf ] */ \
592 __ASM_EMIT("vblendvps %%xmm2, %%xmm0, %%xmm4, %%xmm0") /* xmm0 = (s & c) | (r & !c) */ \
593 __ASM_EMIT("vmovups %%xmm0, 0x00(%[" DST "], %[off])") \
594 __ASM_EMIT("sub $4, %[count]") \
595 __ASM_EMIT("add $0x10, %[off]") \
596 __ASM_EMIT("6:") \
597 /* x1 block */ \
598 __ASM_EMIT("add $3, %[count]") \
599 __ASM_EMIT("jl 8f") \
600 __ASM_EMIT("7:") \
601 __ASM_EMIT("vmovss 0x00(%[" SRC "], %[off]), %%xmm0") /* xmm0 = s */ \
602 __ASM_EMIT("vmovups 0x00(%[" SRC "], %[off]), %%xmm0") /* xmm0 = s */ \
603 __ASM_EMIT("vmovaps 0x80(%[CVAL]), %%xmm2") /* xmm2 = +1 */ \
604 __ASM_EMIT("vandps 0x00(%[CVAL]), %%xmm0, %%xmm6") /* xmm6 = abs(s) */ \
605 __ASM_EMIT("vcmpps $2, %%xmm2, %%xmm6, %%xmm2") /* xmm2 = c = [ (abs(s) <= +1) & !isnan(s) ] */ \
606 __ASM_EMIT("vandps 0x40(%[CVAL]), %%xmm0, %%xmm4") /* xmm4 = sign(s) */ \
607 __ASM_EMIT("vpcmpgtd 0xc0(%[CVAL]), %%xmm6, %%xmm6") /* xmm6 = [ abs(s) > +Inf ] */ \
608 __ASM_EMIT("vorps 0x80(%[CVAL]), %%xmm4, %%xmm4") /* xmm4 = +1 * sign(s) */ \
609 __ASM_EMIT("vandnps %%xmm4, %%xmm6, %%xmm4") /* xmm4 = r = +1 * sign(s) & [ abs(s) <= +Inf ] */ \
610 __ASM_EMIT("vblendvps %%xmm2, %%xmm0, %%xmm4, %%xmm0") /* xmm0 = (s & c) | (r & !c) */ \
611 __ASM_EMIT("vmovss %%xmm0, 0x00(%[" DST "], %[off])") \
612 __ASM_EMIT("add $0x04, %[off]") \
613 __ASM_EMIT("dec %[count]") \
614 __ASM_EMIT("jge 7b") \
615 __ASM_EMIT("8:")
616
617 IF_ARCH_X86(
618 static uint32_t XLIM_SAT[] __lsp_aligned64 =
619 {
620 LSP_DSP_VEC16(0x7fffffff), // abs
621 LSP_DSP_VEC16(0x80000000), // sign
622 LSP_DSP_VEC16(0x3f800000), // +1
623 LSP_DSP_VEC16(0x7f800000) // +Inf
624 };
625 )
626
627 void limit_saturate1(float *dst, size_t count)
628 {
629 IF_ARCH_X86(size_t off);
630 ARCH_X86_ASM(
631 LIMIT_SAT_BODY("dst", "dst")
632 : [off] "=&r" (off), [count] "+r" (count)
633 : [dst] "r" (dst),
634 [CVAL] "r" (XLIM_SAT)
635 : "cc", "memory",
636 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
637 "%xmm4", "%xmm5", "%xmm6", "%xmm7",
638 "%k4", "%k5", "%k6", "%k7"
639 );
640 }
641
642 void limit_saturate2(float *dst, const float *src, size_t count)
643 {
644 IF_ARCH_X86(size_t off);
645 ARCH_X86_ASM(
646 LIMIT_SAT_BODY("dst", "src")
647 : [off] "=&r" (off), [count] "+r" (count)
648 : [dst] "r" (dst), [src] "r" (src),
649 [CVAL] "r" (XLIM_SAT)
650 : "cc", "memory",
651 "%xmm0", "%xmm1", "%xmm2", "%xmm3",
652 "%xmm4", "%xmm5", "%xmm6", "%xmm7",
653 "%k4", "%k5", "%k6", "%k7"
654 );
655 }
656
657 #undef LIMIT_SAT_BODY
658
659 } /* namespace avx512 */
660 } /* namespace lsp */
661
662 #endif /* PRIVATE_DSP_ARCH_X86_AVX512_FLOAT_H_ */