Debian Codegraph @

gmp4/mpn/x86_64/fastsse/com-palignr.asm

raw
1 dnl AMD64 mpn_com optimised for CPUs with fast SSE copying and SSSE3.
2
3 dnl Copyright 2012, 2013 Free Software Foundation, Inc.
4
5 dnl Contributed to the GNU project by Torbjorn Granlund.
6
7 dnl This file is part of the GNU MP Library.
8 dnl
9 dnl The GNU MP Library is free software; you can redistribute it and/or modify
10 dnl it under the terms of either:
11 dnl
12 dnl * the GNU Lesser General Public License as published by the Free
13 dnl Software Foundation; either version 3 of the License, or (at your
14 dnl option) any later version.
15 dnl
16 dnl or
17 dnl
18 dnl * the GNU General Public License as published by the Free Software
19 dnl Foundation; either version 2 of the License, or (at your option) any
20 dnl later version.
21 dnl
22 dnl or both in parallel, as here.
23 dnl
24 dnl The GNU MP Library is distributed in the hope that it will be useful, but
25 dnl WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
26 dnl or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
27 dnl for more details.
28 dnl
29 dnl You should have received copies of the GNU General Public License and the
30 dnl GNU Lesser General Public License along with the GNU MP Library. If not,
31 dnl see https://www.gnu.org/licenses/.
32
33 include(`../config.m4')
34
35 C cycles/limb cycles/limb cycles/limb good
36 C aligned unaligned best seen for cpu?
37 C AMD K8,K9 2.0 illop 1.0/1.0 N
38 C AMD K10 0.85 illop Y/N
39 C AMD bd1 1.39 ? 1.45 Y/N
40 C AMD bobcat 1.97 ? 8.17 1.5/1.5 N
41 C Intel P4 2.26 illop Y/N
42 C Intel core2 0.52 0.82 opt/0.74 Y
43 C Intel NHM 0.52 0.65 opt/opt Y
44 C Intel SBR 0.51 0.55 opt/0.51 Y
45 C Intel atom 1.16 1.70 opt/opt Y
46 C VIA nano 1.09 1.10 opt/opt Y
47
48 C We use only 16-byte operations, except for unaligned top-most and bottom-most
49 C limbs. We use the SSSE3 palignr instruction when rp - up = 8 (mod 16). That
50 C instruction is better adapted to mpn_copyd's needs, we need to contort the
51 C code to use it here.
52 C
53 C For operands of < COM_SSE_THRESHOLD limbs, we use a plain 64-bit loop, taken
54 C from the x86_64 default code.
55
56 C INPUT PARAMETERS
57 define(`rp', `%rdi')
58 define(`up', `%rsi')
59 define(`n', `%rdx')
60
61 C There are three instructions for loading an aligned 128-bit quantity. We use
62 C movaps, since it has the shortest coding.
63 define(`movdqa', ``movaps'')
64
65 ifdef(`COM_SSE_THRESHOLD',`',`define(`COM_SSE_THRESHOLD', 7)')
66
67 ASM_START()
68 TEXT
69 ALIGN(64)
70 PROLOGUE(mpn_com)
71 FUNC_ENTRY(3)
72
73 cmp $COM_SSE_THRESHOLD, n
74 jbe L(bc)
75
76 pcmpeqb %xmm7, %xmm7 C set to 111...111
77
78 test $8, R8(rp) C is rp 16-byte aligned?
79 jz L(rp_aligned) C jump if rp aligned
80
81 mov (up), %r8
82 lea 8(up), up
83 not %r8
84 mov %r8, (rp)
85 lea 8(rp), rp
86 dec n
87
88 L(rp_aligned):
89 test $8, R8(up)
90 jnz L(uent)
91
92 ifelse(eval(COM_SSE_THRESHOLD >= 8),1,
93 ` sub $8, n',
94 ` jmp L(am)')
95
96 ALIGN(16)
97 L(atop):movdqa 0(up), %xmm0
98 movdqa 16(up), %xmm1
99 movdqa 32(up), %xmm2
100 movdqa 48(up), %xmm3
101 lea 64(up), up
102 pxor %xmm7, %xmm0
103 pxor %xmm7, %xmm1
104 pxor %xmm7, %xmm2
105 pxor %xmm7, %xmm3
106 movdqa %xmm0, (rp)
107 movdqa %xmm1, 16(rp)
108 movdqa %xmm2, 32(rp)
109 movdqa %xmm3, 48(rp)
110 lea 64(rp), rp
111 L(am): sub $8, n
112 jnc L(atop)
113
114 test $4, R8(n)
115 jz 1f
116 movdqa (up), %xmm0
117 movdqa 16(up), %xmm1
118 lea 32(up), up
119 pxor %xmm7, %xmm0
120 pxor %xmm7, %xmm1
121 movdqa %xmm0, (rp)
122 movdqa %xmm1, 16(rp)
123 lea 32(rp), rp
124
125 1: test $2, R8(n)
126 jz 1f
127 movdqa (up), %xmm0
128 lea 16(up), up
129 pxor %xmm7, %xmm0
130 movdqa %xmm0, (rp)
131 lea 16(rp), rp
132
133 1: test $1, R8(n)
134 jz 1f
135 mov (up), %r8
136 not %r8
137 mov %r8, (rp)
138
139 1: FUNC_EXIT()
140 ret
141
142 L(uent):
143 C Code handling up - rp = 8 (mod 16)
144
145 C FIXME: The code below only handles overlap if it is close to complete, or
146 C quite separate: up-rp < 5 or up-up > 15 limbs
147 lea -40(up), %rax C 40 = 5 * GMP_LIMB_BYTES
148 sub rp, %rax
149 cmp $80, %rax C 80 = (15-5) * GMP_LIMB_BYTES
150 jbe L(bc) C deflect to plain loop
151
152 sub $16, n
153 jc L(uend)
154
155 movdqa 120(up), %xmm3
156
157 sub $16, n
158 jmp L(um)
159
160 ALIGN(16)
161 L(utop):movdqa 120(up), %xmm3
162 pxor %xmm7, %xmm0
163 movdqa %xmm0, -128(rp)
164 sub $16, n
165 L(um): movdqa 104(up), %xmm2
166 palignr($8, %xmm2, %xmm3)
167 movdqa 88(up), %xmm1
168 pxor %xmm7, %xmm3
169 movdqa %xmm3, 112(rp)
170 palignr($8, %xmm1, %xmm2)
171 movdqa 72(up), %xmm0
172 pxor %xmm7, %xmm2
173 movdqa %xmm2, 96(rp)
174 palignr($8, %xmm0, %xmm1)
175 movdqa 56(up), %xmm3
176 pxor %xmm7, %xmm1
177 movdqa %xmm1, 80(rp)
178 palignr($8, %xmm3, %xmm0)
179 movdqa 40(up), %xmm2
180 pxor %xmm7, %xmm0
181 movdqa %xmm0, 64(rp)
182 palignr($8, %xmm2, %xmm3)
183 movdqa 24(up), %xmm1
184 pxor %xmm7, %xmm3
185 movdqa %xmm3, 48(rp)
186 palignr($8, %xmm1, %xmm2)
187 movdqa 8(up), %xmm0
188 pxor %xmm7, %xmm2
189 movdqa %xmm2, 32(rp)
190 palignr($8, %xmm0, %xmm1)
191 movdqa -8(up), %xmm3
192 pxor %xmm7, %xmm1
193 movdqa %xmm1, 16(rp)
194 palignr($8, %xmm3, %xmm0)
195 lea 128(up), up
196 lea 128(rp), rp
197 jnc L(utop)
198
199 pxor %xmm7, %xmm0
200 movdqa %xmm0, -128(rp)
201
202 L(uend):test $8, R8(n)
203 jz 1f
204 movdqa 56(up), %xmm3
205 movdqa 40(up), %xmm2
206 palignr($8, %xmm2, %xmm3)
207 movdqa 24(up), %xmm1
208 pxor %xmm7, %xmm3
209 movdqa %xmm3, 48(rp)
210 palignr($8, %xmm1, %xmm2)
211 movdqa 8(up), %xmm0
212 pxor %xmm7, %xmm2
213 movdqa %xmm2, 32(rp)
214 palignr($8, %xmm0, %xmm1)
215 movdqa -8(up), %xmm3
216 pxor %xmm7, %xmm1
217 movdqa %xmm1, 16(rp)
218 palignr($8, %xmm3, %xmm0)
219 lea 64(up), up
220 pxor %xmm7, %xmm0
221 movdqa %xmm0, (rp)
222 lea 64(rp), rp
223
224 1: test $4, R8(n)
225 jz 1f
226 movdqa 24(up), %xmm1
227 movdqa 8(up), %xmm0
228 palignr($8, %xmm0, %xmm1)
229 movdqa -8(up), %xmm3
230 pxor %xmm7, %xmm1
231 movdqa %xmm1, 16(rp)
232 palignr($8, %xmm3, %xmm0)
233 lea 32(up), up
234 pxor %xmm7, %xmm0
235 movdqa %xmm0, (rp)
236 lea 32(rp), rp
237
238 1: test $2, R8(n)
239 jz 1f
240 movdqa 8(up), %xmm0
241 movdqa -8(up), %xmm3
242 palignr($8, %xmm3, %xmm0)
243 lea 16(up), up
244 pxor %xmm7, %xmm0
245 movdqa %xmm0, (rp)
246 lea 16(rp), rp
247
248 1: test $1, R8(n)
249 jz 1f
250 mov (up), %r8
251 not %r8
252 mov %r8, (rp)
253
254 1: FUNC_EXIT()
255 ret
256
257 C Basecase code. Needed for good small operands speed, not for
258 C correctness as the above code is currently written.
259
260 L(bc): lea -8(rp), rp
261 sub $4, R32(n)
262 jc L(end)
263
264 ifelse(eval(1 || COM_SSE_THRESHOLD >= 8),1,
265 ` ALIGN(16)')
266 L(top): mov (up), %r8
267 mov 8(up), %r9
268 lea 32(rp), rp
269 mov 16(up), %r10
270 mov 24(up), %r11
271 lea 32(up), up
272 not %r8
273 not %r9
274 not %r10
275 not %r11
276 mov %r8, -24(rp)
277 mov %r9, -16(rp)
278 ifelse(eval(1 || COM_SSE_THRESHOLD >= 8),1,
279 ` sub $4, R32(n)')
280 mov %r10, -8(rp)
281 mov %r11, (rp)
282 ifelse(eval(1 || COM_SSE_THRESHOLD >= 8),1,
283 ` jnc L(top)')
284
285 L(end): test $1, R8(n)
286 jz 1f
287 mov (up), %r8
288 not %r8
289 mov %r8, 8(rp)
290 lea 8(rp), rp
291 lea 8(up), up
292 1: test $2, R8(n)
293 jz 1f
294 mov (up), %r8
295 mov 8(up), %r9
296 not %r8
297 not %r9
298 mov %r8, 8(rp)
299 mov %r9, 16(rp)
300 1: FUNC_EXIT()
301 ret
302 EPILOGUE()