Debian Codegraph @

gmp4/mpn/x86_64/fastsse/copyi-palignr.asm

raw
1 dnl AMD64 mpn_copyi optimised for CPUs with fast SSE copying and SSSE3.
2
3 dnl Copyright 2012, 2013 Free Software Foundation, Inc.
4
5 dnl Contributed to the GNU project by Torbjörn Granlund.
6
7 dnl This file is part of the GNU MP Library.
8 dnl
9 dnl The GNU MP Library is free software; you can redistribute it and/or modify
10 dnl it under the terms of either:
11 dnl
12 dnl * the GNU Lesser General Public License as published by the Free
13 dnl Software Foundation; either version 3 of the License, or (at your
14 dnl option) any later version.
15 dnl
16 dnl or
17 dnl
18 dnl * the GNU General Public License as published by the Free Software
19 dnl Foundation; either version 2 of the License, or (at your option) any
20 dnl later version.
21 dnl
22 dnl or both in parallel, as here.
23 dnl
24 dnl The GNU MP Library is distributed in the hope that it will be useful, but
25 dnl WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
26 dnl or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
27 dnl for more details.
28 dnl
29 dnl You should have received copies of the GNU General Public License and the
30 dnl GNU Lesser General Public License along with the GNU MP Library. If not,
31 dnl see https://www.gnu.org/licenses/.
32
33 include(`../config.m4')
34
35 C cycles/limb cycles/limb cycles/limb good
36 C aligned unaligned best seen for cpu?
37 C AMD K8,K9 2.0 illop 1.0/1.0 N
38 C AMD K10 0.85 illop Y/N
39 C AMD bull 0.70 0.66 Y
40 C AMD pile 0.68 0.66 Y
41 C AMD steam ? ?
42 C AMD bobcat 1.97 8.16 1.5/1.5 N
43 C AMD jaguar ? ?
44 C Intel P4 2.26 illop Y/N
45 C Intel core 0.52 0.64 opt/opt Y
46 C Intel NHM 0.52 0.71 opt/opt Y
47 C Intel SBR 0.51 0.54 opt/0.51 Y
48 C Intel IBR ? ? Y
49 C Intel HWL 0.51 0.52 0.25/0.25 N
50 C Intel atom 1.16 1.61 opt/opt Y
51 C VIA nano 1.09 1.08 opt/opt Y
52
53 C We use only 16-byte operations, except for unaligned top-most and bottom-most
54 C limbs. We use the SSSE3 palignr instruction when rp - up = 8 (mod 16). That
55 C instruction is better adapted to mpn_copyd's needs, we need to contort the
56 C code to use it here.
57 C
58 C For operands of < COPYI_SSE_THRESHOLD limbs, we use a plain 64-bit loop,
59 C taken from the x86_64 default code.
60
61 C INPUT PARAMETERS
62 define(`rp', `%rdi')
63 define(`up', `%rsi')
64 define(`n', `%rdx')
65
66 C There are three instructions for loading an aligned 128-bit quantity. We use
67 C movaps, since it has the shortest coding.
68 dnl define(`movdqa', ``movaps'')
69
70 ifdef(`COPYI_SSE_THRESHOLD',`',`define(`COPYI_SSE_THRESHOLD', 7)')
71
72 ASM_START()
73 TEXT
74 ALIGN(64)
75 PROLOGUE(mpn_copyi)
76 FUNC_ENTRY(3)
77
78 cmp $COPYI_SSE_THRESHOLD, n
79 jbe L(bc)
80
81 test $8, R8(rp) C is rp 16-byte aligned?
82 jz L(rp_aligned) C jump if rp aligned
83
84 movsq C copy one limb
85 dec n
86
87 L(rp_aligned):
88 test $8, R8(up)
89 jnz L(uent)
90
91 ifelse(eval(COPYI_SSE_THRESHOLD >= 8),1,
92 ` sub $8, n',
93 ` jmp L(am)')
94
95 ALIGN(16)
96 L(atop):movdqa 0(up), %xmm0
97 movdqa 16(up), %xmm1
98 movdqa 32(up), %xmm2
99 movdqa 48(up), %xmm3
100 lea 64(up), up
101 movdqa %xmm0, (rp)
102 movdqa %xmm1, 16(rp)
103 movdqa %xmm2, 32(rp)
104 movdqa %xmm3, 48(rp)
105 lea 64(rp), rp
106 L(am): sub $8, n
107 jnc L(atop)
108
109 test $4, R8(n)
110 jz 1f
111 movdqa (up), %xmm0
112 movdqa 16(up), %xmm1
113 lea 32(up), up
114 movdqa %xmm0, (rp)
115 movdqa %xmm1, 16(rp)
116 lea 32(rp), rp
117
118 1: test $2, R8(n)
119 jz 1f
120 movdqa (up), %xmm0
121 lea 16(up), up
122 movdqa %xmm0, (rp)
123 lea 16(rp), rp
124
125 1: test $1, R8(n)
126 jz 1f
127 mov (up), %r8
128 mov %r8, (rp)
129
130 1: FUNC_EXIT()
131 ret
132
133 L(uent):
134 C Code handling up - rp = 8 (mod 16)
135
136 cmp $16, n
137 jc L(ued0)
138
139 IFDOS(` add $-56, %rsp ')
140 IFDOS(` movdqa %xmm6, (%rsp) ')
141 IFDOS(` movdqa %xmm7, 16(%rsp) ')
142 IFDOS(` movdqa %xmm8, 32(%rsp) ')
143
144 movaps 120(up), %xmm7
145 movaps 104(up), %xmm6
146 movaps 88(up), %xmm5
147 movaps 72(up), %xmm4
148 movaps 56(up), %xmm3
149 movaps 40(up), %xmm2
150 lea 128(up), up
151 sub $32, n
152 jc L(ued1)
153
154 ALIGN(16)
155 L(utop):movaps -104(up), %xmm1
156 sub $16, n
157 movaps -120(up), %xmm0
158 palignr($8, %xmm6, %xmm7)
159 movaps -136(up), %xmm8
160 movdqa %xmm7, 112(rp)
161 palignr($8, %xmm5, %xmm6)
162 movaps 120(up), %xmm7
163 movdqa %xmm6, 96(rp)
164 palignr($8, %xmm4, %xmm5)
165 movaps 104(up), %xmm6
166 movdqa %xmm5, 80(rp)
167 palignr($8, %xmm3, %xmm4)
168 movaps 88(up), %xmm5
169 movdqa %xmm4, 64(rp)
170 palignr($8, %xmm2, %xmm3)
171 movaps 72(up), %xmm4
172 movdqa %xmm3, 48(rp)
173 palignr($8, %xmm1, %xmm2)
174 movaps 56(up), %xmm3
175 movdqa %xmm2, 32(rp)
176 palignr($8, %xmm0, %xmm1)
177 movaps 40(up), %xmm2
178 movdqa %xmm1, 16(rp)
179 palignr($8, %xmm8, %xmm0)
180 lea 128(up), up
181 movdqa %xmm0, (rp)
182 lea 128(rp), rp
183 jnc L(utop)
184
185 L(ued1):movaps -104(up), %xmm1
186 movaps -120(up), %xmm0
187 movaps -136(up), %xmm8
188 palignr($8, %xmm6, %xmm7)
189 movdqa %xmm7, 112(rp)
190 palignr($8, %xmm5, %xmm6)
191 movdqa %xmm6, 96(rp)
192 palignr($8, %xmm4, %xmm5)
193 movdqa %xmm5, 80(rp)
194 palignr($8, %xmm3, %xmm4)
195 movdqa %xmm4, 64(rp)
196 palignr($8, %xmm2, %xmm3)
197 movdqa %xmm3, 48(rp)
198 palignr($8, %xmm1, %xmm2)
199 movdqa %xmm2, 32(rp)
200 palignr($8, %xmm0, %xmm1)
201 movdqa %xmm1, 16(rp)
202 palignr($8, %xmm8, %xmm0)
203 movdqa %xmm0, (rp)
204 lea 128(rp), rp
205
206 IFDOS(` movdqa (%rsp), %xmm6 ')
207 IFDOS(` movdqa 16(%rsp), %xmm7 ')
208 IFDOS(` movdqa 32(%rsp), %xmm8 ')
209 IFDOS(` add $56, %rsp ')
210
211 L(ued0):test $8, R8(n)
212 jz 1f
213 movaps 56(up), %xmm3
214 movaps 40(up), %xmm2
215 movaps 24(up), %xmm1
216 movaps 8(up), %xmm0
217 movaps -8(up), %xmm4
218 palignr($8, %xmm2, %xmm3)
219 movdqa %xmm3, 48(rp)
220 palignr($8, %xmm1, %xmm2)
221 movdqa %xmm2, 32(rp)
222 palignr($8, %xmm0, %xmm1)
223 movdqa %xmm1, 16(rp)
224 palignr($8, %xmm4, %xmm0)
225 lea 64(up), up
226 movdqa %xmm0, (rp)
227 lea 64(rp), rp
228
229 1: test $4, R8(n)
230 jz 1f
231 movaps 24(up), %xmm1
232 movaps 8(up), %xmm0
233 palignr($8, %xmm0, %xmm1)
234 movaps -8(up), %xmm3
235 movdqa %xmm1, 16(rp)
236 palignr($8, %xmm3, %xmm0)
237 lea 32(up), up
238 movdqa %xmm0, (rp)
239 lea 32(rp), rp
240
241 1: test $2, R8(n)
242 jz 1f
243 movdqa 8(up), %xmm0
244 movdqa -8(up), %xmm3
245 palignr($8, %xmm3, %xmm0)
246 lea 16(up), up
247 movdqa %xmm0, (rp)
248 lea 16(rp), rp
249
250 1: test $1, R8(n)
251 jz 1f
252 mov (up), %r8
253 mov %r8, (rp)
254
255 1: FUNC_EXIT()
256 ret
257
258 C Basecase code. Needed for good small operands speed, not for
259 C correctness as the above code is currently written.
260
261 L(bc): lea -8(rp), rp
262 sub $4, R32(n)
263 jc L(end)
264
265 ALIGN(16)
266 L(top): mov (up), %r8
267 mov 8(up), %r9
268 lea 32(rp), rp
269 mov 16(up), %r10
270 mov 24(up), %r11
271 lea 32(up), up
272 mov %r8, -24(rp)
273 mov %r9, -16(rp)
274 ifelse(eval(COPYI_SSE_THRESHOLD >= 8),1,
275 ` sub $4, R32(n)')
276 mov %r10, -8(rp)
277 mov %r11, (rp)
278 ifelse(eval(COPYI_SSE_THRESHOLD >= 8),1,
279 ` jnc L(top)')
280
281 L(end): test $1, R8(n)
282 jz 1f
283 mov (up), %r8
284 mov %r8, 8(rp)
285 lea 8(rp), rp
286 lea 8(up), up
287 1: test $2, R8(n)
288 jz 1f
289 mov (up), %r8
290 mov 8(up), %r9
291 mov %r8, 8(rp)
292 mov %r9, 16(rp)
293 1: FUNC_EXIT()
294 ret
295 EPILOGUE()