Debian Codegraph @

gmp4/mpn/x86_64/fastsse/copyd-palignr.asm

raw
1 dnl AMD64 mpn_copyd optimised for CPUs with fast SSE copying and SSSE3.
2
3 dnl Copyright 2012 Free Software Foundation, Inc.
4
5 dnl Contributed to the GNU project by Torbjorn Granlund.
6
7 dnl This file is part of the GNU MP Library.
8 dnl
9 dnl The GNU MP Library is free software; you can redistribute it and/or modify
10 dnl it under the terms of either:
11 dnl
12 dnl * the GNU Lesser General Public License as published by the Free
13 dnl Software Foundation; either version 3 of the License, or (at your
14 dnl option) any later version.
15 dnl
16 dnl or
17 dnl
18 dnl * the GNU General Public License as published by the Free Software
19 dnl Foundation; either version 2 of the License, or (at your option) any
20 dnl later version.
21 dnl
22 dnl or both in parallel, as here.
23 dnl
24 dnl The GNU MP Library is distributed in the hope that it will be useful, but
25 dnl WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
26 dnl or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
27 dnl for more details.
28 dnl
29 dnl You should have received copies of the GNU General Public License and the
30 dnl GNU Lesser General Public License along with the GNU MP Library. If not,
31 dnl see https://www.gnu.org/licenses/.
32
33 include(`../config.m4')
34
35 C cycles/limb cycles/limb cycles/limb good
36 C aligned unaligned best seen for cpu?
37 C AMD K8,K9 2.0 illop 1.0/1.0 N
38 C AMD K10 0.85 illop Y/N
39 C AMD bull 0.70 0.70 Y
40 C AMD pile 0.68 0.68 Y
41 C AMD steam ? ?
42 C AMD bobcat 1.97 8.24 1.5/1.5 N
43 C AMD jaguar ? ?
44 C Intel P4 2.26 illop Y/N
45 C Intel core 0.52 0.68-0.80 opt/0.64 Y
46 C Intel NHM 0.52 0.64 opt/opt Y
47 C Intel SBR 0.51 0.51 opt/0.51 Y
48 C Intel IBR ? ? Y
49 C Intel HWL 0.51 0.51 0.25/0.25 N
50 C Intel atom 1.16 1.66 opt/opt Y
51 C VIA nano 1.08 1.06 opt/opt Y
52
53 C We use only 16-byte operations, except for unaligned top-most and bottom-most
54 C limbs. We use the SSSE3 palignr instruction when rp - up = 8 (mod 16).
55 C
56 C For operands of < COPYD_SSE_THRESHOLD limbs, we use a plain 64-bit loop,
57 C taken from the x86_64 default code.
58
59 C INPUT PARAMETERS
60 define(`rp', `%rdi')
61 define(`up', `%rsi')
62 define(`n', `%rdx')
63
64 C There are three instructions for loading an aligned 128-bit quantity. We use
65 C movaps, since it has the shortest coding.
66 define(`movdqa', ``movaps'')
67
68 ifdef(`COPYD_SSE_THRESHOLD',`',`define(`COPYD_SSE_THRESHOLD', 7)')
69
70 ASM_START()
71 TEXT
72 ALIGN(64)
73 PROLOGUE(mpn_copyd)
74 FUNC_ENTRY(3)
75
76 lea -8(up,n,8), up
77 lea -8(rp,n,8), rp
78
79 cmp $COPYD_SSE_THRESHOLD, n
80 jbe L(bc)
81
82 test $8, R8(rp) C is rp 16-byte aligned?
83 jnz L(rp_aligned) C jump if rp aligned
84
85 mov (up), %rax C copy one limb
86 mov %rax, (rp)
87 lea -8(up), up
88 lea -8(rp), rp
89 dec n
90
91 L(rp_aligned):
92 test $8, R8(up)
93 jz L(uent)
94
95 ifelse(eval(COPYD_SSE_THRESHOLD >= 8),1,
96 ` sub $8, n',
97 ` jmp L(am)')
98
99 ALIGN(16)
100 L(atop):movdqa -8(up), %xmm0
101 movdqa -24(up), %xmm1
102 movdqa -40(up), %xmm2
103 movdqa -56(up), %xmm3
104 lea -64(up), up
105 movdqa %xmm0, -8(rp)
106 movdqa %xmm1, -24(rp)
107 movdqa %xmm2, -40(rp)
108 movdqa %xmm3, -56(rp)
109 lea -64(rp), rp
110 L(am): sub $8, n
111 jnc L(atop)
112
113 test $4, R8(n)
114 jz 1f
115 movdqa -8(up), %xmm0
116 movdqa -24(up), %xmm1
117 lea -32(up), up
118 movdqa %xmm0, -8(rp)
119 movdqa %xmm1, -24(rp)
120 lea -32(rp), rp
121
122 1: test $2, R8(n)
123 jz 1f
124 movdqa -8(up), %xmm0
125 lea -16(up), up
126 movdqa %xmm0, -8(rp)
127 lea -16(rp), rp
128
129 1: test $1, R8(n)
130 jz 1f
131 mov (up), %r8
132 mov %r8, (rp)
133
134 1: FUNC_EXIT()
135 ret
136
137 L(uent):sub $16, n
138 movdqa (up), %xmm0
139 jc L(uend)
140
141 ALIGN(16)
142 L(utop):sub $16, n
143 movdqa -16(up), %xmm1
144 palignr($8, %xmm1, %xmm0)
145 movdqa %xmm0, -8(rp)
146 movdqa -32(up), %xmm2
147 palignr($8, %xmm2, %xmm1)
148 movdqa %xmm1, -24(rp)
149 movdqa -48(up), %xmm3
150 palignr($8, %xmm3, %xmm2)
151 movdqa %xmm2, -40(rp)
152 movdqa -64(up), %xmm0
153 palignr($8, %xmm0, %xmm3)
154 movdqa %xmm3, -56(rp)
155 movdqa -80(up), %xmm1
156 palignr($8, %xmm1, %xmm0)
157 movdqa %xmm0, -72(rp)
158 movdqa -96(up), %xmm2
159 palignr($8, %xmm2, %xmm1)
160 movdqa %xmm1, -88(rp)
161 movdqa -112(up), %xmm3
162 palignr($8, %xmm3, %xmm2)
163 movdqa %xmm2, -104(rp)
164 movdqa -128(up), %xmm0
165 palignr($8, %xmm0, %xmm3)
166 movdqa %xmm3, -120(rp)
167 lea -128(up), up
168 lea -128(rp), rp
169 jnc L(utop)
170
171 L(uend):test $8, R8(n)
172 jz 1f
173 movdqa -16(up), %xmm1
174 palignr($8, %xmm1, %xmm0)
175 movdqa %xmm0, -8(rp)
176 movdqa -32(up), %xmm0
177 palignr($8, %xmm0, %xmm1)
178 movdqa %xmm1, -24(rp)
179 movdqa -48(up), %xmm1
180 palignr($8, %xmm1, %xmm0)
181 movdqa %xmm0, -40(rp)
182 movdqa -64(up), %xmm0
183 palignr($8, %xmm0, %xmm1)
184 movdqa %xmm1, -56(rp)
185 lea -64(up), up
186 lea -64(rp), rp
187
188 1: test $4, R8(n)
189 jz 1f
190 movdqa -16(up), %xmm1
191 palignr($8, %xmm1, %xmm0)
192 movdqa %xmm0, -8(rp)
193 movdqa -32(up), %xmm0
194 palignr($8, %xmm0, %xmm1)
195 movdqa %xmm1, -24(rp)
196 lea -32(up), up
197 lea -32(rp), rp
198
199 1: test $2, R8(n)
200 jz 1f
201 movdqa -16(up), %xmm1
202 palignr($8, %xmm1, %xmm0)
203 movdqa %xmm0, -8(rp)
204 lea -16(up), up
205 lea -16(rp), rp
206
207 1: test $1, R8(n)
208 jz 1f
209 mov (up), %r8
210 mov %r8, (rp)
211
212 1: FUNC_EXIT()
213 ret
214
215 C Basecase code. Needed for good small operands speed, not for
216 C correctness as the above code is currently written.
217
218 L(bc): sub $4, R32(n)
219 jc L(end)
220
221 ALIGN(16)
222 L(top): mov (up), %r8
223 mov -8(up), %r9
224 lea -32(rp), rp
225 mov -16(up), %r10
226 mov -24(up), %r11
227 lea -32(up), up
228 mov %r8, 32(rp)
229 mov %r9, 24(rp)
230 ifelse(eval(COPYD_SSE_THRESHOLD >= 8),1,
231 ` sub $4, R32(n)')
232 mov %r10, 16(rp)
233 mov %r11, 8(rp)
234 ifelse(eval(COPYD_SSE_THRESHOLD >= 8),1,
235 ` jnc L(top)')
236
237 L(end): test $1, R8(n)
238 jz 1f
239 mov (up), %r8
240 mov %r8, (rp)
241 lea -8(rp), rp
242 lea -8(up), up
243 1: test $2, R8(n)
244 jz 1f
245 mov (up), %r8
246 mov -8(up), %r9
247 mov %r8, (rp)
248 mov %r9, -8(rp)
249 1: FUNC_EXIT()
250 ret
251 EPILOGUE()