Debian Codegraph @

gmp4/mpn/x86_64/fastsse/copyi.asm

raw
1 dnl AMD64 mpn_copyi optimised for CPUs with fast SSE.
2
3 dnl Contributed to the GNU project by Torbjörn Granlund.
4
5 dnl Copyright 2003, 2005, 2007, 2011, 2012 Free Software Foundation, Inc.
6
7 dnl This file is part of the GNU MP Library.
8 dnl
9 dnl The GNU MP Library is free software; you can redistribute it and/or modify
10 dnl it under the terms of either:
11 dnl
12 dnl * the GNU Lesser General Public License as published by the Free
13 dnl Software Foundation; either version 3 of the License, or (at your
14 dnl option) any later version.
15 dnl
16 dnl or
17 dnl
18 dnl * the GNU General Public License as published by the Free Software
19 dnl Foundation; either version 2 of the License, or (at your option) any
20 dnl later version.
21 dnl
22 dnl or both in parallel, as here.
23 dnl
24 dnl The GNU MP Library is distributed in the hope that it will be useful, but
25 dnl WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
26 dnl or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
27 dnl for more details.
28 dnl
29 dnl You should have received copies of the GNU General Public License and the
30 dnl GNU Lesser General Public License along with the GNU MP Library. If not,
31 dnl see https://www.gnu.org/licenses/.
32
33 include(`../config.m4')
34
35
36 C cycles/limb good for cpu?
37 C AMD K8,K9
38 C AMD K10 0.85 1.64 Y/N
39 C AMD bd1 1.4 1.4 Y
40 C AMD bobcat
41 C Intel P4 2.3 2.3 Y
42 C Intel core2 1.0 1.0
43 C Intel NHM 0.5 0.67 Y
44 C Intel SBR 0.5 0.75 Y
45 C Intel atom
46 C VIA nano 1.16 5.16 Y/N
47
48 C We try to do as many 16-byte operations as possible. The top-most and
49 C bottom-most writes might need 8-byte operations. We can always write using
50 C aligned 16-byte operations, we read with both aligned and unaligned 16-byte
51 C operations.
52
53 C Instead of having separate loops for reading aligned and unaligned, we read
54 C using MOVDQU. This seems to work great except for core2; there performance
55 C doubles when reading using MOVDQA (for aligned source). It is unclear how to
56 C best handle the unaligned case there.
57
58 C INPUT PARAMETERS
59 define(`rp', `%rdi')
60 define(`up', `%rsi')
61 define(`n', `%rdx')
62
63 ABI_SUPPORT(DOS64)
64 ABI_SUPPORT(STD64)
65
66 dnl define(`movdqu', lddqu)
67
68 ASM_START()
69 TEXT
70 ALIGN(64)
71 PROLOGUE(mpn_copyi)
72 FUNC_ENTRY(3)
73
74 cmp $3, n
75 jc L(bc)
76
77 test $8, R8(rp) C is rp 16-byte aligned?
78 jz L(ali) C jump if rp aligned
79 movsq C copy single limb
80 dec n
81
82 sub $16, n
83 jc L(sma)
84
85 ALIGN(16)
86 L(top): movdqu (up), %xmm0
87 movdqu 16(up), %xmm1
88 movdqu 32(up), %xmm2
89 movdqu 48(up), %xmm3
90 movdqu 64(up), %xmm4
91 movdqu 80(up), %xmm5
92 movdqu 96(up), %xmm6
93 movdqu 112(up), %xmm7
94 lea 128(up), up
95 movdqa %xmm0, (rp)
96 movdqa %xmm1, 16(rp)
97 movdqa %xmm2, 32(rp)
98 movdqa %xmm3, 48(rp)
99 movdqa %xmm4, 64(rp)
100 movdqa %xmm5, 80(rp)
101 movdqa %xmm6, 96(rp)
102 movdqa %xmm7, 112(rp)
103 lea 128(rp), rp
104 L(ali): sub $16, n
105 jnc L(top)
106
107 L(sma): test $8, R8(n)
108 jz 1f
109 movdqu (up), %xmm0
110 movdqu 16(up), %xmm1
111 movdqu 32(up), %xmm2
112 movdqu 48(up), %xmm3
113 lea 64(up), up
114 movdqa %xmm0, (rp)
115 movdqa %xmm1, 16(rp)
116 movdqa %xmm2, 32(rp)
117 movdqa %xmm3, 48(rp)
118 lea 64(rp), rp
119 1:
120 test $4, R8(n)
121 jz 1f
122 movdqu (up), %xmm0
123 movdqu 16(up), %xmm1
124 lea 32(up), up
125 movdqa %xmm0, (rp)
126 movdqa %xmm1, 16(rp)
127 lea 32(rp), rp
128 1:
129 test $2, R8(n)
130 jz 1f
131 movdqu (up), %xmm0
132 lea 16(up), up
133 movdqa %xmm0, (rp)
134 lea 16(rp), rp
135 ALIGN(16)
136 1:
137 L(end): test $1, R8(n)
138 jz 1f
139 mov (up), %r8
140 mov %r8, (rp)
141 1:
142 FUNC_EXIT()
143 ret
144
145 C Basecase code. Needed for good small operands speed, not for
146 C correctness as the above code is currently written.
147
148 L(bc): sub $2, n
149 jc L(end)
150 ALIGN(16)
151 1: mov (up), %rax
152 mov 8(up), %rcx
153 lea 16(up), up
154 mov %rax, (rp)
155 mov %rcx, 8(rp)
156 lea 16(rp), rp
157 sub $2, n
158 jnc 1b
159
160 test $1, R8(n)
161 jz L(ret)
162 mov (up), %rax
163 mov %rax, (rp)
164 L(ret): FUNC_EXIT()
165 ret
166 EPILOGUE()