Debian Codegraph @

gmp4/mpn/x86_64/fastavx/copyd.asm

raw
1 dnl AMD64 mpn_copyd optimised for CPUs with fast AVX.
2
3 dnl Contributed to the GNU project by Torbjörn Granlund.
4
5 dnl Copyright 2003, 2005, 2007, 2011-2013 Free Software Foundation, Inc.
6
7 dnl This file is part of the GNU MP Library.
8 dnl
9 dnl The GNU MP Library is free software; you can redistribute it and/or modify
10 dnl it under the terms of either:
11 dnl
12 dnl * the GNU Lesser General Public License as published by the Free
13 dnl Software Foundation; either version 3 of the License, or (at your
14 dnl option) any later version.
15 dnl
16 dnl or
17 dnl
18 dnl * the GNU General Public License as published by the Free Software
19 dnl Foundation; either version 2 of the License, or (at your option) any
20 dnl later version.
21 dnl
22 dnl or both in parallel, as here.
23 dnl
24 dnl The GNU MP Library is distributed in the hope that it will be useful, but
25 dnl WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
26 dnl or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
27 dnl for more details.
28 dnl
29 dnl You should have received copies of the GNU General Public License and the
30 dnl GNU Lesser General Public License along with the GNU MP Library. If not,
31 dnl see https://www.gnu.org/licenses/.
32
33 include(`../config.m4')
34
35 C cycles/limb aligned unaligned best seen for cpu?
36 C AMD K8,K9 n/a
37 C AMD K10 n/a
38 C AMD bull n/a
39 C AMD pile 4.87 4.87 N
40 C AMD steam ? ?
41 C AMD bobcat n/a
42 C AMD jaguar n/a
43 C Intel P4 n/a
44 C Intel core n/a
45 C Intel NHM n/a
46 C Intel SBR 0.50 0.91 N
47 C Intel IBR ?
48 C Intel HWL 0.25 0.30 Y
49 C Intel BWL ?
50 C Intel atom n/a
51 C VIA nano n/a
52
53 C We try to do as many 32-byte operations as possible. The top-most and
54 C bottom-most writes might need 8-byte operations. For the bulk copying, we
55 C write using aligned 32-byte operations, but we read with both aligned and
56 C unaligned 32-byte operations.
57
58 define(`rp', `%rdi')
59 define(`up', `%rsi')
60 define(`n', `%rdx')
61
62 ABI_SUPPORT(DOS64)
63 ABI_SUPPORT(STD64)
64
65 dnl define(`vmovdqu', vlddqu)
66
67 ASM_START()
68 TEXT
69 ALIGN(32)
70 PROLOGUE(mpn_copyd)
71 FUNC_ENTRY(3)
72
73 lea -32(rp,n,8), rp
74 lea -32(up,n,8), up
75
76 cmp $7, n C basecase needed for correctness
77 jbe L(bc)
78
79 test $8, R8(rp) C is rp 16-byte aligned?
80 jz L(a2) C jump if rp aligned
81 mov 24(up), %rax
82 lea -8(up), up
83 mov %rax, 24(rp)
84 lea -8(rp), rp
85 dec n
86 L(a2): test $16, R8(rp) C is rp 32-byte aligned?
87 jz L(a3) C jump if rp aligned
88 vmovdqu 16(up), %xmm0
89 lea -16(up), up
90 vmovdqa %xmm0, 16(rp)
91 lea -16(rp), rp
92 sub $2, n
93 L(a3): sub $16, n
94 jc L(sma)
95
96 ALIGN(16)
97 L(top): vmovdqu (up), %ymm0
98 vmovdqu -32(up), %ymm1
99 vmovdqu -64(up), %ymm2
100 vmovdqu -96(up), %ymm3
101 lea -128(up), up
102 vmovdqa %ymm0, (rp)
103 vmovdqa %ymm1, -32(rp)
104 vmovdqa %ymm2, -64(rp)
105 vmovdqa %ymm3, -96(rp)
106 lea -128(rp), rp
107 L(ali): sub $16, n
108 jnc L(top)
109
110 L(sma): test $8, R8(n)
111 jz 1f
112 vmovdqu (up), %ymm0
113 vmovdqu -32(up), %ymm1
114 lea -64(up), up
115 vmovdqa %ymm0, (rp)
116 vmovdqa %ymm1, -32(rp)
117 lea -64(rp), rp
118 1:
119 test $4, R8(n)
120 jz 1f
121 vmovdqu (up), %ymm0
122 lea -32(up), up
123 vmovdqa %ymm0, (rp)
124 lea -32(rp), rp
125 1:
126 test $2, R8(n)
127 jz 1f
128 vmovdqu 16(up), %xmm0
129 lea -16(up), up
130 vmovdqa %xmm0, 16(rp)
131 lea -16(rp), rp
132 1:
133 test $1, R8(n)
134 jz 1f
135 mov 24(up), %r8
136 mov %r8, 24(rp)
137 1:
138 FUNC_EXIT()
139 ret
140
141 ALIGN(16)
142 L(bc): test $4, R8(n)
143 jz 1f
144 mov 24(up), %rax
145 mov 16(up), %rcx
146 mov 8(up), %r8
147 mov (up), %r9
148 lea -32(up), up
149 mov %rax, 24(rp)
150 mov %rcx, 16(rp)
151 mov %r8, 8(rp)
152 mov %r9, (rp)
153 lea -32(rp), rp
154 1:
155 test $2, R8(n)
156 jz 1f
157 mov 24(up), %rax
158 mov 16(up), %rcx
159 lea -16(up), up
160 mov %rax, 24(rp)
161 mov %rcx, 16(rp)
162 lea -16(rp), rp
163 1:
164 test $1, R8(n)
165 jz 1f
166 mov 24(up), %rax
167 mov %rax, 24(rp)
168 1:
169 FUNC_EXIT()
170 ret
171 EPILOGUE()