Debian Codegraph @

gmp4/mpn/x86_64/fastavx/copyi.asm

raw
1 dnl AMD64 mpn_copyi optimised for CPUs with fast AVX.
2
3 dnl Contributed to the GNU project by Torbjörn Granlund.
4
5 dnl Copyright 2003, 2005, 2007, 2011-2013 Free Software Foundation, Inc.
6
7 dnl This file is part of the GNU MP Library.
8 dnl
9 dnl The GNU MP Library is free software; you can redistribute it and/or modify
10 dnl it under the terms of either:
11 dnl
12 dnl * the GNU Lesser General Public License as published by the Free
13 dnl Software Foundation; either version 3 of the License, or (at your
14 dnl option) any later version.
15 dnl
16 dnl or
17 dnl
18 dnl * the GNU General Public License as published by the Free Software
19 dnl Foundation; either version 2 of the License, or (at your option) any
20 dnl later version.
21 dnl
22 dnl or both in parallel, as here.
23 dnl
24 dnl The GNU MP Library is distributed in the hope that it will be useful, but
25 dnl WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
26 dnl or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
27 dnl for more details.
28 dnl
29 dnl You should have received copies of the GNU General Public License and the
30 dnl GNU Lesser General Public License along with the GNU MP Library. If not,
31 dnl see https://www.gnu.org/licenses/.
32
33 include(`../config.m4')
34
35 C cycles/limb aligned unaligned best seen for cpu?
36 C AMD K8,K9 n/a
37 C AMD K10 n/a
38 C AMD bull n/a
39 C AMD pile 4.87 4.87 N
40 C AMD steam ? ?
41 C AMD bobcat n/a
42 C AMD jaguar n/a
43 C Intel P4 n/a
44 C Intel core n/a
45 C Intel NHM n/a
46 C Intel SBR 0.50 0.91 N
47 C Intel IBR ?
48 C Intel HWL 0.25 0.30 Y
49 C Intel BWL ?
50 C Intel atom n/a
51 C VIA nano n/a
52
53 C We try to do as many 32-byte operations as possible. The top-most and
54 C bottom-most writes might need 8-byte operations. For the bulk copying, we
55 C write using aligned 32-byte operations, but we read with both aligned and
56 C unaligned 32-byte operations.
57
58 define(`rp', `%rdi')
59 define(`up', `%rsi')
60 define(`n', `%rdx')
61
62 ABI_SUPPORT(DOS64)
63 ABI_SUPPORT(STD64)
64
65 dnl define(`vmovdqu', vlddqu)
66
67 ASM_START()
68 TEXT
69 ALIGN(32)
70 PROLOGUE(mpn_copyi)
71 FUNC_ENTRY(3)
72
73 cmp $7, n
74 jbe L(bc)
75
76 test $8, R8(rp) C is rp 16-byte aligned?
77 jz L(a2) C jump if rp aligned
78 mov (up), %rax
79 lea 8(up), up
80 mov %rax, (rp)
81 lea 8(rp), rp
82 dec n
83 L(a2): test $16, R8(rp) C is rp 32-byte aligned?
84 jz L(a3) C jump if rp aligned
85 vmovdqu (up), %xmm0
86 lea 16(up), up
87 vmovdqa %xmm0, (rp)
88 lea 16(rp), rp
89 sub $2, n
90 L(a3): sub $16, n
91 jc L(sma)
92
93 ALIGN(16)
94 L(top): vmovdqu (up), %ymm0
95 vmovdqu 32(up), %ymm1
96 vmovdqu 64(up), %ymm2
97 vmovdqu 96(up), %ymm3
98 lea 128(up), up
99 vmovdqa %ymm0, (rp)
100 vmovdqa %ymm1, 32(rp)
101 vmovdqa %ymm2, 64(rp)
102 vmovdqa %ymm3, 96(rp)
103 lea 128(rp), rp
104 L(ali): sub $16, n
105 jnc L(top)
106
107 L(sma): test $8, R8(n)
108 jz 1f
109 vmovdqu (up), %ymm0
110 vmovdqu 32(up), %ymm1
111 lea 64(up), up
112 vmovdqa %ymm0, (rp)
113 vmovdqa %ymm1, 32(rp)
114 lea 64(rp), rp
115 1:
116 test $4, R8(n)
117 jz 1f
118 vmovdqu (up), %ymm0
119 lea 32(up), up
120 vmovdqa %ymm0, (rp)
121 lea 32(rp), rp
122 1:
123 test $2, R8(n)
124 jz 1f
125 vmovdqu (up), %xmm0
126 lea 16(up), up
127 vmovdqa %xmm0, (rp)
128 lea 16(rp), rp
129 1:
130 L(end): test $1, R8(n)
131 jz 1f
132 mov (up), %r8
133 mov %r8, (rp)
134 1:
135 FUNC_EXIT()
136 ret
137
138 ALIGN(16)
139 L(bc): test $4, R8(n)
140 jz 1f
141 mov (up), %rax
142 mov 8(up), %rcx
143 mov 16(up), %r8
144 mov 24(up), %r9
145 lea 32(up), up
146 mov %rax, (rp)
147 mov %rcx, 8(rp)
148 mov %r8, 16(rp)
149 mov %r9, 24(rp)
150 lea 32(rp), rp
151 1:
152 test $2, R8(n)
153 jz 1f
154 mov (up), %rax
155 mov 8(up), %rcx
156 lea 16(up), up
157 mov %rax, (rp)
158 mov %rcx, 8(rp)
159 lea 16(rp), rp
160 1:
161 test $1, R8(n)
162 jz 1f
163 mov (up), %rax
164 mov %rax, (rp)
165 1:
166 FUNC_EXIT()
167 ret
168 EPILOGUE()