Debian Codegraph @

gmp4/mpn/x86_64/coreihwl/mullo_basecase.asm

raw
1 dnl AMD64 mpn_mullo_basecase optimised for Intel Haswell.
2
3 dnl Contributed to the GNU project by Torbjörn Granlund.
4
5 dnl Copyright 2008, 2009, 2011-2013 Free Software Foundation, Inc.
6
7 dnl This file is part of the GNU MP Library.
8 dnl
9 dnl The GNU MP Library is free software; you can redistribute it and/or modify
10 dnl it under the terms of either:
11 dnl
12 dnl * the GNU Lesser General Public License as published by the Free
13 dnl Software Foundation; either version 3 of the License, or (at your
14 dnl option) any later version.
15 dnl
16 dnl or
17 dnl
18 dnl * the GNU General Public License as published by the Free Software
19 dnl Foundation; either version 2 of the License, or (at your option) any
20 dnl later version.
21 dnl
22 dnl or both in parallel, as here.
23 dnl
24 dnl The GNU MP Library is distributed in the hope that it will be useful, but
25 dnl WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
26 dnl or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
27 dnl for more details.
28 dnl
29 dnl You should have received copies of the GNU General Public License and the
30 dnl GNU Lesser General Public License along with the GNU MP Library. If not,
31 dnl see https://www.gnu.org/licenses/.
32
33 include(`../config.m4')
34
35 C cycles/limb mul_2 addmul_2
36 C AMD K8,K9 n/a n/a
37 C AMD K10 n/a n/a
38 C AMD bull n/a n/a
39 C AMD pile n/a n/a
40 C AMD steam ? ?
41 C AMD bobcat n/a n/a
42 C AMD jaguar ? ?
43 C Intel P4 n/a n/a
44 C Intel core n/a n/a
45 C Intel NHM n/a n/a
46 C Intel SBR n/a n/a
47 C Intel IBR n/a n/a
48 C Intel HWL 1.86 2.15
49 C Intel BWL ? ?
50 C Intel atom n/a n/a
51 C VIA nano n/a n/a
52
53 C The inner loops of this code are the result of running a code generation and
54 C optimisation tool suite written by David Harvey and Torbjörn Granlund.
55
56 C TODO
57 C * Implement proper cor2, replacing current cor0.
58 C * Micro-optimise.
59
60 C When playing with pointers, set this to $2 to fall back to conservative
61 C indexing in wind-down code.
62 define(`I',`$1')
63
64 define(`rp', `%rdi')
65 define(`up', `%rsi')
66 define(`vp_param', `%rdx')
67 define(`n', `%rcx')
68
69 define(`vp', `%r8')
70 define(`X0', `%r14')
71 define(`X1', `%r15')
72
73 define(`w0', `%r10')
74 define(`w1', `%r11')
75 define(`w2', `%r12')
76 define(`w3', `%r13')
77 define(`i', `%rbp')
78 define(`v0', `%r9')
79 define(`v1', `%rbx')
80
81 C rax rbx rcx rdx rdi rsi rbp r8 r9 r10 r11 r12 r13 r14 r15
82
83 ABI_SUPPORT(DOS64)
84 ABI_SUPPORT(STD64)
85
86 ASM_START()
87 TEXT
88 ALIGN(32)
89 PROLOGUE(mpn_mullo_basecase)
90 FUNC_ENTRY(4)
91
92 mov vp_param, vp
93 mov (up), %rdx
94
95 cmp $4, n
96 jb L(small)
97
98 push %rbx
99 push %rbp
100 push %r12
101 push %r13
102
103 mov (vp), v0
104 mov 8(vp), v1
105
106 lea 2(n), i
107 shr $2, i
108 neg n
109 add $2, n
110
111 push up C put entry `up' on stack
112
113 test $1, R8(n)
114 jnz L(m2x1)
115
116 L(m2x0):mulx( v0, w0, w3)
117 xor R32(w2), R32(w2)
118 test $2, R8(n)
119 jz L(m2b2)
120
121 L(m2b0):lea -8(rp), rp
122 lea -8(up), up
123 jmp L(m2e0)
124
125 L(m2b2):lea -24(rp), rp
126 lea 8(up), up
127 jmp L(m2e2)
128
129 L(m2x1):mulx( v0, w2, w1)
130 xor R32(w0), R32(w0)
131 test $2, R8(n)
132 jnz L(m2b3)
133
134 L(m2b1):jmp L(m2e1)
135
136 L(m2b3):lea -16(rp), rp
137 lea -16(up), up
138 jmp L(m2e3)
139
140 ALIGN(16)
141 L(m2tp):mulx( v1, %rax, w0)
142 add %rax, w2
143 mov (up), %rdx
144 mulx( v0, %rax, w1)
145 adc $0, w0
146 add %rax, w2
147 adc $0, w1
148 add w3, w2
149 L(m2e1):mov w2, (rp)
150 adc $0, w1
151 mulx( v1, %rax, w2)
152 add %rax, w0
153 mov 8(up), %rdx
154 adc $0, w2
155 mulx( v0, %rax, w3)
156 add %rax, w0
157 adc $0, w3
158 add w1, w0
159 L(m2e0):mov w0, 8(rp)
160 adc $0, w3
161 mulx( v1, %rax, w0)
162 add %rax, w2
163 mov 16(up), %rdx
164 mulx( v0, %rax, w1)
165 adc $0, w0
166 add %rax, w2
167 adc $0, w1
168 add w3, w2
169 L(m2e3):mov w2, 16(rp)
170 adc $0, w1
171 mulx( v1, %rax, w2)
172 add %rax, w0
173 mov 24(up), %rdx
174 adc $0, w2
175 mulx( v0, %rax, w3)
176 add %rax, w0
177 adc $0, w3
178 add w1, w0
179 lea 32(up), up
180 L(m2e2):mov w0, 24(rp)
181 adc $0, w3
182 dec i
183 lea 32(rp), rp
184 jnz L(m2tp)
185
186 L(m2ed):mulx( v1, %rax, w0)
187 add %rax, w2
188 mov (up), %rdx
189 mulx( v0, %rax, w1)
190 add w2, %rax
191 add w3, %rax
192 mov %rax, (rp)
193
194 mov (%rsp), up C restore `up' to beginning
195 lea 16(vp), vp
196 lea 8(rp,n,8), rp C put back rp to old rp + 2
197 add $2, n
198 jge L(cor1)
199
200 push %r14
201 push %r15
202
203 L(outer):
204 mov (vp), v0
205 mov 8(vp), v1
206
207 lea (n), i
208 sar $2, i
209
210 mov (up), %rdx
211 test $1, R8(n)
212 jnz L(bx1)
213
214 L(bx0): mov (rp), X1
215 mov 8(rp), X0
216 mulx( v0, %rax, w3)
217 add %rax, X1
218 adc $0, w3
219 mulx( v1, %rax, w0)
220 add %rax, X0
221 adc $0, w0
222 mov 8(up), %rdx
223 mov X1, (rp)
224 mulx( v0, %rax, w1)
225 test $2, R8(n)
226 jz L(b2)
227
228 L(b0): lea 8(rp), rp
229 lea 8(up), up
230 jmp L(lo0)
231
232 L(b2): mov 16(rp), X1
233 lea 24(rp), rp
234 lea 24(up), up
235 jmp L(lo2)
236
237 L(bx1): mov (rp), X0
238 mov 8(rp), X1
239 mulx( v0, %rax, w1)
240 add %rax, X0
241 mulx( v1, %rax, w2)
242 adc $0, w1
243 mov X0, (rp)
244 add %rax, X1
245 adc $0, w2
246 mov 8(up), %rdx
247 test $2, R8(n)
248 jnz L(b3)
249
250 L(b1): lea 16(up), up
251 lea 16(rp), rp
252 jmp L(lo1)
253
254 L(b3): mov 16(rp), X0
255 lea 32(up), up
256 mulx( v0, %rax, w3)
257 inc i
258 jz L(cj3)
259 jmp L(lo3)
260
261 ALIGN(16)
262 L(top): mulx( v0, %rax, w3)
263 add w0, X1
264 adc $0, w2
265 L(lo3): add %rax, X1
266 adc $0, w3
267 mulx( v1, %rax, w0)
268 add %rax, X0
269 adc $0, w0
270 lea 32(rp), rp
271 add w1, X1
272 mov -16(up), %rdx
273 mov X1, -24(rp)
274 adc $0, w3
275 add w2, X0
276 mov -8(rp), X1
277 mulx( v0, %rax, w1)
278 adc $0, w0
279 L(lo2): add %rax, X0
280 mulx( v1, %rax, w2)
281 adc $0, w1
282 add w3, X0
283 mov X0, -16(rp)
284 adc $0, w1
285 add %rax, X1
286 adc $0, w2
287 add w0, X1
288 mov -8(up), %rdx
289 adc $0, w2
290 L(lo1): mulx( v0, %rax, w3)
291 add %rax, X1
292 adc $0, w3
293 mov (rp), X0
294 mulx( v1, %rax, w0)
295 add %rax, X0
296 adc $0, w0
297 add w1, X1
298 mov X1, -8(rp)
299 adc $0, w3
300 mov (up), %rdx
301 add w2, X0
302 mulx( v0, %rax, w1)
303 adc $0, w0
304 L(lo0): add %rax, X0
305 adc $0, w1
306 mulx( v1, %rax, w2)
307 add w3, X0
308 mov 8(rp), X1
309 mov X0, (rp)
310 mov 16(rp), X0
311 adc $0, w1
312 add %rax, X1
313 adc $0, w2
314 mov 8(up), %rdx
315 lea 32(up), up
316 inc i
317 jnz L(top)
318
319 L(end): mulx( v0, %rax, w3)
320 add w0, X1
321 adc $0, w2
322 L(cj3): add %rax, X1
323 adc $0, w3
324 mulx( v1, %rax, w0)
325 add %rax, X0
326 add w1, X1
327 mov -16(up), %rdx
328 mov X1, 8(rp)
329 adc $0, w3
330 add w2, X0
331 mulx( v0, %rax, w1)
332 add X0, %rax
333 add w3, %rax
334 mov %rax, 16(rp)
335
336 mov 16(%rsp), up C restore `up' to beginning
337 lea 16(vp), vp
338 lea 24(rp,n,8), rp C put back rp to old rp + 2
339 add $2, n
340 jl L(outer)
341
342 pop %r15
343 pop %r14
344
345 jnz L(cor0)
346
347 L(cor1):mov (vp), v0
348 mov 8(vp), v1
349 mov (up), %rdx
350 mulx( v0, %r12, %rbp) C u0 x v2
351 add (rp), %r12 C FIXME: rp[0] still available in reg?
352 adc %rax, %rbp
353 mov 8(up), %r10
354 imul v0, %r10
355 imul v1, %rdx
356 mov %r12, (rp)
357 add %r10, %rdx
358 add %rbp, %rdx
359 mov %rdx, 8(rp)
360 pop %rax C deallocate `up' copy
361 pop %r13
362 pop %r12
363 pop %rbp
364 pop %rbx
365 FUNC_EXIT()
366 ret
367
368 L(cor0):mov (vp), %r11
369 imul (up), %r11
370 add %rax, %r11
371 mov %r11, (rp)
372 pop %rax C deallocate `up' copy
373 pop %r13
374 pop %r12
375 pop %rbp
376 pop %rbx
377 FUNC_EXIT()
378 ret
379
380 ALIGN(16)
381 L(small):
382 cmp $2, n
383 jae L(gt1)
384 L(n1): imul (vp), %rdx
385 mov %rdx, (rp)
386 FUNC_EXIT()
387 ret
388 L(gt1): ja L(gt2)
389 L(n2): mov (vp), %r9
390 mulx( %r9, %rax, %rdx)
391 mov %rax, (rp)
392 mov 8(up), %rax
393 imul %r9, %rax
394 add %rax, %rdx
395 mov 8(vp), %r9
396 mov (up), %rcx
397 imul %r9, %rcx
398 add %rcx, %rdx
399 mov %rdx, 8(rp)
400 FUNC_EXIT()
401 ret
402 L(gt2):
403 L(n3): mov (vp), %r9
404 mulx( %r9, %rax, %r10) C u0 x v0
405 mov %rax, (rp)
406 mov 8(up), %rdx
407 mulx( %r9, %rax, %rdx) C u1 x v0
408 imul 16(up), %r9 C u2 x v0
409 add %rax, %r10
410 adc %rdx, %r9
411 mov 8(vp), %r11
412 mov (up), %rdx
413 mulx( %r11, %rax, %rdx) C u0 x v1
414 add %rax, %r10
415 adc %rdx, %r9
416 imul 8(up), %r11 C u1 x v1
417 add %r11, %r9
418 mov %r10, 8(rp)
419 mov 16(vp), %r10
420 mov (up), %rax
421 imul %rax, %r10 C u0 x v2
422 add %r10, %r9
423 mov %r9, 16(rp)
424 FUNC_EXIT()
425 ret
426 EPILOGUE()