Debian Codegraph @

gmp4/mpn/x86_64/coreinhm/redc_1.asm

raw
1 dnl X86-64 mpn_redc_1 optimised for Intel Nehalem and Westmere.
2
3 dnl Contributed to the GNU project by Torbjörn Granlund.
4
5 dnl Copyright 2003-2005, 2007, 2008, 2011-2013 Free Software Foundation, Inc.
6
7 dnl This file is part of the GNU MP Library.
8 dnl
9 dnl The GNU MP Library is free software; you can redistribute it and/or modify
10 dnl it under the terms of either:
11 dnl
12 dnl * the GNU Lesser General Public License as published by the Free
13 dnl Software Foundation; either version 3 of the License, or (at your
14 dnl option) any later version.
15 dnl
16 dnl or
17 dnl
18 dnl * the GNU General Public License as published by the Free Software
19 dnl Foundation; either version 2 of the License, or (at your option) any
20 dnl later version.
21 dnl
22 dnl or both in parallel, as here.
23 dnl
24 dnl The GNU MP Library is distributed in the hope that it will be useful, but
25 dnl WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
26 dnl or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
27 dnl for more details.
28 dnl
29 dnl You should have received copies of the GNU General Public License and the
30 dnl GNU Lesser General Public License along with the GNU MP Library. If not,
31 dnl see https://www.gnu.org/licenses/.
32
33 include(`../config.m4')
34
35 C cycles/limb
36 C AMD K8,K9 ?
37 C AMD K10 ?
38 C AMD bull ?
39 C AMD pile ?
40 C AMD steam ?
41 C AMD bobcat ?
42 C AMD jaguar ?
43 C Intel P4 ?
44 C Intel core ?
45 C Intel NHM ?
46 C Intel SBR ?
47 C Intel IBR ?
48 C Intel HWL ?
49 C Intel BWL ?
50 C Intel atom ?
51 C VIA nano ?
52
53 C The inner loops of this code are the result of running a code generation and
54 C optimisation tool suite written by David Harvey and Torbjörn Granlund.
55
56 C TODO
57 C * Micro-optimise, none performed thus far.
58 C * Consider inlining mpn_add_n.
59 C * Single basecases out before the pushes.
60
61 C When playing with pointers, set this to $2 to fall back to conservative
62 C indexing in wind-down code.
63 define(`I',`$1')
64
65 define(`rp', `%rdi') C rcx
66 define(`up', `%rsi') C rdx
67 define(`mp_param', `%rdx') C r8
68 define(`n', `%rcx') C r9
69 define(`u0inv', `%r8') C stack
70
71 define(`i', `%r14')
72 define(`j', `%r15')
73 define(`mp', `%r12')
74 define(`q0', `%r13')
75
76 C rax rbx rcx rdx rdi rsi rbp r8 r9 r10 r11 r12 r13 r14 r15
77
78 ABI_SUPPORT(DOS64)
79 ABI_SUPPORT(STD64)
80
81 define(`ALIGNx', `ALIGN(16)')
82
83 ASM_START()
84 TEXT
85 ALIGN(32)
86 PROLOGUE(mpn_redc_1)
87 FUNC_ENTRY(4)
88 IFDOS(` mov 56(%rsp), %r8 ')
89 push %rbx
90 push %rbp
91 push %r12
92 push %r13
93 push %r14
94 push %r15
95
96 mov (up), q0
97 mov n, j C outer loop induction var
98 lea (mp_param,n,8), mp
99 lea (up,n,8), up
100 neg n
101 imul u0inv, q0 C first iteration q0
102
103 test $1, R8(n)
104 jz L(bx0)
105
106 L(bx1): test $2, R8(n)
107 jz L(b3)
108
109 L(b1): cmp $-1, R32(n)
110 jz L(n1)
111
112 L(otp1):lea 3(n), i
113 mov (mp,n,8), %rax
114 mov (up,n,8), %rbp
115 mul q0
116 add %rax, %rbp
117 mov $0, R32(%r9)
118 mov 8(mp,n,8), %rax
119 adc %rdx, %r9
120 mul q0
121 mov $0, R32(%r11)
122 mov 8(up,n,8), %rbx
123 add %rax, %rbx
124 mov 16(mp,n,8), %rax
125 adc %rdx, %r11
126 add %r9, %rbx
127 adc $0, %r11
128 mov 16(up,n,8), %rbp
129 mul q0
130 add %rax, %rbp
131 mov $0, R32(%r9)
132 mov 24(mp,n,8), %rax
133 adc %rdx, %r9
134 mov %rbx, 8(up,n,8)
135 imul u0inv, %rbx C next q limb
136 jmp L(e1)
137
138 ALIGNx
139 L(tp1): mul q0
140 add %rax, %rbp
141 mov $0, R32(%r9)
142 mov -16(mp,i,8), %rax
143 adc %rdx, %r9
144 mul q0
145 add %r11, %rbp
146 mov $0, R32(%r11)
147 mov -16(up,i,8), %r10
148 adc $0, %r9
149 add %rax, %r10
150 mov -8(mp,i,8), %rax
151 adc %rdx, %r11
152 mov %rbp, -24(up,i,8)
153 add %r9, %r10
154 adc $0, %r11
155 mov -8(up,i,8), %rbp
156 mul q0
157 add %rax, %rbp
158 mov $0, R32(%r9)
159 mov (mp,i,8), %rax
160 adc %rdx, %r9
161 mov %r10, -16(up,i,8)
162 L(e1): add %r11, %rbp
163 adc $0, %r9
164 mul q0
165 mov (up,i,8), %r10
166 mov $0, R32(%r11)
167 add %rax, %r10
168 mov 8(mp,i,8), %rax
169 adc %rdx, %r11
170 mov %rbp, -8(up,i,8)
171 add %r9, %r10
172 adc $0, %r11
173 mov 8(up,i,8), %rbp
174 mov %r10, (up,i,8)
175 add $4, i
176 jnc L(tp1)
177
178 L(ed1): mul q0
179 add %rax, %rbp
180 adc $0, %rdx
181 add %r11, %rbp
182 adc $0, %rdx
183 mov %rbp, I(-8(up),-24(up,i,8))
184 mov %rdx, (up,n,8) C up[0]
185 mov %rbx, q0 C previously computed q limb -> q0
186 lea 8(up), up C up++
187 dec j
188 jnz L(otp1)
189 jmp L(cj)
190
191 L(b3): cmp $-3, R32(n)
192 jz L(n3)
193
194 L(otp3):lea 5(n), i
195 mov (mp,n,8), %rax
196 mov (up,n,8), %rbp
197 mul q0
198 add %rax, %rbp
199 mov $0, R32(%r9)
200 mov 8(mp,n,8), %rax
201 adc %rdx, %r9
202 mul q0
203 mov 8(up,n,8), %rbx
204 mov $0, R32(%r11)
205 add %rax, %rbx
206 mov 16(mp,n,8), %rax
207 adc %rdx, %r11
208 add %r9, %rbx
209 adc $0, %r11
210 mov 16(up,n,8), %rbp
211 mov %rbx, 8(up,n,8)
212 imul u0inv, %rbx C next q limb
213 C jmp L(tp3)
214
215 ALIGNx
216 L(tp3): mul q0
217 add %rax, %rbp
218 mov $0, R32(%r9)
219 mov -16(mp,i,8), %rax
220 adc %rdx, %r9
221 mul q0
222 add %r11, %rbp
223 mov $0, R32(%r11)
224 mov -16(up,i,8), %r10
225 adc $0, %r9
226 add %rax, %r10
227 mov -8(mp,i,8), %rax
228 adc %rdx, %r11
229 mov %rbp, -24(up,i,8)
230 add %r9, %r10
231 adc $0, %r11
232 mov -8(up,i,8), %rbp
233 mul q0
234 add %rax, %rbp
235 mov $0, R32(%r9)
236 mov (mp,i,8), %rax
237 adc %rdx, %r9
238 mov %r10, -16(up,i,8)
239 add %r11, %rbp
240 adc $0, %r9
241 mul q0
242 mov (up,i,8), %r10
243 mov $0, R32(%r11)
244 add %rax, %r10
245 mov 8(mp,i,8), %rax
246 adc %rdx, %r11
247 mov %rbp, -8(up,i,8)
248 add %r9, %r10
249 adc $0, %r11
250 mov 8(up,i,8), %rbp
251 mov %r10, (up,i,8)
252 add $4, i
253 jnc L(tp3)
254
255 L(ed3): mul q0
256 add %rax, %rbp
257 adc $0, %rdx
258 add %r11, %rbp
259 adc $0, %rdx
260 mov %rbp, I(-8(up),-24(up,i,8))
261 mov %rdx, (up,n,8) C up[0]
262 mov %rbx, q0 C previously computed q limb -> q0
263 lea 8(up), up C up++
264 dec j
265 jnz L(otp3)
266 C jmp L(cj)
267
268 L(cj):
269 IFSTD(` lea (up,n,8), up C param 2: up
270 lea (up,n,8), %rdx C param 3: up - n
271 neg R32(n) ') C param 4: n
272
273 IFDOS(` lea (up,n,8), %rdx C param 2: up
274 lea (%rdx,n,8), %r8 C param 3: up - n
275 neg R32(n)
276 mov n, %r9 C param 4: n
277 mov rp, %rcx ') C param 1: rp
278
279 CALL( mpn_add_n)
280
281 L(ret): pop %r15
282 pop %r14
283 pop %r13
284 pop %r12
285 pop %rbp
286 pop %rbx
287 FUNC_EXIT()
288 ret
289
290 L(bx0): test $2, R8(n)
291 jnz L(b2)
292
293 L(b0):
294 L(otp0):lea 2(n), i
295 mov (mp,n,8), %rax
296 mul q0
297 mov $0, R32(%r11)
298 mov (up,n,8), %r10
299 add %rax, %r10
300 mov 8(mp,n,8), %rax
301 adc %rdx, %r11
302 mov 8(up,n,8), %rbx
303 mul q0
304 add %rax, %rbx
305 mov $0, R32(%r9)
306 mov 16(mp,n,8), %rax
307 adc %rdx, %r9
308 add %r11, %rbx
309 adc $0, %r9
310 mul q0
311 mov 16(up,n,8), %r10
312 mov $0, R32(%r11)
313 add %rax, %r10
314 mov 24(mp,n,8), %rax
315 adc %rdx, %r11
316 mov %rbx, 8(up,n,8)
317 imul u0inv, %rbx C next q limb
318 jmp L(e0)
319
320 ALIGNx
321 L(tp0): mul q0
322 add %rax, %rbp
323 mov $0, R32(%r9)
324 mov -16(mp,i,8), %rax
325 adc %rdx, %r9
326 mul q0
327 add %r11, %rbp
328 mov $0, R32(%r11)
329 mov -16(up,i,8), %r10
330 adc $0, %r9
331 add %rax, %r10
332 mov -8(mp,i,8), %rax
333 adc %rdx, %r11
334 mov %rbp, -24(up,i,8)
335 add %r9, %r10
336 adc $0, %r11
337 mov -8(up,i,8), %rbp
338 mul q0
339 add %rax, %rbp
340 mov $0, R32(%r9)
341 mov (mp,i,8), %rax
342 adc %rdx, %r9
343 mov %r10, -16(up,i,8)
344 add %r11, %rbp
345 adc $0, %r9
346 mul q0
347 mov (up,i,8), %r10
348 mov $0, R32(%r11)
349 add %rax, %r10
350 mov 8(mp,i,8), %rax
351 adc %rdx, %r11
352 mov %rbp, -8(up,i,8)
353 L(e0): add %r9, %r10
354 adc $0, %r11
355 mov 8(up,i,8), %rbp
356 mov %r10, (up,i,8)
357 add $4, i
358 jnc L(tp0)
359
360 L(ed0): mul q0
361 add %rax, %rbp
362 adc $0, %rdx
363 add %r11, %rbp
364 adc $0, %rdx
365 mov %rbp, I(-8(up),-24(up,i,8))
366 mov %rdx, (up,n,8) C up[0]
367 mov %rbx, q0 C previously computed q limb -> q0
368 lea 8(up), up C up++
369 dec j
370 jnz L(otp0)
371 jmp L(cj)
372
373 L(b2): cmp $-2, R32(n)
374 jz L(n2)
375
376 L(otp2):lea 4(n), i
377 mov (mp,n,8), %rax
378 mul q0
379 mov (up,n,8), %r10
380 mov $0, R32(%r11)
381 add %rax, %r10
382 mov 8(mp,n,8), %rax
383 adc %rdx, %r11
384 mov 8(up,n,8), %rbx
385 mul q0
386 add %rax, %rbx
387 mov $0, R32(%r9)
388 mov 16(mp,n,8), %rax
389 adc %rdx, %r9
390 mul q0
391 add %r11, %rbx
392 mov $0, R32(%r11)
393 mov 16(up,n,8), %r10
394 adc $0, %r9
395 add %rax, %r10
396 mov 24(mp,n,8), %rax
397 adc %rdx, %r11
398 mov %rbx, 8(up,n,8)
399 imul u0inv, %rbx C next q limb
400 jmp L(e2)
401
402 ALIGNx
403 L(tp2): mul q0
404 add %rax, %rbp
405 mov $0, R32(%r9)
406 mov -16(mp,i,8), %rax
407 adc %rdx, %r9
408 mul q0
409 add %r11, %rbp
410 mov $0, R32(%r11)
411 mov -16(up,i,8), %r10
412 adc $0, %r9
413 add %rax, %r10
414 mov -8(mp,i,8), %rax
415 adc %rdx, %r11
416 mov %rbp, -24(up,i,8)
417 L(e2): add %r9, %r10
418 adc $0, %r11
419 mov -8(up,i,8), %rbp
420 mul q0
421 add %rax, %rbp
422 mov $0, R32(%r9)
423 mov (mp,i,8), %rax
424 adc %rdx, %r9
425 mov %r10, -16(up,i,8)
426 add %r11, %rbp
427 adc $0, %r9
428 mul q0
429 mov (up,i,8), %r10
430 mov $0, R32(%r11)
431 add %rax, %r10
432 mov 8(mp,i,8), %rax
433 adc %rdx, %r11
434 mov %rbp, -8(up,i,8)
435 add %r9, %r10
436 adc $0, %r11
437 mov 8(up,i,8), %rbp
438 mov %r10, (up,i,8)
439 add $4, i
440 jnc L(tp2)
441
442 L(ed2): mul q0
443 add %rax, %rbp
444 adc $0, %rdx
445 add %r11, %rbp
446 adc $0, %rdx
447 mov %rbp, I(-8(up),-24(up,i,8))
448 mov %rdx, (up,n,8) C up[0]
449 mov %rbx, q0 C previously computed q limb -> q0
450 lea 8(up), up C up++
451 dec j
452 jnz L(otp2)
453 jmp L(cj)
454
455 L(n1): mov (mp_param), %rax
456 mul q0
457 add -8(up), %rax
458 adc (up), %rdx
459 mov %rdx, (rp)
460 mov $0, R32(%rax)
461 adc R32(%rax), R32(%rax)
462 jmp L(ret)
463
464 L(n2): mov (mp_param), %rax
465 mov -16(up), %rbp
466 mul q0
467 add %rax, %rbp
468 mov %rdx, %r9
469 adc $0, %r9
470 mov -8(mp), %rax
471 mov -8(up), %r10
472 mul q0
473 add %rax, %r10
474 mov %rdx, %r11
475 adc $0, %r11
476 add %r9, %r10
477 adc $0, %r11
478 mov %r10, q0
479 imul u0inv, q0 C next q0
480 mov -16(mp), %rax
481 mul q0
482 add %rax, %r10
483 mov %rdx, %r9
484 adc $0, %r9
485 mov -8(mp), %rax
486 mov (up), %r14
487 mul q0
488 add %rax, %r14
489 adc $0, %rdx
490 add %r9, %r14
491 adc $0, %rdx
492 xor R32(%rax), R32(%rax)
493 add %r11, %r14
494 adc 8(up), %rdx
495 mov %r14, (rp)
496 mov %rdx, 8(rp)
497 adc R32(%rax), R32(%rax)
498 jmp L(ret)
499
500 ALIGNx
501 L(n3): mov -24(mp), %rax
502 mov -24(up), %r10
503 mul q0
504 add %rax, %r10
505 mov -16(mp), %rax
506 mov %rdx, %r11
507 adc $0, %r11
508 mov -16(up), %rbp
509 mul q0
510 add %rax, %rbp
511 mov %rdx, %r9
512 adc $0, %r9
513 mov -8(mp), %rax
514 add %r11, %rbp
515 mov -8(up), %r10
516 adc $0, %r9
517 mul q0
518 mov %rbp, q0
519 imul u0inv, q0 C next q0
520 add %rax, %r10
521 mov %rdx, %r11
522 adc $0, %r11
523 mov %rbp, -16(up)
524 add %r9, %r10
525 adc $0, %r11
526 mov %r10, -8(up)
527 mov %r11, -24(up) C up[0]
528 lea 8(up), up C up++
529 dec j
530 jnz L(n3)
531
532 mov -48(up), %rdx
533 mov -40(up), %rbx
534 xor R32(%rax), R32(%rax)
535 add %rbp, %rdx
536 adc %r10, %rbx
537 adc -8(up), %r11
538 mov %rdx, (rp)
539 mov %rbx, 8(rp)
540 mov %r11, 16(rp)
541 adc R32(%rax), R32(%rax)
542 jmp L(ret)
543 EPILOGUE()
544 ASM_END()