Debian Codegraph @

gmp4/mpn/x86_64/bobcat/redc_1.asm

raw
1 dnl X86-64 mpn_redc_1 optimised for AMD bobcat.
2
3 dnl Contributed to the GNU project by Torbjörn Granlund.
4
5 dnl Copyright 2003-2005, 2007, 2008, 2011-2013 Free Software Foundation, Inc.
6
7 dnl This file is part of the GNU MP Library.
8 dnl
9 dnl The GNU MP Library is free software; you can redistribute it and/or modify
10 dnl it under the terms of either:
11 dnl
12 dnl * the GNU Lesser General Public License as published by the Free
13 dnl Software Foundation; either version 3 of the License, or (at your
14 dnl option) any later version.
15 dnl
16 dnl or
17 dnl
18 dnl * the GNU General Public License as published by the Free Software
19 dnl Foundation; either version 2 of the License, or (at your option) any
20 dnl later version.
21 dnl
22 dnl or both in parallel, as here.
23 dnl
24 dnl The GNU MP Library is distributed in the hope that it will be useful, but
25 dnl WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
26 dnl or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
27 dnl for more details.
28 dnl
29 dnl You should have received copies of the GNU General Public License and the
30 dnl GNU Lesser General Public License along with the GNU MP Library. If not,
31 dnl see https://www.gnu.org/licenses/.
32
33 include(`../config.m4')
34
35 C cycles/limb
36 C AMD K8,K9 ?
37 C AMD K10 ?
38 C AMD bull ?
39 C AMD pile ?
40 C AMD steam ?
41 C AMD bobcat 5.0
42 C AMD jaguar ?
43 C Intel P4 ?
44 C Intel core ?
45 C Intel NHM ?
46 C Intel SBR ?
47 C Intel IBR ?
48 C Intel HWL ?
49 C Intel BWL ?
50 C Intel atom ?
51 C VIA nano ?
52
53 C TODO
54 C * Micro-optimise, none performed thus far.
55 C * Consider inlining mpn_add_n.
56 C * Single basecases out before the pushes.
57
58 C When playing with pointers, set this to $2 to fall back to conservative
59 C indexing in wind-down code.
60 define(`I',`$1')
61
62 define(`rp', `%rdi') C rcx
63 define(`up', `%rsi') C rdx
64 define(`mp_param', `%rdx') C r8
65 define(`n', `%rcx') C r9
66 define(`u0inv', `%r8') C stack
67
68 define(`i', `%r14')
69 define(`j', `%r15')
70 define(`mp', `%r12')
71 define(`q0', `%r13')
72 define(`w0', `%rbp')
73 define(`w1', `%r9')
74 define(`w2', `%r10')
75 define(`w3', `%r11')
76
77 C rax rbx rcx rdx rdi rsi rbp r8 r9 r10 r11 r12 r13 r14 r15
78
79 ABI_SUPPORT(DOS64)
80 ABI_SUPPORT(STD64)
81
82 define(`ALIGNx', `ALIGN(16)')
83
84 ASM_START()
85 TEXT
86 ALIGN(32)
87 PROLOGUE(mpn_redc_1)
88 FUNC_ENTRY(4)
89 IFDOS(` mov 56(%rsp), %r8 ')
90 push %rbx
91 push %rbp
92 push %r12
93 push %r13
94 push %r14
95 push %r15
96
97 mov (up), q0
98 mov n, j C outer loop induction var
99 lea (mp_param,n,8), mp
100 lea (up,n,8), up
101 neg n
102 imul u0inv, q0 C first iteration q0
103
104 test $1, R8(n)
105 jz L(bx0)
106
107 L(bx1): test $2, R8(n)
108 jz L(b3)
109
110 L(b1): cmp $-1, R32(n)
111 jz L(n1)
112
113 L(otp1):lea 1(n), i
114 mov (mp,n,8), %rax
115 mul q0
116 mov %rax, w2
117 mov %rdx, w3
118 mov 8(mp,n,8), %rax
119 mul q0
120 mov %rax, %rbx
121 mov %rdx, w1
122 add (up,n,8), w2
123 adc w3, %rbx
124 adc $0, w1
125 mov 16(mp,n,8), %rax
126 mul q0
127 mov %rax, w2
128 mov %rdx, w3
129 add 8(up,n,8), %rbx
130 mov %rbx, 8(up,n,8)
131 adc w1, w2
132 adc $0, w3
133 imul u0inv, %rbx C next q limb
134 jmp L(e1)
135
136 ALIGNx
137 L(tp1): add w0, -16(up,i,8)
138 adc w1, w2
139 adc $0, w3
140 mov (mp,i,8), %rax
141 mul q0
142 mov %rax, w0
143 mov %rdx, w1
144 add w2, -8(up,i,8)
145 adc w3, w0
146 adc $0, w1
147 mov 8(mp,i,8), %rax
148 mul q0
149 mov %rax, w2
150 mov %rdx, w3
151 add w0, (up,i,8)
152 adc w1, w2
153 adc $0, w3
154 L(e1): mov 16(mp,i,8), %rax
155 mul q0
156 mov %rax, w0
157 mov %rdx, w1
158 add w2, 8(up,i,8)
159 adc w3, w0
160 adc $0, w1
161 mov 24(mp,i,8), %rax
162 mul q0
163 mov %rax, w2
164 mov %rdx, w3
165 add $4, i
166 js L(tp1)
167
168 L(ed1): add w0, I(-16(up),-16(up,i,8))
169 adc w1, w2
170 adc $0, w3
171 add w2, I(-8(up),-8(up,i,8))
172 adc $0, w3
173 mov w3, (up,n,8) C up[0]
174 mov %rbx, q0 C previously computed q limb -> q0
175 lea 8(up), up C up++
176 dec j
177 jnz L(otp1)
178 jmp L(cj)
179
180 L(b3): cmp $-3, R32(n)
181 jz L(n3)
182
183 L(otp3):lea 3(n), i
184 mov (mp,n,8), %rax
185 mul q0
186 mov %rax, w2
187 mov %rdx, w3
188 mov 8(mp,n,8), %rax
189 mul q0
190 mov %rax, %rbx
191 mov %rdx, w1
192 add (up,n,8), w2
193 adc w3, %rbx
194 adc $0, w1
195 mov 16(mp,n,8), %rax
196 mul q0
197 mov %rax, w2
198 mov %rdx, w3
199 add 8(up,n,8), %rbx
200 mov %rbx, 8(up,n,8)
201 adc w1, w2
202 adc $0, w3
203 imul u0inv, %rbx C next q limb
204 jmp L(e3)
205
206 ALIGNx
207 L(tp3): add w0, -16(up,i,8)
208 adc w1, w2
209 adc $0, w3
210 L(e3): mov (mp,i,8), %rax
211 mul q0
212 mov %rax, w0
213 mov %rdx, w1
214 add w2, -8(up,i,8)
215 adc w3, w0
216 adc $0, w1
217 mov 8(mp,i,8), %rax
218 mul q0
219 mov %rax, w2
220 mov %rdx, w3
221 add w0, (up,i,8)
222 adc w1, w2
223 adc $0, w3
224 mov 16(mp,i,8), %rax
225 mul q0
226 mov %rax, w0
227 mov %rdx, w1
228 add w2, 8(up,i,8)
229 adc w3, w0
230 adc $0, w1
231 mov 24(mp,i,8), %rax
232 mul q0
233 mov %rax, w2
234 mov %rdx, w3
235 add $4, i
236 js L(tp3)
237
238 L(ed3): add w0, I(-16(up),-16(up,i,8))
239 adc w1, w2
240 adc $0, w3
241 add w2, I(-8(up),-8(up,i,8))
242 adc $0, w3
243 mov w3, (up,n,8) C up[0]
244 mov %rbx, q0 C previously computed q limb -> q0
245 lea 8(up), up C up++
246 dec j
247 jnz L(otp3)
248 C jmp L(cj)
249
250 L(cj):
251 IFSTD(` lea (up,n,8), up C param 2: up
252 lea (up,n,8), %rdx C param 3: up - n
253 neg R32(n) ') C param 4: n
254
255 IFDOS(` lea (up,n,8), %rdx C param 2: up
256 lea (%rdx,n,8), %r8 C param 3: up - n
257 neg R32(n)
258 mov n, %r9 C param 4: n
259 mov rp, %rcx ') C param 1: rp
260
261 CALL( mpn_add_n)
262
263 L(ret): pop %r15
264 pop %r14
265 pop %r13
266 pop %r12
267 pop %rbp
268 pop %rbx
269 FUNC_EXIT()
270 ret
271
272 L(bx0): test $2, R8(n)
273 jnz L(b2)
274
275 L(b0):
276 L(otp0):lea (n), i
277 mov (mp,n,8), %rax
278 mul q0
279 mov %rax, w0
280 mov %rdx, w1
281 mov 8(mp,n,8), %rax
282 mul q0
283 mov %rax, %rbx
284 mov %rdx, w3
285 add (up,n,8), w0
286 adc w1, %rbx
287 adc $0, w3
288 mov 16(mp,n,8), %rax
289 mul q0
290 mov %rax, w0
291 mov %rdx, w1
292 add 8(up,n,8), %rbx
293 mov %rbx, 8(up,n,8)
294 adc w3, w0
295 adc $0, w1
296 imul u0inv, %rbx C next q limb
297 jmp L(e0)
298
299 ALIGNx
300 L(tp0): add w0, -16(up,i,8)
301 adc w1, w2
302 adc $0, w3
303 mov (mp,i,8), %rax
304 mul q0
305 mov %rax, w0
306 mov %rdx, w1
307 add w2, -8(up,i,8)
308 adc w3, w0
309 adc $0, w1
310 mov 8(mp,i,8), %rax
311 mul q0
312 mov %rax, w2
313 mov %rdx, w3
314 add w0, (up,i,8)
315 adc w1, w2
316 adc $0, w3
317 mov 16(mp,i,8), %rax
318 mul q0
319 mov %rax, w0
320 mov %rdx, w1
321 add w2, 8(up,i,8)
322 adc w3, w0
323 adc $0, w1
324 L(e0): mov 24(mp,i,8), %rax
325 mul q0
326 mov %rax, w2
327 mov %rdx, w3
328 add $4, i
329 js L(tp0)
330
331 L(ed0): add w0, I(-16(up),-16(up,i,8))
332 adc w1, w2
333 adc $0, w3
334 add w2, I(-8(up),-8(up,i,8))
335 adc $0, w3
336 mov w3, (up,n,8) C up[0]
337 mov %rbx, q0 C previously computed q limb -> q0
338 lea 8(up), up C up++
339 dec j
340 jnz L(otp0)
341 jmp L(cj)
342
343 L(b2): cmp $-2, R32(n)
344 jz L(n2)
345
346 L(otp2):lea 2(n), i
347 mov (mp,n,8), %rax
348 mul q0
349 mov %rax, w0
350 mov %rdx, w1
351 mov 8(mp,n,8), %rax
352 mul q0
353 mov %rax, %rbx
354 mov %rdx, w3
355 add (up,n,8), w0
356 adc w1, %rbx
357 adc $0, w3
358 mov 16(mp,n,8), %rax
359 mul q0
360 mov %rax, w0
361 mov %rdx, w1
362 add 8(up,n,8), %rbx
363 mov %rbx, 8(up,n,8)
364 adc w3, w0
365 adc $0, w1
366 imul u0inv, %rbx C next q limb
367 jmp L(e2)
368
369 ALIGNx
370 L(tp2): add w0, -16(up,i,8)
371 adc w1, w2
372 adc $0, w3
373 mov (mp,i,8), %rax
374 mul q0
375 mov %rax, w0
376 mov %rdx, w1
377 add w2, -8(up,i,8)
378 adc w3, w0
379 adc $0, w1
380 L(e2): mov 8(mp,i,8), %rax
381 mul q0
382 mov %rax, w2
383 mov %rdx, w3
384 add w0, (up,i,8)
385 adc w1, w2
386 adc $0, w3
387 mov 16(mp,i,8), %rax
388 mul q0
389 mov %rax, w0
390 mov %rdx, w1
391 add w2, 8(up,i,8)
392 adc w3, w0
393 adc $0, w1
394 mov 24(mp,i,8), %rax
395 mul q0
396 mov %rax, w2
397 mov %rdx, w3
398 add $4, i
399 js L(tp2)
400
401 L(ed2): add w0, I(-16(up),-16(up,i,8))
402 adc w1, w2
403 adc $0, w3
404 add w2, I(-8(up),-8(up,i,8))
405 adc $0, w3
406 mov w3, (up,n,8) C up[0]
407 mov %rbx, q0 C previously computed q limb -> q0
408 lea 8(up), up C up++
409 dec j
410 jnz L(otp2)
411 jmp L(cj)
412
413 L(n1): mov (mp_param), %rax
414 mul q0
415 add -8(up), %rax
416 adc (up), %rdx
417 mov %rdx, (rp)
418 mov $0, R32(%rax)
419 adc R32(%rax), R32(%rax)
420 jmp L(ret)
421
422 L(n2): mov (mp_param), %rax
423 mov -16(up), %rbp
424 mul q0
425 add %rax, %rbp
426 mov %rdx, %r9
427 adc $0, %r9
428 mov -8(mp), %rax
429 mov -8(up), %r10
430 mul q0
431 add %rax, %r10
432 mov %rdx, %r11
433 adc $0, %r11
434 add %r9, %r10
435 adc $0, %r11
436 mov %r10, q0
437 imul u0inv, q0 C next q0
438 mov -16(mp), %rax
439 mul q0
440 add %rax, %r10
441 mov %rdx, %r9
442 adc $0, %r9
443 mov -8(mp), %rax
444 mov (up), %r14
445 mul q0
446 add %rax, %r14
447 adc $0, %rdx
448 add %r9, %r14
449 adc $0, %rdx
450 xor R32(%rax), R32(%rax)
451 add %r11, %r14
452 adc 8(up), %rdx
453 mov %r14, (rp)
454 mov %rdx, 8(rp)
455 adc R32(%rax), R32(%rax)
456 jmp L(ret)
457
458 ALIGNx
459 L(n3): mov -24(mp), %rax
460 mov -24(up), %r10
461 mul q0
462 add %rax, %r10
463 mov -16(mp), %rax
464 mov %rdx, %r11
465 adc $0, %r11
466 mov -16(up), %rbp
467 mul q0
468 add %rax, %rbp
469 mov %rdx, %r9
470 adc $0, %r9
471 mov -8(mp), %rax
472 add %r11, %rbp
473 mov -8(up), %r10
474 adc $0, %r9
475 mul q0
476 mov %rbp, q0
477 imul u0inv, q0 C next q0
478 add %rax, %r10
479 mov %rdx, %r11
480 adc $0, %r11
481 mov %rbp, -16(up)
482 add %r9, %r10
483 adc $0, %r11
484 mov %r10, -8(up)
485 mov %r11, -24(up) C up[0]
486 lea 8(up), up C up++
487 dec j
488 jnz L(n3)
489
490 mov -48(up), %rdx
491 mov -40(up), %rbx
492 xor R32(%rax), R32(%rax)
493 add %rbp, %rdx
494 adc %r10, %rbx
495 adc -8(up), %r11
496 mov %rdx, (rp)
497 mov %rbx, 8(rp)
498 mov %r11, 16(rp)
499 adc R32(%rax), R32(%rax)
500 jmp L(ret)
501 EPILOGUE()
502 ASM_END()