Debian Codegraph @

gmp4/mpn/x86_64/core2/redc_1.asm

raw
1 dnl X86-64 mpn_redc_1 optimised for Intel Conroe and Wolfdale.
2
3 dnl Contributed to the GNU project by Torbjörn Granlund.
4
5 dnl Copyright 2003-2005, 2007, 2008, 2011-2013 Free Software Foundation, Inc.
6
7 dnl This file is part of the GNU MP Library.
8 dnl
9 dnl The GNU MP Library is free software; you can redistribute it and/or modify
10 dnl it under the terms of either:
11 dnl
12 dnl * the GNU Lesser General Public License as published by the Free
13 dnl Software Foundation; either version 3 of the License, or (at your
14 dnl option) any later version.
15 dnl
16 dnl or
17 dnl
18 dnl * the GNU General Public License as published by the Free Software
19 dnl Foundation; either version 2 of the License, or (at your option) any
20 dnl later version.
21 dnl
22 dnl or both in parallel, as here.
23 dnl
24 dnl The GNU MP Library is distributed in the hope that it will be useful, but
25 dnl WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
26 dnl or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
27 dnl for more details.
28 dnl
29 dnl You should have received copies of the GNU General Public License and the
30 dnl GNU Lesser General Public License along with the GNU MP Library. If not,
31 dnl see https://www.gnu.org/licenses/.
32
33 include(`../config.m4')
34
35 C cycles/limb
36 C AMD K8,K9 ?
37 C AMD K10 ?
38 C AMD bull ?
39 C AMD pile ?
40 C AMD steam ?
41 C AMD bobcat ?
42 C AMD jaguar ?
43 C Intel P4 ?
44 C Intel core 4.5 (fluctuating)
45 C Intel NHM ?
46 C Intel SBR ?
47 C Intel IBR ?
48 C Intel HWL ?
49 C Intel BWL ?
50 C Intel atom ?
51 C VIA nano ?
52
53 C The inner loops of this code are the result of running a code generation and
54 C optimisation tool suite written by David Harvey and Torbjörn Granlund.
55
56 C TODO
57 C * Micro-optimise, none performed thus far.
58 C * Consider inlining mpn_add_n.
59 C * Single basecases out before the pushes.
60 C * Keep up[i] in registers for basecases (might require pushes).
61
62 C When playing with pointers, set this to $2 to fall back to conservative
63 C indexing in wind-down code.
64 define(`I',`$1')
65
66 define(`rp', `%rdi') C rcx
67 define(`up', `%rsi') C rdx
68 define(`mp_param', `%rdx') C r8
69 define(`n', `%rcx') C r9
70 define(`u0inv', `%r8') C stack
71
72 define(`i', `%r14')
73 define(`j', `%r15')
74 define(`mp', `%r12')
75 define(`q0', `%r13')
76
77 C rax rbx rcx rdx rdi rsi rbp r8 r9 r10 r11 r12 r13 r14 r15
78 C X q0' n X rp up u0i mp q0 i j
79
80 ABI_SUPPORT(DOS64)
81 ABI_SUPPORT(STD64)
82
83 define(`ALIGNx', `ALIGN(16)')
84
85 ASM_START()
86 TEXT
87 ALIGN(32)
88 PROLOGUE(mpn_redc_1)
89 FUNC_ENTRY(4)
90 IFDOS(` mov 56(%rsp), %r8 ')
91 push %rbx
92 push %rbp
93 push %r12
94 push %r13
95 push %r14
96 push %r15
97
98 mov (up), q0
99 mov n, j C outer loop induction var
100 lea (mp_param,n,8), mp
101 lea -16(up,n,8), up
102 neg n
103 imul u0inv, q0 C first iteration q0
104
105 test $1, R8(n)
106 jz L(b0)
107
108 L(b1): cmp $-1, R32(n)
109 jz L(n1)
110 cmp $-3, R32(n)
111 jz L(n3)
112
113 push rp
114
115 L(otp1):lea 3(n), i
116 mov (mp,n,8), %rax
117 mul q0
118 lea (%rax), %rbp
119 mov 8(mp,n,8), %rax
120 lea (%rdx), %r9
121 mul q0
122 lea (%rax), %r11
123 mov 16(mp,n,8), %rax
124 mov 16(up,n,8), %r10
125 lea (%rdx), %rdi
126 mul q0
127 add %rbp, %r10
128 lea (%rax), %rbp
129 mov 24(mp,n,8), %rax
130 adc %r9, %r11
131 mov 24(up,n,8), %rbx
132 lea (%rdx), %r9
133 adc $0, %rdi
134 mul q0
135 add %r11, %rbx
136 lea (%rax), %r11
137 mov 32(mp,n,8), %rax
138 adc %rdi, %rbp
139 mov %rbx, 24(up,n,8)
140 mov 32(up,n,8), %r10
141 lea (%rdx), %rdi
142 adc $0, %r9
143 imul u0inv, %rbx C next q limb
144 add $2, i
145 jns L(ed1)
146
147 ALIGNx
148 L(tp1): mul q0
149 add %rbp, %r10
150 lea (%rax), %rbp
151 mov (mp,i,8), %rax
152 adc %r9, %r11
153 mov %r10, -8(up,i,8)
154 mov (up,i,8), %r10
155 lea (%rdx), %r9
156 adc $0, %rdi
157 mul q0
158 add %r11, %r10
159 lea (%rax), %r11
160 mov 8(mp,i,8), %rax
161 adc %rdi, %rbp
162 mov %r10, (up,i,8)
163 mov 8(up,i,8), %r10
164 lea (%rdx), %rdi
165 adc $0, %r9
166 add $2, i
167 js L(tp1)
168
169 L(ed1): mul q0
170 add %rbp, %r10
171 adc %r9, %r11
172 mov %r10, I(-8(up),-8(up,i,8))
173 mov I((up),(up,i,8)), %r10
174 adc $0, %rdi
175 add %r11, %r10
176 adc %rdi, %rax
177 mov %r10, I((up),(up,i,8))
178 mov I(8(up),8(up,i,8)), %r10
179 adc $0, %rdx
180 add %rax, %r10
181 mov %r10, I(8(up),8(up,i,8))
182 adc $0, %rdx
183 mov %rdx, 16(up,n,8) C up[0]
184 mov %rbx, q0 C previously computed q limb -> q0
185 lea 8(up), up C up++
186 dec j
187 jnz L(otp1)
188 jmp L(cj)
189
190 L(b0): cmp $-2, R32(n)
191 jz L(n2)
192 cmp $-4, R32(n)
193 jz L(n4)
194
195 push rp
196
197 L(otp0):lea 4(n), i
198 mov (mp,n,8), %rax
199 mul q0
200 lea (%rax), %r11
201 mov 8(mp,n,8), %rax
202 lea (%rdx), %rdi
203 mul q0
204 lea (%rax), %rbp
205 mov 16(mp,n,8), %rax
206 mov 16(up,n,8), %r10
207 lea (%rdx), %r9
208 mul q0
209 add %r11, %r10
210 lea (%rax), %r11
211 mov 24(mp,n,8), %rax
212 adc %rdi, %rbp
213 mov 24(up,n,8), %rbx
214 lea (%rdx), %rdi
215 adc $0, %r9
216 mul q0
217 add %rbp, %rbx
218 lea (%rax), %rbp
219 mov 32(mp,n,8), %rax
220 adc %r9, %r11
221 mov %rbx, 24(up,n,8)
222 mov 32(up,n,8), %r10
223 lea (%rdx), %r9
224 adc $0, %rdi
225 imul u0inv, %rbx C next q limb
226 jmp L(e0)
227
228 ALIGNx
229 L(tp0): mul q0
230 add %rbp, %r10
231 lea (%rax), %rbp
232 mov (mp,i,8), %rax
233 adc %r9, %r11
234 mov %r10, -8(up,i,8)
235 mov (up,i,8), %r10
236 lea (%rdx), %r9
237 adc $0, %rdi
238 L(e0): mul q0
239 add %r11, %r10
240 lea (%rax), %r11
241 mov 8(mp,i,8), %rax
242 adc %rdi, %rbp
243 mov %r10, (up,i,8)
244 mov 8(up,i,8), %r10
245 lea (%rdx), %rdi
246 adc $0, %r9
247 add $2, i
248 js L(tp0)
249
250 L(ed0): mul q0
251 add %rbp, %r10
252 adc %r9, %r11
253 mov %r10, I(-8(up),-8(up,i,8))
254 mov I((up),(up,i,8)), %r10
255 adc $0, %rdi
256 add %r11, %r10
257 adc %rdi, %rax
258 mov %r10, I((up),(up,i,8))
259 mov I(8(up),8(up,i,8)), %r10
260 adc $0, %rdx
261 add %rax, %r10
262 mov %r10, I(8(up),8(up,i,8))
263 adc $0, %rdx
264 mov %rdx, 16(up,n,8) C up[0]
265 mov %rbx, q0 C previously computed q limb -> q0
266 lea 8(up), up C up++
267 dec j
268 jnz L(otp0)
269
270 L(cj): lea 16(up), up C FIXME
271 pop rp
272 L(add_n):
273 IFSTD(` lea (up,n,8), up C param 2: up
274 lea (up,n,8), %rdx C param 3: up - n
275 neg R32(n) ') C param 4: n
276
277 IFDOS(` lea (up,n,8), %rdx C param 2: up
278 lea (%rdx,n,8), %r8 C param 3: up - n
279 neg R32(n)
280 mov n, %r9 C param 4: n
281 mov rp, %rcx ') C param 1: rp
282
283 CALL( mpn_add_n)
284
285 L(ret): pop %r15
286 pop %r14
287 pop %r13
288 pop %r12
289 pop %rbp
290 pop %rbx
291 FUNC_EXIT()
292 ret
293
294 L(n1): mov (mp_param), %rax
295 mul q0
296 add 8(up), %rax
297 adc 16(up), %rdx
298 mov %rdx, (rp)
299 mov $0, R32(%rax)
300 adc R32(%rax), R32(%rax)
301 jmp L(ret)
302
303 L(n2): mov (mp_param), %rax
304 mov (up), %rbp
305 mul q0
306 add %rax, %rbp
307 mov %rdx, %r9
308 adc $0, %r9
309 mov -8(mp), %rax
310 mov 8(up), %r10
311 mul q0
312 add %rax, %r10
313 mov %rdx, %r11
314 adc $0, %r11
315 add %r9, %r10
316 adc $0, %r11
317 mov %r10, q0
318 imul u0inv, q0 C next q0
319 mov -16(mp), %rax
320 mul q0
321 add %rax, %r10
322 mov %rdx, %r9
323 adc $0, %r9
324 mov -8(mp), %rax
325 mov 16(up), %r14
326 mul q0
327 add %rax, %r14
328 adc $0, %rdx
329 add %r9, %r14
330 adc $0, %rdx
331 xor R32(%rax), R32(%rax)
332 add %r11, %r14
333 adc 24(up), %rdx
334 mov %r14, (rp)
335 mov %rdx, 8(rp)
336 adc R32(%rax), R32(%rax)
337 jmp L(ret)
338
339 ALIGNx
340 L(n3): mov -24(mp), %rax
341 mov -8(up), %r10
342 mul q0
343 add %rax, %r10
344 mov -16(mp), %rax
345 mov %rdx, %r11
346 adc $0, %r11
347 mov (up), %rbp
348 mul q0
349 add %rax, %rbp
350 mov %rdx, %r9
351 adc $0, %r9
352 mov -8(mp), %rax
353 add %r11, %rbp
354 mov 8(up), %r10
355 adc $0, %r9
356 mul q0
357 mov %rbp, q0
358 imul u0inv, q0 C next q0
359 add %rax, %r10
360 mov %rdx, %r11
361 adc $0, %r11
362 mov %rbp, (up)
363 add %r9, %r10
364 adc $0, %r11
365 mov %r10, 8(up)
366 mov %r11, -8(up) C up[0]
367 lea 8(up), up C up++
368 dec j
369 jnz L(n3)
370
371 mov -32(up), %rdx
372 mov -24(up), %rbx
373 xor R32(%rax), R32(%rax)
374 add %rbp, %rdx
375 adc %r10, %rbx
376 adc 8(up), %r11
377 mov %rdx, (rp)
378 mov %rbx, 8(rp)
379 mov %r11, 16(rp)
380 adc R32(%rax), R32(%rax)
381 jmp L(ret)
382
383 ALIGNx
384 L(n4): mov -32(mp), %rax
385 mul q0
386 lea (%rax), %r11
387 mov -24(mp), %rax
388 lea (%rdx), %r14
389 mul q0
390 lea (%rax), %rbp
391 mov -16(mp), %rax
392 mov -16(up), %r10
393 lea (%rdx), %r9
394 mul q0
395 add %r11, %r10
396 lea (%rax), %r11
397 mov -8(mp), %rax
398 adc %r14, %rbp
399 mov -8(up), %rbx
400 lea (%rdx), %r14
401 adc $0, %r9
402 mul q0
403 add %rbp, %rbx
404 adc %r9, %r11
405 mov %rbx, -8(up)
406 mov (up), %r10
407 adc $0, %r14
408 imul u0inv, %rbx C next q limb
409 add %r11, %r10
410 adc %r14, %rax
411 mov %r10, (up)
412 mov 8(up), %r10
413 adc $0, %rdx
414 add %rax, %r10
415 mov %r10, 8(up)
416 adc $0, %rdx
417 mov %rdx, -16(up) C up[0]
418 mov %rbx, q0 C previously computed q limb -> q0
419 lea 8(up), up C up++
420 dec j
421 jnz L(n4)
422 lea 16(up), up
423 jmp L(add_n)
424 EPILOGUE()
425 ASM_END()