Debian Codegraph @

mpn/powerpc64/mode64/p9/sqr_basecase.asm

raw
1 dnl Power9 mpn_sqr_basecase.
2
3 dnl Copyright 1999-2001, 2003-2006, 2008, 2017-2018 Free Software Foundation,
4 dnl Inc.
5
6 dnl This file is part of the GNU MP Library.
7 dnl
8 dnl The GNU MP Library is free software; you can redistribute it and/or modify
9 dnl it under the terms of either:
10 dnl
11 dnl * the GNU Lesser General Public License as published by the Free
12 dnl Software Foundation; either version 3 of the License, or (at your
13 dnl option) any later version.
14 dnl
15 dnl or
16 dnl
17 dnl * the GNU General Public License as published by the Free Software
18 dnl Foundation; either version 2 of the License, or (at your option) any
19 dnl later version.
20 dnl
21 dnl or both in parallel, as here.
22 dnl
23 dnl The GNU MP Library is distributed in the hope that it will be useful, but
24 dnl WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
25 dnl or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
26 dnl for more details.
27 dnl
28 dnl You should have received copies of the GNU General Public License and the
29 dnl GNU Lesser General Public License along with the GNU MP Library. If not,
30 dnl see https://www.gnu.org/licenses/.
31
32 include(`../config.m4')
33
34 C cycles/limb
35 C POWER3/PPC630 -
36 C POWER4/PPC970 -
37 C POWER5 -
38 C POWER6 -
39 C POWER7 -
40 C POWER8 -
41 C POWER9 1.62
42
43 C TODO
44 C * Completely separate evn and odd code into two outer loops. Also consider
45 C unrolling these two outer loops and thereby eliminate all branches.
46 C * Avoid the reloading of u1 before every loop start.
47 C * Reduce register usage.
48 C * Consider getting rid of cy and instead load 3 u limbs, use addc+adde+adde.
49 C * Consider skewing conditional adjustments to allow mask creation with subfe
50 C like in the un=3 code. It might streamline the adjustments (or not).
51
52 C INPUT PARAMETERS
53 define(`rp', `r3')
54 define(`up', `r4')
55 define(`un', `r5')
56
57 define(`u0', `r0')
58 define(`u1', `r7')
59 define(`rp2', `r24')
60 define(`up2', `r25')
61 define(`cy', `r6')
62
63 define(`LSHU1U0',`
64 addc u0, u0, u0
65 adde u1, u1, u1
66 li cy, 0
67 addze cy, cy
68 ')
69 define(`LSHU1U',`
70 addc u0, u0, u0
71 add u0, u0, cy
72 adde u1, u1, u1
73 li cy, 0
74 addze cy, cy
75 ')
76 define(`LSHU1UF',`
77 addc u0, u0, u0
78 add u0, u0, cy
79 adde u1, u1, u1
80 ')
81 define(`LSHU1UHF',`
82 add u0, u0, u0
83 add u0, u0, cy
84 ')
85 C These are cleverer replacements, but they tend to leave CA set, disturbing
86 C the main accumulation code! Breaking that false dependency might have a
87 C positive performance impact. Note that the subfe here results in a mask for
88 C our adjustments.
89 define(`xLSHU1U0',`
90 addc u0, u0, u0
91 adde u1, u1, u1
92 subfe cy, cy, cy
93 ')
94 define(`xLSHU1U',`
95 subfic cy, cy, 0
96 adde u0, u0, u0
97 adde u1, u1, u1
98 subfe cy, cy, cy
99 ')
100 define(`xLSHU1U',`
101 subfic cy, cy, 0
102 adde u0, u0, u0
103 ')
104
105 ASM_START()
106 PROLOGUE(mpn_sqr_basecase)
107 ld r0, 0(up) C n = 1
108 mulld r8, r0, r0 C weight 0
109 mulhdu r9, r0, r0 C weight 1
110 std r8, 0(rp)
111 cmpdi cr0, un, 2
112 bge cr0, L(ge2)
113 std r9, 8(rp)
114 blr
115
116 L(ge2): bgt cr0, L(gt2)
117 ld r6, 8(up)
118 mulld r10, r6, r6 C u1 * u1
119 mulhdu r11, r6, r6 C u1 * u1
120 mulld r4, r6, r0 C u1 * u0
121 mulhdu r5, r6, r0 C u1 * u0
122 addc r4, r4, r4
123 adde r5, r5, r5
124 addze r11, r11
125 addc r9, r9, r4
126 adde r10, r10, r5
127 addze r11, r11
128 std r9, 8(rp)
129 std r10, 16(rp)
130 std r11, 24(rp)
131 blr
132
133 L(gt2): cmpdi cr0, un, 3
134 bgt cr0, L(gt3)
135 std r30, -16(r1)
136 std r31, -8(r1)
137 subfo r12, r12, r12 C clear OV (and result register)
138 ld r8, 8(r4)
139 mulld r5, r8, r8 C W2
140 mulhdu r10, r8, r8 C W3
141 sradi r11, u0, 63 C CAUTION: clobbers CA
142 and r11, r11, r8 C W3
143 addc u0, u0, u0
144 adde u1, r8, r8
145 subfe r6, r6, r6 C mask
146 ld r4, 16(r4) C W2
147 mulld r12, r8, u0 C W1 u1 x u0
148 mulhdu r8, r8, u0 C W2 u1 x u0
149 maddld( r31, r4, u0, r11) C W2
150 maddhdu(r30, r4, u0, r11) C W3
151 andc r6, r4, r6 C W4
152 addc r9, r12, r9 C W1
153 std r9, 8(rp) C W1
154 mulld r9, r4, u1 C W3
155 mulhdu r11, r4, u1 C W4
156 addex( r5, r5, r8, 0) C W2
157 adde r5, r31, r5 C W2
158 std r5, 16(rp) C W2
159 maddld( r5, r4, r4, r6) C W4 u2^2
160 maddhdu(r6, r4, r4, r6) C W5 u2^2
161 addex( r9, r9, r30, 0) C W3
162 adde r9, r9, r10 C W3
163 std r9, 24(rp) C W3
164 adde r5, r5, r11 C W4
165 addze r6, r6 C W5
166 li r8, 0
167 addex( r5, r5, r8, 0) C W4
168 std r5, 32(rp) C W4
169 addex( r6, r6, r8, 0) C W5
170 std r6, 40(rp) C W5
171 ld r30, -16(r1)
172 ld r31, -8(r1)
173 blr
174
175 L(gt3): std r22, -80(r1)
176 std r23, -72(r1)
177 std r24, -64(r1)
178 std r25, -56(r1)
179 std r26, -48(r1)
180 std r27, -40(r1)
181 std r28, -32(r1)
182 std r29, -24(r1)
183 std r30, -16(r1)
184 std r31, -8(r1)
185
186 mr rp2, rp
187 mr up2, up
188 addi r22, un, -1 C count for loop FIXME: Adjust
189 subfo r0, r0, r0 C clear OV (and r0)
190 rldicl r0, un, 0, 63 C r0 = un & 1
191 cmpdi cr7, r0, 0
192
193 ld u0, 0(up2)
194 ld u1, 8(up2)
195
196 cmpdi cr5, r22, 4
197 srdi r31, r22, 2
198 addi r22, r22, -2
199 mtctr r31
200
201 beq cr7, L(m2_evn)
202 L(m2_odd):
203 rldicl. r31, r22, 63, 63 C r22 & 2
204 mulld r23, u0, u0
205 mulhdu r12, u0, u0
206 mulld r5, u1, u1
207 mulhdu r10, u1, u1
208
209 sradi r11, u0, 63
210 and r11, r11, u1
211
212 LSHU1U0
213
214 ld r8, 8(up2)
215 ld r9, 16(up2)
216 mulld r28, r8, u0 C W u1 x u0
217 mulhdu r31, r8, u0 C W u1 x u0
218 std r23, 0(rp2)
219
220 bne cr0, L(m2_11)
221 L(m2_01):
222 addi up, up2, 16
223 addi rp, rp2, 0
224 b L(m2_lo2)
225 L(m2_11):
226 addi up, up2, 0
227 addi rp, rp2, -16
228 b L(m2_lo0)
229
230 L(m2_evn):
231 rldicl. r31, r22, 63, 63 C r22 & 2
232 mulld r23, u0, u0
233 mulhdu r5, u0, u0
234 mulld r12, u1, u1
235 mulhdu r11, u1, u1
236
237 sradi r10, u0, 63
238 and r10, r10, u1
239
240 LSHU1U0
241
242 ld r9, 8(up2)
243 ld r8, 16(up2)
244 mulld r29, r9, u0 C W u1 x u0
245 mulhdu r30, r9, u0 C W u1 x u0
246 std r23, 0(rp2)
247
248 beq cr0, L(m2_10)
249 L(m2_00):
250 addi up, up2, 8
251 addi rp, rp2, -8
252 b L(m2_lo1)
253 L(m2_10):
254 addi up, up2, 24
255 addi rp, rp2, 8
256 ble cr5, L(m2_end)
257
258 L(m2_top):
259 ld r9, 0(up)
260 maddld( r28, r8, u0, r10)
261 maddhdu(r31, r8, u0, r10)
262 adde r5, r29, r5
263 std r5, 0(rp)
264 mulld r5, r8, u1
265 mulhdu r10, r8, u1
266 addex( r12, r12, r30, 0)
267 L(m2_lo2):
268 ld r8, 8(up)
269 maddld( r29, r9, u0, r11)
270 maddhdu(r30, r9, u0, r11)
271 adde r12, r28, r12
272 std r12, 8(rp)
273 mulld r12, r9, u1
274 mulhdu r11, r9, u1
275 addex( r5, r5, r31, 0)
276 L(m2_lo1):
277 ld r9, 16(up)
278 maddld( r28, r8, u0, r10)
279 maddhdu(r31, r8, u0, r10)
280 adde r5, r29, r5
281 std r5, 16(rp)
282 mulld r5, r8, u1
283 mulhdu r10, r8, u1
284 addex( r12, r12, r30, 0)
285 L(m2_lo0):
286 ld r8, 24(up)
287 maddld( r29, r9, u0, r11)
288 maddhdu(r30, r9, u0, r11)
289 adde r12, r28, r12
290 std r12, 24(rp)
291 mulld r12, r9, u1
292 mulhdu r11, r9, u1
293 addex( r5, r5, r31, 0)
294 addi up, up, 32
295 addi rp, rp, 32
296 bdnz L(m2_top)
297
298 L(m2_end):
299 ld r9, 0(up)
300 maddld( r28, r8, u0, r10)
301 maddhdu(r31, r8, u0, r10)
302 adde r5, r29, r5
303 std r5, 0(rp)
304 mulld r5, r8, u1
305 mulhdu r10, r8, u1
306 b L(cj) C jump to addmul_2 tail
307
308 L(outer):
309 addi up2, up2, 16
310 addi rp2, rp2, 32
311
312 ld u0, 0(up2)
313 ld u1, 8(up2)
314
315 cmpdi cr5, r22, 4
316 srdi r31, r22, 2
317 addi r22, r22, -2
318 mtctr r31
319
320 ld r26, 0(rp2)
321 ld r27, 16(rp2)
322
323 rldicl. r31, r22, 63, 63 C r22 & 2
324 beq cr7, L(evn)
325
326 L(odd): maddld( r23, u0, u0, r26) C W u2^2
327 maddhdu(r12, u0, u0, r26) C W u2^2
328 maddld( r5, u1, u1, r27) C W u3^2
329 maddhdu(r10, u1, u1, r27) C W u3^2
330 ld r26, 8(rp2)
331
332 ld r8, -8(up2)
333 sradi r8, r8, 63 C CAUTION: clobbers CA
334 and r8, r8, u0
335 sradi r11, u0, 63 C CAUTION: clobbers CA
336 and r11, r11, u1
337
338 LSHU1U
339
340 addc r23, r23, r8
341
342 ld r8, 8(up2)
343 ld r9, 16(up2)
344 maddld( r28, r8, u0, r26) C W u3 x u2
345 maddhdu(r31, r8, u0, r26) C W u3 x u2
346 ld r26, 24(rp2)
347 std r23, 0(rp2) C W0
348
349 bne cr0, L(11)
350 L(01):
351 addi up, up2, 16
352 addi rp, rp2, 0
353 b L(lo2)
354 L(11):
355 addi up, up2, 0
356 addi rp, rp2, -16
357 b L(lo0)
358
359 L(evn): maddld( r23, u0, u0, r26) C W u2^2
360 maddhdu(r5, u0, u0, r26) C W u2^2
361 maddld( r12, u1, u1, r27) C W u3^2
362 maddhdu(r11, u1, u1, r27) C W u3^2
363 ld r27, 8(rp2)
364
365 ld r9, -8(up2)
366 sradi r9, r9, 63 C CAUTION: clobbers CA
367 and r9, r9, u0
368 sradi r10, u0, 63 C CAUTION: clobbers CA
369 and r10, r10, u1
370
371 LSHU1U
372
373 addc r23, r23, r9
374
375 ld r9, 8(up2)
376 ld r8, 16(up2)
377 maddld( r29, r9, u0, r27) C W u3 x u2
378 maddhdu(r30, r9, u0, r27) C W u3 x u2
379 ld r27, 24(rp2)
380 std r23, 0(rp2) C W0
381
382 beq cr0, L(10)
383 L(00):
384 addi up, up2, 8
385 addi rp, rp2, -8
386 b L(lo1)
387 L(10):
388 addi up, up2, 24
389 addi rp, rp2, 8
390 ble cr5, L(end)
391
392 L(top): ld r9, 0(up)
393 maddld( r28, r8, u0, r10)
394 maddhdu(r31, r8, u0, r10)
395 adde r5, r29, r5
396 ld r26, 24(rp)
397 std r5, 0(rp)
398 maddld( r5, r8, u1, r27)
399 maddhdu(r10, r8, u1, r27)
400 addex( r12, r12, r30, 0)
401 L(lo2): ld r8, 8(up)
402 maddld( r29, r9, u0, r11)
403 maddhdu(r30, r9, u0, r11)
404 adde r12, r28, r12
405 ld r27, 32(rp)
406 std r12, 8(rp)
407 maddld( r12, r9, u1, r26)
408 maddhdu(r11, r9, u1, r26)
409 addex( r5, r5, r31, 0)
410 L(lo1): ld r9, 16(up)
411 maddld( r28, r8, u0, r10)
412 maddhdu(r31, r8, u0, r10)
413 adde r5, r29, r5
414 ld r26, 40(rp)
415 std r5, 16(rp)
416 maddld( r5, r8, u1, r27)
417 maddhdu(r10, r8, u1, r27)
418 addex( r12, r12, r30, 0)
419 L(lo0): ld r8, 24(up)
420 maddld( r29, r9, u0, r11)
421 maddhdu(r30, r9, u0, r11)
422 adde r12, r28, r12
423 ld r27, 48(rp)
424 std r12, 24(rp)
425 maddld( r12, r9, u1, r26)
426 maddhdu(r11, r9, u1, r26)
427 addex( r5, r5, r31, 0)
428 addi up, up, 32
429 addi rp, rp, 32
430 bdnz L(top)
431
432 L(end): ld r9, 0(up)
433 maddld( r28, r8, u0, r10)
434 maddhdu(r31, r8, u0, r10)
435 adde r5, r29, r5
436 std r5, 0(rp)
437 maddld( r5, r8, u1, r27)
438 maddhdu(r10, r8, u1, r27)
439 L(cj): addex( r12, r12, r30, 0)
440 maddld( r29, r9, u0, r11)
441 maddhdu(r30, r9, u0, r11)
442 adde r12, r28, r12
443 std r12, 8(rp)
444 mulld r12, r9, u1
445 mulhdu r11, r9, u1
446 addex( r5, r5, r31, 0)
447 adde r5, r29, r5
448 std r5, 16(rp)
449 addex( r12, r12, r30, 0)
450 adde r12, r12, r10
451 std r12, 24(rp)
452 li r4, 0
453 addze r5, r11
454 addex( r5, r5, r4, 0)
455 std r5, 32(rp)
456 bgt cr5, L(outer)
457
458 L(corner):
459 ld u0, 16(up2)
460 ld u1, 24(up2)
461 ld r26, 32(rp2)
462 bne cr7, L(corner_odd)
463
464 L(corner_evn):
465 ld r27, 40(rp2)
466 maddld( r23, u0, u0, r26) C W u2^2
467 maddhdu(r5, u0, u0, r26) C W u2^2
468 mulld r12, u1, u1 C W u3^2
469 mulhdu r11, u1, u1 C W u3^2
470
471 ld r9, 8(up2)
472 sradi r9, r9, 63 C CAUTION: clobbers CA
473 and r9, r9, u0
474 sradi r10, u0, 63 C CAUTION: clobbers CA
475 and r10, r10, u1
476
477 LSHU1UHF
478
479 addc r23, r23, r9
480
481 ld r9, 24(up2)
482 maddld( r29, r9, u0, r27) C W u3 x u2
483 maddhdu(r30, r9, u0, r27) C W u3 x u2
484 std r23, 32(rp2)
485 adde r5, r29, r5
486 std r5, 40(rp2)
487 addex( r12, r12, r30, 0)
488 adde r12, r12, r10 C W FIXME can this co?
489 std r12, 48(rp2)
490 li r4, 0
491 addex( r5, r11, r4, 0)
492 addze r5, r5
493 std r5, 56(rp2)
494 b L(ret)
495
496 L(corner_odd):
497 ld r27, 48(rp2)
498 maddld( r23, u0, u0, r26) C W u2^2
499 maddhdu(r12, u0, u0, r26) C W u2^2
500 maddld( r5, u1, u1, r27) C W u3^2
501 maddhdu(r10, u1, u1, r27) C W u3^2
502 ld r26, 40(rp2)
503
504 ld r8, 8(up2)
505 sradi r8, r8, 63 C CAUTION: clobbers CA
506 and r8, r8, u0
507 sradi r11, u0, 63 C CAUTION: clobbers CA
508 and r11, r11, u1
509
510 LSHU1UF
511
512 addc r23, r23, r8
513
514 ld r8, 24(up2)
515 ld r9, 32(up2)
516 maddld( r28, r8, u0, r26) C W u3 x u2
517 maddhdu(r31, r8, u0, r26) C W u3 x u2
518 std r23, 32(rp2)
519 maddld( r29, r9, u0, r11)
520 maddhdu(r30, r9, u0, r11)
521 adde r12, r28, r12
522 std r12, 40(rp2)
523 mulld r12, r9, u1
524 mulhdu r11, r9, u1
525 addex( r5, r5, r31, 0)
526 adde r5, r29, r5
527 std r5, 48(rp2)
528 addex( r12, r12, r30, 0)
529 adde r12, r12, r10
530 std r12, 56(rp2)
531 mulld r23, r9, r9 C W u2^2
532 mulhdu r12, r9, r9 C W u2^2
533 adde r23, r23, r11
534 addze r12, r12
535 sradi r4, r8, 63 C CAUTION: clobbers CA
536 and r4, r4, r9
537 addex( r23, r23, r4, 0)
538 std r23, 64(rp2)
539 li r4, 0
540 addex( r12, r12, r4, 0)
541 std r12, 72(rp2)
542
543 L(ret): ld r22, -80(r1)
544 ld r23, -72(r1)
545 ld r24, -64(r1)
546 ld r25, -56(r1)
547 ld r26, -48(r1)
548 ld r27, -40(r1)
549 ld r28, -32(r1)
550 ld r29, -24(r1)
551 ld r30, -16(r1)
552 ld r31, -8(r1)
553 blr
554 EPILOGUE()
555 ASM_END()