| 1 |
dnl ARM64 mpn_sqr_basecase |
| 2 |
|
| 3 |
dnl Contributed to the GNU project by Torbjörn Granlund. |
| 4 |
|
| 5 |
dnl Copyright 2020 Free Software Foundation, Inc. |
| 6 |
|
| 7 |
dnl This file is part of the GNU MP Library. |
| 8 |
dnl |
| 9 |
dnl The GNU MP Library is free software; you can redistribute it and/or modify |
| 10 |
dnl it under the terms of either: |
| 11 |
dnl |
| 12 |
dnl * the GNU Lesser General Public License as published by the Free |
| 13 |
dnl Software Foundation; either version 3 of the License, or (at your |
| 14 |
dnl option) any later version. |
| 15 |
dnl |
| 16 |
dnl or |
| 17 |
dnl |
| 18 |
dnl * the GNU General Public License as published by the Free Software |
| 19 |
dnl Foundation; either version 2 of the License, or (at your option) any |
| 20 |
dnl later version. |
| 21 |
dnl |
| 22 |
dnl or both in parallel, as here. |
| 23 |
dnl |
| 24 |
dnl The GNU MP Library is distributed in the hope that it will be useful, but |
| 25 |
dnl WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY |
| 26 |
dnl or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
| 27 |
dnl for more details. |
| 28 |
dnl |
| 29 |
dnl You should have received copies of the GNU General Public License and the |
| 30 |
dnl GNU Lesser General Public License along with the GNU MP Library. If not, |
| 31 |
dnl see https://www.gnu.org/licenses/. |
| 32 |
|
| 33 |
include(`../config.m4') |
| 34 |
|
| 35 |
dnl TODO |
| 36 |
dnl * Replace the mul_1 code with less scheduled and thus simpler code. If |
| 37 |
dnl we base it on the addmul_1 loop, the corner code could benefit from |
| 38 |
dnl similar incoming register state, which could eliminate some loads. |
| 39 |
dnl * Handle n = 4 early. |
| 40 |
dnl * Duplicate addmul loop into 4 loops which fall into each other. Perhaps |
| 41 |
dnl stick to one mul_1 loop, but do the (mod 4) stuff at its end instead of |
| 42 |
dnl its beginning. |
| 43 |
|
| 44 |
define(`rp', `x0') |
| 45 |
define(`up', `x1') |
| 46 |
define(`n', `x2') |
| 47 |
|
| 48 |
define(`v0', `x3') |
| 49 |
define(`CY', `x17') |
| 50 |
|
| 51 |
PROLOGUE(mpn_sqr_basecase) |
| 52 |
cmp n, #3 |
| 53 |
b.ls L(le3) |
| 54 |
|
| 55 |
ldr v0, [up],#8 |
| 56 |
sub n, n, #1 |
| 57 |
mul x6, v0, v0 |
| 58 |
umulh x4, v0, v0 |
| 59 |
str x6, [rp],#8 |
| 60 |
lsl v0, v0, 1 |
| 61 |
lsl n, n, #3 |
| 62 |
lsr x16, n, #5 |
| 63 |
tbnz n, #3, L(mbx1) |
| 64 |
|
| 65 |
L(mbx0):adds x11, x4, xzr C move and clear cy |
| 66 |
tbz n, #4, L(mb00) |
| 67 |
|
| 68 |
L(mb10):ldp x4, x5, [up],#16 |
| 69 |
mul x8, x4, v0 |
| 70 |
umulh x10, x4, v0 |
| 71 |
cbz x16, L(m2e) |
| 72 |
ldp x6, x7, [up],#16 |
| 73 |
mul x9, x5, v0 |
| 74 |
b L(mmid)-8 |
| 75 |
|
| 76 |
L(mbx1):ldr x7, [up],#8 |
| 77 |
mul x9, x7, v0 |
| 78 |
umulh x11, x7, v0 |
| 79 |
adds x9, x9, x4 |
| 80 |
str x9, [rp],#8 |
| 81 |
tbnz n, #4, L(mb10) |
| 82 |
L(mb00):ldp x6, x7, [up],#16 |
| 83 |
mul x8, x6, v0 |
| 84 |
umulh x10, x6, v0 |
| 85 |
ldp x4, x5, [up],#16 |
| 86 |
mul x9, x7, v0 |
| 87 |
adcs x12, x8, x11 |
| 88 |
umulh x11, x7, v0 |
| 89 |
sub x16, x16, #1 |
| 90 |
cbz x16, L(mend) |
| 91 |
|
| 92 |
ALIGN(16) |
| 93 |
L(mtop):mul x8, x4, v0 |
| 94 |
ldp x6, x7, [up],#16 |
| 95 |
adcs x13, x9, x10 |
| 96 |
umulh x10, x4, v0 |
| 97 |
mul x9, x5, v0 |
| 98 |
stp x12, x13, [rp],#16 |
| 99 |
adcs x12, x8, x11 |
| 100 |
umulh x11, x5, v0 |
| 101 |
L(mmid):mul x8, x6, v0 |
| 102 |
ldp x4, x5, [up],#16 |
| 103 |
adcs x13, x9, x10 |
| 104 |
umulh x10, x6, v0 |
| 105 |
mul x9, x7, v0 |
| 106 |
stp x12, x13, [rp],#16 |
| 107 |
adcs x12, x8, x11 |
| 108 |
umulh x11, x7, v0 |
| 109 |
sub x16, x16, #1 |
| 110 |
cbnz x16, L(mtop) |
| 111 |
|
| 112 |
L(mend):mul x8, x4, v0 |
| 113 |
adcs x13, x9, x10 |
| 114 |
umulh x10, x4, v0 |
| 115 |
stp x12, x13, [rp],#16 |
| 116 |
L(m2e): mul x9, x5, v0 |
| 117 |
adcs x12, x8, x11 |
| 118 |
umulh x11, x5, v0 |
| 119 |
adcs x13, x9, x10 |
| 120 |
stp x12, x13, [rp],#16 |
| 121 |
adc x11, x11, xzr |
| 122 |
str x11, [rp],#8 |
| 123 |
|
| 124 |
L(outer): |
| 125 |
sub n, n, #8 |
| 126 |
sub rp, rp, n |
| 127 |
sub up, up, n |
| 128 |
ldp x6, x7, [up,#-16] |
| 129 |
ldr v0, [rp,#-8] |
| 130 |
and x8, x7, x6, asr 63 |
| 131 |
mul x9, x7, x7 |
| 132 |
adds v0, v0, x8 |
| 133 |
umulh x4, x7, x7 |
| 134 |
adc x4, x4, xzr |
| 135 |
adds v0, v0, x9 |
| 136 |
str v0, [rp,#-8] |
| 137 |
adc CY, x4, xzr |
| 138 |
adds xzr, x6, x6 |
| 139 |
adc v0, x7, x7 |
| 140 |
cmp n, #16 |
| 141 |
beq L(cor2) |
| 142 |
|
| 143 |
lsr x16, n, #5 |
| 144 |
tbz n, #3, L(bx0) |
| 145 |
|
| 146 |
L(bx1): ldr x4, [up],#8 |
| 147 |
mul x8, x4, v0 |
| 148 |
umulh x4, x4, v0 |
| 149 |
tbz n, #4, L(b01) |
| 150 |
|
| 151 |
L(b11): ldp x5, x6, [up],#16 |
| 152 |
ldp x12, x13, [rp] |
| 153 |
ldr x14, [rp,#16] |
| 154 |
mul x9, x5, v0 |
| 155 |
umulh x5, x5, v0 |
| 156 |
mul x10, x6, v0 |
| 157 |
umulh x6, x6, v0 |
| 158 |
adds x8, x12, x8 |
| 159 |
adcs x4, x13, x4 |
| 160 |
adcs x5, x14, x5 |
| 161 |
adc x6, x6, xzr |
| 162 |
adds x8, x8, CY |
| 163 |
adcs x4, x4, x9 |
| 164 |
adcs x5, x5, x10 |
| 165 |
adc CY, x6, xzr |
| 166 |
stp x8, x4, [rp],#16 |
| 167 |
str x5, [rp],#8 |
| 168 |
cbnz x16, L(top) |
| 169 |
b L(end) |
| 170 |
|
| 171 |
L(b01): ldr x12, [rp] |
| 172 |
adds x8, x12, x8 |
| 173 |
adc x4, x4, xzr |
| 174 |
adds x8, x8, CY |
| 175 |
adc CY, x4, xzr |
| 176 |
str x8, [rp],#8 |
| 177 |
b L(top) |
| 178 |
|
| 179 |
L(bx0): ldp x4, x5, [up],#16 |
| 180 |
tbz n, #4, L(top)+4 |
| 181 |
|
| 182 |
L(b10): ldp x12, x13, [rp] |
| 183 |
mul x8, x4, v0 |
| 184 |
umulh x4, x4, v0 |
| 185 |
mul x9, x5, v0 |
| 186 |
umulh x5, x5, v0 |
| 187 |
adds x8, x12, x8 |
| 188 |
adcs x4, x13, x4 |
| 189 |
adc x5, x5, xzr |
| 190 |
adds x8, x8, CY |
| 191 |
adcs x4, x4, x9 |
| 192 |
adc CY, x5, xzr |
| 193 |
stp x8, x4, [rp],#16 |
| 194 |
|
| 195 |
ALIGN(16) |
| 196 |
L(top): ldp x4, x5, [up],#16 |
| 197 |
ldp x6, x7, [up],#16 |
| 198 |
ldp x12, x13, [rp] |
| 199 |
ldp x14, x15, [rp,#16] |
| 200 |
mul x8, x4, v0 |
| 201 |
umulh x4, x4, v0 |
| 202 |
mul x9, x5, v0 |
| 203 |
umulh x5, x5, v0 |
| 204 |
mul x10, x6, v0 |
| 205 |
umulh x6, x6, v0 |
| 206 |
mul x11, x7, v0 |
| 207 |
umulh x7, x7, v0 |
| 208 |
adds x8, x12, x8 |
| 209 |
adcs x4, x13, x4 |
| 210 |
adcs x5, x14, x5 |
| 211 |
adcs x6, x15, x6 |
| 212 |
adc x7, x7, xzr |
| 213 |
adds x8, x8, CY |
| 214 |
adcs x4, x4, x9 |
| 215 |
adcs x5, x5, x10 |
| 216 |
adcs x6, x6, x11 |
| 217 |
adc CY, x7, xzr |
| 218 |
stp x8, x4, [rp],#16 |
| 219 |
stp x5, x6, [rp],#16 |
| 220 |
sub x16, x16, #1 |
| 221 |
cbnz x16, L(top) |
| 222 |
|
| 223 |
L(end): str CY, [rp],#8 |
| 224 |
b L(outer) |
| 225 |
|
| 226 |
L(cor2):ldp x10, x11, [up] |
| 227 |
ldp x12, x13, [rp] |
| 228 |
mul x8, x10, v0 |
| 229 |
umulh x4, x10, v0 |
| 230 |
mul x9, x11, v0 |
| 231 |
umulh x5, x11, v0 |
| 232 |
adds x8, x12, x8 |
| 233 |
adcs x4, x13, x4 |
| 234 |
adc x5, x5, xzr |
| 235 |
adds x8, x8, CY |
| 236 |
adcs x13, x4, x9 |
| 237 |
adc x12, x5, xzr |
| 238 |
str x8, [rp] |
| 239 |
and x8, x10, x7, asr 63 |
| 240 |
mul x9, x10, x10 |
| 241 |
adds x13, x13, x8 |
| 242 |
umulh x4, x10, x10 |
| 243 |
adc x4, x4, xzr |
| 244 |
adds x13, x13, x9 |
| 245 |
adc CY, x4, xzr |
| 246 |
adds xzr, x7, x7 |
| 247 |
adc v0, x10, x10 |
| 248 |
mul x8, x11, v0 |
| 249 |
umulh x4, x11, v0 |
| 250 |
adds x8, x12, x8 |
| 251 |
adc x4, x4, xzr |
| 252 |
adds x8, x8, CY |
| 253 |
adc v0, x4, xzr |
| 254 |
stp x13, x8, [rp,#8] |
| 255 |
and x2, x11, x10, asr 63 |
| 256 |
mul x5, x11, x11 |
| 257 |
adds v0, v0, x2 |
| 258 |
umulh x4, x11, x11 |
| 259 |
adc x4, x4, xzr |
| 260 |
adds v0, v0, x5 |
| 261 |
adc x4, x4, xzr |
| 262 |
stp v0, x4, [rp,#24] |
| 263 |
ret |
| 264 |
|
| 265 |
L(le3): ldr v0, [up] |
| 266 |
mul x4, v0, v0 C W0 |
| 267 |
umulh x5, v0, v0 C W1 |
| 268 |
cmp n, #2 |
| 269 |
b.hs L(2o3) |
| 270 |
stp x4, x5, [rp] |
| 271 |
ret |
| 272 |
|
| 273 |
L(2o3): ldr x6, [up,#8] |
| 274 |
mul x7, x6, x6 C W2 |
| 275 |
umulh x8, x6, x6 C W3 |
| 276 |
mul x9, v0, x6 C W1+1/64 |
| 277 |
umulh x10, v0, x6 C W2+1/64 |
| 278 |
b.hi L(3) |
| 279 |
adds x5, x5, x9 C W1 |
| 280 |
adcs x7, x7, x10 C W2 |
| 281 |
adc x8, x8, xzr C W3 |
| 282 |
adds x5, x5, x9 C W1 |
| 283 |
adcs x7, x7, x10 C W2 |
| 284 |
adc x8, x8, xzr C W3 |
| 285 |
stp x4, x5, [rp] |
| 286 |
stp x7, x8, [rp,#16] |
| 287 |
ret |
| 288 |
|
| 289 |
L(3): ldr x11, [up,#16] |
| 290 |
mul x12, x11, x11 C W4 |
| 291 |
umulh x13, x11, x11 C W5 |
| 292 |
mul x14, v0, x11 C W2+1/64 |
| 293 |
umulh x15, v0, x11 C W3+1/64 |
| 294 |
mul x16, x6, x11 C W3+1/64 |
| 295 |
umulh x17, x6, x11 C W4+1/64 |
| 296 |
adds x5, x5, x9 |
| 297 |
adcs x7, x7, x10 |
| 298 |
adcs x8, x8, x15 |
| 299 |
adcs x12, x12, x17 |
| 300 |
adc x13, x13, xzr |
| 301 |
adds x5, x5, x9 |
| 302 |
adcs x7, x7, x10 |
| 303 |
adcs x8, x8, x15 |
| 304 |
adcs x12, x12, x17 |
| 305 |
adc x13, x13, xzr |
| 306 |
adds x7, x7, x14 |
| 307 |
adcs x8, x8, x16 |
| 308 |
adcs x12, x12, xzr |
| 309 |
adc x13, x13, xzr |
| 310 |
adds x7, x7, x14 |
| 311 |
adcs x8, x8, x16 |
| 312 |
adcs x12, x12, xzr |
| 313 |
adc x13, x13, xzr |
| 314 |
stp x4, x5, [rp] |
| 315 |
stp x7, x8, [rp,#16] |
| 316 |
stp x12, x13, [rp,#32] |
| 317 |
ret |
| 318 |
EPILOGUE() |