Debian Codegraph @

mpn/arm64/applem1/sqr_basecase.asm

raw
1 dnl ARM64 mpn_sqr_basecase
2
3 dnl Contributed to the GNU project by Torbjörn Granlund.
4
5 dnl Copyright 2020 Free Software Foundation, Inc.
6
7 dnl This file is part of the GNU MP Library.
8 dnl
9 dnl The GNU MP Library is free software; you can redistribute it and/or modify
10 dnl it under the terms of either:
11 dnl
12 dnl * the GNU Lesser General Public License as published by the Free
13 dnl Software Foundation; either version 3 of the License, or (at your
14 dnl option) any later version.
15 dnl
16 dnl or
17 dnl
18 dnl * the GNU General Public License as published by the Free Software
19 dnl Foundation; either version 2 of the License, or (at your option) any
20 dnl later version.
21 dnl
22 dnl or both in parallel, as here.
23 dnl
24 dnl The GNU MP Library is distributed in the hope that it will be useful, but
25 dnl WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
26 dnl or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
27 dnl for more details.
28 dnl
29 dnl You should have received copies of the GNU General Public License and the
30 dnl GNU Lesser General Public License along with the GNU MP Library. If not,
31 dnl see https://www.gnu.org/licenses/.
32
33 include(`../config.m4')
34
35 dnl TODO
36 dnl * Replace the mul_1 code with less scheduled and thus simpler code. If
37 dnl we base it on the addmul_1 loop, the corner code could benefit from
38 dnl similar incoming register state, which could eliminate some loads.
39 dnl * Handle n = 4 early.
40 dnl * Duplicate addmul loop into 4 loops which fall into each other. Perhaps
41 dnl stick to one mul_1 loop, but do the (mod 4) stuff at its end instead of
42 dnl its beginning.
43
44 define(`rp', `x0')
45 define(`up', `x1')
46 define(`n', `x2')
47
48 define(`v0', `x3')
49 define(`CY', `x17')
50
51 PROLOGUE(mpn_sqr_basecase)
52 cmp n, #3
53 b.ls L(le3)
54
55 ldr v0, [up],#8
56 sub n, n, #1
57 mul x6, v0, v0
58 umulh x4, v0, v0
59 str x6, [rp],#8
60 lsl v0, v0, 1
61 lsl n, n, #3
62 lsr x16, n, #5
63 tbnz n, #3, L(mbx1)
64
65 L(mbx0):adds x11, x4, xzr C move and clear cy
66 tbz n, #4, L(mb00)
67
68 L(mb10):ldp x4, x5, [up],#16
69 mul x8, x4, v0
70 umulh x10, x4, v0
71 cbz x16, L(m2e)
72 ldp x6, x7, [up],#16
73 mul x9, x5, v0
74 b L(mmid)-8
75
76 L(mbx1):ldr x7, [up],#8
77 mul x9, x7, v0
78 umulh x11, x7, v0
79 adds x9, x9, x4
80 str x9, [rp],#8
81 tbnz n, #4, L(mb10)
82 L(mb00):ldp x6, x7, [up],#16
83 mul x8, x6, v0
84 umulh x10, x6, v0
85 ldp x4, x5, [up],#16
86 mul x9, x7, v0
87 adcs x12, x8, x11
88 umulh x11, x7, v0
89 sub x16, x16, #1
90 cbz x16, L(mend)
91
92 ALIGN(16)
93 L(mtop):mul x8, x4, v0
94 ldp x6, x7, [up],#16
95 adcs x13, x9, x10
96 umulh x10, x4, v0
97 mul x9, x5, v0
98 stp x12, x13, [rp],#16
99 adcs x12, x8, x11
100 umulh x11, x5, v0
101 L(mmid):mul x8, x6, v0
102 ldp x4, x5, [up],#16
103 adcs x13, x9, x10
104 umulh x10, x6, v0
105 mul x9, x7, v0
106 stp x12, x13, [rp],#16
107 adcs x12, x8, x11
108 umulh x11, x7, v0
109 sub x16, x16, #1
110 cbnz x16, L(mtop)
111
112 L(mend):mul x8, x4, v0
113 adcs x13, x9, x10
114 umulh x10, x4, v0
115 stp x12, x13, [rp],#16
116 L(m2e): mul x9, x5, v0
117 adcs x12, x8, x11
118 umulh x11, x5, v0
119 adcs x13, x9, x10
120 stp x12, x13, [rp],#16
121 adc x11, x11, xzr
122 str x11, [rp],#8
123
124 L(outer):
125 sub n, n, #8
126 sub rp, rp, n
127 sub up, up, n
128 ldp x6, x7, [up,#-16]
129 ldr v0, [rp,#-8]
130 and x8, x7, x6, asr 63
131 mul x9, x7, x7
132 adds v0, v0, x8
133 umulh x4, x7, x7
134 adc x4, x4, xzr
135 adds v0, v0, x9
136 str v0, [rp,#-8]
137 adc CY, x4, xzr
138 adds xzr, x6, x6
139 adc v0, x7, x7
140 cmp n, #16
141 beq L(cor2)
142
143 lsr x16, n, #5
144 tbz n, #3, L(bx0)
145
146 L(bx1): ldr x4, [up],#8
147 mul x8, x4, v0
148 umulh x4, x4, v0
149 tbz n, #4, L(b01)
150
151 L(b11): ldp x5, x6, [up],#16
152 ldp x12, x13, [rp]
153 ldr x14, [rp,#16]
154 mul x9, x5, v0
155 umulh x5, x5, v0
156 mul x10, x6, v0
157 umulh x6, x6, v0
158 adds x8, x12, x8
159 adcs x4, x13, x4
160 adcs x5, x14, x5
161 adc x6, x6, xzr
162 adds x8, x8, CY
163 adcs x4, x4, x9
164 adcs x5, x5, x10
165 adc CY, x6, xzr
166 stp x8, x4, [rp],#16
167 str x5, [rp],#8
168 cbnz x16, L(top)
169 b L(end)
170
171 L(b01): ldr x12, [rp]
172 adds x8, x12, x8
173 adc x4, x4, xzr
174 adds x8, x8, CY
175 adc CY, x4, xzr
176 str x8, [rp],#8
177 b L(top)
178
179 L(bx0): ldp x4, x5, [up],#16
180 tbz n, #4, L(top)+4
181
182 L(b10): ldp x12, x13, [rp]
183 mul x8, x4, v0
184 umulh x4, x4, v0
185 mul x9, x5, v0
186 umulh x5, x5, v0
187 adds x8, x12, x8
188 adcs x4, x13, x4
189 adc x5, x5, xzr
190 adds x8, x8, CY
191 adcs x4, x4, x9
192 adc CY, x5, xzr
193 stp x8, x4, [rp],#16
194
195 ALIGN(16)
196 L(top): ldp x4, x5, [up],#16
197 ldp x6, x7, [up],#16
198 ldp x12, x13, [rp]
199 ldp x14, x15, [rp,#16]
200 mul x8, x4, v0
201 umulh x4, x4, v0
202 mul x9, x5, v0
203 umulh x5, x5, v0
204 mul x10, x6, v0
205 umulh x6, x6, v0
206 mul x11, x7, v0
207 umulh x7, x7, v0
208 adds x8, x12, x8
209 adcs x4, x13, x4
210 adcs x5, x14, x5
211 adcs x6, x15, x6
212 adc x7, x7, xzr
213 adds x8, x8, CY
214 adcs x4, x4, x9
215 adcs x5, x5, x10
216 adcs x6, x6, x11
217 adc CY, x7, xzr
218 stp x8, x4, [rp],#16
219 stp x5, x6, [rp],#16
220 sub x16, x16, #1
221 cbnz x16, L(top)
222
223 L(end): str CY, [rp],#8
224 b L(outer)
225
226 L(cor2):ldp x10, x11, [up]
227 ldp x12, x13, [rp]
228 mul x8, x10, v0
229 umulh x4, x10, v0
230 mul x9, x11, v0
231 umulh x5, x11, v0
232 adds x8, x12, x8
233 adcs x4, x13, x4
234 adc x5, x5, xzr
235 adds x8, x8, CY
236 adcs x13, x4, x9
237 adc x12, x5, xzr
238 str x8, [rp]
239 and x8, x10, x7, asr 63
240 mul x9, x10, x10
241 adds x13, x13, x8
242 umulh x4, x10, x10
243 adc x4, x4, xzr
244 adds x13, x13, x9
245 adc CY, x4, xzr
246 adds xzr, x7, x7
247 adc v0, x10, x10
248 mul x8, x11, v0
249 umulh x4, x11, v0
250 adds x8, x12, x8
251 adc x4, x4, xzr
252 adds x8, x8, CY
253 adc v0, x4, xzr
254 stp x13, x8, [rp,#8]
255 and x2, x11, x10, asr 63
256 mul x5, x11, x11
257 adds v0, v0, x2
258 umulh x4, x11, x11
259 adc x4, x4, xzr
260 adds v0, v0, x5
261 adc x4, x4, xzr
262 stp v0, x4, [rp,#24]
263 ret
264
265 L(le3): ldr v0, [up]
266 mul x4, v0, v0 C W0
267 umulh x5, v0, v0 C W1
268 cmp n, #2
269 b.hs L(2o3)
270 stp x4, x5, [rp]
271 ret
272
273 L(2o3): ldr x6, [up,#8]
274 mul x7, x6, x6 C W2
275 umulh x8, x6, x6 C W3
276 mul x9, v0, x6 C W1+1/64
277 umulh x10, v0, x6 C W2+1/64
278 b.hi L(3)
279 adds x5, x5, x9 C W1
280 adcs x7, x7, x10 C W2
281 adc x8, x8, xzr C W3
282 adds x5, x5, x9 C W1
283 adcs x7, x7, x10 C W2
284 adc x8, x8, xzr C W3
285 stp x4, x5, [rp]
286 stp x7, x8, [rp,#16]
287 ret
288
289 L(3): ldr x11, [up,#16]
290 mul x12, x11, x11 C W4
291 umulh x13, x11, x11 C W5
292 mul x14, v0, x11 C W2+1/64
293 umulh x15, v0, x11 C W3+1/64
294 mul x16, x6, x11 C W3+1/64
295 umulh x17, x6, x11 C W4+1/64
296 adds x5, x5, x9
297 adcs x7, x7, x10
298 adcs x8, x8, x15
299 adcs x12, x12, x17
300 adc x13, x13, xzr
301 adds x5, x5, x9
302 adcs x7, x7, x10
303 adcs x8, x8, x15
304 adcs x12, x12, x17
305 adc x13, x13, xzr
306 adds x7, x7, x14
307 adcs x8, x8, x16
308 adcs x12, x12, xzr
309 adc x13, x13, xzr
310 adds x7, x7, x14
311 adcs x8, x8, x16
312 adcs x12, x12, xzr
313 adc x13, x13, xzr
314 stp x4, x5, [rp]
315 stp x7, x8, [rp,#16]
316 stp x12, x13, [rp,#32]
317 ret
318 EPILOGUE()