Debian Codegraph @

src/qd8-f32-qc8w-gemm/gen/qd8-f32-qc8w-gemm-4x16c4-minmax-asm-aarch64-neondot-cortex-a55.S

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-gemm/4x16c4-aarch64-neondot-cortex-a55.S.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2020 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include "xnnpack/assembly.h"
12
13 # void xnn_qd8_f32_qc8w_gemm_minmax_ukernel_4x16c4__asm_aarch64_neondot_cortex_a55(
14 # size_t mr, x0
15 # size_t nc, x1
16 # size_t kc, x2 / x0
17 # const int8_t* restrict a, x3
18 # size_t a_stride, x4
19 # const void* restrict w, x5
20 # int8_t* restrict c, x6
21 # size_t cm_stride, x7
22 # size_t cn_stride, [sp] -> x12
23 # const union xnn_f32_minmax_params *params, [sp + 8] -> x11
24 # const struct xnn_qd8_quantization_params *quantization_params) [sp + 16] -> x16
25
26 # d8-d15, x19-x30 need to be preserved if used. x18 is reserved by the OS.
27
28 // Register usage
29 // A0 x3 v0 v4
30 // A1 x15 v1 v5
31 // A2 x13 v2 v6
32 // A3 x4 v3 v7
33 // B x5 v8 v9 v10 v11
34 // C0 x6 v16 v20 v24 v28
35 // C1 x8 v17 v21 v25 v29
36 // C2 x9 v18 v22 v26 v30
37 // C3 x7 v19 v23 v27 v31
38 // temp x14 for Cortex-A55 loads
39 // unused v14 v15
40
41 BEGIN_FUNCTION xnn_qd8_f32_qc8w_gemm_minmax_ukernel_4x16c4__asm_aarch64_neondot_cortex_a55
42
43 # Clamp A and C pointers
44 CMP x0, 2 // if mr < 2
45
46 LDP x12, x11, [sp] // cn_stride, params
47
48 ADD x15, x3, x4 // a1 = a0 + a_stride
49 ADD x8, x6, x7 // c1 = c0 + cm_stride
50
51 STP d8, d9, [sp, -48]!
52 STP d12, d13, [sp, 32]
53 LDR x16, [sp, 64] // &quantization_params[0].zero_point
54 LD2 {v12.4s, v13.4s}, [x16] // v12 zero_point, v13 scale
55
56 CSEL x15, x3, x15, LO // a1 = a0
57 CSEL x8, x6, x8, LO // c1 = c0
58 ADD x2, x2, 3 // kc = (kc + 3) & ~3
59
60 ADD x13, x15, x4 // a2 = a1 + a_stride
61 ADD x9, x8, x7 // c2 = c1 + cm_stride
62 // if mr <= 2
63 CSEL x13, x15, x13, LS // a2 = a1
64 CSEL x9, x8, x9, LS // c2 = c1
65 BIC x2, x2, 3
66
67 STP d10, d11, [sp, 16]
68
69 CMP x0, 4 // if mr < 4
70 ADD x4, x13, x4 // a3 = a2 + a_stride
71 ADD x7, x9, x7 // c3 = c2 + cm_stride
72 CSEL x4, x13, x4, LO // a3 = a2
73 CSEL x7, x9, x7, LO // c3 = c2
74
75 .p2align 3
76 0:
77 # Load initial bias from w into accumulators
78 LDP q16, q20, [x5], 32
79 SUBS x0, x2, 16 // k = kc - 16
80 MUL v17.4s, v16.4s, v12.s[1]
81 MUL v18.4s, v16.4s, v12.s[2]
82 LDP q24, q28, [x5], 32
83 MUL v19.4s, v16.4s, v12.s[3]
84 MUL v21.4s, v20.4s, v12.s[1]
85 MUL v22.4s, v20.4s, v12.s[2]
86 MUL v23.4s, v20.4s, v12.s[3]
87 MUL v25.4s, v24.4s, v12.s[1]
88 MUL v26.4s, v24.4s, v12.s[2]
89 MUL v27.4s, v24.4s, v12.s[3]
90 MUL v29.4s, v28.4s, v12.s[1]
91 MUL v30.4s, v28.4s, v12.s[2]
92 MUL v31.4s, v28.4s, v12.s[3]
93 MUL v24.4s, v24.4s, v12.s[0]
94 MUL v28.4s, v28.4s, v12.s[0]
95 MUL v16.4s, v16.4s, v12.s[0]
96 MUL v20.4s, v20.4s, v12.s[0]
97
98 # Is there at least 16 bytes for prologue/epilogue?
99 B.LO 4f
100
101 # prologue - read A and B values for block 0 and 1
102 LDR d0, [x3], 8
103 LDR q8, [x5], 16
104 LDR d1, [x15], 8
105 LDR d2, [x13], 8
106 LDR d3, [x4], 8
107 SUBS x0, x0, 16 // is there 16 for main loop?
108 LDR d9, [x5], 8
109 LDR x14, [x5], 8
110 # Is there at least 16 bytes for main loop?
111 B.LO 2f
112
113 # Main loop - 16 bytes of A in 4 groups.
114 # 4 row of 4 vectors wide = 16 sdot instructions for 4 channels
115 # 4 LD64 for A
116 # 4 LD128 for W. = 2 LD64 + INS.
117 # for each 4 sdot, 1 LD64 for A, 2 LD64 for W + INS.
118
119 .p2align 3
120 1:
121 # BLOCK 0
122 SDOT v16.4s, v8.16b, v0.4b[0]
123 LDR d10, [x5], 8
124 SDOT v17.4s, v8.16b, v1.4b[0]
125 INS v9.d[1], x14
126 SDOT v18.4s, v8.16b, v2.4b[0]
127 LDR x14, [x5], 8
128 SDOT v19.4s, v8.16b, v3.4b[0]
129 LDR d4, [x3], 8
130
131 # BLOCK 1
132 SDOT v20.4s, v9.16b, v0.4b[0]
133 LDR d11, [x5], 8
134 SDOT v21.4s, v9.16b, v1.4b[0]
135 INS v10.d[1], x14
136 SDOT v22.4s, v9.16b, v2.4b[0]
137 LDR x14, [x5], 8
138 SDOT v23.4s, v9.16b, v3.4b[0]
139 LDR d5, [x15], 8
140
141 # BLOCK 2
142 SDOT v24.4s, v10.16b, v0.4b[0]
143 LDR d8, [x5], 8
144 SDOT v25.4s, v10.16b, v1.4b[0]
145 INS v11.d[1], x14
146 SDOT v26.4s, v10.16b, v2.4b[0]
147 LDR x14, [x5], 8
148 SDOT v27.4s, v10.16b, v3.4b[0]
149 LDR d6, [x13], 8
150
151 # BLOCK 3
152 SDOT v28.4s, v11.16b, v0.4b[0]
153 LDR d9, [x5], 8
154 SDOT v29.4s, v11.16b, v1.4b[0]
155 INS v8.d[1], x14
156 SDOT v30.4s, v11.16b, v2.4b[0]
157 LDR x14, [x5], 8
158 SDOT v31.4s, v11.16b, v3.4b[0]
159 LDR d7, [x4], 8
160
161 # BLOCK 0
162 SDOT v16.4s, v8.16b, v0.4b[1]
163 LDR d10, [x5], 8
164 SDOT v17.4s, v8.16b, v1.4b[1]
165 INS v9.d[1], x14
166 SDOT v18.4s, v8.16b, v2.4b[1]
167 LDR x14, [x5], 8
168 SDOT v19.4s, v8.16b, v3.4b[1]
169
170 # BLOCK 1
171 SDOT v20.4s, v9.16b, v0.4b[1]
172 LDR d11, [x5], 8
173 SDOT v21.4s, v9.16b, v1.4b[1]
174 INS v10.d[1], x14
175 SDOT v22.4s, v9.16b, v2.4b[1]
176 LDR x14, [x5], 8
177 SDOT v23.4s, v9.16b, v3.4b[1]
178
179 # BLOCK 2
180 SDOT v24.4s, v10.16b, v0.4b[1]
181 LDR d8, [x5], 8
182 SDOT v25.4s, v10.16b, v1.4b[1]
183 INS v11.d[1], x14
184 SDOT v26.4s, v10.16b, v2.4b[1]
185 LDR x14, [x5], 8
186 SDOT v27.4s, v10.16b, v3.4b[1]
187
188 # BLOCK 3
189 SDOT v28.4s, v11.16b, v0.4b[1]
190 LDR d9, [x5], 8
191 SDOT v29.4s, v11.16b, v1.4b[1]
192 INS v8.d[1], x14
193 SDOT v30.4s, v11.16b, v2.4b[1]
194 LDR x14, [x5], 8
195 SDOT v31.4s, v11.16b, v3.4b[1]
196
197 # BLOCK 0
198 SDOT v16.4s, v8.16b, v4.4b[0]
199 LDR d10, [x5], 8
200 SDOT v17.4s, v8.16b, v5.4b[0]
201 INS v9.d[1], x14
202 SDOT v18.4s, v8.16b, v6.4b[0]
203 LDR x14, [x5], 8
204 SDOT v19.4s, v8.16b, v7.4b[0]
205 LDR d0, [x3], 8
206
207 # BLOCK 1
208 SDOT v20.4s, v9.16b, v4.4b[0]
209 LDR d11, [x5], 8
210 SDOT v21.4s, v9.16b, v5.4b[0]
211 INS v10.d[1], x14
212 SDOT v22.4s, v9.16b, v6.4b[0]
213 LDR x14, [x5], 8
214 SDOT v23.4s, v9.16b, v7.4b[0]
215 LDR d1, [x15], 8
216
217 # BLOCK 2
218 SDOT v24.4s, v10.16b, v4.4b[0]
219 LDR d8, [x5], 8
220 SDOT v25.4s, v10.16b, v5.4b[0]
221 INS v11.d[1], x14
222 SDOT v26.4s, v10.16b, v6.4b[0]
223 LDR x14, [x5], 8
224 SDOT v27.4s, v10.16b, v7.4b[0]
225 LDR d2, [x13], 8
226
227 # BLOCK 3
228 SDOT v28.4s, v11.16b, v4.4b[0]
229 LDR d9, [x5], 8
230 SDOT v29.4s, v11.16b, v5.4b[0]
231 INS v8.d[1], x14
232 SDOT v30.4s, v11.16b, v6.4b[0]
233 LDR x14, [x5], 8
234 SDOT v31.4s, v11.16b, v7.4b[0]
235 LDR d3, [x4], 8
236
237 # BLOCK 0
238 SDOT v16.4s, v8.16b, v4.4b[1]
239 LDR d10, [x5], 8
240 SDOT v17.4s, v8.16b, v5.4b[1]
241 INS v9.d[1], x14
242 SDOT v18.4s, v8.16b, v6.4b[1]
243 LDR x14, [x5], 8
244 SDOT v19.4s, v8.16b, v7.4b[1]
245
246 # BLOCK 1
247 SDOT v20.4s, v9.16b, v4.4b[1]
248 LDR d11, [x5], 8
249 SDOT v21.4s, v9.16b, v5.4b[1]
250 INS v10.d[1], x14
251 SDOT v22.4s, v9.16b, v6.4b[1]
252 LDR x14, [x5], 8
253 SDOT v23.4s, v9.16b, v7.4b[1]
254
255 # BLOCK 2
256 SDOT v24.4s, v10.16b, v4.4b[1]
257 LDR d8, [x5], 8 // First B values for block 0 and 1
258 SDOT v25.4s, v10.16b, v5.4b[1]
259 INS v11.d[1], x14
260 SDOT v26.4s, v10.16b, v6.4b[1]
261 LDR x14, [x5], 8
262 SDOT v27.4s, v10.16b, v7.4b[1]
263 SUBS x0, x0, 16
264
265 # BLOCK 3
266 SDOT v28.4s, v11.16b, v4.4b[1]
267 LDR d9, [x5], 8
268 SDOT v29.4s, v11.16b, v5.4b[1]
269 INS v8.d[1], x14
270 SDOT v30.4s, v11.16b, v6.4b[1]
271 LDR x14, [x5], 8
272 SDOT v31.4s, v11.16b, v7.4b[1]
273 B.HS 1b
274
275 # Epilogue. Same as main loop but no preloads in final group
276 2:
277 # BLOCK 0
278 SDOT v16.4s, v8.16b, v0.4b[0]
279 LDR d10, [x5], 8
280 SDOT v17.4s, v8.16b, v1.4b[0]
281 INS v9.d[1], x14
282 SDOT v18.4s, v8.16b, v2.4b[0]
283 LDR x14, [x5], 8
284 SDOT v19.4s, v8.16b, v3.4b[0]
285 LDR d4, [x3], 8
286
287 # BLOCK 1
288 SDOT v20.4s, v9.16b, v0.4b[0]
289 LDR d11, [x5], 8
290 SDOT v21.4s, v9.16b, v1.4b[0]
291 INS v10.d[1], x14
292 SDOT v22.4s, v9.16b, v2.4b[0]
293 LDR x14, [x5], 8
294 SDOT v23.4s, v9.16b, v3.4b[0]
295 LDR d5, [x15], 8
296
297 # BLOCK 2
298 SDOT v24.4s, v10.16b, v0.4b[0]
299 LDR d8, [x5], 8
300 SDOT v25.4s, v10.16b, v1.4b[0]
301 INS v11.d[1], x14
302 SDOT v26.4s, v10.16b, v2.4b[0]
303 LDR x14, [x5], 8
304 SDOT v27.4s, v10.16b, v3.4b[0]
305 LDR d6, [x13], 8
306
307 # BLOCK 3
308 SDOT v28.4s, v11.16b, v0.4b[0]
309 LDR d9, [x5], 8
310 SDOT v29.4s, v11.16b, v1.4b[0]
311 INS v8.d[1], x14
312 SDOT v30.4s, v11.16b, v2.4b[0]
313 LDR x14, [x5], 8
314 SDOT v31.4s, v11.16b, v3.4b[0]
315 LDR d7, [x4], 8
316
317 # BLOCK 0
318 SDOT v16.4s, v8.16b, v0.4b[1]
319 LDR d10, [x5], 8
320 SDOT v17.4s, v8.16b, v1.4b[1]
321 INS v9.d[1], x14
322 SDOT v18.4s, v8.16b, v2.4b[1]
323 LDR x14, [x5], 8
324 SDOT v19.4s, v8.16b, v3.4b[1]
325
326 # BLOCK 1
327 SDOT v20.4s, v9.16b, v0.4b[1]
328 LDR d11, [x5], 8
329 SDOT v21.4s, v9.16b, v1.4b[1]
330 INS v10.d[1], x14
331 SDOT v22.4s, v9.16b, v2.4b[1]
332 LDR x14, [x5], 8
333 SDOT v23.4s, v9.16b, v3.4b[1]
334
335 # BLOCK 2
336 SDOT v24.4s, v10.16b, v0.4b[1]
337 LDR d8, [x5], 8
338 SDOT v25.4s, v10.16b, v1.4b[1]
339 INS v11.d[1], x14
340 SDOT v26.4s, v10.16b, v2.4b[1]
341 LDR x14, [x5], 8
342 SDOT v27.4s, v10.16b, v3.4b[1]
343
344 # BLOCK 3
345 SDOT v28.4s, v11.16b, v0.4b[1]
346 LDR d9, [x5], 8
347 SDOT v29.4s, v11.16b, v1.4b[1]
348 INS v8.d[1], x14
349 SDOT v30.4s, v11.16b, v2.4b[1]
350 LDR x14, [x5], 8
351 SDOT v31.4s, v11.16b, v3.4b[1]
352
353 # BLOCK 0
354 SDOT v16.4s, v8.16b, v4.4b[0]
355 LDR d10, [x5], 8
356 SDOT v17.4s, v8.16b, v5.4b[0]
357 INS v9.d[1], x14
358 SDOT v18.4s, v8.16b, v6.4b[0]
359 LDR x14, [x5], 8
360 SDOT v19.4s, v8.16b, v7.4b[0]
361
362 # BLOCK 1
363 SDOT v20.4s, v9.16b, v4.4b[0]
364 LDR d11, [x5], 8
365 SDOT v21.4s, v9.16b, v5.4b[0]
366 INS v10.d[1], x14
367 SDOT v22.4s, v9.16b, v6.4b[0]
368 LDR x14, [x5], 8
369 SDOT v23.4s, v9.16b, v7.4b[0]
370
371 # BLOCK 2
372 SDOT v24.4s, v10.16b, v4.4b[0]
373 LDR d8, [x5], 8
374 SDOT v25.4s, v10.16b, v5.4b[0]
375 INS v11.d[1], x14
376 SDOT v26.4s, v10.16b, v6.4b[0]
377 LDR x14, [x5], 8
378 SDOT v27.4s, v10.16b, v7.4b[0]
379
380 # BLOCK 3
381 SDOT v28.4s, v11.16b, v4.4b[0]
382 LDR d9, [x5], 8
383 SDOT v29.4s, v11.16b, v5.4b[0]
384 INS v8.d[1], x14
385 SDOT v30.4s, v11.16b, v6.4b[0]
386 LDR x14, [x5], 8
387 SDOT v31.4s, v11.16b, v7.4b[0]
388
389 # BLOCK 0
390 SDOT v16.4s, v8.16b, v4.4b[1]
391 LDR d10, [x5], 8
392 SDOT v17.4s, v8.16b, v5.4b[1]
393 INS v9.d[1], x14
394 SDOT v18.4s, v8.16b, v6.4b[1]
395 LDR x14, [x5], 8
396 SDOT v19.4s, v8.16b, v7.4b[1]
397
398 # BLOCK 1
399 SDOT v20.4s, v9.16b, v4.4b[1]
400 LDR d11, [x5], 8
401 SDOT v21.4s, v9.16b, v5.4b[1]
402 INS v10.d[1], x14
403 SDOT v22.4s, v9.16b, v6.4b[1]
404 LDR x14, [x5], 8
405 SDOT v23.4s, v9.16b, v7.4b[1]
406
407 # BLOCK 2
408 SDOT v24.4s, v10.16b, v4.4b[1]
409 SDOT v25.4s, v10.16b, v5.4b[1]
410 INS v11.d[1], x14
411 SDOT v26.4s, v10.16b, v6.4b[1]
412 SDOT v27.4s, v10.16b, v7.4b[1]
413 AND x0, x2, 15 // kc remainder 0 to 12
414
415 # BLOCK 3
416 SDOT v28.4s, v11.16b, v4.4b[1]
417 SDOT v29.4s, v11.16b, v5.4b[1]
418 SDOT v30.4s, v11.16b, v6.4b[1]
419 SDOT v31.4s, v11.16b, v7.4b[1]
420
421 # Is there a remainder?- 4 to 12 bytes of A
422 CBNZ x0, 5f
423
424 .p2align 3
425 3:
426 SCVTF v16.4s, v16.4s
427 SCVTF v17.4s, v17.4s
428 SCVTF v18.4s, v18.4s
429 SCVTF v19.4s, v19.4s
430 SCVTF v20.4s, v20.4s
431 SCVTF v21.4s, v21.4s
432 SCVTF v22.4s, v22.4s
433 SCVTF v23.4s, v23.4s
434 LDP q0, q1, [x5], 32 // kernel_scale
435 SCVTF v24.4s, v24.4s
436 SCVTF v25.4s, v25.4s
437 SCVTF v26.4s, v26.4s
438 SCVTF v27.4s, v27.4s
439 SCVTF v28.4s, v28.4s
440 SCVTF v29.4s, v29.4s
441 SCVTF v30.4s, v30.4s
442 SCVTF v31.4s, v31.4s
443 LDP q2, q3, [x5], 32
444 FMUL v4.4s, v0.4s, v13.s[0] // kernel_scale * scale
445 FMUL v5.4s, v1.4s, v13.s[0]
446 FMUL v6.4s, v2.4s, v13.s[0]
447 FMUL v7.4s, v3.4s, v13.s[0]
448 FMUL v8.4s, v0.4s, v13.s[1]
449 FMUL v9.4s, v1.4s, v13.s[1]
450 FMUL v10.4s, v2.4s, v13.s[1]
451 FMUL v11.4s, v3.4s, v13.s[1]
452 FMUL v16.4s, v16.4s, v4.4s
453 FMUL v20.4s, v20.4s, v5.4s
454 FMUL v24.4s, v24.4s, v6.4s
455 FMUL v28.4s, v28.4s, v7.4s
456 FMUL v17.4s, v17.4s, v8.4s
457 FMUL v21.4s, v21.4s, v9.4s
458 FMUL v25.4s, v25.4s, v10.4s
459 FMUL v29.4s, v29.4s, v11.4s
460 FMUL v4.4s, v0.4s, v13.s[2]
461 FMUL v5.4s, v1.4s, v13.s[2]
462 FMUL v6.4s, v2.4s, v13.s[2]
463 FMUL v7.4s, v3.4s, v13.s[2]
464 FMUL v8.4s, v0.4s, v13.s[3]
465 FMUL v9.4s, v1.4s, v13.s[3]
466 FMUL v10.4s, v2.4s, v13.s[3]
467 FMUL v11.4s, v3.4s, v13.s[3]
468 LDP q0, q1, [x5], 32 // bias
469 FMUL v18.4s, v18.4s, v4.4s
470 FMUL v22.4s, v22.4s, v5.4s
471 FMUL v26.4s, v26.4s, v6.4s
472 FMUL v30.4s, v30.4s, v7.4s
473 FMUL v19.4s, v19.4s, v8.4s
474 FMUL v23.4s, v23.4s, v9.4s
475 FMUL v27.4s, v27.4s, v10.4s
476 FMUL v31.4s, v31.4s, v11.4s
477 LDP q2, q3, [x5], 32
478 FADD v16.4s, v16.4s, v0.4s
479 FADD v17.4s, v17.4s, v0.4s
480 FADD v18.4s, v18.4s, v0.4s
481 FADD v19.4s, v19.4s, v0.4s
482 FADD v20.4s, v20.4s, v1.4s
483 FADD v21.4s, v21.4s, v1.4s
484 FADD v22.4s, v22.4s, v1.4s
485 FADD v23.4s, v23.4s, v1.4s
486 LD2R {v0.4s, v1.4s}, [x11] // min max
487 FADD v24.4s, v24.4s, v2.4s
488 FADD v25.4s, v25.4s, v2.4s
489 FADD v26.4s, v26.4s, v2.4s
490 FADD v27.4s, v27.4s, v2.4s
491 FADD v28.4s, v28.4s, v3.4s
492 FADD v29.4s, v29.4s, v3.4s
493 FADD v30.4s, v30.4s, v3.4s
494 FADD v31.4s, v31.4s, v3.4s
495 FMAX v16.4s, v16.4s, v0.4s
496 FMAX v17.4s, v17.4s, v0.4s
497 FMAX v18.4s, v18.4s, v0.4s
498 FMAX v19.4s, v19.4s, v0.4s
499 FMAX v20.4s, v20.4s, v0.4s
500 FMAX v21.4s, v21.4s, v0.4s
501 FMAX v22.4s, v22.4s, v0.4s
502 FMAX v23.4s, v23.4s, v0.4s
503 FMAX v24.4s, v24.4s, v0.4s
504 FMAX v25.4s, v25.4s, v0.4s
505 FMAX v26.4s, v26.4s, v0.4s
506 FMAX v27.4s, v27.4s, v0.4s
507 FMAX v28.4s, v28.4s, v0.4s
508 FMAX v29.4s, v29.4s, v0.4s
509 FMAX v30.4s, v30.4s, v0.4s
510 FMAX v31.4s, v31.4s, v0.4s
511 FMIN v16.4s, v16.4s, v1.4s
512 FMIN v17.4s, v17.4s, v1.4s
513 FMIN v18.4s, v18.4s, v1.4s
514 FMIN v19.4s, v19.4s, v1.4s
515 FMIN v20.4s, v20.4s, v1.4s
516 FMIN v21.4s, v21.4s, v1.4s
517 FMIN v22.4s, v22.4s, v1.4s
518 FMIN v23.4s, v23.4s, v1.4s
519 FMIN v24.4s, v24.4s, v1.4s
520 FMIN v25.4s, v25.4s, v1.4s
521 FMIN v26.4s, v26.4s, v1.4s
522 FMIN v27.4s, v27.4s, v1.4s
523 FMIN v28.4s, v28.4s, v1.4s
524 FMIN v29.4s, v29.4s, v1.4s
525 FMIN v30.4s, v30.4s, v1.4s
526 FMIN v31.4s, v31.4s, v1.4s
527
528 SUBS x1, x1, 16
529 B.LO 6f
530 STP q19, q23, [x7]
531 STP q27, q31, [x7, #32]
532 ADD x7, x7, x12
533 STP q18, q22, [x9]
534 STP q26, q30, [x9, #32]
535 ADD x9, x9, x12
536 STP q17, q21, [x8]
537 STP q25, q29, [x8, #32]
538 ADD x8, x8, x12
539 STP q16, q20, [x6]
540 STP q24, q28, [x6, #32]
541 ADD x6, x6, x12
542 SUB x3, x3, x2 // a0 -= kc
543 SUB x15, x15, x2 // a1 -= kc
544 SUB x13, x13, x2 // a2 -= kc
545 SUB x4, x4, x2 // a3 -= kc
546 B.NE 0b
547
548
549 # Restore d8-d13 from stack
550 LDP d12, d13, [sp, 32]
551 LDP d10, d11, [sp, 16]
552 LDP d8, d9, [sp], 48
553 RET
554
555 # Remainder- 4 to 12 bytes of A
556 # Although C4, its safe to read 16 bytes.
557 .p2align 3
558 4:
559 AND x0, x2, 15 // kc remainder 4 to 12
560 5:
561 LDP q8, q9, [x5], 32
562 LDP q10, q11, [x5], 32
563 LD1 {v0.16b}, [x3], x0
564 LD1 {v1.16b}, [x15], x0
565 LD1 {v2.16b}, [x13], x0
566 LD1 {v3.16b}, [x4], x0
567 SDOT v16.4s, v8.16b, v0.4b[0]
568 SDOT v17.4s, v8.16b, v1.4b[0]
569 SDOT v18.4s, v8.16b, v2.4b[0]
570 SDOT v19.4s, v8.16b, v3.4b[0]
571 SDOT v20.4s, v9.16b, v0.4b[0]
572 SDOT v21.4s, v9.16b, v1.4b[0]
573 SDOT v22.4s, v9.16b, v2.4b[0]
574 SDOT v23.4s, v9.16b, v3.4b[0]
575 SDOT v24.4s, v10.16b, v0.4b[0]
576 SDOT v25.4s, v10.16b, v1.4b[0]
577 SDOT v26.4s, v10.16b, v2.4b[0]
578 SDOT v27.4s, v10.16b, v3.4b[0]
579 SDOT v28.4s, v11.16b, v0.4b[0]
580 SDOT v29.4s, v11.16b, v1.4b[0]
581 SDOT v30.4s, v11.16b, v2.4b[0]
582 SDOT v31.4s, v11.16b, v3.4b[0]
583 CMP x0, 4
584 B.LS 3b
585 LDP q8, q9, [x5], 32
586 LDP q10, q11, [x5], 32
587 SDOT v16.4s, v8.16b, v0.4b[1]
588 SDOT v17.4s, v8.16b, v1.4b[1]
589 SDOT v18.4s, v8.16b, v2.4b[1]
590 SDOT v19.4s, v8.16b, v3.4b[1]
591 SDOT v20.4s, v9.16b, v0.4b[1]
592 SDOT v21.4s, v9.16b, v1.4b[1]
593 SDOT v22.4s, v9.16b, v2.4b[1]
594 SDOT v23.4s, v9.16b, v3.4b[1]
595 SDOT v24.4s, v10.16b, v0.4b[1]
596 SDOT v25.4s, v10.16b, v1.4b[1]
597 SDOT v26.4s, v10.16b, v2.4b[1]
598 SDOT v27.4s, v10.16b, v3.4b[1]
599 SDOT v28.4s, v11.16b, v0.4b[1]
600 SDOT v29.4s, v11.16b, v1.4b[1]
601 SDOT v30.4s, v11.16b, v2.4b[1]
602 SDOT v31.4s, v11.16b, v3.4b[1]
603 CMP x0, 8
604 B.LS 3b
605 LDP q8, q9, [x5], 32
606 LDP q10, q11, [x5], 32
607 SDOT v16.4s, v8.16b, v0.4b[2]
608 SDOT v17.4s, v8.16b, v1.4b[2]
609 SDOT v18.4s, v8.16b, v2.4b[2]
610 SDOT v19.4s, v8.16b, v3.4b[2]
611 SDOT v20.4s, v9.16b, v0.4b[2]
612 SDOT v21.4s, v9.16b, v1.4b[2]
613 SDOT v22.4s, v9.16b, v2.4b[2]
614 SDOT v23.4s, v9.16b, v3.4b[2]
615 SDOT v24.4s, v10.16b, v0.4b[2]
616 SDOT v25.4s, v10.16b, v1.4b[2]
617 SDOT v26.4s, v10.16b, v2.4b[2]
618 SDOT v27.4s, v10.16b, v3.4b[2]
619 SDOT v28.4s, v11.16b, v0.4b[2]
620 SDOT v29.4s, v11.16b, v1.4b[2]
621 SDOT v30.4s, v11.16b, v2.4b[2]
622 SDOT v31.4s, v11.16b, v3.4b[2]
623 B 3b
624
625 # Store odd width
626 .p2align 3
627 6:
628 TBZ x1, 3, 7f
629 STP q19, q23, [x7]
630 STP q18, q22, [x9]
631 STP q17, q21, [x8]
632 STP q16, q20, [x6]
633 MOV v16.16b, v24.16b
634 MOV v17.16b, v25.16b
635 MOV v18.16b, v26.16b
636 MOV v19.16b, v27.16b
637 MOV v20.16b, v28.16b
638 MOV v21.16b, v29.16b
639 MOV v22.16b, v30.16b
640 MOV v23.16b, v31.16b
641 ADD x6, x6, #32
642 ADD x7, x7, #32
643 ADD x8, x8, #32
644 ADD x9, x9, #32
645 7:
646 TBZ x1, 2, 8f
647 STR q19, [x7]
648 STR q18, [x9]
649 STR q17, [x8]
650 STR q16, [x6]
651 MOV v16.16b, v20.16b
652 MOV v17.16b, v21.16b
653 MOV v18.16b, v22.16b
654 MOV v19.16b, v23.16b
655 ADD x6, x6, #16
656 ADD x7, x7, #16
657 ADD x8, x8, #16
658 ADD x9, x9, #16
659 8:
660 TBZ x1, 1, 9f
661 ST1 {v19.2s}, [x7]
662 ST1 {v18.2s}, [x9]
663 ST1 {v17.2s}, [x8]
664 ST1 {v16.2s}, [x6]
665 DUP d16, v16.d[1]
666 DUP d17, v17.d[1]
667 DUP d18, v18.d[1]
668 DUP d19, v19.d[1]
669 ADD x6, x6, #8
670 ADD x7, x7, #8
671 ADD x8, x8, #8
672 ADD x9, x9, #8
673 9:
674 TBZ x1, 0, 10f
675 STR s19, [x7]
676 STR s18, [x9]
677 STR s17, [x8]
678 STR s16, [x6]
679 10:
680 # Restore d8-d13 from stack
681 LDP d12, d13, [sp, 32]
682 LDP d10, d11, [sp, 16]
683 LDP d8, d9, [sp], 48
684 RET
685
686 END_FUNCTION xnn_qd8_f32_qc8w_gemm_minmax_ukernel_4x16c4__asm_aarch64_neondot_cortex_a55
687
688 #ifdef __ELF__
689 .section ".note.GNU-stack","",%progbits
690 #endif