Debian Codegraph @

src/qs8-qc8w-igemm/gen/qs8-qc8w-igemm-4x16c4-minmax-fp32-asm-aarch64-neondot-cortex-a55.S

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-igemm/4x16c4-aarch64-neondot-cortex-a55.S.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2020 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include "xnnpack/assembly.h"
12
13 # void xnn_qs8_qc8w_igemm_minmax_fp32_ukernel_4x16c4__asm_aarch64_neondot_cortex_a55(
14 # size_t mr, x0
15 # size_t nc, x1
16 # size_t kc, x2 / x0
17 # size_t ks, x3 / x9
18 # const int8_t** restrict a, x4
19 # const int8_t* restrict w, x5
20 # int8_t* restrict c, x6
21 # size_t cm_stride, x7
22 # size_t cn_stride, [sp] -> (x0)
23 # size_t a_offset, [sp + 8] -> x8
24 # const int8_t* zero, [sp + 16] -> x12
25 # const union xnn_qs8_qc8w_conv_minmax_params *params) [sp + 24] -> x11
26
27 # d8-d15, x19-x30 need to be preserved if used. x18 is reserved by the OS.
28
29 // Register usage
30 // A0 x13 v0 v4
31 // A1 x14 v1 v5
32 // A2 x15 v2 v6
33 // A3 x10 v3 v7
34 // B x5 v8 v9 v10 v11
35 // C0 x6 v16 v20 v24 v28
36 // C1 x16 v17 v21 v25 v29
37 // C2 x17 v18 v22 v26 v30
38 // C3 x7 v19 v23 v27 v31
39 // unused v12 v13 v14 v15
40
41 // x11 temp for Cortex-A55 loads
42
43 BEGIN_FUNCTION xnn_qs8_qc8w_igemm_minmax_fp32_ukernel_4x16c4__asm_aarch64_neondot_cortex_a55
44
45 # Clamp C pointers
46 CMP x0, 2 // if mr < 2
47 LDR x8, [sp, 8] // Load a_offset
48 ADD x16, x6, x7 // c1 = c0 + cm_stride
49 LDR x12, [sp, 16] // Load zero
50 LDR x11, [sp, 24] // Load params pointer
51 CSEL x16, x6, x16, LO // c1 = c0
52 ADD x2, x2, 3 // kc = (kc + 3) & ~3
53 STP d8, d9, [sp, -32]! // Save d8-d11 on stack
54
55 ADD x17, x16, x7 // c2 = c1 + cm_stride
56 STP d10, d11, [sp, 16]
57 // if mr <= 2
58 CSEL x17, x16, x17, LS // c2 = c1
59 BIC x2, x2, 3
60
61 CMP x0, 4 // if mr < 4
62 ADD x7, x17, x7 // c3 = c2 + cm_stride
63 CSEL x7, x17, x7, LO // c3 = c2
64
65 .p2align 3
66 0:
67 # Load initial bias from w into accumulators
68 LDP q16, q20, [x5], 32
69 MOV v17.16b, v16.16b
70 MOV v18.16b, v16.16b
71 LDP q24, q28, [x5], 32
72 MOV v19.16b, v16.16b
73 MOV v21.16b, v20.16b
74 MOV v22.16b, v20.16b
75 MOV v23.16b, v20.16b
76 MOV v25.16b, v24.16b
77 MOV v26.16b, v24.16b
78 MOV v27.16b, v24.16b
79 MOV v29.16b, v28.16b
80 MOV v30.16b, v28.16b
81 MOV v31.16b, v28.16b
82 MOV x9, x3 // p = ks
83
84 .p2align 3
85 1:
86 # Load next 4 A pointers
87 LDP x13, x14, [x4], 16
88 LDP x15, x10, [x4], 16
89
90 CMP x13, x12 // if a0 == zero
91 ADD x13, x13, x8 // a0 += a_offset
92 CSEL x13, x12, x13, EQ // a0 = zero, else a0 += a_offset
93 CMP x14, x12 // if a1 == zero
94 ADD x14, x14, x8 // a1 += a_offset
95 CSEL x14, x12, x14, EQ // a1 = zero, else a1 += a_offset
96 CMP x15, x12 // if a2 == zero
97 ADD x15, x15, x8 // a2 += a_offset
98 CSEL x15, x12, x15, EQ // a2 = zero, else a2 += a_offset
99 CMP x10, x12 // if a3 == zero
100 ADD x10, x10, x8 // a3 += a_offset
101 CSEL x10, x12, x10, EQ // a3 = zero, else a3 += a_offset
102
103 # Is there at least 16 bytes for prologue/epilogue?
104 SUBS x0, x2, 16 // k = kc - 16
105 B.LO 5f
106
107 # prologue - read A and B values for block 0 and 1
108 LDR d0, [x13], 8
109 LDR q8, [x5], 16
110 LDR d1, [x14], 8
111 LDR d2, [x15], 8
112 LDR d3, [x10], 8
113 SUBS x0, x0, 16 // is there 16 for main loop?
114 LDR d9, [x5], 8
115 LDR x11, [x5], 8
116 # Is there at least 16 bytes for main loop?
117 B.LO 3f
118
119 # Main loop - 16 bytes of A in 4 groups.
120 # 4 row of 4 vectors wide = 16 sdot instructions for 4 channels
121 # 4 LD64 for A
122 # 4 LD128 for W. = 2 LD64 + INS.
123 # for each 4 sdot, 1 LD64 for A, 2 LD64 for W + INS.
124
125 .p2align 3
126 2:
127 # BLOCK 0
128 SDOT v16.4s, v8.16b, v0.4b[0]
129 LDR d10, [x5], 8
130 SDOT v17.4s, v8.16b, v1.4b[0]
131 INS v9.d[1], x11
132 SDOT v18.4s, v8.16b, v2.4b[0]
133 LDR x11, [x5], 8
134 SDOT v19.4s, v8.16b, v3.4b[0]
135 LDR d4, [x13], 8
136
137 # BLOCK 1
138 SDOT v20.4s, v9.16b, v0.4b[0]
139 LDR d11, [x5], 8
140 SDOT v21.4s, v9.16b, v1.4b[0]
141 INS v10.d[1], x11
142 SDOT v22.4s, v9.16b, v2.4b[0]
143 LDR x11, [x5], 8
144 SDOT v23.4s, v9.16b, v3.4b[0]
145 LDR d5, [x14], 8
146
147 # BLOCK 2
148 SDOT v24.4s, v10.16b, v0.4b[0]
149 LDR d8, [x5], 8
150 SDOT v25.4s, v10.16b, v1.4b[0]
151 INS v11.d[1], x11
152 SDOT v26.4s, v10.16b, v2.4b[0]
153 LDR x11, [x5], 8
154 SDOT v27.4s, v10.16b, v3.4b[0]
155 LDR d6, [x15], 8
156
157 # BLOCK 3
158 SDOT v28.4s, v11.16b, v0.4b[0]
159 LDR d9, [x5], 8
160 SDOT v29.4s, v11.16b, v1.4b[0]
161 INS v8.d[1], x11
162 SDOT v30.4s, v11.16b, v2.4b[0]
163 LDR x11, [x5], 8
164 SDOT v31.4s, v11.16b, v3.4b[0]
165 LDR d7, [x10], 8
166
167 # BLOCK 0
168 SDOT v16.4s, v8.16b, v0.4b[1]
169 LDR d10, [x5], 8
170 SDOT v17.4s, v8.16b, v1.4b[1]
171 INS v9.d[1], x11
172 SDOT v18.4s, v8.16b, v2.4b[1]
173 LDR x11, [x5], 8
174 SDOT v19.4s, v8.16b, v3.4b[1]
175
176 # BLOCK 1
177 SDOT v20.4s, v9.16b, v0.4b[1]
178 LDR d11, [x5], 8
179 SDOT v21.4s, v9.16b, v1.4b[1]
180 INS v10.d[1], x11
181 SDOT v22.4s, v9.16b, v2.4b[1]
182 LDR x11, [x5], 8
183 SDOT v23.4s, v9.16b, v3.4b[1]
184
185 # BLOCK 2
186 SDOT v24.4s, v10.16b, v0.4b[1]
187 LDR d8, [x5], 8
188 SDOT v25.4s, v10.16b, v1.4b[1]
189 INS v11.d[1], x11
190 SDOT v26.4s, v10.16b, v2.4b[1]
191 LDR x11, [x5], 8
192 SDOT v27.4s, v10.16b, v3.4b[1]
193
194 # BLOCK 3
195 SDOT v28.4s, v11.16b, v0.4b[1]
196 LDR d9, [x5], 8
197 SDOT v29.4s, v11.16b, v1.4b[1]
198 INS v8.d[1], x11
199 SDOT v30.4s, v11.16b, v2.4b[1]
200 LDR x11, [x5], 8
201 SDOT v31.4s, v11.16b, v3.4b[1]
202
203 # BLOCK 0
204 SDOT v16.4s, v8.16b, v4.4b[0]
205 LDR d10, [x5], 8
206 SDOT v17.4s, v8.16b, v5.4b[0]
207 INS v9.d[1], x11
208 SDOT v18.4s, v8.16b, v6.4b[0]
209 LDR x11, [x5], 8
210 SDOT v19.4s, v8.16b, v7.4b[0]
211 LDR d0, [x13], 8
212
213 # BLOCK 1
214 SDOT v20.4s, v9.16b, v4.4b[0]
215 LDR d11, [x5], 8
216 SDOT v21.4s, v9.16b, v5.4b[0]
217 INS v10.d[1], x11
218 SDOT v22.4s, v9.16b, v6.4b[0]
219 LDR x11, [x5], 8
220 SDOT v23.4s, v9.16b, v7.4b[0]
221 LDR d1, [x14], 8
222
223 # BLOCK 2
224 SDOT v24.4s, v10.16b, v4.4b[0]
225 LDR d8, [x5], 8
226 SDOT v25.4s, v10.16b, v5.4b[0]
227 INS v11.d[1], x11
228 SDOT v26.4s, v10.16b, v6.4b[0]
229 LDR x11, [x5], 8
230 SDOT v27.4s, v10.16b, v7.4b[0]
231 LDR d2, [x15], 8
232
233 # BLOCK 3
234 SDOT v28.4s, v11.16b, v4.4b[0]
235 LDR d9, [x5], 8
236 SDOT v29.4s, v11.16b, v5.4b[0]
237 INS v8.d[1], x11
238 SDOT v30.4s, v11.16b, v6.4b[0]
239 LDR x11, [x5], 8
240 SDOT v31.4s, v11.16b, v7.4b[0]
241 LDR d3, [x10], 8
242
243 # BLOCK 0
244 SDOT v16.4s, v8.16b, v4.4b[1]
245 LDR d10, [x5], 8
246 SDOT v17.4s, v8.16b, v5.4b[1]
247 INS v9.d[1], x11
248 SDOT v18.4s, v8.16b, v6.4b[1]
249 LDR x11, [x5], 8
250 SDOT v19.4s, v8.16b, v7.4b[1]
251
252 # BLOCK 1
253 SDOT v20.4s, v9.16b, v4.4b[1]
254 LDR d11, [x5], 8
255 SDOT v21.4s, v9.16b, v5.4b[1]
256 INS v10.d[1], x11
257 SDOT v22.4s, v9.16b, v6.4b[1]
258 LDR x11, [x5], 8
259 SDOT v23.4s, v9.16b, v7.4b[1]
260
261 # BLOCK 2
262 SDOT v24.4s, v10.16b, v4.4b[1]
263 LDR d8, [x5], 8 // First B values for block 0 and 1
264 SDOT v25.4s, v10.16b, v5.4b[1]
265 INS v11.d[1], x11
266 SDOT v26.4s, v10.16b, v6.4b[1]
267 LDR x11, [x5], 8
268 SDOT v27.4s, v10.16b, v7.4b[1]
269 SUBS x0, x0, 16
270
271 # BLOCK 3
272 SDOT v28.4s, v11.16b, v4.4b[1]
273 LDR d9, [x5], 8
274 SDOT v29.4s, v11.16b, v5.4b[1]
275 INS v8.d[1], x11
276 SDOT v30.4s, v11.16b, v6.4b[1]
277 LDR x11, [x5], 8
278 SDOT v31.4s, v11.16b, v7.4b[1]
279 B.HS 2b
280
281 # Epilogue. Same as main loop but no preloads in final group
282 3:
283 # BLOCK 0
284 SDOT v16.4s, v8.16b, v0.4b[0]
285 LDR d10, [x5], 8
286 SDOT v17.4s, v8.16b, v1.4b[0]
287 INS v9.d[1], x11
288 SDOT v18.4s, v8.16b, v2.4b[0]
289 LDR x11, [x5], 8
290 SDOT v19.4s, v8.16b, v3.4b[0]
291 LDR d4, [x13], 8
292
293 # BLOCK 1
294 SDOT v20.4s, v9.16b, v0.4b[0]
295 LDR d11, [x5], 8
296 SDOT v21.4s, v9.16b, v1.4b[0]
297 INS v10.d[1], x11
298 SDOT v22.4s, v9.16b, v2.4b[0]
299 LDR x11, [x5], 8
300 SDOT v23.4s, v9.16b, v3.4b[0]
301 LDR d5, [x14], 8
302
303 # BLOCK 2
304 SDOT v24.4s, v10.16b, v0.4b[0]
305 LDR d8, [x5], 8
306 SDOT v25.4s, v10.16b, v1.4b[0]
307 INS v11.d[1], x11
308 SDOT v26.4s, v10.16b, v2.4b[0]
309 LDR x11, [x5], 8
310 SDOT v27.4s, v10.16b, v3.4b[0]
311 LDR d6, [x15], 8
312
313 # BLOCK 3
314 SDOT v28.4s, v11.16b, v0.4b[0]
315 LDR d9, [x5], 8
316 SDOT v29.4s, v11.16b, v1.4b[0]
317 INS v8.d[1], x11
318 SDOT v30.4s, v11.16b, v2.4b[0]
319 LDR x11, [x5], 8
320 SDOT v31.4s, v11.16b, v3.4b[0]
321 LDR d7, [x10], 8
322
323 # BLOCK 0
324 SDOT v16.4s, v8.16b, v0.4b[1]
325 LDR d10, [x5], 8
326 SDOT v17.4s, v8.16b, v1.4b[1]
327 INS v9.d[1], x11
328 SDOT v18.4s, v8.16b, v2.4b[1]
329 LDR x11, [x5], 8
330 SDOT v19.4s, v8.16b, v3.4b[1]
331
332 # BLOCK 1
333 SDOT v20.4s, v9.16b, v0.4b[1]
334 LDR d11, [x5], 8
335 SDOT v21.4s, v9.16b, v1.4b[1]
336 INS v10.d[1], x11
337 SDOT v22.4s, v9.16b, v2.4b[1]
338 LDR x11, [x5], 8
339 SDOT v23.4s, v9.16b, v3.4b[1]
340
341 # BLOCK 2
342 SDOT v24.4s, v10.16b, v0.4b[1]
343 LDR d8, [x5], 8
344 SDOT v25.4s, v10.16b, v1.4b[1]
345 INS v11.d[1], x11
346 SDOT v26.4s, v10.16b, v2.4b[1]
347 LDR x11, [x5], 8
348 SDOT v27.4s, v10.16b, v3.4b[1]
349
350 # BLOCK 3
351 SDOT v28.4s, v11.16b, v0.4b[1]
352 LDR d9, [x5], 8
353 SDOT v29.4s, v11.16b, v1.4b[1]
354 INS v8.d[1], x11
355 SDOT v30.4s, v11.16b, v2.4b[1]
356 LDR x11, [x5], 8
357 SDOT v31.4s, v11.16b, v3.4b[1]
358
359 # BLOCK 0
360 SDOT v16.4s, v8.16b, v4.4b[0]
361 LDR d10, [x5], 8
362 SDOT v17.4s, v8.16b, v5.4b[0]
363 INS v9.d[1], x11
364 SDOT v18.4s, v8.16b, v6.4b[0]
365 LDR x11, [x5], 8
366 SDOT v19.4s, v8.16b, v7.4b[0]
367
368 # BLOCK 1
369 SDOT v20.4s, v9.16b, v4.4b[0]
370 LDR d11, [x5], 8
371 SDOT v21.4s, v9.16b, v5.4b[0]
372 INS v10.d[1], x11
373 SDOT v22.4s, v9.16b, v6.4b[0]
374 LDR x11, [x5], 8
375 SDOT v23.4s, v9.16b, v7.4b[0]
376
377 # BLOCK 2
378 SDOT v24.4s, v10.16b, v4.4b[0]
379 LDR d8, [x5], 8
380 SDOT v25.4s, v10.16b, v5.4b[0]
381 INS v11.d[1], x11
382 SDOT v26.4s, v10.16b, v6.4b[0]
383 LDR x11, [x5], 8
384 SDOT v27.4s, v10.16b, v7.4b[0]
385
386 # BLOCK 3
387 SDOT v28.4s, v11.16b, v4.4b[0]
388 LDR d9, [x5], 8
389 SDOT v29.4s, v11.16b, v5.4b[0]
390 INS v8.d[1], x11
391 SDOT v30.4s, v11.16b, v6.4b[0]
392 LDR x11, [x5], 8
393 SDOT v31.4s, v11.16b, v7.4b[0]
394
395 # BLOCK 0
396 SDOT v16.4s, v8.16b, v4.4b[1]
397 LDR d10, [x5], 8
398 SDOT v17.4s, v8.16b, v5.4b[1]
399 INS v9.d[1], x11
400 SDOT v18.4s, v8.16b, v6.4b[1]
401 LDR x11, [x5], 8
402 SDOT v19.4s, v8.16b, v7.4b[1]
403
404 # BLOCK 1
405 SDOT v20.4s, v9.16b, v4.4b[1]
406 LDR d11, [x5], 8
407 SDOT v21.4s, v9.16b, v5.4b[1]
408 INS v10.d[1], x11
409 SDOT v22.4s, v9.16b, v6.4b[1]
410 LDR x11, [x5], 8
411 SDOT v23.4s, v9.16b, v7.4b[1]
412
413 # BLOCK 2
414 SDOT v24.4s, v10.16b, v4.4b[1]
415 SDOT v25.4s, v10.16b, v5.4b[1]
416 INS v11.d[1], x11
417 SDOT v26.4s, v10.16b, v6.4b[1]
418 SDOT v27.4s, v10.16b, v7.4b[1]
419 AND x0, x2, 15 // kc remainder 0 to 12
420
421 # BLOCK 3
422 SDOT v28.4s, v11.16b, v4.4b[1]
423 SDOT v29.4s, v11.16b, v5.4b[1]
424 LDR x11, [sp, 56] // reload params pointer
425 SDOT v30.4s, v11.16b, v6.4b[1]
426 SDOT v31.4s, v11.16b, v7.4b[1]
427
428 # Is there a remainder?- 4 to 12 bytes of A
429 CBNZ x0, 6f
430
431 .p2align 3
432 4:
433 # ks loop
434 SUBS x9, x9, 32 // ks -= MR * sizeof(int8_t*)
435 B.HI 1b
436
437 SCVTF v16.4s, v16.4s
438 SCVTF v17.4s, v17.4s
439 # Load per channel scale values from weights
440 LDR q4, [x5], 16
441 SCVTF v18.4s, v18.4s
442 SCVTF v19.4s, v19.4s
443 LDR q5, [x5], 16
444 SCVTF v20.4s, v20.4s
445 SCVTF v21.4s, v21.4s
446 SCVTF v22.4s, v22.4s
447 SCVTF v23.4s, v23.4s
448 SCVTF v24.4s, v24.4s
449 SCVTF v25.4s, v25.4s
450 SCVTF v26.4s, v26.4s
451 SCVTF v27.4s, v27.4s
452 SCVTF v28.4s, v28.4s
453 SCVTF v29.4s, v29.4s
454 SCVTF v30.4s, v30.4s
455 SCVTF v31.4s, v31.4s
456
457 LDR q6, [x5], 16
458 FMUL v16.4s, v16.4s, v4.4s
459 FMUL v17.4s, v17.4s, v4.4s
460 FMUL v18.4s, v18.4s, v4.4s
461 FMUL v19.4s, v19.4s, v4.4s
462 FMUL v20.4s, v20.4s, v5.4s
463 LDR q4, [x5], 16
464 FMUL v21.4s, v21.4s, v5.4s
465 FMUL v22.4s, v22.4s, v5.4s
466 FMUL v23.4s, v23.4s, v5.4s
467 FMUL v24.4s, v24.4s, v6.4s
468 FMUL v25.4s, v25.4s, v6.4s
469 FMUL v26.4s, v26.4s, v6.4s
470 FMUL v27.4s, v27.4s, v6.4s
471 FMUL v28.4s, v28.4s, v4.4s
472 FMUL v29.4s, v29.4s, v4.4s
473 FMUL v30.4s, v30.4s, v4.4s
474 FMUL v31.4s, v31.4s, v4.4s
475
476 FCVTNS v16.4s, v16.4s
477 FCVTNS v17.4s, v17.4s
478 FCVTNS v18.4s, v18.4s
479 FCVTNS v19.4s, v19.4s
480 FCVTNS v20.4s, v20.4s
481 FCVTNS v21.4s, v21.4s
482 FCVTNS v22.4s, v22.4s
483 FCVTNS v23.4s, v23.4s
484 FCVTNS v24.4s, v24.4s
485 FCVTNS v25.4s, v25.4s
486 FCVTNS v26.4s, v26.4s
487 FCVTNS v27.4s, v27.4s
488 FCVTNS v28.4s, v28.4s
489 FCVTNS v29.4s, v29.4s
490 FCVTNS v30.4s, v30.4s
491 FCVTNS v31.4s, v31.4s
492
493 SQXTN v16.4h, v16.4s
494 SQXTN v17.4h, v17.4s
495 SQXTN v18.4h, v18.4s
496 SQXTN v19.4h, v19.4s
497 SQXTN v24.4h, v24.4s
498 SQXTN v25.4h, v25.4s
499 SQXTN v26.4h, v26.4s
500 SQXTN v27.4h, v27.4s
501 LD1R {v6.8h}, [x11], 2 // add bias
502
503 SQXTN2 v16.8h, v20.4s
504 SQXTN2 v17.8h, v21.4s
505 SQXTN2 v18.8h, v22.4s
506 SQXTN2 v19.8h, v23.4s
507 SQXTN2 v24.8h, v28.4s
508 SQXTN2 v25.8h, v29.4s
509 SQXTN2 v26.8h, v30.4s
510 SQXTN2 v27.8h, v31.4s
511
512 SQADD v16.8h, v16.8h, v6.8h
513 SQADD v17.8h, v17.8h, v6.8h
514 SQADD v18.8h, v18.8h, v6.8h
515 SQADD v19.8h, v19.8h, v6.8h
516 SQADD v24.8h, v24.8h, v6.8h
517 SQADD v25.8h, v25.8h, v6.8h
518 SQADD v26.8h, v26.8h, v6.8h
519 SQADD v27.8h, v27.8h, v6.8h
520 LD1R {v4.16b}, [x11], 1 // clamp min value
521
522 SQXTN v0.8b, v16.8h
523 SQXTN v1.8b, v17.8h
524 SQXTN v2.8b, v18.8h
525 SQXTN v3.8b, v19.8h
526 LD1R {v5.16b}, [x11] // clamp max value
527 SQXTN2 v0.16b, v24.8h
528 SQXTN2 v1.16b, v25.8h
529 SQXTN2 v2.16b, v26.8h
530 SQXTN2 v3.16b, v27.8h
531 LDR x0, [sp, 32] // cn_stride
532 SMAX v0.16b, v0.16b, v4.16b
533 SMAX v1.16b, v1.16b, v4.16b
534 SUB x11, x11, 3 // rewind params pointer
535 SMAX v2.16b, v2.16b, v4.16b
536 SMAX v3.16b, v3.16b, v4.16b
537 SUBS x1, x1, 16
538 SMIN v0.16b, v0.16b, v5.16b
539 SMIN v1.16b, v1.16b, v5.16b
540 SMIN v2.16b, v2.16b, v5.16b
541 SMIN v3.16b, v3.16b, v5.16b
542 B.LO 7f
543
544 # Store full 4 x 16
545 ST1 {v3.16b}, [x7], x0
546 ST1 {v2.16b}, [x17], x0
547 ST1 {v1.16b}, [x16], x0
548 ST1 {v0.16b}, [x6], x0
549
550 SUB x4, x4, x3 // a -= ks
551
552 # nc loop
553 B.HI 0b
554
555 # Restore d8-d11 from stack
556 LDP d10, d11, [sp, 16]
557 LDP d8, d9, [sp], 32
558 RET
559
560 # Remainder- 4 to 12 bytes of A
561 # Although C4, its safe to read 16 bytes.
562 .p2align 3
563 5:
564 AND x0, x2, 15 // kc remainder 4 to 12
565 6:
566 LDR q0, [x13]
567 LDP q8, q9, [x5], 32
568 LDR q1, [x14]
569 LDR q2, [x15]
570 LDR q3, [x10]
571 LDP q10, q11, [x5], 32
572 SDOT v16.4s, v8.16b, v0.4b[0]
573 SDOT v17.4s, v8.16b, v1.4b[0]
574 SDOT v18.4s, v8.16b, v2.4b[0]
575 SDOT v19.4s, v8.16b, v3.4b[0]
576 SDOT v20.4s, v9.16b, v0.4b[0]
577 SDOT v21.4s, v9.16b, v1.4b[0]
578 SDOT v22.4s, v9.16b, v2.4b[0]
579 SDOT v23.4s, v9.16b, v3.4b[0]
580 SDOT v24.4s, v10.16b, v0.4b[0]
581 SDOT v25.4s, v10.16b, v1.4b[0]
582 SDOT v26.4s, v10.16b, v2.4b[0]
583 SDOT v27.4s, v10.16b, v3.4b[0]
584 SDOT v28.4s, v11.16b, v0.4b[0]
585 SDOT v29.4s, v11.16b, v1.4b[0]
586 SDOT v30.4s, v11.16b, v2.4b[0]
587 SDOT v31.4s, v11.16b, v3.4b[0]
588 CMP x0, 4
589 B.LS 4b
590 LDP q8, q9, [x5], 32
591 LDP q10, q11, [x5], 32
592 SDOT v16.4s, v8.16b, v0.4b[1]
593 SDOT v17.4s, v8.16b, v1.4b[1]
594 SDOT v18.4s, v8.16b, v2.4b[1]
595 SDOT v19.4s, v8.16b, v3.4b[1]
596 SDOT v20.4s, v9.16b, v0.4b[1]
597 SDOT v21.4s, v9.16b, v1.4b[1]
598 SDOT v22.4s, v9.16b, v2.4b[1]
599 SDOT v23.4s, v9.16b, v3.4b[1]
600 SDOT v24.4s, v10.16b, v0.4b[1]
601 SDOT v25.4s, v10.16b, v1.4b[1]
602 SDOT v26.4s, v10.16b, v2.4b[1]
603 SDOT v27.4s, v10.16b, v3.4b[1]
604 SDOT v28.4s, v11.16b, v0.4b[1]
605 SDOT v29.4s, v11.16b, v1.4b[1]
606 SDOT v30.4s, v11.16b, v2.4b[1]
607 SDOT v31.4s, v11.16b, v3.4b[1]
608 CMP x0, 8
609 B.LS 4b
610 LDP q8, q9, [x5], 32
611 LDP q10, q11, [x5], 32
612 SDOT v16.4s, v8.16b, v0.4b[2]
613 SDOT v17.4s, v8.16b, v1.4b[2]
614 SDOT v18.4s, v8.16b, v2.4b[2]
615 SDOT v19.4s, v8.16b, v3.4b[2]
616 SDOT v20.4s, v9.16b, v0.4b[2]
617 SDOT v21.4s, v9.16b, v1.4b[2]
618 SDOT v22.4s, v9.16b, v2.4b[2]
619 SDOT v23.4s, v9.16b, v3.4b[2]
620 SDOT v24.4s, v10.16b, v0.4b[2]
621 SDOT v25.4s, v10.16b, v1.4b[2]
622 SDOT v26.4s, v10.16b, v2.4b[2]
623 SDOT v27.4s, v10.16b, v3.4b[2]
624 SDOT v28.4s, v11.16b, v0.4b[2]
625 SDOT v29.4s, v11.16b, v1.4b[2]
626 SDOT v30.4s, v11.16b, v2.4b[2]
627 SDOT v31.4s, v11.16b, v3.4b[2]
628 B 4b
629
630 # Store odd width
631 .p2align 3
632 7:
633 TBZ x1, 3, 8f
634 STR d3, [x7], 8
635 STR d2, [x17], 8
636 DUP d3, v3.d[1]
637 DUP d2, v2.d[1]
638 STR d1, [x16], 8
639 STR d0, [x6], 8
640 DUP d1, v1.d[1]
641 DUP d0, v0.d[1]
642 8:
643 TBZ x1, 2, 9f
644 STR s3, [x7], 4
645 STR s2, [x17], 4
646 DUP s3, v3.s[1]
647 DUP s2, v2.s[1]
648 STR s1, [x16], 4
649 STR s0, [x6], 4
650 DUP s1, v1.s[1]
651 DUP s0, v0.s[1]
652 9:
653 TBZ x1, 1, 10f
654 STR h3, [x7], 2
655 STR h2, [x17], 2
656 DUP h3, v3.h[1]
657 DUP h2, v2.h[1]
658 STR h1, [x16], 2
659 STR h0, [x6], 2
660 DUP h1, v1.h[1]
661 DUP h0, v0.h[1]
662 10:
663 TBZ x1, 0, 11f
664 STR b3, [x7]
665 STR b2, [x17]
666 STR b1, [x16]
667 STR b0, [x6]
668 11:
669 # Restore d8-d11 from stack
670 LDP d10, d11, [sp, 16]
671 LDP d8, d9, [sp], 32
672 RET
673
674 END_FUNCTION xnn_qs8_qc8w_igemm_minmax_fp32_ukernel_4x16c4__asm_aarch64_neondot_cortex_a55
675
676 #ifdef __ELF__
677 .section ".note.GNU-stack","",%progbits
678 #endif