Debian Codegraph @

src/qd8-f16-qc8w-gemm/gen/qd8-f16-qc8w-gemm-4x16c4-minmax-asm-aarch64-neondotfp16arith-cortex-a55.S

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-gemm/4x16c4-aarch64-neondot-cortex-a55.S.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2020 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include "xnnpack/assembly.h"
12
13 # void xnn_qd8_f16_qc8w_gemm_minmax_ukernel_4x16c4__asm_aarch64_neondotfp16arith_cortex_a55(
14 # size_t mr, x0
15 # size_t nc, x1
16 # size_t kc, x2 / x0
17 # const int8_t* restrict a, x3
18 # size_t a_stride, x4
19 # const void* restrict w, x5
20 # int8_t* restrict c, x6
21 # size_t cm_stride, x7
22 # size_t cn_stride, [sp] -> x12
23 # const union xnn_f16_minmax_params *params, [sp + 8] -> x11
24 # const struct xnn_qd8_quantization_params *quantization_params) [sp + 16] -> x16
25
26 # d8-d15, x19-x30 need to be preserved if used. x18 is reserved by the OS.
27
28 // Register usage
29 // A0 x3 v0 v4
30 // A1 x15 v1 v5
31 // A2 x13 v2 v6
32 // A3 x4 v3 v7
33 // B x5 v8 v9 v10 v11
34 // C0 x6 v16 v20 v24 v28
35 // C1 x8 v17 v21 v25 v29
36 // C2 x9 v18 v22 v26 v30
37 // C3 x7 v19 v23 v27 v31
38 // temp x14 for Cortex-A55 loads
39 // unused v14 v15
40
41 BEGIN_FUNCTION xnn_qd8_f16_qc8w_gemm_minmax_ukernel_4x16c4__asm_aarch64_neondotfp16arith_cortex_a55
42
43 # Clamp A and C pointers
44 CMP x0, 2 // if mr < 2
45
46 LDP x12, x11, [sp] // cn_stride, params
47
48 ADD x15, x3, x4 // a1 = a0 + a_stride
49 ADD x8, x6, x7 // c1 = c0 + cm_stride
50
51 STP d8, d9, [sp, -48]!
52 STP d12, d13, [sp, 32]
53 LDR x16, [sp, 64] // &quantization_params[0].zero_point
54 LD2 {v12.4s, v13.4s}, [x16] // v12 zero_point, v13 scale
55
56 CSEL x15, x3, x15, LO // a1 = a0
57 CSEL x8, x6, x8, LO // c1 = c0
58 ADD x2, x2, 3 // kc = (kc + 3) & ~3
59
60 ADD x13, x15, x4 // a2 = a1 + a_stride
61 ADD x9, x8, x7 // c2 = c1 + cm_stride
62 // if mr <= 2
63 CSEL x13, x15, x13, LS // a2 = a1
64 CSEL x9, x8, x9, LS // c2 = c1
65 BIC x2, x2, 3
66
67 STP d10, d11, [sp, 16]
68
69 CMP x0, 4 // if mr < 4
70 ADD x4, x13, x4 // a3 = a2 + a_stride
71 ADD x7, x9, x7 // c3 = c2 + cm_stride
72 CSEL x4, x13, x4, LO // a3 = a2
73 CSEL x7, x9, x7, LO // c3 = c2
74
75 .p2align 3
76 0:
77 # Load initial bias from w into accumulators
78 LDP q16, q20, [x5], 32
79 SUBS x0, x2, 16 // k = kc - 16
80 MUL v17.4s, v16.4s, v12.s[1]
81 MUL v18.4s, v16.4s, v12.s[2]
82 LDP q24, q28, [x5], 32
83 MUL v19.4s, v16.4s, v12.s[3]
84 MUL v21.4s, v20.4s, v12.s[1]
85 MUL v22.4s, v20.4s, v12.s[2]
86 MUL v23.4s, v20.4s, v12.s[3]
87 MUL v25.4s, v24.4s, v12.s[1]
88 MUL v26.4s, v24.4s, v12.s[2]
89 MUL v27.4s, v24.4s, v12.s[3]
90 MUL v29.4s, v28.4s, v12.s[1]
91 MUL v30.4s, v28.4s, v12.s[2]
92 MUL v31.4s, v28.4s, v12.s[3]
93 MUL v24.4s, v24.4s, v12.s[0]
94 MUL v28.4s, v28.4s, v12.s[0]
95 MUL v16.4s, v16.4s, v12.s[0]
96 MUL v20.4s, v20.4s, v12.s[0]
97
98 # Is there at least 16 bytes for prologue/epilogue?
99 B.LO 4f
100
101 # prologue - read A and B values for block 0 and 1
102 LDR d0, [x3], 8
103 LDR q8, [x5], 16
104 LDR d1, [x15], 8
105 LDR d2, [x13], 8
106 LDR d3, [x4], 8
107 SUBS x0, x0, 16 // is there 16 for main loop?
108 LDR d9, [x5], 8
109 LDR x14, [x5], 8
110 # Is there at least 16 bytes for main loop?
111 B.LO 2f
112
113 # Main loop - 16 bytes of A in 4 groups.
114 # 4 row of 4 vectors wide = 16 sdot instructions for 4 channels
115 # 4 LD64 for A
116 # 4 LD128 for W. = 2 LD64 + INS.
117 # for each 4 sdot, 1 LD64 for A, 2 LD64 for W + INS.
118
119 .p2align 3
120 1:
121 # BLOCK 0
122 SDOT v16.4s, v8.16b, v0.4b[0]
123 LDR d10, [x5], 8
124 SDOT v17.4s, v8.16b, v1.4b[0]
125 INS v9.d[1], x14
126 SDOT v18.4s, v8.16b, v2.4b[0]
127 LDR x14, [x5], 8
128 SDOT v19.4s, v8.16b, v3.4b[0]
129 LDR d4, [x3], 8
130
131 # BLOCK 1
132 SDOT v20.4s, v9.16b, v0.4b[0]
133 LDR d11, [x5], 8
134 SDOT v21.4s, v9.16b, v1.4b[0]
135 INS v10.d[1], x14
136 SDOT v22.4s, v9.16b, v2.4b[0]
137 LDR x14, [x5], 8
138 SDOT v23.4s, v9.16b, v3.4b[0]
139 LDR d5, [x15], 8
140
141 # BLOCK 2
142 SDOT v24.4s, v10.16b, v0.4b[0]
143 LDR d8, [x5], 8
144 SDOT v25.4s, v10.16b, v1.4b[0]
145 INS v11.d[1], x14
146 SDOT v26.4s, v10.16b, v2.4b[0]
147 LDR x14, [x5], 8
148 SDOT v27.4s, v10.16b, v3.4b[0]
149 LDR d6, [x13], 8
150
151 # BLOCK 3
152 SDOT v28.4s, v11.16b, v0.4b[0]
153 LDR d9, [x5], 8
154 SDOT v29.4s, v11.16b, v1.4b[0]
155 INS v8.d[1], x14
156 SDOT v30.4s, v11.16b, v2.4b[0]
157 LDR x14, [x5], 8
158 SDOT v31.4s, v11.16b, v3.4b[0]
159 LDR d7, [x4], 8
160
161 # BLOCK 0
162 SDOT v16.4s, v8.16b, v0.4b[1]
163 LDR d10, [x5], 8
164 SDOT v17.4s, v8.16b, v1.4b[1]
165 INS v9.d[1], x14
166 SDOT v18.4s, v8.16b, v2.4b[1]
167 LDR x14, [x5], 8
168 SDOT v19.4s, v8.16b, v3.4b[1]
169
170 # BLOCK 1
171 SDOT v20.4s, v9.16b, v0.4b[1]
172 LDR d11, [x5], 8
173 SDOT v21.4s, v9.16b, v1.4b[1]
174 INS v10.d[1], x14
175 SDOT v22.4s, v9.16b, v2.4b[1]
176 LDR x14, [x5], 8
177 SDOT v23.4s, v9.16b, v3.4b[1]
178
179 # BLOCK 2
180 SDOT v24.4s, v10.16b, v0.4b[1]
181 LDR d8, [x5], 8
182 SDOT v25.4s, v10.16b, v1.4b[1]
183 INS v11.d[1], x14
184 SDOT v26.4s, v10.16b, v2.4b[1]
185 LDR x14, [x5], 8
186 SDOT v27.4s, v10.16b, v3.4b[1]
187
188 # BLOCK 3
189 SDOT v28.4s, v11.16b, v0.4b[1]
190 LDR d9, [x5], 8
191 SDOT v29.4s, v11.16b, v1.4b[1]
192 INS v8.d[1], x14
193 SDOT v30.4s, v11.16b, v2.4b[1]
194 LDR x14, [x5], 8
195 SDOT v31.4s, v11.16b, v3.4b[1]
196
197 # BLOCK 0
198 SDOT v16.4s, v8.16b, v4.4b[0]
199 LDR d10, [x5], 8
200 SDOT v17.4s, v8.16b, v5.4b[0]
201 INS v9.d[1], x14
202 SDOT v18.4s, v8.16b, v6.4b[0]
203 LDR x14, [x5], 8
204 SDOT v19.4s, v8.16b, v7.4b[0]
205 LDR d0, [x3], 8
206
207 # BLOCK 1
208 SDOT v20.4s, v9.16b, v4.4b[0]
209 LDR d11, [x5], 8
210 SDOT v21.4s, v9.16b, v5.4b[0]
211 INS v10.d[1], x14
212 SDOT v22.4s, v9.16b, v6.4b[0]
213 LDR x14, [x5], 8
214 SDOT v23.4s, v9.16b, v7.4b[0]
215 LDR d1, [x15], 8
216
217 # BLOCK 2
218 SDOT v24.4s, v10.16b, v4.4b[0]
219 LDR d8, [x5], 8
220 SDOT v25.4s, v10.16b, v5.4b[0]
221 INS v11.d[1], x14
222 SDOT v26.4s, v10.16b, v6.4b[0]
223 LDR x14, [x5], 8
224 SDOT v27.4s, v10.16b, v7.4b[0]
225 LDR d2, [x13], 8
226
227 # BLOCK 3
228 SDOT v28.4s, v11.16b, v4.4b[0]
229 LDR d9, [x5], 8
230 SDOT v29.4s, v11.16b, v5.4b[0]
231 INS v8.d[1], x14
232 SDOT v30.4s, v11.16b, v6.4b[0]
233 LDR x14, [x5], 8
234 SDOT v31.4s, v11.16b, v7.4b[0]
235 LDR d3, [x4], 8
236
237 # BLOCK 0
238 SDOT v16.4s, v8.16b, v4.4b[1]
239 LDR d10, [x5], 8
240 SDOT v17.4s, v8.16b, v5.4b[1]
241 INS v9.d[1], x14
242 SDOT v18.4s, v8.16b, v6.4b[1]
243 LDR x14, [x5], 8
244 SDOT v19.4s, v8.16b, v7.4b[1]
245
246 # BLOCK 1
247 SDOT v20.4s, v9.16b, v4.4b[1]
248 LDR d11, [x5], 8
249 SDOT v21.4s, v9.16b, v5.4b[1]
250 INS v10.d[1], x14
251 SDOT v22.4s, v9.16b, v6.4b[1]
252 LDR x14, [x5], 8
253 SDOT v23.4s, v9.16b, v7.4b[1]
254
255 # BLOCK 2
256 SDOT v24.4s, v10.16b, v4.4b[1]
257 LDR d8, [x5], 8 // First B values for block 0 and 1
258 SDOT v25.4s, v10.16b, v5.4b[1]
259 INS v11.d[1], x14
260 SDOT v26.4s, v10.16b, v6.4b[1]
261 LDR x14, [x5], 8
262 SDOT v27.4s, v10.16b, v7.4b[1]
263 SUBS x0, x0, 16
264
265 # BLOCK 3
266 SDOT v28.4s, v11.16b, v4.4b[1]
267 LDR d9, [x5], 8
268 SDOT v29.4s, v11.16b, v5.4b[1]
269 INS v8.d[1], x14
270 SDOT v30.4s, v11.16b, v6.4b[1]
271 LDR x14, [x5], 8
272 SDOT v31.4s, v11.16b, v7.4b[1]
273 B.HS 1b
274
275 # Epilogue. Same as main loop but no preloads in final group
276 2:
277 # BLOCK 0
278 SDOT v16.4s, v8.16b, v0.4b[0]
279 LDR d10, [x5], 8
280 SDOT v17.4s, v8.16b, v1.4b[0]
281 INS v9.d[1], x14
282 SDOT v18.4s, v8.16b, v2.4b[0]
283 LDR x14, [x5], 8
284 SDOT v19.4s, v8.16b, v3.4b[0]
285 LDR d4, [x3], 8
286
287 # BLOCK 1
288 SDOT v20.4s, v9.16b, v0.4b[0]
289 LDR d11, [x5], 8
290 SDOT v21.4s, v9.16b, v1.4b[0]
291 INS v10.d[1], x14
292 SDOT v22.4s, v9.16b, v2.4b[0]
293 LDR x14, [x5], 8
294 SDOT v23.4s, v9.16b, v3.4b[0]
295 LDR d5, [x15], 8
296
297 # BLOCK 2
298 SDOT v24.4s, v10.16b, v0.4b[0]
299 LDR d8, [x5], 8
300 SDOT v25.4s, v10.16b, v1.4b[0]
301 INS v11.d[1], x14
302 SDOT v26.4s, v10.16b, v2.4b[0]
303 LDR x14, [x5], 8
304 SDOT v27.4s, v10.16b, v3.4b[0]
305 LDR d6, [x13], 8
306
307 # BLOCK 3
308 SDOT v28.4s, v11.16b, v0.4b[0]
309 LDR d9, [x5], 8
310 SDOT v29.4s, v11.16b, v1.4b[0]
311 INS v8.d[1], x14
312 SDOT v30.4s, v11.16b, v2.4b[0]
313 LDR x14, [x5], 8
314 SDOT v31.4s, v11.16b, v3.4b[0]
315 LDR d7, [x4], 8
316
317 # BLOCK 0
318 SDOT v16.4s, v8.16b, v0.4b[1]
319 LDR d10, [x5], 8
320 SDOT v17.4s, v8.16b, v1.4b[1]
321 INS v9.d[1], x14
322 SDOT v18.4s, v8.16b, v2.4b[1]
323 LDR x14, [x5], 8
324 SDOT v19.4s, v8.16b, v3.4b[1]
325
326 # BLOCK 1
327 SDOT v20.4s, v9.16b, v0.4b[1]
328 LDR d11, [x5], 8
329 SDOT v21.4s, v9.16b, v1.4b[1]
330 INS v10.d[1], x14
331 SDOT v22.4s, v9.16b, v2.4b[1]
332 LDR x14, [x5], 8
333 SDOT v23.4s, v9.16b, v3.4b[1]
334
335 # BLOCK 2
336 SDOT v24.4s, v10.16b, v0.4b[1]
337 LDR d8, [x5], 8
338 SDOT v25.4s, v10.16b, v1.4b[1]
339 INS v11.d[1], x14
340 SDOT v26.4s, v10.16b, v2.4b[1]
341 LDR x14, [x5], 8
342 SDOT v27.4s, v10.16b, v3.4b[1]
343
344 # BLOCK 3
345 SDOT v28.4s, v11.16b, v0.4b[1]
346 LDR d9, [x5], 8
347 SDOT v29.4s, v11.16b, v1.4b[1]
348 INS v8.d[1], x14
349 SDOT v30.4s, v11.16b, v2.4b[1]
350 LDR x14, [x5], 8
351 SDOT v31.4s, v11.16b, v3.4b[1]
352
353 # BLOCK 0
354 SDOT v16.4s, v8.16b, v4.4b[0]
355 LDR d10, [x5], 8
356 SDOT v17.4s, v8.16b, v5.4b[0]
357 INS v9.d[1], x14
358 SDOT v18.4s, v8.16b, v6.4b[0]
359 LDR x14, [x5], 8
360 SDOT v19.4s, v8.16b, v7.4b[0]
361
362 # BLOCK 1
363 SDOT v20.4s, v9.16b, v4.4b[0]
364 LDR d11, [x5], 8
365 SDOT v21.4s, v9.16b, v5.4b[0]
366 INS v10.d[1], x14
367 SDOT v22.4s, v9.16b, v6.4b[0]
368 LDR x14, [x5], 8
369 SDOT v23.4s, v9.16b, v7.4b[0]
370
371 # BLOCK 2
372 SDOT v24.4s, v10.16b, v4.4b[0]
373 LDR d8, [x5], 8
374 SDOT v25.4s, v10.16b, v5.4b[0]
375 INS v11.d[1], x14
376 SDOT v26.4s, v10.16b, v6.4b[0]
377 LDR x14, [x5], 8
378 SDOT v27.4s, v10.16b, v7.4b[0]
379
380 # BLOCK 3
381 SDOT v28.4s, v11.16b, v4.4b[0]
382 LDR d9, [x5], 8
383 SDOT v29.4s, v11.16b, v5.4b[0]
384 INS v8.d[1], x14
385 SDOT v30.4s, v11.16b, v6.4b[0]
386 LDR x14, [x5], 8
387 SDOT v31.4s, v11.16b, v7.4b[0]
388
389 # BLOCK 0
390 SDOT v16.4s, v8.16b, v4.4b[1]
391 LDR d10, [x5], 8
392 SDOT v17.4s, v8.16b, v5.4b[1]
393 INS v9.d[1], x14
394 SDOT v18.4s, v8.16b, v6.4b[1]
395 LDR x14, [x5], 8
396 SDOT v19.4s, v8.16b, v7.4b[1]
397
398 # BLOCK 1
399 SDOT v20.4s, v9.16b, v4.4b[1]
400 LDR d11, [x5], 8
401 SDOT v21.4s, v9.16b, v5.4b[1]
402 INS v10.d[1], x14
403 SDOT v22.4s, v9.16b, v6.4b[1]
404 LDR x14, [x5], 8
405 SDOT v23.4s, v9.16b, v7.4b[1]
406
407 # BLOCK 2
408 SDOT v24.4s, v10.16b, v4.4b[1]
409 SDOT v25.4s, v10.16b, v5.4b[1]
410 INS v11.d[1], x14
411 SDOT v26.4s, v10.16b, v6.4b[1]
412 SDOT v27.4s, v10.16b, v7.4b[1]
413 AND x0, x2, 15 // kc remainder 0 to 12
414
415 # BLOCK 3
416 SDOT v28.4s, v11.16b, v4.4b[1]
417 SDOT v29.4s, v11.16b, v5.4b[1]
418 SDOT v30.4s, v11.16b, v6.4b[1]
419 SDOT v31.4s, v11.16b, v7.4b[1]
420
421 # Is there a remainder?- 4 to 12 bytes of A
422 CBNZ x0, 5f
423
424 .p2align 3
425 3:
426 SCVTF v16.4s, v16.4s
427 SCVTF v17.4s, v17.4s
428 SCVTF v18.4s, v18.4s
429 SCVTF v19.4s, v19.4s
430 SCVTF v20.4s, v20.4s
431 SCVTF v21.4s, v21.4s
432 SCVTF v22.4s, v22.4s
433 SCVTF v23.4s, v23.4s
434 LDP q0, q1, [x5], 32 // kernel_scale
435 SCVTF v24.4s, v24.4s
436 SCVTF v25.4s, v25.4s
437 SCVTF v26.4s, v26.4s
438 SCVTF v27.4s, v27.4s
439 SCVTF v28.4s, v28.4s
440 SCVTF v29.4s, v29.4s
441 SCVTF v30.4s, v30.4s
442 SCVTF v31.4s, v31.4s
443 LDP q2, q3, [x5], 32
444 FMUL v4.4s, v0.4s, v13.s[0] // kernel_scale * scale
445 FMUL v5.4s, v1.4s, v13.s[0]
446 FMUL v6.4s, v2.4s, v13.s[0]
447 FMUL v7.4s, v3.4s, v13.s[0]
448 FMUL v8.4s, v0.4s, v13.s[1]
449 FMUL v9.4s, v1.4s, v13.s[1]
450 FMUL v10.4s, v2.4s, v13.s[1]
451 FMUL v11.4s, v3.4s, v13.s[1]
452 FMUL v16.4s, v16.4s, v4.4s
453 FMUL v20.4s, v20.4s, v5.4s
454 FMUL v24.4s, v24.4s, v6.4s
455 FMUL v28.4s, v28.4s, v7.4s
456 FMUL v17.4s, v17.4s, v8.4s
457 FMUL v21.4s, v21.4s, v9.4s
458 FMUL v25.4s, v25.4s, v10.4s
459 FMUL v29.4s, v29.4s, v11.4s
460 FMUL v4.4s, v0.4s, v13.s[2]
461 FMUL v5.4s, v1.4s, v13.s[2]
462 FMUL v6.4s, v2.4s, v13.s[2]
463 FMUL v7.4s, v3.4s, v13.s[2]
464 FMUL v8.4s, v0.4s, v13.s[3]
465 FMUL v9.4s, v1.4s, v13.s[3]
466 FMUL v10.4s, v2.4s, v13.s[3]
467 FMUL v11.4s, v3.4s, v13.s[3]
468 LDP q0, q1, [x5], 32 // bias
469 FMUL v18.4s, v18.4s, v4.4s
470 FMUL v22.4s, v22.4s, v5.4s
471 FMUL v26.4s, v26.4s, v6.4s
472 FMUL v30.4s, v30.4s, v7.4s
473 FMUL v19.4s, v19.4s, v8.4s
474 FMUL v23.4s, v23.4s, v9.4s
475 FMUL v27.4s, v27.4s, v10.4s
476 FMUL v31.4s, v31.4s, v11.4s
477 LDP q2, q3, [x5], 32
478 FADD v16.4s, v16.4s, v0.4s
479 FADD v17.4s, v17.4s, v0.4s
480 FADD v18.4s, v18.4s, v0.4s
481 FADD v19.4s, v19.4s, v0.4s
482 FADD v20.4s, v20.4s, v1.4s
483 FADD v21.4s, v21.4s, v1.4s
484 FADD v22.4s, v22.4s, v1.4s
485 FADD v23.4s, v23.4s, v1.4s
486 LD2R {v0.8h, v1.8h}, [x11] // min max
487 FADD v24.4s, v24.4s, v2.4s
488 FADD v25.4s, v25.4s, v2.4s
489 FADD v26.4s, v26.4s, v2.4s
490 FADD v27.4s, v27.4s, v2.4s
491 FADD v28.4s, v28.4s, v3.4s
492 FADD v29.4s, v29.4s, v3.4s
493 FADD v30.4s, v30.4s, v3.4s
494 FADD v31.4s, v31.4s, v3.4s
495 FCVTN v19.4h, v19.4s
496 FCVTN v27.4h, v27.4s
497 FCVTN v18.4h, v18.4s
498 FCVTN v26.4h, v26.4s
499 FCVTN v17.4h, v17.4s
500 FCVTN v25.4h, v25.4s
501 FCVTN v16.4h, v16.4s
502 FCVTN v24.4h, v24.4s
503 FCVTN2 v19.8h, v23.4s
504 FCVTN2 v27.8h, v31.4s
505 FCVTN2 v18.8h, v22.4s
506 FCVTN2 v26.8h, v30.4s
507 FCVTN2 v17.8h, v21.4s
508 FCVTN2 v25.8h, v29.4s
509 FCVTN2 v16.8h, v20.4s
510 FCVTN2 v24.8h, v28.4s
511 FMAX v19.8h, v19.8h, v0.8h
512 FMAX v27.8h, v27.8h, v0.8h
513 FMAX v18.8h, v18.8h, v0.8h
514 FMAX v26.8h, v26.8h, v0.8h
515 FMAX v17.8h, v17.8h, v0.8h
516 FMAX v25.8h, v25.8h, v0.8h
517 FMAX v16.8h, v16.8h, v0.8h
518 FMAX v24.8h, v24.8h, v0.8h
519 FMIN v19.8h, v19.8h, v1.8h
520 FMIN v27.8h, v27.8h, v1.8h
521 FMIN v18.8h, v18.8h, v1.8h
522 FMIN v26.8h, v26.8h, v1.8h
523 FMIN v17.8h, v17.8h, v1.8h
524 FMIN v25.8h, v25.8h, v1.8h
525 FMIN v16.8h, v16.8h, v1.8h
526 FMIN v24.8h, v24.8h, v1.8h
527
528 SUBS x1, x1, 16
529 B.LO 6f
530 STP q19, q27, [x7]
531 ADD x7, x7, x12
532 STP q18, q26, [x9]
533 ADD x9, x9, x12
534 STP q17, q25, [x8]
535 ADD x8, x8, x12
536 STP q16, q24, [x6]
537 ADD x6, x6, x12
538 SUB x3, x3, x2 // a0 -= kc
539 SUB x15, x15, x2 // a1 -= kc
540 SUB x13, x13, x2 // a2 -= kc
541 SUB x4, x4, x2 // a3 -= kc
542 B.NE 0b
543
544
545 # Restore d8-d13 from stack
546 LDP d12, d13, [sp, 32]
547 LDP d10, d11, [sp, 16]
548 LDP d8, d9, [sp], 48
549 RET
550
551 # Remainder- 4 to 12 bytes of A
552 # Although C4, its safe to read 16 bytes.
553 .p2align 3
554 4:
555 AND x0, x2, 15 // kc remainder 4 to 12
556 5:
557 LDP q8, q9, [x5], 32
558 LDP q10, q11, [x5], 32
559 LD1 {v0.16b}, [x3], x0
560 LD1 {v1.16b}, [x15], x0
561 LD1 {v2.16b}, [x13], x0
562 LD1 {v3.16b}, [x4], x0
563 SDOT v16.4s, v8.16b, v0.4b[0]
564 SDOT v17.4s, v8.16b, v1.4b[0]
565 SDOT v18.4s, v8.16b, v2.4b[0]
566 SDOT v19.4s, v8.16b, v3.4b[0]
567 SDOT v20.4s, v9.16b, v0.4b[0]
568 SDOT v21.4s, v9.16b, v1.4b[0]
569 SDOT v22.4s, v9.16b, v2.4b[0]
570 SDOT v23.4s, v9.16b, v3.4b[0]
571 SDOT v24.4s, v10.16b, v0.4b[0]
572 SDOT v25.4s, v10.16b, v1.4b[0]
573 SDOT v26.4s, v10.16b, v2.4b[0]
574 SDOT v27.4s, v10.16b, v3.4b[0]
575 SDOT v28.4s, v11.16b, v0.4b[0]
576 SDOT v29.4s, v11.16b, v1.4b[0]
577 SDOT v30.4s, v11.16b, v2.4b[0]
578 SDOT v31.4s, v11.16b, v3.4b[0]
579 CMP x0, 4
580 B.LS 3b
581 LDP q8, q9, [x5], 32
582 LDP q10, q11, [x5], 32
583 SDOT v16.4s, v8.16b, v0.4b[1]
584 SDOT v17.4s, v8.16b, v1.4b[1]
585 SDOT v18.4s, v8.16b, v2.4b[1]
586 SDOT v19.4s, v8.16b, v3.4b[1]
587 SDOT v20.4s, v9.16b, v0.4b[1]
588 SDOT v21.4s, v9.16b, v1.4b[1]
589 SDOT v22.4s, v9.16b, v2.4b[1]
590 SDOT v23.4s, v9.16b, v3.4b[1]
591 SDOT v24.4s, v10.16b, v0.4b[1]
592 SDOT v25.4s, v10.16b, v1.4b[1]
593 SDOT v26.4s, v10.16b, v2.4b[1]
594 SDOT v27.4s, v10.16b, v3.4b[1]
595 SDOT v28.4s, v11.16b, v0.4b[1]
596 SDOT v29.4s, v11.16b, v1.4b[1]
597 SDOT v30.4s, v11.16b, v2.4b[1]
598 SDOT v31.4s, v11.16b, v3.4b[1]
599 CMP x0, 8
600 B.LS 3b
601 LDP q8, q9, [x5], 32
602 LDP q10, q11, [x5], 32
603 SDOT v16.4s, v8.16b, v0.4b[2]
604 SDOT v17.4s, v8.16b, v1.4b[2]
605 SDOT v18.4s, v8.16b, v2.4b[2]
606 SDOT v19.4s, v8.16b, v3.4b[2]
607 SDOT v20.4s, v9.16b, v0.4b[2]
608 SDOT v21.4s, v9.16b, v1.4b[2]
609 SDOT v22.4s, v9.16b, v2.4b[2]
610 SDOT v23.4s, v9.16b, v3.4b[2]
611 SDOT v24.4s, v10.16b, v0.4b[2]
612 SDOT v25.4s, v10.16b, v1.4b[2]
613 SDOT v26.4s, v10.16b, v2.4b[2]
614 SDOT v27.4s, v10.16b, v3.4b[2]
615 SDOT v28.4s, v11.16b, v0.4b[2]
616 SDOT v29.4s, v11.16b, v1.4b[2]
617 SDOT v30.4s, v11.16b, v2.4b[2]
618 SDOT v31.4s, v11.16b, v3.4b[2]
619 B 3b
620
621 # Store odd width
622 .p2align 3
623 6:
624 TBZ x1, 3, 7f
625 STR q19, [x7], 16
626 STR q18, [x9], 16
627 MOV v19.16b, v27.16b
628 MOV v18.16b, v26.16b
629 STR q17, [x8], 16
630 STR q16, [x6], 16
631 MOV v17.16b, v25.16b
632 MOV v16.16b, v24.16b
633 7:
634 TBZ x1, 2, 8f
635 STR d19, [x7], 8
636 STR d18, [x9], 8
637 DUP d19, v19.d[1]
638 DUP d18, v18.d[1]
639 STR d17, [x8], 8
640 STR d16, [x6], 8
641 DUP d17, v17.d[1]
642 DUP d16, v16.d[1]
643 8:
644 TBZ x1, 1, 9f
645 STR s19, [x7], 4
646 STR s18, [x9], 4
647 DUP s19, v19.s[1]
648 DUP s18, v18.s[1]
649 STR s17, [x8], 4
650 STR s16, [x6], 4
651 DUP s17, v17.s[1]
652 DUP s16, v16.s[1]
653 9:
654 TBZ x1, 0, 10f
655 STR h19, [x7]
656 STR h18, [x9]
657 STR h17, [x8]
658 STR h16, [x6]
659 10:
660 # Restore d8-d13 from stack
661 LDP d12, d13, [sp, 32]
662 LDP d10, d11, [sp, 16]
663 LDP d8, d9, [sp], 48
664 RET
665
666 END_FUNCTION xnn_qd8_f16_qc8w_gemm_minmax_ukernel_4x16c4__asm_aarch64_neondotfp16arith_cortex_a55
667
668 #ifdef __ELF__
669 .section ".note.GNU-stack","",%progbits
670 #endif