Debian Codegraph @

src/qd8-f32-qc8w-igemm/gen/qd8-f32-qc8w-igemm-4x16c4-minmax-asm-aarch64-neondot-cortex-a55.S

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-igemm/4x16c4-aarch64-neondot-cortex-a55.S.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2020 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include "xnnpack/assembly.h"
12
13 # void xnn_qd8_f32_qc8w_igemm_minmax__ukernel_4x16c4__asm_aarch64_neondot_cortex_a55(
14 # size_t mr, x0
15 # size_t nc, x1
16 # size_t kc, x2 / x0
17 # size_t ks, x3 / x9
18 # const int8_t** restrict a, x4
19 # const int8_t* restrict w, x5
20 # int8_t* restrict c, x6
21 # size_t cm_stride, x7
22 # size_t cn_stride, [sp] -> (x0)
23 # size_t a_offset, [sp + 8] -> x8
24 # const int8_t* zero, [sp + 16] -> x12
25 # const int8_t* zero_data, [sp + 24] -> x19
26 # const union xnn_f32_minmax_params *params, [sp + 32] -> x11
27 # const struct xnn_qd8_quantization_params *quantization_params) [sp + 40] -> x17
28
29 # d8-d15, x19-x30 need to be preserved if used. x18 is reserved by the OS.
30
31 // Register usage
32 // A0 x13 v0 v4
33 // A1 x14 v1 v5
34 // A2 x15 v2 v6
35 // A3 x10 v3 v7
36 // B x5 v8 v9 v10 v11
37 // C0 x6 v16 v20 v24 v28
38 // C1 x16 v17 v21 v25 v29
39 // C2 x17 v18 v22 v26 v30
40 // C3 x7 v19 v23 v27 v31
41 // unused v13, v14 v15
42
43 // x11 temp for Cortex-A55 loads
44
45 BEGIN_FUNCTION xnn_qd8_f32_qc8w_igemm_minmax_ukernel_4x16c4__asm_aarch64_neondot_cortex_a55
46
47 # Clamp C pointers
48 CMP x0, 2 // if mr < 2
49 LDR x8, [sp, 8] // Load a_offset
50 ADD x16, x6, x7 // c1 = c0 + cm_stride
51 LDR x12, [sp, 16] // Load zero
52 LDR x11, [sp, 32] // Load params pointer
53 CSEL x16, x6, x16, LO // c1 = c0
54 ADD x2, x2, 3 // kc = (kc + 3) & ~3
55 STP d8, d9, [sp, -48]! // Save d8-d11 on stack
56 STR x19, [sp, 40] // Save x19 to stack
57 LDR x19, [sp, 72] // Load zero_data
58 STR d12, [sp, 32]
59 LDR x17, [sp, 88] // &quantization_params.zero_point
60 LD1 {v12.4s}, [x17] // zero point and scale
61
62 ADD x17, x16, x7 // c2 = c1 + cm_stride
63 STP d10, d11, [sp, 16]
64 // if mr <= 2
65 CSEL x17, x16, x17, LS // c2 = c1
66 BIC x2, x2, 3
67
68 CMP x0, 4 // if mr < 4
69 ADD x7, x17, x7 // c3 = c2 + cm_stride
70 CSEL x7, x17, x7, LO // c3 = c2
71
72 .p2align 3
73 0:
74 # Load initial bias from w into accumulators
75 LDP q16, q20, [x5], 32
76 MUL v17.4s, v16.4s, v12.s[0]
77 MUL v18.4s, v16.4s, v12.s[0]
78 LDP q24, q28, [x5], 32
79 MUL v19.4s, v16.4s, v12.s[0]
80 MUL v21.4s, v20.4s, v12.s[0]
81 MUL v22.4s, v20.4s, v12.s[0]
82 MUL v23.4s, v20.4s, v12.s[0]
83 MUL v25.4s, v24.4s, v12.s[0]
84 MUL v26.4s, v24.4s, v12.s[0]
85 MUL v27.4s, v24.4s, v12.s[0]
86 MUL v29.4s, v28.4s, v12.s[0]
87 MUL v30.4s, v28.4s, v12.s[0]
88 MUL v31.4s, v28.4s, v12.s[0]
89 MUL v24.4s, v24.4s, v12.s[0]
90 MUL v28.4s, v28.4s, v12.s[0]
91 MUL v16.4s, v16.4s, v12.s[0]
92 MUL v20.4s, v20.4s, v12.s[0]
93 MOV x9, x3 // p = ks
94
95 .p2align 3
96 1:
97 # Load next 4 A pointers
98 LDP x13, x14, [x4], 16
99 LDP x15, x10, [x4], 16
100
101 CMP x13, x12 // if a0 == zero
102 ADD x13, x13, x8 // a0 += a_offset
103 CSEL x13, x19, x13, EQ // a0 = zero_data, else a0 += a_offset
104 CMP x14, x12 // if a1 == zero
105 ADD x14, x14, x8 // a1 += a_offset
106 CSEL x14, x19, x14, EQ // a1 = zero_data, else a1 += a_offset
107 CMP x15, x12 // if a2 == zero
108 ADD x15, x15, x8 // a2 += a_offset
109 CSEL x15, x19, x15, EQ // a2 = zero_data, else a2 += a_offset
110 CMP x10, x12 // if a3 == zero
111 ADD x10, x10, x8 // a3 += a_offset
112 CSEL x10, x19, x10, EQ // a3 = zero_data, else a3 += a_offset
113
114 # Is there at least 16 bytes for prologue/epilogue?
115 SUBS x0, x2, 16 // k = kc - 16
116 B.LO 5f
117
118 # prologue - read A and B values for block 0 and 1
119 LDR d0, [x13], 8
120 LDR q8, [x5], 16
121 LDR d1, [x14], 8
122 LDR d2, [x15], 8
123 LDR d3, [x10], 8
124 SUBS x0, x0, 16 // is there 16 for main loop?
125 LDR d9, [x5], 8
126 LDR x11, [x5], 8
127 # Is there at least 16 bytes for main loop?
128 B.LO 3f
129
130 # Main loop - 16 bytes of A in 4 groups.
131 # 4 row of 4 vectors wide = 16 sdot instructions for 4 channels
132 # 4 LD64 for A
133 # 4 LD128 for W. = 2 LD64 + INS.
134 # for each 4 sdot, 1 LD64 for A, 2 LD64 for W + INS.
135
136 .p2align 3
137 2:
138 # BLOCK 0
139 SDOT v16.4s, v8.16b, v0.4b[0]
140 LDR d10, [x5], 8
141 SDOT v17.4s, v8.16b, v1.4b[0]
142 INS v9.d[1], x11
143 SDOT v18.4s, v8.16b, v2.4b[0]
144 LDR x11, [x5], 8
145 SDOT v19.4s, v8.16b, v3.4b[0]
146 LDR d4, [x13], 8
147
148 # BLOCK 1
149 SDOT v20.4s, v9.16b, v0.4b[0]
150 LDR d11, [x5], 8
151 SDOT v21.4s, v9.16b, v1.4b[0]
152 INS v10.d[1], x11
153 SDOT v22.4s, v9.16b, v2.4b[0]
154 LDR x11, [x5], 8
155 SDOT v23.4s, v9.16b, v3.4b[0]
156 LDR d5, [x14], 8
157
158 # BLOCK 2
159 SDOT v24.4s, v10.16b, v0.4b[0]
160 LDR d8, [x5], 8
161 SDOT v25.4s, v10.16b, v1.4b[0]
162 INS v11.d[1], x11
163 SDOT v26.4s, v10.16b, v2.4b[0]
164 LDR x11, [x5], 8
165 SDOT v27.4s, v10.16b, v3.4b[0]
166 LDR d6, [x15], 8
167
168 # BLOCK 3
169 SDOT v28.4s, v11.16b, v0.4b[0]
170 LDR d9, [x5], 8
171 SDOT v29.4s, v11.16b, v1.4b[0]
172 INS v8.d[1], x11
173 SDOT v30.4s, v11.16b, v2.4b[0]
174 LDR x11, [x5], 8
175 SDOT v31.4s, v11.16b, v3.4b[0]
176 LDR d7, [x10], 8
177
178 # BLOCK 0
179 SDOT v16.4s, v8.16b, v0.4b[1]
180 LDR d10, [x5], 8
181 SDOT v17.4s, v8.16b, v1.4b[1]
182 INS v9.d[1], x11
183 SDOT v18.4s, v8.16b, v2.4b[1]
184 LDR x11, [x5], 8
185 SDOT v19.4s, v8.16b, v3.4b[1]
186
187 # BLOCK 1
188 SDOT v20.4s, v9.16b, v0.4b[1]
189 LDR d11, [x5], 8
190 SDOT v21.4s, v9.16b, v1.4b[1]
191 INS v10.d[1], x11
192 SDOT v22.4s, v9.16b, v2.4b[1]
193 LDR x11, [x5], 8
194 SDOT v23.4s, v9.16b, v3.4b[1]
195
196 # BLOCK 2
197 SDOT v24.4s, v10.16b, v0.4b[1]
198 LDR d8, [x5], 8
199 SDOT v25.4s, v10.16b, v1.4b[1]
200 INS v11.d[1], x11
201 SDOT v26.4s, v10.16b, v2.4b[1]
202 LDR x11, [x5], 8
203 SDOT v27.4s, v10.16b, v3.4b[1]
204
205 # BLOCK 3
206 SDOT v28.4s, v11.16b, v0.4b[1]
207 LDR d9, [x5], 8
208 SDOT v29.4s, v11.16b, v1.4b[1]
209 INS v8.d[1], x11
210 SDOT v30.4s, v11.16b, v2.4b[1]
211 LDR x11, [x5], 8
212 SDOT v31.4s, v11.16b, v3.4b[1]
213
214 # BLOCK 0
215 SDOT v16.4s, v8.16b, v4.4b[0]
216 LDR d10, [x5], 8
217 SDOT v17.4s, v8.16b, v5.4b[0]
218 INS v9.d[1], x11
219 SDOT v18.4s, v8.16b, v6.4b[0]
220 LDR x11, [x5], 8
221 SDOT v19.4s, v8.16b, v7.4b[0]
222 LDR d0, [x13], 8
223
224 # BLOCK 1
225 SDOT v20.4s, v9.16b, v4.4b[0]
226 LDR d11, [x5], 8
227 SDOT v21.4s, v9.16b, v5.4b[0]
228 INS v10.d[1], x11
229 SDOT v22.4s, v9.16b, v6.4b[0]
230 LDR x11, [x5], 8
231 SDOT v23.4s, v9.16b, v7.4b[0]
232 LDR d1, [x14], 8
233
234 # BLOCK 2
235 SDOT v24.4s, v10.16b, v4.4b[0]
236 LDR d8, [x5], 8
237 SDOT v25.4s, v10.16b, v5.4b[0]
238 INS v11.d[1], x11
239 SDOT v26.4s, v10.16b, v6.4b[0]
240 LDR x11, [x5], 8
241 SDOT v27.4s, v10.16b, v7.4b[0]
242 LDR d2, [x15], 8
243
244 # BLOCK 3
245 SDOT v28.4s, v11.16b, v4.4b[0]
246 LDR d9, [x5], 8
247 SDOT v29.4s, v11.16b, v5.4b[0]
248 INS v8.d[1], x11
249 SDOT v30.4s, v11.16b, v6.4b[0]
250 LDR x11, [x5], 8
251 SDOT v31.4s, v11.16b, v7.4b[0]
252 LDR d3, [x10], 8
253
254 # BLOCK 0
255 SDOT v16.4s, v8.16b, v4.4b[1]
256 LDR d10, [x5], 8
257 SDOT v17.4s, v8.16b, v5.4b[1]
258 INS v9.d[1], x11
259 SDOT v18.4s, v8.16b, v6.4b[1]
260 LDR x11, [x5], 8
261 SDOT v19.4s, v8.16b, v7.4b[1]
262
263 # BLOCK 1
264 SDOT v20.4s, v9.16b, v4.4b[1]
265 LDR d11, [x5], 8
266 SDOT v21.4s, v9.16b, v5.4b[1]
267 INS v10.d[1], x11
268 SDOT v22.4s, v9.16b, v6.4b[1]
269 LDR x11, [x5], 8
270 SDOT v23.4s, v9.16b, v7.4b[1]
271
272 # BLOCK 2
273 SDOT v24.4s, v10.16b, v4.4b[1]
274 LDR d8, [x5], 8 // First B values for block 0 and 1
275 SDOT v25.4s, v10.16b, v5.4b[1]
276 INS v11.d[1], x11
277 SDOT v26.4s, v10.16b, v6.4b[1]
278 LDR x11, [x5], 8
279 SDOT v27.4s, v10.16b, v7.4b[1]
280 SUBS x0, x0, 16
281
282 # BLOCK 3
283 SDOT v28.4s, v11.16b, v4.4b[1]
284 LDR d9, [x5], 8
285 SDOT v29.4s, v11.16b, v5.4b[1]
286 INS v8.d[1], x11
287 SDOT v30.4s, v11.16b, v6.4b[1]
288 LDR x11, [x5], 8
289 SDOT v31.4s, v11.16b, v7.4b[1]
290 B.HS 2b
291
292 # Epilogue. Same as main loop but no preloads in final group
293 3:
294 # BLOCK 0
295 SDOT v16.4s, v8.16b, v0.4b[0]
296 LDR d10, [x5], 8
297 SDOT v17.4s, v8.16b, v1.4b[0]
298 INS v9.d[1], x11
299 SDOT v18.4s, v8.16b, v2.4b[0]
300 LDR x11, [x5], 8
301 SDOT v19.4s, v8.16b, v3.4b[0]
302 LDR d4, [x13], 8
303
304 # BLOCK 1
305 SDOT v20.4s, v9.16b, v0.4b[0]
306 LDR d11, [x5], 8
307 SDOT v21.4s, v9.16b, v1.4b[0]
308 INS v10.d[1], x11
309 SDOT v22.4s, v9.16b, v2.4b[0]
310 LDR x11, [x5], 8
311 SDOT v23.4s, v9.16b, v3.4b[0]
312 LDR d5, [x14], 8
313
314 # BLOCK 2
315 SDOT v24.4s, v10.16b, v0.4b[0]
316 LDR d8, [x5], 8
317 SDOT v25.4s, v10.16b, v1.4b[0]
318 INS v11.d[1], x11
319 SDOT v26.4s, v10.16b, v2.4b[0]
320 LDR x11, [x5], 8
321 SDOT v27.4s, v10.16b, v3.4b[0]
322 LDR d6, [x15], 8
323
324 # BLOCK 3
325 SDOT v28.4s, v11.16b, v0.4b[0]
326 LDR d9, [x5], 8
327 SDOT v29.4s, v11.16b, v1.4b[0]
328 INS v8.d[1], x11
329 SDOT v30.4s, v11.16b, v2.4b[0]
330 LDR x11, [x5], 8
331 SDOT v31.4s, v11.16b, v3.4b[0]
332 LDR d7, [x10], 8
333
334 # BLOCK 0
335 SDOT v16.4s, v8.16b, v0.4b[1]
336 LDR d10, [x5], 8
337 SDOT v17.4s, v8.16b, v1.4b[1]
338 INS v9.d[1], x11
339 SDOT v18.4s, v8.16b, v2.4b[1]
340 LDR x11, [x5], 8
341 SDOT v19.4s, v8.16b, v3.4b[1]
342
343 # BLOCK 1
344 SDOT v20.4s, v9.16b, v0.4b[1]
345 LDR d11, [x5], 8
346 SDOT v21.4s, v9.16b, v1.4b[1]
347 INS v10.d[1], x11
348 SDOT v22.4s, v9.16b, v2.4b[1]
349 LDR x11, [x5], 8
350 SDOT v23.4s, v9.16b, v3.4b[1]
351
352 # BLOCK 2
353 SDOT v24.4s, v10.16b, v0.4b[1]
354 LDR d8, [x5], 8
355 SDOT v25.4s, v10.16b, v1.4b[1]
356 INS v11.d[1], x11
357 SDOT v26.4s, v10.16b, v2.4b[1]
358 LDR x11, [x5], 8
359 SDOT v27.4s, v10.16b, v3.4b[1]
360
361 # BLOCK 3
362 SDOT v28.4s, v11.16b, v0.4b[1]
363 LDR d9, [x5], 8
364 SDOT v29.4s, v11.16b, v1.4b[1]
365 INS v8.d[1], x11
366 SDOT v30.4s, v11.16b, v2.4b[1]
367 LDR x11, [x5], 8
368 SDOT v31.4s, v11.16b, v3.4b[1]
369
370 # BLOCK 0
371 SDOT v16.4s, v8.16b, v4.4b[0]
372 LDR d10, [x5], 8
373 SDOT v17.4s, v8.16b, v5.4b[0]
374 INS v9.d[1], x11
375 SDOT v18.4s, v8.16b, v6.4b[0]
376 LDR x11, [x5], 8
377 SDOT v19.4s, v8.16b, v7.4b[0]
378
379 # BLOCK 1
380 SDOT v20.4s, v9.16b, v4.4b[0]
381 LDR d11, [x5], 8
382 SDOT v21.4s, v9.16b, v5.4b[0]
383 INS v10.d[1], x11
384 SDOT v22.4s, v9.16b, v6.4b[0]
385 LDR x11, [x5], 8
386 SDOT v23.4s, v9.16b, v7.4b[0]
387
388 # BLOCK 2
389 SDOT v24.4s, v10.16b, v4.4b[0]
390 LDR d8, [x5], 8
391 SDOT v25.4s, v10.16b, v5.4b[0]
392 INS v11.d[1], x11
393 SDOT v26.4s, v10.16b, v6.4b[0]
394 LDR x11, [x5], 8
395 SDOT v27.4s, v10.16b, v7.4b[0]
396
397 # BLOCK 3
398 SDOT v28.4s, v11.16b, v4.4b[0]
399 LDR d9, [x5], 8
400 SDOT v29.4s, v11.16b, v5.4b[0]
401 INS v8.d[1], x11
402 SDOT v30.4s, v11.16b, v6.4b[0]
403 LDR x11, [x5], 8
404 SDOT v31.4s, v11.16b, v7.4b[0]
405
406 # BLOCK 0
407 SDOT v16.4s, v8.16b, v4.4b[1]
408 LDR d10, [x5], 8
409 SDOT v17.4s, v8.16b, v5.4b[1]
410 INS v9.d[1], x11
411 SDOT v18.4s, v8.16b, v6.4b[1]
412 LDR x11, [x5], 8
413 SDOT v19.4s, v8.16b, v7.4b[1]
414
415 # BLOCK 1
416 SDOT v20.4s, v9.16b, v4.4b[1]
417 LDR d11, [x5], 8
418 SDOT v21.4s, v9.16b, v5.4b[1]
419 INS v10.d[1], x11
420 SDOT v22.4s, v9.16b, v6.4b[1]
421 LDR x11, [x5], 8
422 SDOT v23.4s, v9.16b, v7.4b[1]
423
424 # BLOCK 2
425 SDOT v24.4s, v10.16b, v4.4b[1]
426 SDOT v25.4s, v10.16b, v5.4b[1]
427 INS v11.d[1], x11
428 SDOT v26.4s, v10.16b, v6.4b[1]
429 SDOT v27.4s, v10.16b, v7.4b[1]
430 AND x0, x2, 15 // kc remainder 0 to 12
431
432 # BLOCK 3
433 SDOT v28.4s, v11.16b, v4.4b[1]
434 SDOT v29.4s, v11.16b, v5.4b[1]
435 LDR x11, [sp, 80] // reload params pointer
436 SDOT v30.4s, v11.16b, v6.4b[1]
437 SDOT v31.4s, v11.16b, v7.4b[1]
438
439 # Is there a remainder?- 4 to 12 bytes of A
440 CBNZ x0, 6f
441
442 .p2align 3
443 4:
444 # ks loop
445 SUBS x9, x9, 32 // ks -= MR * sizeof(int8_t*)
446 B.HI 1b
447
448 SCVTF v16.4s, v16.4s
449 SCVTF v17.4s, v17.4s
450 SCVTF v18.4s, v18.4s
451 SCVTF v19.4s, v19.4s
452 SCVTF v20.4s, v20.4s
453 SCVTF v21.4s, v21.4s
454 SCVTF v22.4s, v22.4s
455 SCVTF v23.4s, v23.4s
456 LDP q0, q1, [x5], 32 // kernel_scale
457 SCVTF v24.4s, v24.4s
458 SCVTF v25.4s, v25.4s
459 SCVTF v26.4s, v26.4s
460 SCVTF v27.4s, v27.4s
461 SCVTF v28.4s, v28.4s
462 SCVTF v29.4s, v29.4s
463 SCVTF v30.4s, v30.4s
464 SCVTF v31.4s, v31.4s
465 LDP q2, q3, [x5], 32
466 FMUL v4.4s, v0.4s, v12.s[1] // kernel_scale * scale
467 FMUL v5.4s, v1.4s, v12.s[1]
468 FMUL v6.4s, v2.4s, v12.s[1]
469 FMUL v7.4s, v3.4s, v12.s[1]
470 FMUL v8.4s, v0.4s, v12.s[1]
471 FMUL v9.4s, v1.4s, v12.s[1]
472 FMUL v10.4s, v2.4s, v12.s[1]
473 FMUL v11.4s, v3.4s, v12.s[1]
474 FMUL v16.4s, v16.4s, v4.4s
475 FMUL v20.4s, v20.4s, v5.4s
476 FMUL v24.4s, v24.4s, v6.4s
477 FMUL v28.4s, v28.4s, v7.4s
478 FMUL v17.4s, v17.4s, v8.4s
479 FMUL v21.4s, v21.4s, v9.4s
480 FMUL v25.4s, v25.4s, v10.4s
481 FMUL v29.4s, v29.4s, v11.4s
482 FMUL v4.4s, v0.4s, v12.s[1]
483 FMUL v5.4s, v1.4s, v12.s[1]
484 FMUL v6.4s, v2.4s, v12.s[1]
485 FMUL v7.4s, v3.4s, v12.s[1]
486 FMUL v8.4s, v0.4s, v12.s[1]
487 FMUL v9.4s, v1.4s, v12.s[1]
488 FMUL v10.4s, v2.4s, v12.s[1]
489 FMUL v11.4s, v3.4s, v12.s[1]
490 LDP q0, q1, [x5], 32 // bias
491 FMUL v18.4s, v18.4s, v4.4s
492 FMUL v22.4s, v22.4s, v5.4s
493 FMUL v26.4s, v26.4s, v6.4s
494 FMUL v30.4s, v30.4s, v7.4s
495 FMUL v19.4s, v19.4s, v8.4s
496 FMUL v23.4s, v23.4s, v9.4s
497 FMUL v27.4s, v27.4s, v10.4s
498 FMUL v31.4s, v31.4s, v11.4s
499 LDP q2, q3, [x5], 32
500 FADD v16.4s, v16.4s, v0.4s
501 FADD v17.4s, v17.4s, v0.4s
502 FADD v18.4s, v18.4s, v0.4s
503 FADD v19.4s, v19.4s, v0.4s
504 FADD v20.4s, v20.4s, v1.4s
505 FADD v21.4s, v21.4s, v1.4s
506 FADD v22.4s, v22.4s, v1.4s
507 FADD v23.4s, v23.4s, v1.4s
508 LD2R {v0.4s, v1.4s}, [x11] // min max
509 FADD v24.4s, v24.4s, v2.4s
510 FADD v25.4s, v25.4s, v2.4s
511 FADD v26.4s, v26.4s, v2.4s
512 FADD v27.4s, v27.4s, v2.4s
513 FADD v28.4s, v28.4s, v3.4s
514 FADD v29.4s, v29.4s, v3.4s
515 FADD v30.4s, v30.4s, v3.4s
516 FADD v31.4s, v31.4s, v3.4s
517 FMAX v16.4s, v16.4s, v0.4s
518 FMAX v17.4s, v17.4s, v0.4s
519 FMAX v18.4s, v18.4s, v0.4s
520 FMAX v19.4s, v19.4s, v0.4s
521 FMAX v20.4s, v20.4s, v0.4s
522 FMAX v21.4s, v21.4s, v0.4s
523 FMAX v22.4s, v22.4s, v0.4s
524 FMAX v23.4s, v23.4s, v0.4s
525 FMAX v24.4s, v24.4s, v0.4s
526 FMAX v25.4s, v25.4s, v0.4s
527 FMAX v26.4s, v26.4s, v0.4s
528 FMAX v27.4s, v27.4s, v0.4s
529 FMAX v28.4s, v28.4s, v0.4s
530 FMAX v29.4s, v29.4s, v0.4s
531 FMAX v30.4s, v30.4s, v0.4s
532 FMAX v31.4s, v31.4s, v0.4s
533 FMIN v16.4s, v16.4s, v1.4s
534 FMIN v17.4s, v17.4s, v1.4s
535 FMIN v18.4s, v18.4s, v1.4s
536 FMIN v19.4s, v19.4s, v1.4s
537 FMIN v20.4s, v20.4s, v1.4s
538 FMIN v21.4s, v21.4s, v1.4s
539 FMIN v22.4s, v22.4s, v1.4s
540 FMIN v23.4s, v23.4s, v1.4s
541 FMIN v24.4s, v24.4s, v1.4s
542 FMIN v25.4s, v25.4s, v1.4s
543 FMIN v26.4s, v26.4s, v1.4s
544 FMIN v27.4s, v27.4s, v1.4s
545 FMIN v28.4s, v28.4s, v1.4s
546 FMIN v29.4s, v29.4s, v1.4s
547 FMIN v30.4s, v30.4s, v1.4s
548 FMIN v31.4s, v31.4s, v1.4s
549 SUBS x1, x1, 16
550 LDR x0, [sp, 48] // cn_stride
551 B.LO 7f
552 STP q19, q23, [x7]
553 STP q27, q31, [x7, #32]
554 ADD x7, x7, x0
555
556 STP q18, q22, [x17]
557 STP q26, q30, [x17, #32]
558 ADD x17, x17, x0
559
560 STP q17, q21, [x16]
561 STP q25, q29, [x16, #32]
562 ADD x16, x16, x0
563
564 STP q16, q20, [x6]
565 STP q24, q28, [x6, #32]
566 ADD x6, x6, x0
567 SUB x4, x4, x3 // a -= ks
568 B.NE 0b
569
570
571 # Restore d8-d12 from stack
572 LDR x19, [sp, 40]
573 LDR d12, [sp, 32]
574 LDP d10, d11, [sp, 16]
575 LDP d8, d9, [sp], 48
576 RET
577
578 # Remainder- 4 to 12 bytes of A
579 # Although C4, its safe to read 16 bytes.
580 .p2align 3
581 5:
582 AND x0, x2, 15 // kc remainder 4 to 12
583 6:
584 LDR q0, [x13]
585 LDP q8, q9, [x5], 32
586 LDR q1, [x14]
587 LDR q2, [x15]
588 LDR q3, [x10]
589 LDP q10, q11, [x5], 32
590 SDOT v16.4s, v8.16b, v0.4b[0]
591 SDOT v17.4s, v8.16b, v1.4b[0]
592 SDOT v18.4s, v8.16b, v2.4b[0]
593 SDOT v19.4s, v8.16b, v3.4b[0]
594 SDOT v20.4s, v9.16b, v0.4b[0]
595 SDOT v21.4s, v9.16b, v1.4b[0]
596 SDOT v22.4s, v9.16b, v2.4b[0]
597 SDOT v23.4s, v9.16b, v3.4b[0]
598 SDOT v24.4s, v10.16b, v0.4b[0]
599 SDOT v25.4s, v10.16b, v1.4b[0]
600 SDOT v26.4s, v10.16b, v2.4b[0]
601 SDOT v27.4s, v10.16b, v3.4b[0]
602 SDOT v28.4s, v11.16b, v0.4b[0]
603 SDOT v29.4s, v11.16b, v1.4b[0]
604 SDOT v30.4s, v11.16b, v2.4b[0]
605 SDOT v31.4s, v11.16b, v3.4b[0]
606 CMP x0, 4
607 B.LS 4b
608 LDP q8, q9, [x5], 32
609 LDP q10, q11, [x5], 32
610 SDOT v16.4s, v8.16b, v0.4b[1]
611 SDOT v17.4s, v8.16b, v1.4b[1]
612 SDOT v18.4s, v8.16b, v2.4b[1]
613 SDOT v19.4s, v8.16b, v3.4b[1]
614 SDOT v20.4s, v9.16b, v0.4b[1]
615 SDOT v21.4s, v9.16b, v1.4b[1]
616 SDOT v22.4s, v9.16b, v2.4b[1]
617 SDOT v23.4s, v9.16b, v3.4b[1]
618 SDOT v24.4s, v10.16b, v0.4b[1]
619 SDOT v25.4s, v10.16b, v1.4b[1]
620 SDOT v26.4s, v10.16b, v2.4b[1]
621 SDOT v27.4s, v10.16b, v3.4b[1]
622 SDOT v28.4s, v11.16b, v0.4b[1]
623 SDOT v29.4s, v11.16b, v1.4b[1]
624 SDOT v30.4s, v11.16b, v2.4b[1]
625 SDOT v31.4s, v11.16b, v3.4b[1]
626 CMP x0, 8
627 B.LS 4b
628 LDP q8, q9, [x5], 32
629 LDP q10, q11, [x5], 32
630 SDOT v16.4s, v8.16b, v0.4b[2]
631 SDOT v17.4s, v8.16b, v1.4b[2]
632 SDOT v18.4s, v8.16b, v2.4b[2]
633 SDOT v19.4s, v8.16b, v3.4b[2]
634 SDOT v20.4s, v9.16b, v0.4b[2]
635 SDOT v21.4s, v9.16b, v1.4b[2]
636 SDOT v22.4s, v9.16b, v2.4b[2]
637 SDOT v23.4s, v9.16b, v3.4b[2]
638 SDOT v24.4s, v10.16b, v0.4b[2]
639 SDOT v25.4s, v10.16b, v1.4b[2]
640 SDOT v26.4s, v10.16b, v2.4b[2]
641 SDOT v27.4s, v10.16b, v3.4b[2]
642 SDOT v28.4s, v11.16b, v0.4b[2]
643 SDOT v29.4s, v11.16b, v1.4b[2]
644 SDOT v30.4s, v11.16b, v2.4b[2]
645 SDOT v31.4s, v11.16b, v3.4b[2]
646 B 4b
647
648 # Store odd width
649 .p2align 3
650 7:
651 TBZ x1, 3, 8f
652 STP q19, q23, [x7]
653 STP q18, q22, [x17]
654 STP q17, q21, [x16]
655 STP q16, q20, [x6]
656 MOV v16.16b, v24.16b
657 MOV v17.16b, v25.16b
658 MOV v18.16b, v26.16b
659 MOV v19.16b, v27.16b
660 MOV v20.16b, v28.16b
661 MOV v21.16b, v29.16b
662 MOV v22.16b, v30.16b
663 MOV v23.16b, v31.16b
664 ADD x7, x7, #32
665 ADD x17, x17, #32
666 ADD x16, x16, #32
667 ADD x6, x6, #32
668 8:
669 TBZ x1, 2, 9f
670 STR q19, [x7]
671 STR q18, [x17]
672 STR q17, [x16]
673 STR q16, [x6]
674 MOV v16.16b, v20.16b
675 MOV v17.16b, v21.16b
676 MOV v18.16b, v22.16b
677 MOV v19.16b, v23.16b
678 ADD x7, x7, #16
679 ADD x17, x17, #16
680 ADD x16, x16, #16
681 ADD x6, x6, #16
682 9:
683 TBZ x1, 1, 10f
684 ST1 {v19.2s}, [x7]
685 ST1 {v18.2s}, [x17]
686 ST1 {v17.2s}, [x16]
687 ST1 {v16.2s}, [x6]
688 DUP d16, v16.d[1]
689 DUP d17, v17.d[1]
690 DUP d18, v18.d[1]
691 DUP d19, v19.d[1]
692 ADD x7, x7, #8
693 ADD x17, x17, #8
694 ADD x16, x16, #8
695 ADD x6, x6, #8
696 10:
697 TBZ x1, 0, 11f
698 STR s19, [x7]
699 STR s18, [x17]
700 STR s17, [x16]
701 STR s16, [x6]
702 11:
703 # Restore d8-d12 from stack
704 LDR x19, [sp, 40]
705 LDR d12, [sp, 32]
706 LDP d10, d11, [sp, 16]
707 LDP d8, d9, [sp], 48
708 RET
709
710 END_FUNCTION xnn_qd8_f32_qc8w_igemm_minmax_ukernel_4x16c4__asm_aarch64_neondot_cortex_a55
711
712 #ifdef __ELF__
713 .section ".note.GNU-stack","",%progbits
714 #endif