Debian Codegraph @

src/qd8-f16-qc8w-igemm/gen/qd8-f16-qc8w-igemm-4x16c4-minmax-asm-aarch64-neondot-cortex-a55.S

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-igemm/4x16c4-aarch64-neondot-cortex-a55.S.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2020 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include "xnnpack/assembly.h"
12
13 # void xnn_qd8_f16_qc8w_igemm_minmax__ukernel_4x16c4__asm_aarch64_neondotfp16arith_cortex_a55(
14 # size_t mr, x0
15 # size_t nc, x1
16 # size_t kc, x2 / x0
17 # size_t ks, x3 / x9
18 # const int8_t** restrict a, x4
19 # const int8_t* restrict w, x5
20 # int8_t* restrict c, x6
21 # size_t cm_stride, x7
22 # size_t cn_stride, [sp] -> (x0)
23 # size_t a_offset, [sp + 8] -> x8
24 # const int8_t* zero, [sp + 16] -> x12
25 # const int8_t* zero_data, [sp + 24] -> x19
26 # const union xnn_f16_minmax_params *params, [sp + 32] -> x11
27 # const struct xnn_qd8_quantization_params *quantization_params) [sp + 40] -> x17
28
29 # d8-d15, x19-x30 need to be preserved if used. x18 is reserved by the OS.
30
31 // Register usage
32 // A0 x13 v0 v4
33 // A1 x14 v1 v5
34 // A2 x15 v2 v6
35 // A3 x10 v3 v7
36 // B x5 v8 v9 v10 v11
37 // C0 x6 v16 v20 v24 v28
38 // C1 x16 v17 v21 v25 v29
39 // C2 x17 v18 v22 v26 v30
40 // C3 x7 v19 v23 v27 v31
41 // unused v13, v14 v15
42
43 // x11 temp for Cortex-A55 loads
44
45 BEGIN_FUNCTION xnn_qd8_f16_qc8w_igemm_minmax_ukernel_4x16c4__asm_aarch64_neondotfp16arith_cortex_a55
46
47 # Clamp C pointers
48 CMP x0, 2 // if mr < 2
49 LDR x8, [sp, 8] // Load a_offset
50 ADD x16, x6, x7 // c1 = c0 + cm_stride
51 LDR x12, [sp, 16] // Load zero
52 LDR x11, [sp, 32] // Load params pointer
53 CSEL x16, x6, x16, LO // c1 = c0
54 ADD x2, x2, 3 // kc = (kc + 3) & ~3
55 STP d8, d9, [sp, -48]! // Save d8-d11 on stack
56 STR x19, [sp, 40] // Save x19 to stack
57 LDR x19, [sp, 72] // Load zero_data
58 STR d12, [sp, 32]
59 LDR x17, [sp, 88] // &quantization_params.zero_point
60 LD1 {v12.4s}, [x17] // zero point and scale
61
62 ADD x17, x16, x7 // c2 = c1 + cm_stride
63 STP d10, d11, [sp, 16]
64 // if mr <= 2
65 CSEL x17, x16, x17, LS // c2 = c1
66 BIC x2, x2, 3
67
68 CMP x0, 4 // if mr < 4
69 ADD x7, x17, x7 // c3 = c2 + cm_stride
70 CSEL x7, x17, x7, LO // c3 = c2
71
72 .p2align 3
73 0:
74 # Load initial bias from w into accumulators
75 LDP q16, q20, [x5], 32
76 MUL v17.4s, v16.4s, v12.s[0]
77 MUL v18.4s, v16.4s, v12.s[0]
78 LDP q24, q28, [x5], 32
79 MUL v19.4s, v16.4s, v12.s[0]
80 MUL v21.4s, v20.4s, v12.s[0]
81 MUL v22.4s, v20.4s, v12.s[0]
82 MUL v23.4s, v20.4s, v12.s[0]
83 MUL v25.4s, v24.4s, v12.s[0]
84 MUL v26.4s, v24.4s, v12.s[0]
85 MUL v27.4s, v24.4s, v12.s[0]
86 MUL v29.4s, v28.4s, v12.s[0]
87 MUL v30.4s, v28.4s, v12.s[0]
88 MUL v31.4s, v28.4s, v12.s[0]
89 MUL v24.4s, v24.4s, v12.s[0]
90 MUL v28.4s, v28.4s, v12.s[0]
91 MUL v16.4s, v16.4s, v12.s[0]
92 MUL v20.4s, v20.4s, v12.s[0]
93 MOV x9, x3 // p = ks
94
95 .p2align 3
96 1:
97 # Load next 4 A pointers
98 LDP x13, x14, [x4], 16
99 LDP x15, x10, [x4], 16
100
101 CMP x13, x12 // if a0 == zero
102 ADD x13, x13, x8 // a0 += a_offset
103 CSEL x13, x19, x13, EQ // a0 = zero_data, else a0 += a_offset
104 CMP x14, x12 // if a1 == zero
105 ADD x14, x14, x8 // a1 += a_offset
106 CSEL x14, x19, x14, EQ // a1 = zero_data, else a1 += a_offset
107 CMP x15, x12 // if a2 == zero
108 ADD x15, x15, x8 // a2 += a_offset
109 CSEL x15, x19, x15, EQ // a2 = zero_data, else a2 += a_offset
110 CMP x10, x12 // if a3 == zero
111 ADD x10, x10, x8 // a3 += a_offset
112 CSEL x10, x19, x10, EQ // a3 = zero_data, else a3 += a_offset
113
114 # Is there at least 16 bytes for prologue/epilogue?
115 SUBS x0, x2, 16 // k = kc - 16
116 B.LO 5f
117
118 # prologue - read A and B values for block 0 and 1
119 LDR d0, [x13], 8
120 LDR q8, [x5], 16
121 LDR d1, [x14], 8
122 LDR d2, [x15], 8
123 LDR d3, [x10], 8
124 SUBS x0, x0, 16 // is there 16 for main loop?
125 LDR d9, [x5], 8
126 LDR x11, [x5], 8
127 # Is there at least 16 bytes for main loop?
128 B.LO 3f
129
130 # Main loop - 16 bytes of A in 4 groups.
131 # 4 row of 4 vectors wide = 16 sdot instructions for 4 channels
132 # 4 LD64 for A
133 # 4 LD128 for W. = 2 LD64 + INS.
134 # for each 4 sdot, 1 LD64 for A, 2 LD64 for W + INS.
135
136 .p2align 3
137 2:
138 # BLOCK 0
139 SDOT v16.4s, v8.16b, v0.4b[0]
140 LDR d10, [x5], 8
141 SDOT v17.4s, v8.16b, v1.4b[0]
142 INS v9.d[1], x11
143 SDOT v18.4s, v8.16b, v2.4b[0]
144 LDR x11, [x5], 8
145 SDOT v19.4s, v8.16b, v3.4b[0]
146 LDR d4, [x13], 8
147
148 # BLOCK 1
149 SDOT v20.4s, v9.16b, v0.4b[0]
150 LDR d11, [x5], 8
151 SDOT v21.4s, v9.16b, v1.4b[0]
152 INS v10.d[1], x11
153 SDOT v22.4s, v9.16b, v2.4b[0]
154 LDR x11, [x5], 8
155 SDOT v23.4s, v9.16b, v3.4b[0]
156 LDR d5, [x14], 8
157
158 # BLOCK 2
159 SDOT v24.4s, v10.16b, v0.4b[0]
160 LDR d8, [x5], 8
161 SDOT v25.4s, v10.16b, v1.4b[0]
162 INS v11.d[1], x11
163 SDOT v26.4s, v10.16b, v2.4b[0]
164 LDR x11, [x5], 8
165 SDOT v27.4s, v10.16b, v3.4b[0]
166 LDR d6, [x15], 8
167
168 # BLOCK 3
169 SDOT v28.4s, v11.16b, v0.4b[0]
170 LDR d9, [x5], 8
171 SDOT v29.4s, v11.16b, v1.4b[0]
172 INS v8.d[1], x11
173 SDOT v30.4s, v11.16b, v2.4b[0]
174 LDR x11, [x5], 8
175 SDOT v31.4s, v11.16b, v3.4b[0]
176 LDR d7, [x10], 8
177
178 # BLOCK 0
179 SDOT v16.4s, v8.16b, v0.4b[1]
180 LDR d10, [x5], 8
181 SDOT v17.4s, v8.16b, v1.4b[1]
182 INS v9.d[1], x11
183 SDOT v18.4s, v8.16b, v2.4b[1]
184 LDR x11, [x5], 8
185 SDOT v19.4s, v8.16b, v3.4b[1]
186
187 # BLOCK 1
188 SDOT v20.4s, v9.16b, v0.4b[1]
189 LDR d11, [x5], 8
190 SDOT v21.4s, v9.16b, v1.4b[1]
191 INS v10.d[1], x11
192 SDOT v22.4s, v9.16b, v2.4b[1]
193 LDR x11, [x5], 8
194 SDOT v23.4s, v9.16b, v3.4b[1]
195
196 # BLOCK 2
197 SDOT v24.4s, v10.16b, v0.4b[1]
198 LDR d8, [x5], 8
199 SDOT v25.4s, v10.16b, v1.4b[1]
200 INS v11.d[1], x11
201 SDOT v26.4s, v10.16b, v2.4b[1]
202 LDR x11, [x5], 8
203 SDOT v27.4s, v10.16b, v3.4b[1]
204
205 # BLOCK 3
206 SDOT v28.4s, v11.16b, v0.4b[1]
207 LDR d9, [x5], 8
208 SDOT v29.4s, v11.16b, v1.4b[1]
209 INS v8.d[1], x11
210 SDOT v30.4s, v11.16b, v2.4b[1]
211 LDR x11, [x5], 8
212 SDOT v31.4s, v11.16b, v3.4b[1]
213
214 # BLOCK 0
215 SDOT v16.4s, v8.16b, v4.4b[0]
216 LDR d10, [x5], 8
217 SDOT v17.4s, v8.16b, v5.4b[0]
218 INS v9.d[1], x11
219 SDOT v18.4s, v8.16b, v6.4b[0]
220 LDR x11, [x5], 8
221 SDOT v19.4s, v8.16b, v7.4b[0]
222 LDR d0, [x13], 8
223
224 # BLOCK 1
225 SDOT v20.4s, v9.16b, v4.4b[0]
226 LDR d11, [x5], 8
227 SDOT v21.4s, v9.16b, v5.4b[0]
228 INS v10.d[1], x11
229 SDOT v22.4s, v9.16b, v6.4b[0]
230 LDR x11, [x5], 8
231 SDOT v23.4s, v9.16b, v7.4b[0]
232 LDR d1, [x14], 8
233
234 # BLOCK 2
235 SDOT v24.4s, v10.16b, v4.4b[0]
236 LDR d8, [x5], 8
237 SDOT v25.4s, v10.16b, v5.4b[0]
238 INS v11.d[1], x11
239 SDOT v26.4s, v10.16b, v6.4b[0]
240 LDR x11, [x5], 8
241 SDOT v27.4s, v10.16b, v7.4b[0]
242 LDR d2, [x15], 8
243
244 # BLOCK 3
245 SDOT v28.4s, v11.16b, v4.4b[0]
246 LDR d9, [x5], 8
247 SDOT v29.4s, v11.16b, v5.4b[0]
248 INS v8.d[1], x11
249 SDOT v30.4s, v11.16b, v6.4b[0]
250 LDR x11, [x5], 8
251 SDOT v31.4s, v11.16b, v7.4b[0]
252 LDR d3, [x10], 8
253
254 # BLOCK 0
255 SDOT v16.4s, v8.16b, v4.4b[1]
256 LDR d10, [x5], 8
257 SDOT v17.4s, v8.16b, v5.4b[1]
258 INS v9.d[1], x11
259 SDOT v18.4s, v8.16b, v6.4b[1]
260 LDR x11, [x5], 8
261 SDOT v19.4s, v8.16b, v7.4b[1]
262
263 # BLOCK 1
264 SDOT v20.4s, v9.16b, v4.4b[1]
265 LDR d11, [x5], 8
266 SDOT v21.4s, v9.16b, v5.4b[1]
267 INS v10.d[1], x11
268 SDOT v22.4s, v9.16b, v6.4b[1]
269 LDR x11, [x5], 8
270 SDOT v23.4s, v9.16b, v7.4b[1]
271
272 # BLOCK 2
273 SDOT v24.4s, v10.16b, v4.4b[1]
274 LDR d8, [x5], 8 // First B values for block 0 and 1
275 SDOT v25.4s, v10.16b, v5.4b[1]
276 INS v11.d[1], x11
277 SDOT v26.4s, v10.16b, v6.4b[1]
278 LDR x11, [x5], 8
279 SDOT v27.4s, v10.16b, v7.4b[1]
280 SUBS x0, x0, 16
281
282 # BLOCK 3
283 SDOT v28.4s, v11.16b, v4.4b[1]
284 LDR d9, [x5], 8
285 SDOT v29.4s, v11.16b, v5.4b[1]
286 INS v8.d[1], x11
287 SDOT v30.4s, v11.16b, v6.4b[1]
288 LDR x11, [x5], 8
289 SDOT v31.4s, v11.16b, v7.4b[1]
290 B.HS 2b
291
292 # Epilogue. Same as main loop but no preloads in final group
293 3:
294 # BLOCK 0
295 SDOT v16.4s, v8.16b, v0.4b[0]
296 LDR d10, [x5], 8
297 SDOT v17.4s, v8.16b, v1.4b[0]
298 INS v9.d[1], x11
299 SDOT v18.4s, v8.16b, v2.4b[0]
300 LDR x11, [x5], 8
301 SDOT v19.4s, v8.16b, v3.4b[0]
302 LDR d4, [x13], 8
303
304 # BLOCK 1
305 SDOT v20.4s, v9.16b, v0.4b[0]
306 LDR d11, [x5], 8
307 SDOT v21.4s, v9.16b, v1.4b[0]
308 INS v10.d[1], x11
309 SDOT v22.4s, v9.16b, v2.4b[0]
310 LDR x11, [x5], 8
311 SDOT v23.4s, v9.16b, v3.4b[0]
312 LDR d5, [x14], 8
313
314 # BLOCK 2
315 SDOT v24.4s, v10.16b, v0.4b[0]
316 LDR d8, [x5], 8
317 SDOT v25.4s, v10.16b, v1.4b[0]
318 INS v11.d[1], x11
319 SDOT v26.4s, v10.16b, v2.4b[0]
320 LDR x11, [x5], 8
321 SDOT v27.4s, v10.16b, v3.4b[0]
322 LDR d6, [x15], 8
323
324 # BLOCK 3
325 SDOT v28.4s, v11.16b, v0.4b[0]
326 LDR d9, [x5], 8
327 SDOT v29.4s, v11.16b, v1.4b[0]
328 INS v8.d[1], x11
329 SDOT v30.4s, v11.16b, v2.4b[0]
330 LDR x11, [x5], 8
331 SDOT v31.4s, v11.16b, v3.4b[0]
332 LDR d7, [x10], 8
333
334 # BLOCK 0
335 SDOT v16.4s, v8.16b, v0.4b[1]
336 LDR d10, [x5], 8
337 SDOT v17.4s, v8.16b, v1.4b[1]
338 INS v9.d[1], x11
339 SDOT v18.4s, v8.16b, v2.4b[1]
340 LDR x11, [x5], 8
341 SDOT v19.4s, v8.16b, v3.4b[1]
342
343 # BLOCK 1
344 SDOT v20.4s, v9.16b, v0.4b[1]
345 LDR d11, [x5], 8
346 SDOT v21.4s, v9.16b, v1.4b[1]
347 INS v10.d[1], x11
348 SDOT v22.4s, v9.16b, v2.4b[1]
349 LDR x11, [x5], 8
350 SDOT v23.4s, v9.16b, v3.4b[1]
351
352 # BLOCK 2
353 SDOT v24.4s, v10.16b, v0.4b[1]
354 LDR d8, [x5], 8
355 SDOT v25.4s, v10.16b, v1.4b[1]
356 INS v11.d[1], x11
357 SDOT v26.4s, v10.16b, v2.4b[1]
358 LDR x11, [x5], 8
359 SDOT v27.4s, v10.16b, v3.4b[1]
360
361 # BLOCK 3
362 SDOT v28.4s, v11.16b, v0.4b[1]
363 LDR d9, [x5], 8
364 SDOT v29.4s, v11.16b, v1.4b[1]
365 INS v8.d[1], x11
366 SDOT v30.4s, v11.16b, v2.4b[1]
367 LDR x11, [x5], 8
368 SDOT v31.4s, v11.16b, v3.4b[1]
369
370 # BLOCK 0
371 SDOT v16.4s, v8.16b, v4.4b[0]
372 LDR d10, [x5], 8
373 SDOT v17.4s, v8.16b, v5.4b[0]
374 INS v9.d[1], x11
375 SDOT v18.4s, v8.16b, v6.4b[0]
376 LDR x11, [x5], 8
377 SDOT v19.4s, v8.16b, v7.4b[0]
378
379 # BLOCK 1
380 SDOT v20.4s, v9.16b, v4.4b[0]
381 LDR d11, [x5], 8
382 SDOT v21.4s, v9.16b, v5.4b[0]
383 INS v10.d[1], x11
384 SDOT v22.4s, v9.16b, v6.4b[0]
385 LDR x11, [x5], 8
386 SDOT v23.4s, v9.16b, v7.4b[0]
387
388 # BLOCK 2
389 SDOT v24.4s, v10.16b, v4.4b[0]
390 LDR d8, [x5], 8
391 SDOT v25.4s, v10.16b, v5.4b[0]
392 INS v11.d[1], x11
393 SDOT v26.4s, v10.16b, v6.4b[0]
394 LDR x11, [x5], 8
395 SDOT v27.4s, v10.16b, v7.4b[0]
396
397 # BLOCK 3
398 SDOT v28.4s, v11.16b, v4.4b[0]
399 LDR d9, [x5], 8
400 SDOT v29.4s, v11.16b, v5.4b[0]
401 INS v8.d[1], x11
402 SDOT v30.4s, v11.16b, v6.4b[0]
403 LDR x11, [x5], 8
404 SDOT v31.4s, v11.16b, v7.4b[0]
405
406 # BLOCK 0
407 SDOT v16.4s, v8.16b, v4.4b[1]
408 LDR d10, [x5], 8
409 SDOT v17.4s, v8.16b, v5.4b[1]
410 INS v9.d[1], x11
411 SDOT v18.4s, v8.16b, v6.4b[1]
412 LDR x11, [x5], 8
413 SDOT v19.4s, v8.16b, v7.4b[1]
414
415 # BLOCK 1
416 SDOT v20.4s, v9.16b, v4.4b[1]
417 LDR d11, [x5], 8
418 SDOT v21.4s, v9.16b, v5.4b[1]
419 INS v10.d[1], x11
420 SDOT v22.4s, v9.16b, v6.4b[1]
421 LDR x11, [x5], 8
422 SDOT v23.4s, v9.16b, v7.4b[1]
423
424 # BLOCK 2
425 SDOT v24.4s, v10.16b, v4.4b[1]
426 SDOT v25.4s, v10.16b, v5.4b[1]
427 INS v11.d[1], x11
428 SDOT v26.4s, v10.16b, v6.4b[1]
429 SDOT v27.4s, v10.16b, v7.4b[1]
430 AND x0, x2, 15 // kc remainder 0 to 12
431
432 # BLOCK 3
433 SDOT v28.4s, v11.16b, v4.4b[1]
434 SDOT v29.4s, v11.16b, v5.4b[1]
435 LDR x11, [sp, 80] // reload params pointer
436 SDOT v30.4s, v11.16b, v6.4b[1]
437 SDOT v31.4s, v11.16b, v7.4b[1]
438
439 # Is there a remainder?- 4 to 12 bytes of A
440 CBNZ x0, 6f
441
442 .p2align 3
443 4:
444 # ks loop
445 SUBS x9, x9, 32 // ks -= MR * sizeof(int8_t*)
446 B.HI 1b
447
448 SCVTF v16.4s, v16.4s
449 SCVTF v17.4s, v17.4s
450 SCVTF v18.4s, v18.4s
451 SCVTF v19.4s, v19.4s
452 SCVTF v20.4s, v20.4s
453 SCVTF v21.4s, v21.4s
454 SCVTF v22.4s, v22.4s
455 SCVTF v23.4s, v23.4s
456 LDP q0, q1, [x5], 32 // kernel_scale
457 SCVTF v24.4s, v24.4s
458 SCVTF v25.4s, v25.4s
459 SCVTF v26.4s, v26.4s
460 SCVTF v27.4s, v27.4s
461 SCVTF v28.4s, v28.4s
462 SCVTF v29.4s, v29.4s
463 SCVTF v30.4s, v30.4s
464 SCVTF v31.4s, v31.4s
465 LDP q2, q3, [x5], 32
466 FMUL v4.4s, v0.4s, v12.s[1] // kernel_scale * scale
467 FMUL v5.4s, v1.4s, v12.s[1]
468 FMUL v6.4s, v2.4s, v12.s[1]
469 FMUL v7.4s, v3.4s, v12.s[1]
470 FMUL v8.4s, v0.4s, v12.s[1]
471 FMUL v9.4s, v1.4s, v12.s[1]
472 FMUL v10.4s, v2.4s, v12.s[1]
473 FMUL v11.4s, v3.4s, v12.s[1]
474 FMUL v16.4s, v16.4s, v4.4s
475 FMUL v20.4s, v20.4s, v5.4s
476 FMUL v24.4s, v24.4s, v6.4s
477 FMUL v28.4s, v28.4s, v7.4s
478 FMUL v17.4s, v17.4s, v8.4s
479 FMUL v21.4s, v21.4s, v9.4s
480 FMUL v25.4s, v25.4s, v10.4s
481 FMUL v29.4s, v29.4s, v11.4s
482 FMUL v4.4s, v0.4s, v12.s[1]
483 FMUL v5.4s, v1.4s, v12.s[1]
484 FMUL v6.4s, v2.4s, v12.s[1]
485 FMUL v7.4s, v3.4s, v12.s[1]
486 FMUL v8.4s, v0.4s, v12.s[1]
487 FMUL v9.4s, v1.4s, v12.s[1]
488 FMUL v10.4s, v2.4s, v12.s[1]
489 FMUL v11.4s, v3.4s, v12.s[1]
490 LDP q0, q1, [x5], 32 // bias
491 FMUL v18.4s, v18.4s, v4.4s
492 FMUL v22.4s, v22.4s, v5.4s
493 FMUL v26.4s, v26.4s, v6.4s
494 FMUL v30.4s, v30.4s, v7.4s
495 FMUL v19.4s, v19.4s, v8.4s
496 FMUL v23.4s, v23.4s, v9.4s
497 FMUL v27.4s, v27.4s, v10.4s
498 FMUL v31.4s, v31.4s, v11.4s
499 LDP q2, q3, [x5], 32
500 FADD v16.4s, v16.4s, v0.4s
501 FADD v17.4s, v17.4s, v0.4s
502 FADD v18.4s, v18.4s, v0.4s
503 FADD v19.4s, v19.4s, v0.4s
504 FADD v20.4s, v20.4s, v1.4s
505 FADD v21.4s, v21.4s, v1.4s
506 FADD v22.4s, v22.4s, v1.4s
507 FADD v23.4s, v23.4s, v1.4s
508 LD2R {v0.8h, v1.8h}, [x11] // min max
509 FADD v24.4s, v24.4s, v2.4s
510 FADD v25.4s, v25.4s, v2.4s
511 FADD v26.4s, v26.4s, v2.4s
512 FADD v27.4s, v27.4s, v2.4s
513 FADD v28.4s, v28.4s, v3.4s
514 FADD v29.4s, v29.4s, v3.4s
515 FADD v30.4s, v30.4s, v3.4s
516 FADD v31.4s, v31.4s, v3.4s
517 FCVTN v16.4h, v16.4s
518 FCVTN v17.4h, v17.4s
519 FCVTN v18.4h, v18.4s
520 FCVTN v19.4h, v19.4s
521 FCVTN v24.4h, v24.4s
522 FCVTN v25.4h, v25.4s
523 FCVTN v26.4h, v26.4s
524 FCVTN v27.4h, v27.4s
525 FCVTN2 v16.8h, v20.4s
526 FCVTN2 v17.8h, v21.4s
527 FCVTN2 v18.8h, v22.4s
528 FCVTN2 v19.8h, v23.4s
529 FCVTN2 v24.8h, v28.4s
530 FCVTN2 v25.8h, v29.4s
531 FCVTN2 v26.8h, v30.4s
532 FCVTN2 v27.8h, v31.4s
533 FMAX v16.8h, v16.8h, v0.8h
534 FMAX v17.8h, v17.8h, v0.8h
535 FMAX v18.8h, v18.8h, v0.8h
536 FMAX v19.8h, v19.8h, v0.8h
537 FMAX v24.8h, v24.8h, v0.8h
538 FMAX v25.8h, v25.8h, v0.8h
539 FMAX v26.8h, v26.8h, v0.8h
540 FMAX v27.8h, v27.8h, v0.8h
541 FMIN v16.8h, v16.8h, v1.8h
542 FMIN v17.8h, v17.8h, v1.8h
543 FMIN v18.8h, v18.8h, v1.8h
544 FMIN v19.8h, v19.8h, v1.8h
545 FMIN v24.8h, v24.8h, v1.8h
546 FMIN v25.8h, v25.8h, v1.8h
547 FMIN v26.8h, v26.8h, v1.8h
548 FMIN v27.8h, v27.8h, v1.8h
549 SUBS x1, x1, 16
550 LDR x0, [sp, 48] // cn_stride
551 B.LO 7f
552 STP q19, q27, [x7]
553 ADD x7, x7, x0
554
555 STP q18, q26, [x17]
556 ADD x17, x17, x0
557
558 STP q17, q25, [x16]
559 ADD x16, x16, x0
560
561 STP q16, q24, [x6]
562 ADD x6, x6, x0
563 SUB x4, x4, x3 // a -= ks
564 B.NE 0b
565
566
567 # Restore d8-d12 from stack
568 LDR x19, [sp, 40]
569 LDR d12, [sp, 32]
570 LDP d10, d11, [sp, 16]
571 LDP d8, d9, [sp], 48
572 RET
573
574 # Remainder- 4 to 12 bytes of A
575 # Although C4, its safe to read 16 bytes.
576 .p2align 3
577 5:
578 AND x0, x2, 15 // kc remainder 4 to 12
579 6:
580 LDR q0, [x13]
581 LDP q8, q9, [x5], 32
582 LDR q1, [x14]
583 LDR q2, [x15]
584 LDR q3, [x10]
585 LDP q10, q11, [x5], 32
586 SDOT v16.4s, v8.16b, v0.4b[0]
587 SDOT v17.4s, v8.16b, v1.4b[0]
588 SDOT v18.4s, v8.16b, v2.4b[0]
589 SDOT v19.4s, v8.16b, v3.4b[0]
590 SDOT v20.4s, v9.16b, v0.4b[0]
591 SDOT v21.4s, v9.16b, v1.4b[0]
592 SDOT v22.4s, v9.16b, v2.4b[0]
593 SDOT v23.4s, v9.16b, v3.4b[0]
594 SDOT v24.4s, v10.16b, v0.4b[0]
595 SDOT v25.4s, v10.16b, v1.4b[0]
596 SDOT v26.4s, v10.16b, v2.4b[0]
597 SDOT v27.4s, v10.16b, v3.4b[0]
598 SDOT v28.4s, v11.16b, v0.4b[0]
599 SDOT v29.4s, v11.16b, v1.4b[0]
600 SDOT v30.4s, v11.16b, v2.4b[0]
601 SDOT v31.4s, v11.16b, v3.4b[0]
602 CMP x0, 4
603 B.LS 4b
604 LDP q8, q9, [x5], 32
605 LDP q10, q11, [x5], 32
606 SDOT v16.4s, v8.16b, v0.4b[1]
607 SDOT v17.4s, v8.16b, v1.4b[1]
608 SDOT v18.4s, v8.16b, v2.4b[1]
609 SDOT v19.4s, v8.16b, v3.4b[1]
610 SDOT v20.4s, v9.16b, v0.4b[1]
611 SDOT v21.4s, v9.16b, v1.4b[1]
612 SDOT v22.4s, v9.16b, v2.4b[1]
613 SDOT v23.4s, v9.16b, v3.4b[1]
614 SDOT v24.4s, v10.16b, v0.4b[1]
615 SDOT v25.4s, v10.16b, v1.4b[1]
616 SDOT v26.4s, v10.16b, v2.4b[1]
617 SDOT v27.4s, v10.16b, v3.4b[1]
618 SDOT v28.4s, v11.16b, v0.4b[1]
619 SDOT v29.4s, v11.16b, v1.4b[1]
620 SDOT v30.4s, v11.16b, v2.4b[1]
621 SDOT v31.4s, v11.16b, v3.4b[1]
622 CMP x0, 8
623 B.LS 4b
624 LDP q8, q9, [x5], 32
625 LDP q10, q11, [x5], 32
626 SDOT v16.4s, v8.16b, v0.4b[2]
627 SDOT v17.4s, v8.16b, v1.4b[2]
628 SDOT v18.4s, v8.16b, v2.4b[2]
629 SDOT v19.4s, v8.16b, v3.4b[2]
630 SDOT v20.4s, v9.16b, v0.4b[2]
631 SDOT v21.4s, v9.16b, v1.4b[2]
632 SDOT v22.4s, v9.16b, v2.4b[2]
633 SDOT v23.4s, v9.16b, v3.4b[2]
634 SDOT v24.4s, v10.16b, v0.4b[2]
635 SDOT v25.4s, v10.16b, v1.4b[2]
636 SDOT v26.4s, v10.16b, v2.4b[2]
637 SDOT v27.4s, v10.16b, v3.4b[2]
638 SDOT v28.4s, v11.16b, v0.4b[2]
639 SDOT v29.4s, v11.16b, v1.4b[2]
640 SDOT v30.4s, v11.16b, v2.4b[2]
641 SDOT v31.4s, v11.16b, v3.4b[2]
642 B 4b
643
644 # Store odd width
645 .p2align 3
646 7:
647 TBZ x1, 3, 8f
648 STR q19, [x7], 16
649 STR q18, [x17], 16
650 STR q17, [x16], 16
651 STR q16, [x6], 16
652 MOV v16.16b, v24.16b
653 MOV v17.16b, v25.16b
654 MOV v18.16b, v26.16b
655 MOV v19.16b, v27.16b
656 8:
657 TBZ x1, 2, 9f
658 STR d19, [x7], 8
659 STR d18, [x17], 8
660 STR d17, [x16], 8
661 STR d16, [x6], 8
662 DUP d16, v16.d[1]
663 DUP d17, v17.d[1]
664 DUP d18, v18.d[1]
665 DUP d19, v19.d[1]
666 9:
667 TBZ x1, 1, 10f
668 STR s19, [x7], 4
669 STR s18, [x17], 4
670 STR s17, [x16], 4
671 STR s16, [x6], 4
672 DUP s16, v16.s[1]
673 DUP s17, v17.s[1]
674 DUP s18, v18.s[1]
675 DUP s19, v19.s[1]
676 10:
677 TBZ x1, 0, 11f
678 STR h19, [x7]
679 STR h18, [x17]
680 STR h17, [x16]
681 STR h16, [x6]
682 11:
683 # Restore d8-d12 from stack
684 LDR x19, [sp, 40]
685 LDR d12, [sp, 32]
686 LDP d10, d11, [sp, 16]
687 LDP d8, d9, [sp], 48
688 RET
689
690 END_FUNCTION xnn_qd8_f16_qc8w_igemm_minmax_ukernel_4x16c4__asm_aarch64_neondotfp16arith_cortex_a55
691
692 #ifdef __ELF__
693 .section ".note.GNU-stack","",%progbits
694 #endif