Debian Codegraph @

src/qs8-qc8w-igemm/gen/qs8-qc8w-igemm-4x16-minmax-fp32-asm-aarch64-neon-mlal-lane-ld64-prfm.S

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-igemm/4x16-aarch64-neon-mlal-lane-ld64.S.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2021 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include "xnnpack/assembly.h"
12
13 # void xnn_qs8_qc8w_igemm_minmax_fp32_ukernel_4x16__asm_aarch64_neon_mlal_lane_ld64_prfm(
14 # size_t mr, x0
15 # size_t nc, x1
16 # size_t kc, x2 / x0
17 # size_t ks, x3 / x9
18 # const int8_t** restrict a, x4
19 # const int8_t* restrict w, x5
20 # int8_t* restrict c, x6
21 # size_t cm_stride, x7
22 # size_t cn_stride, [sp] -> x10
23 # size_t a_offset, [sp + 8] -> x8
24 # const int8_t* zero, [sp + 16] -> x12
25 # const xnn_qs8_conv_minmax_params params [sp + 24] -> (x11)
26
27 # d8-d15, x19-x30 need to be preserved if used. x18 is reserved by the OS.
28
29 // Register usage
30 // A0 x13 v0
31 // A1 x14 v1
32 // A2 x15 v2
33 // A3 x20 v3
34 // B x5 v4 v5
35 // C0 x6 v16 v20 v24 v28
36 // C1 x16 v17 v21 v25 v29
37 // C2 x17 v18 v22 v26 v30
38 // C3 x7 v19 v23 v27 v31
39 # unused v7 v8 v9 v10 v11 v12 v13 v14 v15
40
41 BEGIN_FUNCTION xnn_qs8_qc8w_igemm_minmax_fp32_ukernel_4x16__asm_aarch64_neon_mlal_lane_ld64_prfm
42
43 # Clamp C pointers
44 CMP x0, 2 // if mr < 2
45 LDP x10, x8, [sp] // Load cn_stride, a_offset
46 ADD x16, x6, x7 // c1 = c0 + cm_stride
47 CSEL x16, x6, x16, LO // c1 = c0
48
49 ADD x17, x16, x7 // c2 = c1 + cm_stride
50 LDP x12, x11, [sp, 16] // Load zero, params pointer
51 // if mr <= 2
52 CSEL x17, x16, x17, LS // c2 = c1
53
54 CMP x0, 4 // if mr < 4
55 STR x20, [sp, -16]! // Save x20 on stack
56 ADD x7, x17, x7 // c3 = c2 + cm_stride
57 CSEL x7, x17, x7, LO // c3 = c2
58
59
60 .p2align 3
61 0:
62 # Load initial bias from w into accumulators
63 LDP q16, q20, [x5], 32
64 MOV v17.16b, v16.16b
65 MOV v18.16b, v16.16b
66 LDP q24, q28, [x5], 32
67 MOV v19.16b, v16.16b
68 MOV v21.16b, v20.16b
69 MOV v22.16b, v20.16b
70 MOV v23.16b, v20.16b
71 MOV v25.16b, v24.16b
72 MOV v26.16b, v24.16b
73 MOV v27.16b, v24.16b
74 MOV v29.16b, v28.16b
75 MOV v30.16b, v28.16b
76 MOV v31.16b, v28.16b
77 MOV x9, x3 // p = ks
78
79 .p2align 3
80 1:
81 # Load next 4 A pointers
82 LDP x13, x14, [x4], 16
83 LDP x15, x20, [x4], 16
84
85 CMP x13, x12 // if a0 == zero
86 ADD x13, x13, x8 // a0 += a_offset
87 CSEL x13, x12, x13, EQ // a0 = zero, else += a0 + a_offset
88 CMP x14, x12 // if a1 == zero
89 ADD x14, x14, x8 // a1 += a_offset
90 CSEL x14, x12, x14, EQ // a1 = zero, else += a1 + a_offset
91 CMP x15, x12 // if a2 == zero
92 ADD x15, x15, x8 // a2 += a_offset
93 CSEL x15, x12, x15, EQ // a2 = zero, else += a2 + a_offset
94 CMP x20, x12 // if a3 == zero
95 ADD x20, x20, x8 // a3 += a_offset
96 CSEL x20, x12, x20, EQ // a3 = zero, else += a3 + a_offset
97
98 # Is there at least 8 bytes for main loop?
99 SUBS x0, x2, 8 // k = kc - 8
100 B.LO 4f
101
102 # Main loop - 8 bytes of A
103 .p2align 3
104 2:
105 LD1 {v0.8b}, [x13], 8
106 LDP d4, d5, [x5], 16
107 LD1 {v1.8b}, [x14], 8
108 LD1 {v2.8b}, [x15], 8
109 LD1 {v3.8b}, [x20], 8
110 SXTL v0.8h, v0.8b
111 SXTL v4.8h, v4.8b
112 SXTL v5.8h, v5.8b
113 SXTL v1.8h, v1.8b
114 SXTL v2.8h, v2.8b
115 SXTL v3.8h, v3.8b
116 SMLAL v16.4s, v4.4h, v0.h[0]
117 SMLAL2 v20.4s, v4.8h, v0.h[0]
118 PRFM PLDL1KEEP, [x13, 128]
119 SMLAL v24.4s, v5.4h, v0.h[0]
120 SMLAL2 v28.4s, v5.8h, v0.h[0]
121 PRFM PLDL1KEEP, [x14, 128]
122 SMLAL v17.4s, v4.4h, v1.h[0]
123 SMLAL2 v21.4s, v4.8h, v1.h[0]
124 PRFM PLDL1KEEP, [x15, 128]
125 SMLAL v25.4s, v5.4h, v1.h[0]
126 SMLAL2 v29.4s, v5.8h, v1.h[0]
127 PRFM PLDL1KEEP, [x20, 128]
128 SMLAL v18.4s, v4.4h, v2.h[0]
129 SMLAL2 v22.4s, v4.8h, v2.h[0]
130 PRFM PLDL1KEEP, [x5, 448]
131 SMLAL v26.4s, v5.4h, v2.h[0]
132 SMLAL2 v30.4s, v5.8h, v2.h[0]
133 PRFM PLDL1KEEP, [x5, 512]
134 SMLAL v19.4s, v4.4h, v3.h[0]
135 SMLAL2 v23.4s, v4.8h, v3.h[0]
136 SMLAL v27.4s, v5.4h, v3.h[0]
137 SMLAL2 v31.4s, v5.8h, v3.h[0]
138
139 LDP d4, d5, [x5], 16
140 SXTL v4.8h, v4.8b
141 SXTL v5.8h, v5.8b
142 SMLAL v16.4s, v4.4h, v0.h[1]
143 SMLAL2 v20.4s, v4.8h, v0.h[1]
144 SMLAL v24.4s, v5.4h, v0.h[1]
145 SMLAL2 v28.4s, v5.8h, v0.h[1]
146 SMLAL v17.4s, v4.4h, v1.h[1]
147 SMLAL2 v21.4s, v4.8h, v1.h[1]
148 SMLAL v25.4s, v5.4h, v1.h[1]
149 SMLAL2 v29.4s, v5.8h, v1.h[1]
150 SMLAL v18.4s, v4.4h, v2.h[1]
151 SMLAL2 v22.4s, v4.8h, v2.h[1]
152 SMLAL v26.4s, v5.4h, v2.h[1]
153 SMLAL2 v30.4s, v5.8h, v2.h[1]
154 SMLAL v19.4s, v4.4h, v3.h[1]
155 SMLAL2 v23.4s, v4.8h, v3.h[1]
156 SMLAL v27.4s, v5.4h, v3.h[1]
157 SMLAL2 v31.4s, v5.8h, v3.h[1]
158
159 LDP d4, d5, [x5], 16
160 SXTL v4.8h, v4.8b
161 SXTL v5.8h, v5.8b
162 SMLAL v16.4s, v4.4h, v0.h[2]
163 SMLAL2 v20.4s, v4.8h, v0.h[2]
164 SMLAL v24.4s, v5.4h, v0.h[2]
165 SMLAL2 v28.4s, v5.8h, v0.h[2]
166 SMLAL v17.4s, v4.4h, v1.h[2]
167 SMLAL2 v21.4s, v4.8h, v1.h[2]
168 SMLAL v25.4s, v5.4h, v1.h[2]
169 SMLAL2 v29.4s, v5.8h, v1.h[2]
170 SMLAL v18.4s, v4.4h, v2.h[2]
171 SMLAL2 v22.4s, v4.8h, v2.h[2]
172 SMLAL v26.4s, v5.4h, v2.h[2]
173 SMLAL2 v30.4s, v5.8h, v2.h[2]
174 SMLAL v19.4s, v4.4h, v3.h[2]
175 SMLAL2 v23.4s, v4.8h, v3.h[2]
176 SMLAL v27.4s, v5.4h, v3.h[2]
177 SMLAL2 v31.4s, v5.8h, v3.h[2]
178
179 LDP d4, d5, [x5], 16
180 SXTL v4.8h, v4.8b
181 SXTL v5.8h, v5.8b
182 SMLAL v16.4s, v4.4h, v0.h[3]
183 SMLAL2 v20.4s, v4.8h, v0.h[3]
184 SMLAL v24.4s, v5.4h, v0.h[3]
185 SMLAL2 v28.4s, v5.8h, v0.h[3]
186 SMLAL v17.4s, v4.4h, v1.h[3]
187 SMLAL2 v21.4s, v4.8h, v1.h[3]
188 SMLAL v25.4s, v5.4h, v1.h[3]
189 SMLAL2 v29.4s, v5.8h, v1.h[3]
190 SMLAL v18.4s, v4.4h, v2.h[3]
191 SMLAL2 v22.4s, v4.8h, v2.h[3]
192 SMLAL v26.4s, v5.4h, v2.h[3]
193 SMLAL2 v30.4s, v5.8h, v2.h[3]
194 SMLAL v19.4s, v4.4h, v3.h[3]
195 SMLAL2 v23.4s, v4.8h, v3.h[3]
196 SMLAL v27.4s, v5.4h, v3.h[3]
197 SMLAL2 v31.4s, v5.8h, v3.h[3]
198
199 LDP d4, d5, [x5], 16
200 SXTL v4.8h, v4.8b
201 SXTL v5.8h, v5.8b
202 SMLAL v16.4s, v4.4h, v0.h[4]
203 SMLAL2 v20.4s, v4.8h, v0.h[4]
204 SMLAL v24.4s, v5.4h, v0.h[4]
205 SMLAL2 v28.4s, v5.8h, v0.h[4]
206 SMLAL v17.4s, v4.4h, v1.h[4]
207 SMLAL2 v21.4s, v4.8h, v1.h[4]
208 SMLAL v25.4s, v5.4h, v1.h[4]
209 SMLAL2 v29.4s, v5.8h, v1.h[4]
210 SMLAL v18.4s, v4.4h, v2.h[4]
211 SMLAL2 v22.4s, v4.8h, v2.h[4]
212 SMLAL v26.4s, v5.4h, v2.h[4]
213 SMLAL2 v30.4s, v5.8h, v2.h[4]
214 SMLAL v19.4s, v4.4h, v3.h[4]
215 SMLAL2 v23.4s, v4.8h, v3.h[4]
216 SMLAL v27.4s, v5.4h, v3.h[4]
217 SMLAL2 v31.4s, v5.8h, v3.h[4]
218
219 LDP d4, d5, [x5], 16
220 SXTL v4.8h, v4.8b
221 SXTL v5.8h, v5.8b
222 SMLAL v16.4s, v4.4h, v0.h[5]
223 SMLAL2 v20.4s, v4.8h, v0.h[5]
224 SMLAL v24.4s, v5.4h, v0.h[5]
225 SMLAL2 v28.4s, v5.8h, v0.h[5]
226 SMLAL v17.4s, v4.4h, v1.h[5]
227 SMLAL2 v21.4s, v4.8h, v1.h[5]
228 SMLAL v25.4s, v5.4h, v1.h[5]
229 SMLAL2 v29.4s, v5.8h, v1.h[5]
230 SMLAL v18.4s, v4.4h, v2.h[5]
231 SMLAL2 v22.4s, v4.8h, v2.h[5]
232 SMLAL v26.4s, v5.4h, v2.h[5]
233 SMLAL2 v30.4s, v5.8h, v2.h[5]
234 SMLAL v19.4s, v4.4h, v3.h[5]
235 SMLAL2 v23.4s, v4.8h, v3.h[5]
236 SMLAL v27.4s, v5.4h, v3.h[5]
237 SMLAL2 v31.4s, v5.8h, v3.h[5]
238
239 LDP d4, d5, [x5], 16
240 SXTL v4.8h, v4.8b
241 SXTL v5.8h, v5.8b
242 SMLAL v16.4s, v4.4h, v0.h[6]
243 SMLAL2 v20.4s, v4.8h, v0.h[6]
244 SMLAL v24.4s, v5.4h, v0.h[6]
245 SMLAL2 v28.4s, v5.8h, v0.h[6]
246 SMLAL v17.4s, v4.4h, v1.h[6]
247 SMLAL2 v21.4s, v4.8h, v1.h[6]
248 SMLAL v25.4s, v5.4h, v1.h[6]
249 SMLAL2 v29.4s, v5.8h, v1.h[6]
250 SMLAL v18.4s, v4.4h, v2.h[6]
251 SMLAL2 v22.4s, v4.8h, v2.h[6]
252 SMLAL v26.4s, v5.4h, v2.h[6]
253 SMLAL2 v30.4s, v5.8h, v2.h[6]
254 SMLAL v19.4s, v4.4h, v3.h[6]
255 SMLAL2 v23.4s, v4.8h, v3.h[6]
256 SMLAL v27.4s, v5.4h, v3.h[6]
257 SMLAL2 v31.4s, v5.8h, v3.h[6]
258
259 LDP d4, d5, [x5], 16
260 SXTL v4.8h, v4.8b
261 SXTL v5.8h, v5.8b
262 SMLAL v16.4s, v4.4h, v0.h[7]
263 SMLAL2 v20.4s, v4.8h, v0.h[7]
264 SMLAL v24.4s, v5.4h, v0.h[7]
265 SMLAL2 v28.4s, v5.8h, v0.h[7]
266 SMLAL v17.4s, v4.4h, v1.h[7]
267 SMLAL2 v21.4s, v4.8h, v1.h[7]
268 SMLAL v25.4s, v5.4h, v1.h[7]
269 SMLAL2 v29.4s, v5.8h, v1.h[7]
270 SMLAL v18.4s, v4.4h, v2.h[7]
271 SMLAL2 v22.4s, v4.8h, v2.h[7]
272 SMLAL v26.4s, v5.4h, v2.h[7]
273 SMLAL2 v30.4s, v5.8h, v2.h[7]
274 SMLAL v19.4s, v4.4h, v3.h[7]
275 SMLAL2 v23.4s, v4.8h, v3.h[7]
276 SMLAL v27.4s, v5.4h, v3.h[7]
277 SMLAL2 v31.4s, v5.8h, v3.h[7]
278
279 SUBS x0, x0, 8
280 B.HS 2b
281
282 AND x0, x2, 7 // kc remainder 0 to 7
283 # Is there a remainder?- 1 to 7 bytes of A
284 CBNZ x0, 4f
285
286 3:
287 # ks loop
288 SUBS x9, x9, 32 // ks -= MR * sizeof(int8_t*)
289 B.HI 1b
290
291 SCVTF v16.4s, v16.4s
292 SCVTF v17.4s, v17.4s
293 # Load per channel scale values from weights
294 LDR q4, [x5], 16
295 SCVTF v18.4s, v18.4s
296 SCVTF v19.4s, v19.4s
297 LDR q5, [x5], 16
298 SCVTF v20.4s, v20.4s
299 SCVTF v21.4s, v21.4s
300 SCVTF v22.4s, v22.4s
301 SCVTF v23.4s, v23.4s
302 SCVTF v24.4s, v24.4s
303 SCVTF v25.4s, v25.4s
304 SCVTF v26.4s, v26.4s
305 SCVTF v27.4s, v27.4s
306 SCVTF v28.4s, v28.4s
307 SCVTF v29.4s, v29.4s
308 SCVTF v30.4s, v30.4s
309 SCVTF v31.4s, v31.4s
310
311 LDR q6, [x5], 16
312 FMUL v16.4s, v16.4s, v4.4s
313 FMUL v17.4s, v17.4s, v4.4s
314 FMUL v18.4s, v18.4s, v4.4s
315 FMUL v19.4s, v19.4s, v4.4s
316 FMUL v20.4s, v20.4s, v5.4s
317 LDR q4, [x5], 16
318 FMUL v21.4s, v21.4s, v5.4s
319 FMUL v22.4s, v22.4s, v5.4s
320 FMUL v23.4s, v23.4s, v5.4s
321 FMUL v24.4s, v24.4s, v6.4s
322 FMUL v25.4s, v25.4s, v6.4s
323 FMUL v26.4s, v26.4s, v6.4s
324 FMUL v27.4s, v27.4s, v6.4s
325 FMUL v28.4s, v28.4s, v4.4s
326 FMUL v29.4s, v29.4s, v4.4s
327 FMUL v30.4s, v30.4s, v4.4s
328 FMUL v31.4s, v31.4s, v4.4s
329
330 FCVTNS v16.4s, v16.4s
331 FCVTNS v17.4s, v17.4s
332 FCVTNS v18.4s, v18.4s
333 FCVTNS v19.4s, v19.4s
334 FCVTNS v20.4s, v20.4s
335 FCVTNS v21.4s, v21.4s
336 FCVTNS v22.4s, v22.4s
337 FCVTNS v23.4s, v23.4s
338 FCVTNS v24.4s, v24.4s
339 FCVTNS v25.4s, v25.4s
340 FCVTNS v26.4s, v26.4s
341 FCVTNS v27.4s, v27.4s
342 FCVTNS v28.4s, v28.4s
343 FCVTNS v29.4s, v29.4s
344 FCVTNS v30.4s, v30.4s
345 FCVTNS v31.4s, v31.4s
346
347 SQXTN v16.4h, v16.4s
348 SQXTN v17.4h, v17.4s
349 SQXTN v18.4h, v18.4s
350 SQXTN v19.4h, v19.4s
351 SQXTN v24.4h, v24.4s
352 SQXTN v25.4h, v25.4s
353 SQXTN v26.4h, v26.4s
354 SQXTN v27.4h, v27.4s
355 LD1R {v6.8h}, [x11], 2 // add bias
356
357 SQXTN2 v16.8h, v20.4s
358 SQXTN2 v17.8h, v21.4s
359 SQXTN2 v18.8h, v22.4s
360 SQXTN2 v19.8h, v23.4s
361 SQXTN2 v24.8h, v28.4s
362 SQXTN2 v25.8h, v29.4s
363 SQXTN2 v26.8h, v30.4s
364 SQXTN2 v27.8h, v31.4s
365
366 SQADD v16.8h, v16.8h, v6.8h
367 SQADD v17.8h, v17.8h, v6.8h
368 SQADD v18.8h, v18.8h, v6.8h
369 SQADD v19.8h, v19.8h, v6.8h
370 SQADD v24.8h, v24.8h, v6.8h
371 SQADD v25.8h, v25.8h, v6.8h
372 SQADD v26.8h, v26.8h, v6.8h
373 SQADD v27.8h, v27.8h, v6.8h
374 LD1R {v4.16b}, [x11], 1 // clamp min value
375
376 SQXTN v0.8b, v16.8h
377 SQXTN v1.8b, v17.8h
378 SQXTN v2.8b, v18.8h
379 SQXTN v3.8b, v19.8h
380 LD1R {v5.16b}, [x11] // clamp max value
381 SQXTN2 v0.16b, v24.8h
382 SQXTN2 v1.16b, v25.8h
383 SQXTN2 v2.16b, v26.8h
384 SQXTN2 v3.16b, v27.8h
385 SUB x11, x11, 3 // rewind params pointer
386
387 SMAX v0.16b, v0.16b, v4.16b
388 SMAX v1.16b, v1.16b, v4.16b
389 SMAX v2.16b, v2.16b, v4.16b
390 SMAX v3.16b, v3.16b, v4.16b
391 SUBS x1, x1, 16
392 SMIN v0.16b, v0.16b, v5.16b
393 SMIN v1.16b, v1.16b, v5.16b
394 SMIN v2.16b, v2.16b, v5.16b
395 SMIN v3.16b, v3.16b, v5.16b
396 B.LO 5f
397
398 # Store full 4 x 16
399 ST1 {v3.16b}, [x7], x10
400 ST1 {v2.16b}, [x17], x10
401 ST1 {v1.16b}, [x16], x10
402 ST1 {v0.16b}, [x6], x10
403
404 SUB x4, x4, x3 // a -= ks
405
406 # nc loop
407 B.HI 0b
408
409 # Restore x20 from stack
410 LDR x20, [sp], 16
411 RET
412
413 # Remainder- 1 to 7 bytes of A
414 .p2align 3
415 4:
416 AND x0, x2, 7 // kc remainder 1 to 7
417
418 LD1 {v0.8b}, [x13], x0
419 LDP d4, d5, [x5], 16
420 LD1 {v1.8b}, [x14], x0
421 LD1 {v2.8b}, [x15], x0
422 LD1 {v3.8b}, [x20], x0
423 SXTL v0.8h, v0.8b
424 SXTL v4.8h, v4.8b
425 SXTL v5.8h, v5.8b
426 SXTL v1.8h, v1.8b
427 SXTL v2.8h, v2.8b
428 SXTL v3.8h, v3.8b
429 SMLAL v16.4s, v4.4h, v0.h[0]
430 SMLAL2 v20.4s, v4.8h, v0.h[0]
431 SMLAL v24.4s, v5.4h, v0.h[0]
432 SMLAL2 v28.4s, v5.8h, v0.h[0]
433 SMLAL v17.4s, v4.4h, v1.h[0]
434 SMLAL2 v21.4s, v4.8h, v1.h[0]
435 SMLAL v25.4s, v5.4h, v1.h[0]
436 SMLAL2 v29.4s, v5.8h, v1.h[0]
437 SMLAL v18.4s, v4.4h, v2.h[0]
438 SMLAL2 v22.4s, v4.8h, v2.h[0]
439 SMLAL v26.4s, v5.4h, v2.h[0]
440 SMLAL2 v30.4s, v5.8h, v2.h[0]
441 SMLAL v19.4s, v4.4h, v3.h[0]
442 SMLAL2 v23.4s, v4.8h, v3.h[0]
443 SMLAL v27.4s, v5.4h, v3.h[0]
444 SMLAL2 v31.4s, v5.8h, v3.h[0]
445 CMP x0, 2
446 B.LO 3b
447
448 LDP d4, d5, [x5], 16
449 SXTL v4.8h, v4.8b
450 SXTL v5.8h, v5.8b
451 SMLAL v16.4s, v4.4h, v0.h[1]
452 SMLAL2 v20.4s, v4.8h, v0.h[1]
453 SMLAL v24.4s, v5.4h, v0.h[1]
454 SMLAL2 v28.4s, v5.8h, v0.h[1]
455 SMLAL v17.4s, v4.4h, v1.h[1]
456 SMLAL2 v21.4s, v4.8h, v1.h[1]
457 SMLAL v25.4s, v5.4h, v1.h[1]
458 SMLAL2 v29.4s, v5.8h, v1.h[1]
459 SMLAL v18.4s, v4.4h, v2.h[1]
460 SMLAL2 v22.4s, v4.8h, v2.h[1]
461 SMLAL v26.4s, v5.4h, v2.h[1]
462 SMLAL2 v30.4s, v5.8h, v2.h[1]
463 SMLAL v19.4s, v4.4h, v3.h[1]
464 SMLAL2 v23.4s, v4.8h, v3.h[1]
465 SMLAL v27.4s, v5.4h, v3.h[1]
466 SMLAL2 v31.4s, v5.8h, v3.h[1]
467 B.EQ 3b
468
469 LDP d4, d5, [x5], 16
470 SXTL v4.8h, v4.8b
471 SXTL v5.8h, v5.8b
472 SMLAL v16.4s, v4.4h, v0.h[2]
473 SMLAL2 v20.4s, v4.8h, v0.h[2]
474 SMLAL v24.4s, v5.4h, v0.h[2]
475 SMLAL2 v28.4s, v5.8h, v0.h[2]
476 SMLAL v17.4s, v4.4h, v1.h[2]
477 SMLAL2 v21.4s, v4.8h, v1.h[2]
478 SMLAL v25.4s, v5.4h, v1.h[2]
479 SMLAL2 v29.4s, v5.8h, v1.h[2]
480 SMLAL v18.4s, v4.4h, v2.h[2]
481 SMLAL2 v22.4s, v4.8h, v2.h[2]
482 SMLAL v26.4s, v5.4h, v2.h[2]
483 SMLAL2 v30.4s, v5.8h, v2.h[2]
484 SMLAL v19.4s, v4.4h, v3.h[2]
485 SMLAL2 v23.4s, v4.8h, v3.h[2]
486 SMLAL v27.4s, v5.4h, v3.h[2]
487 SMLAL2 v31.4s, v5.8h, v3.h[2]
488 CMP x0, 4
489 B.LO 3b
490
491 LDP d4, d5, [x5], 16
492 SXTL v4.8h, v4.8b
493 SXTL v5.8h, v5.8b
494 SMLAL v16.4s, v4.4h, v0.h[3]
495 SMLAL2 v20.4s, v4.8h, v0.h[3]
496 SMLAL v24.4s, v5.4h, v0.h[3]
497 SMLAL2 v28.4s, v5.8h, v0.h[3]
498 SMLAL v17.4s, v4.4h, v1.h[3]
499 SMLAL2 v21.4s, v4.8h, v1.h[3]
500 SMLAL v25.4s, v5.4h, v1.h[3]
501 SMLAL2 v29.4s, v5.8h, v1.h[3]
502 SMLAL v18.4s, v4.4h, v2.h[3]
503 SMLAL2 v22.4s, v4.8h, v2.h[3]
504 SMLAL v26.4s, v5.4h, v2.h[3]
505 SMLAL2 v30.4s, v5.8h, v2.h[3]
506 SMLAL v19.4s, v4.4h, v3.h[3]
507 SMLAL2 v23.4s, v4.8h, v3.h[3]
508 SMLAL v27.4s, v5.4h, v3.h[3]
509 SMLAL2 v31.4s, v5.8h, v3.h[3]
510 B.EQ 3b
511
512 LDP d4, d5, [x5], 16
513 SXTL v4.8h, v4.8b
514 SXTL v5.8h, v5.8b
515 SMLAL v16.4s, v4.4h, v0.h[4]
516 SMLAL2 v20.4s, v4.8h, v0.h[4]
517 SMLAL v24.4s, v5.4h, v0.h[4]
518 SMLAL2 v28.4s, v5.8h, v0.h[4]
519 SMLAL v17.4s, v4.4h, v1.h[4]
520 SMLAL2 v21.4s, v4.8h, v1.h[4]
521 SMLAL v25.4s, v5.4h, v1.h[4]
522 SMLAL2 v29.4s, v5.8h, v1.h[4]
523 SMLAL v18.4s, v4.4h, v2.h[4]
524 SMLAL2 v22.4s, v4.8h, v2.h[4]
525 SMLAL v26.4s, v5.4h, v2.h[4]
526 SMLAL2 v30.4s, v5.8h, v2.h[4]
527 SMLAL v19.4s, v4.4h, v3.h[4]
528 SMLAL2 v23.4s, v4.8h, v3.h[4]
529 SMLAL v27.4s, v5.4h, v3.h[4]
530 SMLAL2 v31.4s, v5.8h, v3.h[4]
531 CMP x0, 6
532 B.LO 3b
533
534 LDP d4, d5, [x5], 16
535 SXTL v4.8h, v4.8b
536 SXTL v5.8h, v5.8b
537 SMLAL v16.4s, v4.4h, v0.h[5]
538 SMLAL2 v20.4s, v4.8h, v0.h[5]
539 SMLAL v24.4s, v5.4h, v0.h[5]
540 SMLAL2 v28.4s, v5.8h, v0.h[5]
541 SMLAL v17.4s, v4.4h, v1.h[5]
542 SMLAL2 v21.4s, v4.8h, v1.h[5]
543 SMLAL v25.4s, v5.4h, v1.h[5]
544 SMLAL2 v29.4s, v5.8h, v1.h[5]
545 SMLAL v18.4s, v4.4h, v2.h[5]
546 SMLAL2 v22.4s, v4.8h, v2.h[5]
547 SMLAL v26.4s, v5.4h, v2.h[5]
548 SMLAL2 v30.4s, v5.8h, v2.h[5]
549 SMLAL v19.4s, v4.4h, v3.h[5]
550 SMLAL2 v23.4s, v4.8h, v3.h[5]
551 SMLAL v27.4s, v5.4h, v3.h[5]
552 SMLAL2 v31.4s, v5.8h, v3.h[5]
553 B.EQ 3b
554
555 LDP d4, d5, [x5], 16
556 SXTL v4.8h, v4.8b
557 SXTL v5.8h, v5.8b
558 SMLAL v16.4s, v4.4h, v0.h[6]
559 SMLAL2 v20.4s, v4.8h, v0.h[6]
560 SMLAL v24.4s, v5.4h, v0.h[6]
561 SMLAL2 v28.4s, v5.8h, v0.h[6]
562 SMLAL v17.4s, v4.4h, v1.h[6]
563 SMLAL2 v21.4s, v4.8h, v1.h[6]
564 SMLAL v25.4s, v5.4h, v1.h[6]
565 SMLAL2 v29.4s, v5.8h, v1.h[6]
566 SMLAL v18.4s, v4.4h, v2.h[6]
567 SMLAL2 v22.4s, v4.8h, v2.h[6]
568 SMLAL v26.4s, v5.4h, v2.h[6]
569 SMLAL2 v30.4s, v5.8h, v2.h[6]
570 SMLAL v19.4s, v4.4h, v3.h[6]
571 SMLAL2 v23.4s, v4.8h, v3.h[6]
572 SMLAL v27.4s, v5.4h, v3.h[6]
573 SMLAL2 v31.4s, v5.8h, v3.h[6]
574 B 3b
575
576 # Store odd width
577 .p2align 3
578 5:
579 TBZ x1, 3, 6f
580 STR d3, [x7], 8
581 STR d2, [x17], 8
582 DUP d3, v3.d[1]
583 DUP d2, v2.d[1]
584 STR d1, [x16], 8
585 STR d0, [x6], 8
586 DUP d1, v1.d[1]
587 DUP d0, v0.d[1]
588 6:
589 TBZ x1, 2, 7f
590 STR s3, [x7], 4
591 STR s2, [x17], 4
592 DUP s3, v3.s[1]
593 DUP s2, v2.s[1]
594 STR s1, [x16], 4
595 STR s0, [x6], 4
596 DUP s1, v1.s[1]
597 DUP s0, v0.s[1]
598 7:
599 TBZ x1, 1, 8f
600 STR h3, [x7], 2
601 STR h2, [x17], 2
602 DUP h3, v3.h[1]
603 DUP h2, v2.h[1]
604 STR h1, [x16], 2
605 STR h0, [x6], 2
606 DUP h1, v1.h[1]
607 DUP h0, v0.h[1]
608 8:
609 TBZ x1, 0, 9f
610 STR b3, [x7]
611 STR b2, [x17]
612 STR b1, [x16]
613 STR b0, [x6]
614 9:
615 # Restore x20 from stack
616 LDR x20, [sp], 16
617 RET
618
619 END_FUNCTION xnn_qs8_qc8w_igemm_minmax_fp32_ukernel_4x16__asm_aarch64_neon_mlal_lane_ld64_prfm
620
621 #ifdef __ELF__
622 .section ".note.GNU-stack","",%progbits
623 #endif