Debian Codegraph @

src/qd8-f16-qc8w-gemm/gen/qd8-f16-qc8w-gemm-4x16c4-minmax-asm-aarch64-neondot-ld128.S

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-gemm/4x16c4-aarch64-neondot-ld128.S.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2020 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include "xnnpack/assembly.h"
12
13 # void xnn_qd8_f16_qc8w_gemm_minmax_ukernel_4x16c4__asm_aarch64_neondotfp16arith_ld128(
14 # size_t mr, x0
15 # size_t nc, x1
16 # size_t kc, x2 / x0
17 # const int8_t* restrict a, x3
18 # size_t a_stride, x4
19 # const void* restrict w, x5
20 # int8_t* restrict c, x6
21 # size_t cm_stride, x7
22 # size_t cn_stride, [sp] -> x12
23 # const union xnn_f16_minmax_params *params, [sp + 8] -> x11
24 # const struct xnn_qd8_quantization_params *quantization_params) [sp + 16] -> x16
25
26 # params structure is 8 bytes
27 # struct {
28 # int32_t zero_point;
29 # float scale;
30 # } scalar;
31
32 # d8-d15, x19-x30 need to be preserved if used. x18 is reserved by the OS.
33
34 // Register usage
35 // A0 x3 v0
36 // A1 x15 v1
37 // A2 x13 v2
38 // A3 x4 v3
39 // B x5 v4 v5 v6 v7
40 // C0 x6 v16 v20 v24 v28
41 // C1 x8 v17 v21 v25 v29
42 // C2 x9 v18 v22 v26 v30
43 // C3 x7 v19 v23 v27 v31
44 // unused v14 v15
45
46 BEGIN_FUNCTION xnn_qd8_f16_qc8w_gemm_minmax_ukernel_4x16c4__asm_aarch64_neondotfp16arith_ld128
47
48 # Clamp A and C pointers
49 CMP x0, 2 // if mr < 2
50 ADD x2, x2, 3 // kc = (kc + 3) & ~3
51 ADD x15, x3, x4 // a1 = a0 + a_stride
52 ADD x8, x6, x7 // c1 = c0 + cm_stride
53 CSEL x15, x3, x15, LO // a1 = a0
54 CSEL x8, x6, x8, LO // c1 = c0
55 BIC x2, x2, 3
56
57 ADD x13, x15, x4 // a2 = a1 + a_stride
58 ADD x9, x8, x7 // c2 = c1 + cm_stride
59 // if mr <= 2
60 CSEL x13, x15, x13, LS // a2 = a1
61 CSEL x9, x8, x9, LS // c2 = c1
62
63 LDP x12, x11, [sp] // cn_stride, params
64 LDR x16, [sp, 16] // &quantization_params[0].zero_point
65 STP d8, d9, [sp, -48]!
66 STP d10, d11, [sp, 16]
67 STP d12, d13, [sp, 32]
68 LDP q12, q13, [x16] // v12 & v13 interleaved zero_point & scale
69
70 CMP x0, 4 // if mr < 4
71 ADD x4, x13, x4 // a3 = a2 + a_stride
72 ADD x7, x9, x7 // c3 = c2 + cm_stride
73 CSEL x4, x13, x4, LO // a3 = a2
74 CSEL x7, x9, x7, LO // c3 = c2
75
76 .p2align 3
77 0:
78 # Load initial bias from w into accumulators
79 SUBS x0, x2, 16 // k = kc - 16
80 LDP q0, q1, [x5], 32
81 MUL v16.4s, v0.4s, v12.s[0]
82 MUL v17.4s, v0.4s, v12.s[2]
83 MUL v18.4s, v0.4s, v13.s[0]
84 LDP q2, q3, [x5], 32
85 MUL v19.4s, v0.4s, v13.s[2]
86 MUL v20.4s, v1.4s, v12.s[0]
87 MUL v21.4s, v1.4s, v12.s[2]
88 MUL v22.4s, v1.4s, v13.s[0]
89 MUL v23.4s, v1.4s, v13.s[2]
90 MUL v24.4s, v2.4s, v12.s[0]
91 MUL v25.4s, v2.4s, v12.s[2]
92 MUL v26.4s, v2.4s, v13.s[0]
93 MUL v27.4s, v2.4s, v13.s[2]
94 MUL v28.4s, v3.4s, v12.s[0]
95 MUL v29.4s, v3.4s, v12.s[2]
96 MUL v30.4s, v3.4s, v13.s[0]
97 MUL v31.4s, v3.4s, v13.s[2]
98
99 # Is there at least 16 bytes?
100 B.LO 3f
101
102 # Main loop - 16 bytes of A
103 .p2align 3
104 1:
105 LDR q0, [x3], 16
106 LDR q4, [x5], 16
107 LDR q1, [x15], 16
108 LDR q2, [x13], 16
109 LDR q3, [x4], 16
110 LDR q5, [x5], 16
111 SDOT v16.4s, v4.16b, v0.4b[0]
112 SDOT v17.4s, v4.16b, v1.4b[0]
113 LDP q6, q7, [x5], 32
114 SDOT v18.4s, v4.16b, v2.4b[0]
115 SDOT v19.4s, v4.16b, v3.4b[0]
116 SDOT v20.4s, v5.16b, v0.4b[0]
117 SDOT v21.4s, v5.16b, v1.4b[0]
118 SDOT v22.4s, v5.16b, v2.4b[0]
119 SDOT v23.4s, v5.16b, v3.4b[0]
120 SDOT v24.4s, v6.16b, v0.4b[0]
121 SDOT v25.4s, v6.16b, v1.4b[0]
122 LDP q4, q5, [x5], 32
123 SDOT v26.4s, v6.16b, v2.4b[0]
124 SDOT v27.4s, v6.16b, v3.4b[0]
125 SDOT v28.4s, v7.16b, v0.4b[0]
126 SDOT v29.4s, v7.16b, v1.4b[0]
127 SDOT v30.4s, v7.16b, v2.4b[0]
128 SDOT v31.4s, v7.16b, v3.4b[0]
129
130 SDOT v16.4s, v4.16b, v0.4b[1]
131 SDOT v17.4s, v4.16b, v1.4b[1]
132 LDP q6, q7, [x5], 32
133 SDOT v18.4s, v4.16b, v2.4b[1]
134 SDOT v19.4s, v4.16b, v3.4b[1]
135 SDOT v20.4s, v5.16b, v0.4b[1]
136 SDOT v21.4s, v5.16b, v1.4b[1]
137 SDOT v22.4s, v5.16b, v2.4b[1]
138 SDOT v23.4s, v5.16b, v3.4b[1]
139 SDOT v24.4s, v6.16b, v0.4b[1]
140 SDOT v25.4s, v6.16b, v1.4b[1]
141 LDP q4, q5, [x5], 32
142 SDOT v26.4s, v6.16b, v2.4b[1]
143 SDOT v27.4s, v6.16b, v3.4b[1]
144 SDOT v28.4s, v7.16b, v0.4b[1]
145 SDOT v29.4s, v7.16b, v1.4b[1]
146 SDOT v30.4s, v7.16b, v2.4b[1]
147 SDOT v31.4s, v7.16b, v3.4b[1]
148
149 SDOT v16.4s, v4.16b, v0.4b[2]
150 SDOT v17.4s, v4.16b, v1.4b[2]
151 LDP q6, q7, [x5], 32
152 SDOT v18.4s, v4.16b, v2.4b[2]
153 SDOT v19.4s, v4.16b, v3.4b[2]
154 SDOT v20.4s, v5.16b, v0.4b[2]
155 SDOT v21.4s, v5.16b, v1.4b[2]
156 SDOT v22.4s, v5.16b, v2.4b[2]
157 SDOT v23.4s, v5.16b, v3.4b[2]
158 SDOT v24.4s, v6.16b, v0.4b[2]
159 SDOT v25.4s, v6.16b, v1.4b[2]
160 LDP q4, q5, [x5], 32
161 SDOT v26.4s, v6.16b, v2.4b[2]
162 SDOT v27.4s, v6.16b, v3.4b[2]
163 SDOT v28.4s, v7.16b, v0.4b[2]
164 SDOT v29.4s, v7.16b, v1.4b[2]
165 SDOT v30.4s, v7.16b, v2.4b[2]
166 SDOT v31.4s, v7.16b, v3.4b[2]
167
168 SDOT v16.4s, v4.16b, v0.4b[3]
169 SDOT v17.4s, v4.16b, v1.4b[3]
170 LDP q6, q7, [x5], 32
171 SDOT v18.4s, v4.16b, v2.4b[3]
172 SDOT v19.4s, v4.16b, v3.4b[3]
173 SDOT v20.4s, v5.16b, v0.4b[3]
174 SDOT v21.4s, v5.16b, v1.4b[3]
175 SDOT v22.4s, v5.16b, v2.4b[3]
176 SDOT v23.4s, v5.16b, v3.4b[3]
177 SDOT v24.4s, v6.16b, v0.4b[3]
178 SDOT v25.4s, v6.16b, v1.4b[3]
179 SDOT v26.4s, v6.16b, v2.4b[3]
180 SDOT v27.4s, v6.16b, v3.4b[3]
181 SUBS x0, x0, 16
182 SDOT v28.4s, v7.16b, v0.4b[3]
183 SDOT v29.4s, v7.16b, v1.4b[3]
184 SDOT v30.4s, v7.16b, v2.4b[3]
185 SDOT v31.4s, v7.16b, v3.4b[3]
186 B.HS 1b
187
188 # Is there a remainder?- 4 to 12 bytes of A
189 TST x0, 15
190 B.NE 3f
191
192 2:
193 LDP q0, q1, [x5], 32 // kernel_scale
194 SCVTF v19.4s, v19.4s
195 SCVTF v23.4s, v23.4s
196 SCVTF v27.4s, v27.4s
197 SCVTF v31.4s, v31.4s
198 SCVTF v18.4s, v18.4s
199 SCVTF v22.4s, v22.4s
200 SCVTF v26.4s, v26.4s
201 LDP q2, q3, [x5], 32
202 SCVTF v30.4s, v30.4s
203 SCVTF v17.4s, v17.4s
204 SCVTF v21.4s, v21.4s
205 SCVTF v25.4s, v25.4s
206 SCVTF v29.4s, v29.4s
207 SCVTF v16.4s, v16.4s
208 SCVTF v20.4s, v20.4s
209 SCVTF v24.4s, v24.4s
210 SCVTF v28.4s, v28.4s
211 FMUL v8.4s, v0.4s, v13.s[3] // kernel_scale * scale
212 FMUL v9.4s, v1.4s, v13.s[3]
213 FMUL v10.4s, v2.4s, v13.s[3]
214 FMUL v11.4s, v3.4s, v13.s[3]
215 FMUL v4.4s, v0.4s, v13.s[1]
216 FMUL v5.4s, v1.4s, v13.s[1]
217 FMUL v6.4s, v2.4s, v13.s[1]
218 FMUL v7.4s, v3.4s, v13.s[1]
219 FMUL v19.4s, v19.4s, v8.4s
220 FMUL v8.4s, v0.4s, v12.s[3]
221 FMUL v23.4s, v23.4s, v9.4s
222 FMUL v9.4s, v1.4s, v12.s[3]
223 FMUL v27.4s, v27.4s, v10.4s
224 FMUL v10.4s, v2.4s, v12.s[3]
225 FMUL v31.4s, v31.4s, v11.4s
226 FMUL v11.4s, v3.4s, v12.s[3]
227 FMUL v18.4s, v18.4s, v4.4s
228 FMUL v4.4s, v0.4s, v12.s[1]
229 FMUL v22.4s, v22.4s, v5.4s
230 FMUL v5.4s, v1.4s, v12.s[1]
231 LDP q0, q1, [x5], 32 // bias
232 FMUL v26.4s, v26.4s, v6.4s
233 FMUL v6.4s, v2.4s, v12.s[1]
234 FMUL v30.4s, v30.4s, v7.4s
235 FMUL v7.4s, v3.4s, v12.s[1]
236 FMUL v17.4s, v17.4s, v8.4s
237 FMUL v21.4s, v21.4s, v9.4s
238 FMUL v25.4s, v25.4s, v10.4s
239 FMUL v29.4s, v29.4s, v11.4s
240 LDP q2, q3, [x5], 32
241 FMUL v16.4s, v16.4s, v4.4s
242 FMUL v20.4s, v20.4s, v5.4s
243 FMUL v24.4s, v24.4s, v6.4s
244 FMUL v28.4s, v28.4s, v7.4s
245 LD2R {v4.8h, v5.8h}, [x11] // min max
246 FADD v19.4s, v19.4s, v0.4s
247 FADD v23.4s, v23.4s, v1.4s
248 FADD v27.4s, v27.4s, v2.4s
249 FADD v31.4s, v31.4s, v3.4s
250 FADD v18.4s, v18.4s, v0.4s
251 FADD v22.4s, v22.4s, v1.4s
252 FADD v26.4s, v26.4s, v2.4s
253 FADD v30.4s, v30.4s, v3.4s
254 FADD v17.4s, v17.4s, v0.4s
255 FADD v21.4s, v21.4s, v1.4s
256 FADD v25.4s, v25.4s, v2.4s
257 FADD v29.4s, v29.4s, v3.4s
258 FADD v16.4s, v16.4s, v0.4s
259 FADD v20.4s, v20.4s, v1.4s
260 FADD v24.4s, v24.4s, v2.4s
261 FADD v28.4s, v28.4s, v3.4s
262 FCVTN v19.4h, v19.4s
263 FCVTN v27.4h, v27.4s
264 FCVTN v18.4h, v18.4s
265 FCVTN v26.4h, v26.4s
266 FCVTN v17.4h, v17.4s
267 FCVTN v25.4h, v25.4s
268 FCVTN v16.4h, v16.4s
269 FCVTN v24.4h, v24.4s
270 FCVTN2 v19.8h, v23.4s
271 FCVTN2 v27.8h, v31.4s
272 FCVTN2 v18.8h, v22.4s
273 FCVTN2 v26.8h, v30.4s
274 FCVTN2 v17.8h, v21.4s
275 FCVTN2 v25.8h, v29.4s
276 FCVTN2 v16.8h, v20.4s
277 FCVTN2 v24.8h, v28.4s
278 FMAX v19.8h, v19.8h, v4.8h
279 FMAX v27.8h, v27.8h, v4.8h
280 FMAX v18.8h, v18.8h, v4.8h
281 FMAX v26.8h, v26.8h, v4.8h
282 FMAX v17.8h, v17.8h, v4.8h
283 FMAX v25.8h, v25.8h, v4.8h
284 FMAX v16.8h, v16.8h, v4.8h
285 FMAX v24.8h, v24.8h, v4.8h
286 FMIN v19.8h, v19.8h, v5.8h
287 FMIN v27.8h, v27.8h, v5.8h
288 FMIN v18.8h, v18.8h, v5.8h
289 FMIN v26.8h, v26.8h, v5.8h
290 FMIN v17.8h, v17.8h, v5.8h
291 FMIN v25.8h, v25.8h, v5.8h
292 FMIN v16.8h, v16.8h, v5.8h
293 FMIN v24.8h, v24.8h, v5.8h
294
295 SUBS x1, x1, 16
296 B.LO 5f
297 STP q19, q27, [x7]
298 ADD x7, x7, x12
299 STP q18, q26, [x9]
300 ADD x9, x9, x12
301 STP q17, q25, [x8]
302 ADD x8, x8, x12
303 STP q16, q24, [x6]
304 ADD x6, x6, x12
305 SUB x3, x3, x2 // a0 -= kc
306 SUB x15, x15, x2 // a1 -= kc
307 SUB x13, x13, x2 // a2 -= kc
308 SUB x4, x4, x2 // a3 -= kc
309 B.NE 0b
310
311 # Restore d8-d13 from stack
312 LDP d12, d13, [sp, 32]
313 LDP d10, d11, [sp, 16]
314 LDP d8, d9, [sp], 48
315 RET
316
317
318 # Remainder- 8 bytes of A
319 .p2align 3
320 3:
321 # Is there a remainder?- 8 bytes of A
322 TBZ x0, 3, 4f
323
324 LDR d0, [x3], 8
325 LDR q4, [x5], 16
326 LDR d1, [x15], 8
327 LDR d2, [x13], 8
328 LDR d3, [x4], 8
329 LDR q5, [x5], 16
330 SDOT v16.4s, v4.16b, v0.4b[0]
331 SDOT v17.4s, v4.16b, v1.4b[0]
332 LDP q6, q7, [x5], 32
333 SDOT v18.4s, v4.16b, v2.4b[0]
334 SDOT v19.4s, v4.16b, v3.4b[0]
335 SDOT v20.4s, v5.16b, v0.4b[0]
336 SDOT v21.4s, v5.16b, v1.4b[0]
337 SDOT v22.4s, v5.16b, v2.4b[0]
338 SDOT v23.4s, v5.16b, v3.4b[0]
339 SDOT v24.4s, v6.16b, v0.4b[0]
340 SDOT v25.4s, v6.16b, v1.4b[0]
341 LDP q4, q5, [x5], 32
342 SDOT v26.4s, v6.16b, v2.4b[0]
343 SDOT v27.4s, v6.16b, v3.4b[0]
344 SDOT v28.4s, v7.16b, v0.4b[0]
345 SDOT v29.4s, v7.16b, v1.4b[0]
346 SDOT v30.4s, v7.16b, v2.4b[0]
347 SDOT v31.4s, v7.16b, v3.4b[0]
348 SDOT v16.4s, v4.16b, v0.4b[1]
349 SDOT v17.4s, v4.16b, v1.4b[1]
350 LDP q6, q7, [x5], 32
351 SDOT v18.4s, v4.16b, v2.4b[1]
352 SDOT v19.4s, v4.16b, v3.4b[1]
353 SDOT v20.4s, v5.16b, v0.4b[1]
354 SDOT v21.4s, v5.16b, v1.4b[1]
355 SDOT v22.4s, v5.16b, v2.4b[1]
356 SDOT v23.4s, v5.16b, v3.4b[1]
357 SDOT v24.4s, v6.16b, v0.4b[1]
358 SDOT v25.4s, v6.16b, v1.4b[1]
359 SDOT v26.4s, v6.16b, v2.4b[1]
360 SDOT v27.4s, v6.16b, v3.4b[1]
361 SDOT v28.4s, v7.16b, v0.4b[1]
362 SDOT v29.4s, v7.16b, v1.4b[1]
363 SDOT v30.4s, v7.16b, v2.4b[1]
364 SDOT v31.4s, v7.16b, v3.4b[1]
365 # Is there a remainder?- 4 bytes of A
366 TBZ x0, 2, 2b
367
368 # Remainder- 4 bytes of A
369 4:
370 LDR s0, [x3], 4
371 LDR q4, [x5], 16
372 LDR s1, [x15], 4
373 LDR s2, [x13], 4
374 LDR s3, [x4], 4
375 SDOT v16.4s, v4.16b, v0.4b[0]
376 LDR q5, [x5], 16
377 SDOT v17.4s, v4.16b, v1.4b[0]
378 SDOT v18.4s, v4.16b, v2.4b[0]
379 SDOT v19.4s, v4.16b, v3.4b[0]
380 SDOT v20.4s, v5.16b, v0.4b[0]
381 LDP q6, q7, [x5], 32
382 SDOT v21.4s, v5.16b, v1.4b[0]
383 SDOT v22.4s, v5.16b, v2.4b[0]
384 SDOT v23.4s, v5.16b, v3.4b[0]
385 SDOT v24.4s, v6.16b, v0.4b[0]
386 SDOT v25.4s, v6.16b, v1.4b[0]
387 SDOT v26.4s, v6.16b, v2.4b[0]
388 SDOT v27.4s, v6.16b, v3.4b[0]
389 SDOT v28.4s, v7.16b, v0.4b[0]
390 SDOT v29.4s, v7.16b, v1.4b[0]
391 SDOT v30.4s, v7.16b, v2.4b[0]
392 SDOT v31.4s, v7.16b, v3.4b[0]
393 B 2b
394
395 # Store odd width
396 .p2align 3
397 5:
398 TBZ x1, 3, 6f
399 STR q19, [x7], 16
400 STR q18, [x9], 16
401 MOV v19.16b, v27.16b
402 MOV v18.16b, v26.16b
403 STR q17, [x8], 16
404 STR q16, [x6], 16
405 MOV v17.16b, v25.16b
406 MOV v16.16b, v24.16b
407 6:
408 TBZ x1, 2, 7f
409 STR d19, [x7], 8
410 STR d18, [x9], 8
411 DUP d19, v19.d[1]
412 DUP d18, v18.d[1]
413 STR d17, [x8], 8
414 STR d16, [x6], 8
415 DUP d17, v17.d[1]
416 DUP d16, v16.d[1]
417 7:
418 TBZ x1, 1, 8f
419 STR s19, [x7], 4
420 STR s18, [x9], 4
421 DUP s19, v19.s[1]
422 DUP s18, v18.s[1]
423 STR s17, [x8], 4
424 STR s16, [x6], 4
425 DUP s17, v17.s[1]
426 DUP s16, v16.s[1]
427 8:
428 TBZ x1, 0, 9f
429 STR h19, [x7]
430 STR h18, [x9]
431 STR h17, [x8]
432 STR h16, [x6]
433 9:
434 # Restore d8-d13 from stack
435 LDP d12, d13, [sp, 32]
436 LDP d10, d11, [sp, 16]
437 LDP d8, d9, [sp], 48
438 RET
439
440 END_FUNCTION xnn_qd8_f16_qc8w_gemm_minmax_ukernel_4x16c4__asm_aarch64_neondotfp16arith_ld128
441
442 #ifdef __ELF__
443 .section ".note.GNU-stack","",%progbits
444 #endif