Debian Codegraph @

src/qd8-f16-qc8w-igemm/gen/qd8-f16-qc8w-igemm-4x16c4-minmax-asm-aarch64-neondot-ld128.S

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-igemm/4x16c4-aarch64-neondot-ld128.S.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2021 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include "xnnpack/assembly.h"
12
13 # void xnn_qd8_f16_qc8w_igemm_minmax_ukernel_4x16c4__asm_aarch64_neondotfp16arith_ld128(
14 # size_t mr, x0
15 # size_t nc, x1
16 # size_t kc, x2 / x0
17 # size_t ks, x3 / x9
18 # const int8_t** restrict a, x4
19 # const int8_t* restrict w, x5
20 # int8_t* restrict c, x6
21 # size_t cm_stride, x7
22 # size_t cn_stride, [sp] -> (x0)
23 # size_t a_offset, [sp + 8] -> x8
24 # const int8_t* zero, [sp + 16] -> x12
25 # const int8_t* zero_data, [sp + 24] -> x19
26 # const union xnn_f16_minmax_params *params, [sp + 32] -> x11
27 # const struct xnn_qd8_quantization_params *quantization_params) [sp + 40] -> x16
28
29 # d8-d15, x19-x30 need to be preserved if used. x18 is reserved by the OS.
30
31 // Register usage
32 // A0 x13 v0
33 // A1 x14 v1
34 // A2 x15 v2
35 // A3 x10 v3
36 // B x5 v4 v5 v6 v7
37 // C0 x6 v16 v20 v24 v28
38 // C1 x16 v17 v21 v25 v29
39 // C2 x17 v18 v22 v26 v30
40 // C3 x7 v19 v23 v27 v31
41 // unused v8 v9 v10 v11 v12 v13 v14 v15
42
43 BEGIN_FUNCTION xnn_qd8_f16_qc8w_igemm_minmax_ukernel_4x16c4__asm_aarch64_neondotfp16arith_ld128
44
45 # Clamp C pointers
46 CMP x0, 2 // if mr < 2
47 LDR x8, [sp, 8] // Load a_offset
48 ADD x16, x6, x7 // c1 = c0 + cm_stride
49 CSEL x16, x6, x16, LO // c1 = c0
50 ADD x2, x2, 3 // kc = (kc + 3) & ~3
51
52 ADD x17, x16, x7 // c2 = c1 + cm_stride
53 LDR x12, [sp, 16] // Load zero
54 LDR x11, [sp, 32] // Load params pointer
55 // if mr <= 2
56 CSEL x17, x16, x17, LS // c2 = c1
57 BIC x2, x2, 3
58 STR x19, [sp, -56] // Push x19 to the stack
59 LDR x19, [sp, 24] // Load zero_data
60 LDR x15, [sp, 40] // &quantization_params[0].zero_point
61 STP d8, d9, [sp, -48]!
62 STP d10, d11, [sp, 16]
63 STP d12, d13, [sp, 32]
64 LD1 {v12.4s}, [x15] // v12 & v13 interleaved zero_point & scale
65
66 CMP x0, 4 // if mr < 4
67 ADD x7, x17, x7 // c3 = c2 + cm_stride
68 CSEL x7, x17, x7, LO // c3 = c2
69
70 .p2align 3
71 0:
72 # Load initial bias from w into accumulators
73 LDP q16, q20, [x5], 32
74 MUL v16.4s, v16.4s, v12.s[0]
75 MUL v20.4s, v20.4s, v12.s[0]
76 MOV v17.16b, v16.16b
77 MOV v18.16b, v16.16b
78 LDP q24, q28, [x5], 32
79 MUL v24.4s, v24.4s, v12.s[0]
80 MUL v28.4s, v28.4s, v12.s[0]
81 MOV v19.16b, v16.16b
82 MOV v21.16b, v20.16b
83 MOV v22.16b, v20.16b
84 MOV v23.16b, v20.16b
85 MOV v25.16b, v24.16b
86 MOV v26.16b, v24.16b
87 MOV v27.16b, v24.16b
88 MOV v29.16b, v28.16b
89 MOV v30.16b, v28.16b
90 MOV v31.16b, v28.16b
91 MOV x9, x3 // p = ks
92
93 .p2align 3
94 1:
95 # Load next 4 A pointers
96 LDP x13, x14, [x4], 16
97 LDP x15, x10, [x4], 16
98
99 CMP x13, x12 // if a0 == zero
100 ADD x13, x13, x8 // a0 += a_offset
101 CSEL x13, x19, x13, EQ // a0 = zero_data, else a0 += a_offset
102 CMP x14, x12 // if a1 == zero
103 ADD x14, x14, x8 // a1 += a_offset
104 CSEL x14, x19, x14, EQ // a1 = zero_data, else a1 += a_offset
105 CMP x15, x12 // if a2 == zero
106 ADD x15, x15, x8 // a2 += a_offset
107 CSEL x15, x19, x15, EQ // a2 = zero_data, else a2 += a_offset
108 CMP x10, x12 // if a3 == zero
109 ADD x10, x10, x8 // a3 += a_offset
110 CSEL x10, x19, x10, EQ // a3 = zero_data, else a3 += a_offset
111
112 # Is there at least 16 bytes for main loop?
113 SUBS x0, x2, 16 // k = kc - 16
114 B.LO 4f
115
116 # Main loop - 16 bytes of A
117 .p2align 3
118 2:
119 LDR q0, [x13], 16
120 LDR q4, [x5], 16
121 LDR q1, [x14], 16
122 LDR q2, [x15], 16
123 LDR q3, [x10], 16
124 LDR q5, [x5], 16
125 SDOT v16.4s, v4.16b, v0.4b[0]
126 SDOT v17.4s, v4.16b, v1.4b[0]
127 LDP q6, q7, [x5], 32
128 SDOT v18.4s, v4.16b, v2.4b[0]
129 SDOT v19.4s, v4.16b, v3.4b[0]
130 SDOT v20.4s, v5.16b, v0.4b[0]
131 SDOT v21.4s, v5.16b, v1.4b[0]
132 SDOT v22.4s, v5.16b, v2.4b[0]
133 SDOT v23.4s, v5.16b, v3.4b[0]
134 SDOT v24.4s, v6.16b, v0.4b[0]
135 SDOT v25.4s, v6.16b, v1.4b[0]
136 LDP q4, q5, [x5], 32
137 SDOT v26.4s, v6.16b, v2.4b[0]
138 SDOT v27.4s, v6.16b, v3.4b[0]
139 SDOT v28.4s, v7.16b, v0.4b[0]
140 SDOT v29.4s, v7.16b, v1.4b[0]
141 SDOT v30.4s, v7.16b, v2.4b[0]
142 SDOT v31.4s, v7.16b, v3.4b[0]
143
144 SDOT v16.4s, v4.16b, v0.4b[1]
145 SDOT v17.4s, v4.16b, v1.4b[1]
146 LDP q6, q7, [x5], 32
147 SDOT v18.4s, v4.16b, v2.4b[1]
148 SDOT v19.4s, v4.16b, v3.4b[1]
149 SDOT v20.4s, v5.16b, v0.4b[1]
150 SDOT v21.4s, v5.16b, v1.4b[1]
151 SDOT v22.4s, v5.16b, v2.4b[1]
152 SDOT v23.4s, v5.16b, v3.4b[1]
153 SDOT v24.4s, v6.16b, v0.4b[1]
154 SDOT v25.4s, v6.16b, v1.4b[1]
155 LDP q4, q5, [x5], 32
156 SDOT v26.4s, v6.16b, v2.4b[1]
157 SDOT v27.4s, v6.16b, v3.4b[1]
158 SDOT v28.4s, v7.16b, v0.4b[1]
159 SDOT v29.4s, v7.16b, v1.4b[1]
160 SDOT v30.4s, v7.16b, v2.4b[1]
161 SDOT v31.4s, v7.16b, v3.4b[1]
162
163 SDOT v16.4s, v4.16b, v0.4b[2]
164 SDOT v17.4s, v4.16b, v1.4b[2]
165 LDP q6, q7, [x5], 32
166 SDOT v18.4s, v4.16b, v2.4b[2]
167 SDOT v19.4s, v4.16b, v3.4b[2]
168 SDOT v20.4s, v5.16b, v0.4b[2]
169 SDOT v21.4s, v5.16b, v1.4b[2]
170 SDOT v22.4s, v5.16b, v2.4b[2]
171 SDOT v23.4s, v5.16b, v3.4b[2]
172 SDOT v24.4s, v6.16b, v0.4b[2]
173 SDOT v25.4s, v6.16b, v1.4b[2]
174 LDP q4, q5, [x5], 32
175 SDOT v26.4s, v6.16b, v2.4b[2]
176 SDOT v27.4s, v6.16b, v3.4b[2]
177 SDOT v28.4s, v7.16b, v0.4b[2]
178 SDOT v29.4s, v7.16b, v1.4b[2]
179 SDOT v30.4s, v7.16b, v2.4b[2]
180 SDOT v31.4s, v7.16b, v3.4b[2]
181
182 SDOT v16.4s, v4.16b, v0.4b[3]
183 SDOT v17.4s, v4.16b, v1.4b[3]
184 LDP q6, q7, [x5], 32
185 SDOT v18.4s, v4.16b, v2.4b[3]
186 SDOT v19.4s, v4.16b, v3.4b[3]
187 SDOT v20.4s, v5.16b, v0.4b[3]
188 SDOT v21.4s, v5.16b, v1.4b[3]
189 SDOT v22.4s, v5.16b, v2.4b[3]
190 SDOT v23.4s, v5.16b, v3.4b[3]
191 SDOT v24.4s, v6.16b, v0.4b[3]
192 SDOT v25.4s, v6.16b, v1.4b[3]
193 SDOT v26.4s, v6.16b, v2.4b[3]
194 SDOT v27.4s, v6.16b, v3.4b[3]
195 SUBS x0, x0, 16
196 SDOT v28.4s, v7.16b, v0.4b[3]
197 SDOT v29.4s, v7.16b, v1.4b[3]
198 SDOT v30.4s, v7.16b, v2.4b[3]
199 SDOT v31.4s, v7.16b, v3.4b[3]
200 B.HS 2b
201
202 # Is there a remainder?- 4 to 12 bytes of A
203 TST x0, 15
204 B.NE 4f
205
206 3:
207 # ks loop
208 SUBS x9, x9, 32 // ks -= MR * sizeof(int8_t*)
209 B.HI 1b
210
211 LDP q0, q1, [x5], 32 // kernel_scale
212 SCVTF v19.4s, v19.4s
213 SCVTF v23.4s, v23.4s
214 SCVTF v27.4s, v27.4s
215 SCVTF v31.4s, v31.4s
216 SCVTF v18.4s, v18.4s
217 SCVTF v22.4s, v22.4s
218 SCVTF v26.4s, v26.4s
219 LDP q2, q3, [x5], 32
220 SCVTF v30.4s, v30.4s
221 SCVTF v17.4s, v17.4s
222 SCVTF v21.4s, v21.4s
223 SCVTF v25.4s, v25.4s
224 SCVTF v29.4s, v29.4s
225 SCVTF v16.4s, v16.4s
226 SCVTF v20.4s, v20.4s
227 SCVTF v24.4s, v24.4s
228 SCVTF v28.4s, v28.4s
229 FMUL v8.4s, v0.4s, v12.s[1] // kernel_scale * scale
230 FMUL v9.4s, v1.4s, v12.s[1]
231 FMUL v10.4s, v2.4s, v12.s[1]
232 FMUL v11.4s, v3.4s, v12.s[1]
233 FMUL v4.4s, v0.4s, v12.s[1]
234 FMUL v5.4s, v1.4s, v12.s[1]
235 FMUL v6.4s, v2.4s, v12.s[1]
236 FMUL v7.4s, v3.4s, v12.s[1]
237 LDP q0, q1, [x5], 32 // bias
238 FMUL v19.4s, v19.4s, v8.4s
239 FMUL v23.4s, v23.4s, v9.4s
240 FMUL v27.4s, v27.4s, v10.4s
241 FMUL v31.4s, v31.4s, v11.4s
242 FMUL v18.4s, v18.4s, v4.4s
243 FMUL v22.4s, v22.4s, v5.4s
244 FMUL v26.4s, v26.4s, v6.4s
245 FMUL v30.4s, v30.4s, v7.4s
246 LDP q2, q3, [x5], 32
247 FMUL v17.4s, v17.4s, v8.4s
248 FMUL v21.4s, v21.4s, v9.4s
249 FMUL v25.4s, v25.4s, v10.4s
250 FMUL v29.4s, v29.4s, v11.4s
251 FMUL v16.4s, v16.4s, v4.4s
252 FMUL v20.4s, v20.4s, v5.4s
253 FMUL v24.4s, v24.4s, v6.4s
254 FMUL v28.4s, v28.4s, v7.4s
255 LD2R {v4.8h, v5.8h}, [x11] // min max
256 FADD v19.4s, v19.4s, v0.4s
257 FADD v23.4s, v23.4s, v1.4s
258 FADD v27.4s, v27.4s, v2.4s
259 FADD v31.4s, v31.4s, v3.4s
260 FADD v18.4s, v18.4s, v0.4s
261 FADD v22.4s, v22.4s, v1.4s
262 FADD v26.4s, v26.4s, v2.4s
263 FADD v30.4s, v30.4s, v3.4s
264 FADD v17.4s, v17.4s, v0.4s
265 FADD v21.4s, v21.4s, v1.4s
266 FADD v25.4s, v25.4s, v2.4s
267 FADD v29.4s, v29.4s, v3.4s
268 FADD v16.4s, v16.4s, v0.4s
269 FADD v20.4s, v20.4s, v1.4s
270 FADD v24.4s, v24.4s, v2.4s
271 FADD v28.4s, v28.4s, v3.4s
272 FCVTN v19.4h, v19.4s
273 FCVTN v27.4h, v27.4s
274 FCVTN v18.4h, v18.4s
275 FCVTN v26.4h, v26.4s
276 FCVTN v17.4h, v17.4s
277 FCVTN v25.4h, v25.4s
278 FCVTN v16.4h, v16.4s
279 FCVTN v24.4h, v24.4s
280 FCVTN2 v19.8h, v23.4s
281 FCVTN2 v27.8h, v31.4s
282 FCVTN2 v18.8h, v22.4s
283 FCVTN2 v26.8h, v30.4s
284 FCVTN2 v17.8h, v21.4s
285 FCVTN2 v25.8h, v29.4s
286 FCVTN2 v16.8h, v20.4s
287 FCVTN2 v24.8h, v28.4s
288 LDR x0, [sp, 48] // cn_stride
289 FMAX v19.8h, v19.8h, v4.8h
290 FMAX v27.8h, v27.8h, v4.8h
291 FMAX v18.8h, v18.8h, v4.8h
292 FMAX v26.8h, v26.8h, v4.8h
293 FMAX v17.8h, v17.8h, v4.8h
294 FMAX v25.8h, v25.8h, v4.8h
295 FMAX v16.8h, v16.8h, v4.8h
296 FMAX v24.8h, v24.8h, v4.8h
297 SUBS x1, x1, 16
298 FMIN v19.8h, v19.8h, v5.8h
299 FMIN v27.8h, v27.8h, v5.8h
300 FMIN v18.8h, v18.8h, v5.8h
301 FMIN v26.8h, v26.8h, v5.8h
302 FMIN v17.8h, v17.8h, v5.8h
303 FMIN v25.8h, v25.8h, v5.8h
304 FMIN v16.8h, v16.8h, v5.8h
305 FMIN v24.8h, v24.8h, v5.8h
306 B.LO 6f
307
308 STP q19, q27, [x7]
309 ADD x7, x7, x0
310 STP q18, q26, [x17]
311 ADD x17, x17, x0
312 STP q17, q25, [x16]
313 ADD x16, x16, x0
314 STP q16, q24, [x6]
315 ADD x6, x6, x0
316 SUB x4, x4, x3 // a -= ks
317
318 # nc loop
319 B.HI 0b
320 # Restore d8-d13 from stack
321 LDR x19, [sp, -8]
322 LDP d12, d13, [sp, 32]
323 LDP d10, d11, [sp, 16]
324 LDP d8, d9, [sp], 48
325 RET
326
327
328 # Remainder- 8 bytes of A
329 .p2align 3
330 4:
331 # Is there a remainder?- 8 bytes of A
332 TBZ x0, 3, 5f
333
334 LDR d0, [x13], 8
335 LDR q4, [x5], 16
336 LDR d1, [x14], 8
337 LDR d2, [x15], 8
338 LDR d3, [x10], 8
339 LDR q5, [x5], 16
340 SDOT v16.4s, v4.16b, v0.4b[0]
341 SDOT v17.4s, v4.16b, v1.4b[0]
342 LDP q6, q7, [x5], 32
343 SDOT v18.4s, v4.16b, v2.4b[0]
344 SDOT v19.4s, v4.16b, v3.4b[0]
345 SDOT v20.4s, v5.16b, v0.4b[0]
346 SDOT v21.4s, v5.16b, v1.4b[0]
347 SDOT v22.4s, v5.16b, v2.4b[0]
348 SDOT v23.4s, v5.16b, v3.4b[0]
349 SDOT v24.4s, v6.16b, v0.4b[0]
350 SDOT v25.4s, v6.16b, v1.4b[0]
351 LDP q4, q5, [x5], 32
352 SDOT v26.4s, v6.16b, v2.4b[0]
353 SDOT v27.4s, v6.16b, v3.4b[0]
354 SDOT v28.4s, v7.16b, v0.4b[0]
355 SDOT v29.4s, v7.16b, v1.4b[0]
356 SDOT v30.4s, v7.16b, v2.4b[0]
357 SDOT v31.4s, v7.16b, v3.4b[0]
358 SDOT v16.4s, v4.16b, v0.4b[1]
359 SDOT v17.4s, v4.16b, v1.4b[1]
360 LDP q6, q7, [x5], 32
361 SDOT v18.4s, v4.16b, v2.4b[1]
362 SDOT v19.4s, v4.16b, v3.4b[1]
363 SDOT v20.4s, v5.16b, v0.4b[1]
364 SDOT v21.4s, v5.16b, v1.4b[1]
365 SDOT v22.4s, v5.16b, v2.4b[1]
366 SDOT v23.4s, v5.16b, v3.4b[1]
367 SDOT v24.4s, v6.16b, v0.4b[1]
368 SDOT v25.4s, v6.16b, v1.4b[1]
369 SDOT v26.4s, v6.16b, v2.4b[1]
370 SDOT v27.4s, v6.16b, v3.4b[1]
371 SDOT v28.4s, v7.16b, v0.4b[1]
372 SDOT v29.4s, v7.16b, v1.4b[1]
373 SDOT v30.4s, v7.16b, v2.4b[1]
374 SDOT v31.4s, v7.16b, v3.4b[1]
375 # Is there a remainder?- 4 bytes of A
376 TBZ x0, 2, 3b
377
378 # Remainder- 4 bytes of A
379 5:
380 LDR s0, [x13], 4
381 LDR q4, [x5], 16
382 LDR s1, [x14], 4
383 LDR s2, [x15], 4
384 LDR s3, [x10], 4
385 LDR q5, [x5], 16
386 SDOT v16.4s, v4.16b, v0.4b[0]
387 SDOT v17.4s, v4.16b, v1.4b[0]
388 LDP q6, q7, [x5], 32
389 SDOT v18.4s, v4.16b, v2.4b[0]
390 SDOT v19.4s, v4.16b, v3.4b[0]
391 SDOT v20.4s, v5.16b, v0.4b[0]
392 SDOT v21.4s, v5.16b, v1.4b[0]
393 SDOT v22.4s, v5.16b, v2.4b[0]
394 SDOT v23.4s, v5.16b, v3.4b[0]
395 SDOT v24.4s, v6.16b, v0.4b[0]
396 SDOT v25.4s, v6.16b, v1.4b[0]
397 SDOT v26.4s, v6.16b, v2.4b[0]
398 SDOT v27.4s, v6.16b, v3.4b[0]
399 SDOT v28.4s, v7.16b, v0.4b[0]
400 SDOT v29.4s, v7.16b, v1.4b[0]
401 SDOT v30.4s, v7.16b, v2.4b[0]
402 SDOT v31.4s, v7.16b, v3.4b[0]
403 B 3b
404
405 # Store odd width
406 .p2align 3
407 6:
408 TBZ x1, 3, 7f
409 STR q19, [x7], 16
410 STR q18, [x17], 16
411 MOV v19.16b, v27.16b
412 MOV v18.16b, v26.16b
413 STR q17, [x16], 16
414 STR q16, [x6], 16
415 MOV v17.16b, v25.16b
416 MOV v16.16b, v24.16b
417 7:
418 TBZ x1, 2, 8f
419 STR d19, [x7], 8
420 STR d18, [x17], 8
421 DUP d19, v19.d[1]
422 DUP d18, v18.d[1]
423 STR d17, [x16], 8
424 STR d16, [x6], 8
425 DUP d17, v17.d[1]
426 DUP d16, v16.d[1]
427 8:
428 TBZ x1, 1, 9f
429 STR s19, [x7], 4
430 STR s18, [x17], 4
431 DUP s19, v19.s[1]
432 DUP s18, v18.s[1]
433 STR s17, [x16], 4
434 STR s16, [x6], 4
435 DUP s17, v17.s[1]
436 DUP s16, v16.s[1]
437 9:
438 TBZ x1, 0, 10f
439 STR h19, [x7]
440 STR h18, [x17]
441 STR h17, [x16]
442 STR h16, [x6]
443 10:
444 # Restore d8-d13 from stack
445 LDR x19, [sp, -8]
446 LDP d12, d13, [sp, 32]
447 LDP d10, d11, [sp, 16]
448 LDP d8, d9, [sp], 48
449 RET
450
451 END_FUNCTION xnn_qd8_f16_qc8w_igemm_minmax_ukernel_4x16c4__asm_aarch64_neondotfp16arith_ld128
452
453 #ifdef __ELF__
454 .section ".note.GNU-stack","",%progbits
455 #endif