Debian Codegraph @

src/qd8-f32-qc8w-igemm/gen/qd8-f32-qc8w-igemm-4x16c4-minmax-asm-aarch64-neondot-ld128.S

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-igemm/4x16c4-aarch64-neondot-ld128.S.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2021 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include "xnnpack/assembly.h"
12
13 # void xnn_qd8_f32_qc8w_igemm_minmax_ukernel_4x16c4__asm_aarch64_neondot_ld128(
14 # size_t mr, x0
15 # size_t nc, x1
16 # size_t kc, x2 / x0
17 # size_t ks, x3 / x9
18 # const int8_t** restrict a, x4
19 # const int8_t* restrict w, x5
20 # int8_t* restrict c, x6
21 # size_t cm_stride, x7
22 # size_t cn_stride, [sp] -> (x0)
23 # size_t a_offset, [sp + 8] -> x8
24 # const int8_t* zero, [sp + 16] -> x12
25 # const int8_t* zero_data, [sp + 24] -> x19
26 # const union xnn_f32_minmax_params *params, [sp + 32] -> x11
27 # const struct xnn_qd8_quantization_params *quantization_params) [sp + 40] -> x16
28
29 # d8-d15, x19-x30 need to be preserved if used. x18 is reserved by the OS.
30
31 // Register usage
32 // A0 x13 v0
33 // A1 x14 v1
34 // A2 x15 v2
35 // A3 x10 v3
36 // B x5 v4 v5 v6 v7
37 // C0 x6 v16 v20 v24 v28
38 // C1 x16 v17 v21 v25 v29
39 // C2 x17 v18 v22 v26 v30
40 // C3 x7 v19 v23 v27 v31
41 // unused v8 v9 v10 v11 v12 v13 v14 v15
42
43 BEGIN_FUNCTION xnn_qd8_f32_qc8w_igemm_minmax_ukernel_4x16c4__asm_aarch64_neondot_ld128
44
45 # Clamp C pointers
46 CMP x0, 2 // if mr < 2
47 LDR x8, [sp, 8] // Load a_offset
48 ADD x16, x6, x7 // c1 = c0 + cm_stride
49 CSEL x16, x6, x16, LO // c1 = c0
50 ADD x2, x2, 3 // kc = (kc + 3) & ~3
51
52 ADD x17, x16, x7 // c2 = c1 + cm_stride
53 LDR x12, [sp, 16] // Load zero
54 LDR x11, [sp, 32] // Load params pointer
55 // if mr <= 2
56 CSEL x17, x16, x17, LS // c2 = c1
57 BIC x2, x2, 3
58 STR x19, [sp, -56] // Push x19 to the stack
59 LDR x19, [sp, 24] // Load zero_data
60 LDR x15, [sp, 40] // &quantization_params[0].zero_point
61 STP d8, d9, [sp, -48]!
62 STP d10, d11, [sp, 16]
63 STP d12, d13, [sp, 32]
64 LD1 {v12.4s}, [x15] // v12 & v13 interleaved zero_point & scale
65
66 CMP x0, 4 // if mr < 4
67 ADD x7, x17, x7 // c3 = c2 + cm_stride
68 CSEL x7, x17, x7, LO // c3 = c2
69
70 .p2align 3
71 0:
72 # Load initial bias from w into accumulators
73 LDP q16, q20, [x5], 32
74 MUL v16.4s, v16.4s, v12.s[0]
75 MUL v20.4s, v20.4s, v12.s[0]
76 MOV v17.16b, v16.16b
77 MOV v18.16b, v16.16b
78 LDP q24, q28, [x5], 32
79 MUL v24.4s, v24.4s, v12.s[0]
80 MUL v28.4s, v28.4s, v12.s[0]
81 MOV v19.16b, v16.16b
82 MOV v21.16b, v20.16b
83 MOV v22.16b, v20.16b
84 MOV v23.16b, v20.16b
85 MOV v25.16b, v24.16b
86 MOV v26.16b, v24.16b
87 MOV v27.16b, v24.16b
88 MOV v29.16b, v28.16b
89 MOV v30.16b, v28.16b
90 MOV v31.16b, v28.16b
91 MOV x9, x3 // p = ks
92
93 .p2align 3
94 1:
95 # Load next 4 A pointers
96 LDP x13, x14, [x4], 16
97 LDP x15, x10, [x4], 16
98
99 CMP x13, x12 // if a0 == zero
100 ADD x13, x13, x8 // a0 += a_offset
101 CSEL x13, x19, x13, EQ // a0 = zero_data, else a0 += a_offset
102 CMP x14, x12 // if a1 == zero
103 ADD x14, x14, x8 // a1 += a_offset
104 CSEL x14, x19, x14, EQ // a1 = zero_data, else a1 += a_offset
105 CMP x15, x12 // if a2 == zero
106 ADD x15, x15, x8 // a2 += a_offset
107 CSEL x15, x19, x15, EQ // a2 = zero_data, else a2 += a_offset
108 CMP x10, x12 // if a3 == zero
109 ADD x10, x10, x8 // a3 += a_offset
110 CSEL x10, x19, x10, EQ // a3 = zero_data, else a3 += a_offset
111
112 # Is there at least 16 bytes for main loop?
113 SUBS x0, x2, 16 // k = kc - 16
114 B.LO 4f
115
116 # Main loop - 16 bytes of A
117 .p2align 3
118 2:
119 LDR q0, [x13], 16
120 LDR q4, [x5], 16
121 LDR q1, [x14], 16
122 LDR q2, [x15], 16
123 LDR q3, [x10], 16
124 LDR q5, [x5], 16
125 SDOT v16.4s, v4.16b, v0.4b[0]
126 SDOT v17.4s, v4.16b, v1.4b[0]
127 LDP q6, q7, [x5], 32
128 SDOT v18.4s, v4.16b, v2.4b[0]
129 SDOT v19.4s, v4.16b, v3.4b[0]
130 SDOT v20.4s, v5.16b, v0.4b[0]
131 SDOT v21.4s, v5.16b, v1.4b[0]
132 SDOT v22.4s, v5.16b, v2.4b[0]
133 SDOT v23.4s, v5.16b, v3.4b[0]
134 SDOT v24.4s, v6.16b, v0.4b[0]
135 SDOT v25.4s, v6.16b, v1.4b[0]
136 LDP q4, q5, [x5], 32
137 SDOT v26.4s, v6.16b, v2.4b[0]
138 SDOT v27.4s, v6.16b, v3.4b[0]
139 SDOT v28.4s, v7.16b, v0.4b[0]
140 SDOT v29.4s, v7.16b, v1.4b[0]
141 SDOT v30.4s, v7.16b, v2.4b[0]
142 SDOT v31.4s, v7.16b, v3.4b[0]
143
144 SDOT v16.4s, v4.16b, v0.4b[1]
145 SDOT v17.4s, v4.16b, v1.4b[1]
146 LDP q6, q7, [x5], 32
147 SDOT v18.4s, v4.16b, v2.4b[1]
148 SDOT v19.4s, v4.16b, v3.4b[1]
149 SDOT v20.4s, v5.16b, v0.4b[1]
150 SDOT v21.4s, v5.16b, v1.4b[1]
151 SDOT v22.4s, v5.16b, v2.4b[1]
152 SDOT v23.4s, v5.16b, v3.4b[1]
153 SDOT v24.4s, v6.16b, v0.4b[1]
154 SDOT v25.4s, v6.16b, v1.4b[1]
155 LDP q4, q5, [x5], 32
156 SDOT v26.4s, v6.16b, v2.4b[1]
157 SDOT v27.4s, v6.16b, v3.4b[1]
158 SDOT v28.4s, v7.16b, v0.4b[1]
159 SDOT v29.4s, v7.16b, v1.4b[1]
160 SDOT v30.4s, v7.16b, v2.4b[1]
161 SDOT v31.4s, v7.16b, v3.4b[1]
162
163 SDOT v16.4s, v4.16b, v0.4b[2]
164 SDOT v17.4s, v4.16b, v1.4b[2]
165 LDP q6, q7, [x5], 32
166 SDOT v18.4s, v4.16b, v2.4b[2]
167 SDOT v19.4s, v4.16b, v3.4b[2]
168 SDOT v20.4s, v5.16b, v0.4b[2]
169 SDOT v21.4s, v5.16b, v1.4b[2]
170 SDOT v22.4s, v5.16b, v2.4b[2]
171 SDOT v23.4s, v5.16b, v3.4b[2]
172 SDOT v24.4s, v6.16b, v0.4b[2]
173 SDOT v25.4s, v6.16b, v1.4b[2]
174 LDP q4, q5, [x5], 32
175 SDOT v26.4s, v6.16b, v2.4b[2]
176 SDOT v27.4s, v6.16b, v3.4b[2]
177 SDOT v28.4s, v7.16b, v0.4b[2]
178 SDOT v29.4s, v7.16b, v1.4b[2]
179 SDOT v30.4s, v7.16b, v2.4b[2]
180 SDOT v31.4s, v7.16b, v3.4b[2]
181
182 SDOT v16.4s, v4.16b, v0.4b[3]
183 SDOT v17.4s, v4.16b, v1.4b[3]
184 LDP q6, q7, [x5], 32
185 SDOT v18.4s, v4.16b, v2.4b[3]
186 SDOT v19.4s, v4.16b, v3.4b[3]
187 SDOT v20.4s, v5.16b, v0.4b[3]
188 SDOT v21.4s, v5.16b, v1.4b[3]
189 SDOT v22.4s, v5.16b, v2.4b[3]
190 SDOT v23.4s, v5.16b, v3.4b[3]
191 SDOT v24.4s, v6.16b, v0.4b[3]
192 SDOT v25.4s, v6.16b, v1.4b[3]
193 SDOT v26.4s, v6.16b, v2.4b[3]
194 SDOT v27.4s, v6.16b, v3.4b[3]
195 SUBS x0, x0, 16
196 SDOT v28.4s, v7.16b, v0.4b[3]
197 SDOT v29.4s, v7.16b, v1.4b[3]
198 SDOT v30.4s, v7.16b, v2.4b[3]
199 SDOT v31.4s, v7.16b, v3.4b[3]
200 B.HS 2b
201
202 # Is there a remainder?- 4 to 12 bytes of A
203 TST x0, 15
204 B.NE 4f
205
206 3:
207 # ks loop
208 SUBS x9, x9, 32 // ks -= MR * sizeof(int8_t*)
209 B.HI 1b
210
211 LDP q0, q1, [x5], 32 // kernel_scale
212 SCVTF v19.4s, v19.4s
213 SCVTF v23.4s, v23.4s
214 SCVTF v27.4s, v27.4s
215 SCVTF v31.4s, v31.4s
216 SCVTF v18.4s, v18.4s
217 SCVTF v22.4s, v22.4s
218 SCVTF v26.4s, v26.4s
219 LDP q2, q3, [x5], 32
220 SCVTF v30.4s, v30.4s
221 SCVTF v17.4s, v17.4s
222 SCVTF v21.4s, v21.4s
223 SCVTF v25.4s, v25.4s
224 SCVTF v29.4s, v29.4s
225 SCVTF v16.4s, v16.4s
226 SCVTF v20.4s, v20.4s
227 SCVTF v24.4s, v24.4s
228 SCVTF v28.4s, v28.4s
229 FMUL v8.4s, v0.4s, v12.s[1] // kernel_scale * scale
230 FMUL v9.4s, v1.4s, v12.s[1]
231 FMUL v10.4s, v2.4s, v12.s[1]
232 FMUL v11.4s, v3.4s, v12.s[1]
233 FMUL v4.4s, v0.4s, v12.s[1]
234 FMUL v5.4s, v1.4s, v12.s[1]
235 FMUL v6.4s, v2.4s, v12.s[1]
236 FMUL v7.4s, v3.4s, v12.s[1]
237 LDP q0, q1, [x5], 32 // bias
238 FMUL v19.4s, v19.4s, v8.4s
239 FMUL v23.4s, v23.4s, v9.4s
240 FMUL v27.4s, v27.4s, v10.4s
241 FMUL v31.4s, v31.4s, v11.4s
242 FMUL v18.4s, v18.4s, v4.4s
243 FMUL v22.4s, v22.4s, v5.4s
244 FMUL v26.4s, v26.4s, v6.4s
245 FMUL v30.4s, v30.4s, v7.4s
246 LDP q2, q3, [x5], 32
247 FMUL v17.4s, v17.4s, v8.4s
248 FMUL v21.4s, v21.4s, v9.4s
249 FMUL v25.4s, v25.4s, v10.4s
250 FMUL v29.4s, v29.4s, v11.4s
251 FMUL v16.4s, v16.4s, v4.4s
252 FMUL v20.4s, v20.4s, v5.4s
253 FMUL v24.4s, v24.4s, v6.4s
254 FMUL v28.4s, v28.4s, v7.4s
255 LD2R {v4.4s, v5.4s}, [x11] // min max
256 FADD v19.4s, v19.4s, v0.4s
257 FADD v23.4s, v23.4s, v1.4s
258 FADD v27.4s, v27.4s, v2.4s
259 FADD v31.4s, v31.4s, v3.4s
260 FADD v18.4s, v18.4s, v0.4s
261 FADD v22.4s, v22.4s, v1.4s
262 FADD v26.4s, v26.4s, v2.4s
263 FADD v30.4s, v30.4s, v3.4s
264 FADD v17.4s, v17.4s, v0.4s
265 FADD v21.4s, v21.4s, v1.4s
266 FADD v25.4s, v25.4s, v2.4s
267 FADD v29.4s, v29.4s, v3.4s
268 FADD v16.4s, v16.4s, v0.4s
269 FADD v20.4s, v20.4s, v1.4s
270 FADD v24.4s, v24.4s, v2.4s
271 FADD v28.4s, v28.4s, v3.4s
272 FMAX v19.4s, v19.4s, v4.4s
273 FMAX v23.4s, v23.4s, v4.4s
274 FMAX v27.4s, v27.4s, v4.4s
275 FMAX v31.4s, v31.4s, v4.4s
276 FMAX v18.4s, v18.4s, v4.4s
277 FMAX v22.4s, v22.4s, v4.4s
278 FMAX v26.4s, v26.4s, v4.4s
279 FMAX v30.4s, v30.4s, v4.4s
280 FMAX v17.4s, v17.4s, v4.4s
281 FMAX v21.4s, v21.4s, v4.4s
282 FMAX v25.4s, v25.4s, v4.4s
283 FMAX v29.4s, v29.4s, v4.4s
284 FMAX v16.4s, v16.4s, v4.4s
285 FMAX v20.4s, v20.4s, v4.4s
286 FMAX v24.4s, v24.4s, v4.4s
287 FMAX v28.4s, v28.4s, v4.4s
288 FMIN v19.4s, v19.4s, v5.4s
289 FMIN v23.4s, v23.4s, v5.4s
290 FMIN v27.4s, v27.4s, v5.4s
291 FMIN v31.4s, v31.4s, v5.4s
292 FMIN v18.4s, v18.4s, v5.4s
293 FMIN v22.4s, v22.4s, v5.4s
294 FMIN v26.4s, v26.4s, v5.4s
295 LDR x0, [sp, 48] // cn_stride
296 FMIN v30.4s, v30.4s, v5.4s
297 FMIN v17.4s, v17.4s, v5.4s
298 FMIN v21.4s, v21.4s, v5.4s
299 FMIN v25.4s, v25.4s, v5.4s
300 FMIN v29.4s, v29.4s, v5.4s
301 SUBS x1, x1, 16
302 FMIN v16.4s, v16.4s, v5.4s
303 FMIN v20.4s, v20.4s, v5.4s
304 FMIN v24.4s, v24.4s, v5.4s
305 FMIN v28.4s, v28.4s, v5.4s
306 B.LO 6f
307
308 STP q19, q23, [x7]
309 STP q27, q31, [x7, #32]
310 ADD x7, x7, x0
311 STP q18, q22, [x17]
312 STP q26, q30, [x17, #32]
313 ADD x17, x17, x0
314 STP q17, q21, [x16]
315 STP q25, q29, [x16, #32]
316 ADD x16, x16, x0
317 STP q16, q20, [x6]
318 STP q24, q28, [x6, #32]
319 ADD x6, x6, x0
320 SUB x4, x4, x3 // a -= ks
321
322 # nc loop
323 B.HI 0b
324 # Restore d8-d13 from stack
325 LDR x19, [sp, -8]
326 LDP d12, d13, [sp, 32]
327 LDP d10, d11, [sp, 16]
328 LDP d8, d9, [sp], 48
329 RET
330
331
332 # Remainder- 8 bytes of A
333 .p2align 3
334 4:
335 # Is there a remainder?- 8 bytes of A
336 TBZ x0, 3, 5f
337
338 LDR d0, [x13], 8
339 LDR q4, [x5], 16
340 LDR d1, [x14], 8
341 LDR d2, [x15], 8
342 LDR d3, [x10], 8
343 LDR q5, [x5], 16
344 SDOT v16.4s, v4.16b, v0.4b[0]
345 SDOT v17.4s, v4.16b, v1.4b[0]
346 LDP q6, q7, [x5], 32
347 SDOT v18.4s, v4.16b, v2.4b[0]
348 SDOT v19.4s, v4.16b, v3.4b[0]
349 SDOT v20.4s, v5.16b, v0.4b[0]
350 SDOT v21.4s, v5.16b, v1.4b[0]
351 SDOT v22.4s, v5.16b, v2.4b[0]
352 SDOT v23.4s, v5.16b, v3.4b[0]
353 SDOT v24.4s, v6.16b, v0.4b[0]
354 SDOT v25.4s, v6.16b, v1.4b[0]
355 LDP q4, q5, [x5], 32
356 SDOT v26.4s, v6.16b, v2.4b[0]
357 SDOT v27.4s, v6.16b, v3.4b[0]
358 SDOT v28.4s, v7.16b, v0.4b[0]
359 SDOT v29.4s, v7.16b, v1.4b[0]
360 SDOT v30.4s, v7.16b, v2.4b[0]
361 SDOT v31.4s, v7.16b, v3.4b[0]
362 SDOT v16.4s, v4.16b, v0.4b[1]
363 SDOT v17.4s, v4.16b, v1.4b[1]
364 LDP q6, q7, [x5], 32
365 SDOT v18.4s, v4.16b, v2.4b[1]
366 SDOT v19.4s, v4.16b, v3.4b[1]
367 SDOT v20.4s, v5.16b, v0.4b[1]
368 SDOT v21.4s, v5.16b, v1.4b[1]
369 SDOT v22.4s, v5.16b, v2.4b[1]
370 SDOT v23.4s, v5.16b, v3.4b[1]
371 SDOT v24.4s, v6.16b, v0.4b[1]
372 SDOT v25.4s, v6.16b, v1.4b[1]
373 SDOT v26.4s, v6.16b, v2.4b[1]
374 SDOT v27.4s, v6.16b, v3.4b[1]
375 SDOT v28.4s, v7.16b, v0.4b[1]
376 SDOT v29.4s, v7.16b, v1.4b[1]
377 SDOT v30.4s, v7.16b, v2.4b[1]
378 SDOT v31.4s, v7.16b, v3.4b[1]
379 # Is there a remainder?- 4 bytes of A
380 TBZ x0, 2, 3b
381
382 # Remainder- 4 bytes of A
383 5:
384 LDR s0, [x13], 4
385 LDR q4, [x5], 16
386 LDR s1, [x14], 4
387 LDR s2, [x15], 4
388 LDR s3, [x10], 4
389 LDR q5, [x5], 16
390 SDOT v16.4s, v4.16b, v0.4b[0]
391 SDOT v17.4s, v4.16b, v1.4b[0]
392 LDP q6, q7, [x5], 32
393 SDOT v18.4s, v4.16b, v2.4b[0]
394 SDOT v19.4s, v4.16b, v3.4b[0]
395 SDOT v20.4s, v5.16b, v0.4b[0]
396 SDOT v21.4s, v5.16b, v1.4b[0]
397 SDOT v22.4s, v5.16b, v2.4b[0]
398 SDOT v23.4s, v5.16b, v3.4b[0]
399 SDOT v24.4s, v6.16b, v0.4b[0]
400 SDOT v25.4s, v6.16b, v1.4b[0]
401 SDOT v26.4s, v6.16b, v2.4b[0]
402 SDOT v27.4s, v6.16b, v3.4b[0]
403 SDOT v28.4s, v7.16b, v0.4b[0]
404 SDOT v29.4s, v7.16b, v1.4b[0]
405 SDOT v30.4s, v7.16b, v2.4b[0]
406 SDOT v31.4s, v7.16b, v3.4b[0]
407 B 3b
408
409 # Store odd width
410 .p2align 3
411 6:
412 TBZ x1, 3, 7f
413 STP q19, q23, [x7]
414 STP q18, q22, [x17]
415 STP q17, q21, [x16]
416 STP q16, q20, [x6]
417 MOV v16.16b, v24.16b
418 MOV v17.16b, v25.16b
419 MOV v18.16b, v26.16b
420 MOV v19.16b, v27.16b
421 MOV v20.16b, v28.16b
422 MOV v21.16b, v29.16b
423 MOV v22.16b, v30.16b
424 MOV v23.16b, v31.16b
425 ADD x7, x7, #32
426 ADD x17, x17, #32
427 ADD x16, x16, #32
428 ADD x6, x6, #32
429 7:
430 TBZ x1, 2, 8f
431 STR q19, [x7]
432 STR q18, [x17]
433 STR q17, [x16]
434 STR q16, [x6]
435 MOV v16.16b, v20.16b
436 MOV v17.16b, v21.16b
437 MOV v18.16b, v22.16b
438 MOV v19.16b, v23.16b
439 ADD x7, x7, #16
440 ADD x17, x17, #16
441 ADD x16, x16, #16
442 ADD x6, x6, #16
443 8:
444 TBZ x1, 1, 9f
445 ST1 {v19.2s}, [x7]
446 ST1 {v18.2s}, [x17]
447 ST1 {v17.2s}, [x16]
448 ST1 {v16.2s}, [x6]
449 DUP d16, v16.d[1]
450 DUP d17, v17.d[1]
451 DUP d18, v18.d[1]
452 DUP d19, v19.d[1]
453 ADD x7, x7, #8
454 ADD x17, x17, #8
455 ADD x16, x16, #8
456 ADD x6, x6, #8
457 9:
458 TBZ x1, 0, 10f
459 STR s19, [x7]
460 STR s18, [x17]
461 STR s17, [x16]
462 STR s16, [x6]
463 10:
464 # Restore d8-d13 from stack
465 LDR x19, [sp, -8]
466 LDP d12, d13, [sp, 32]
467 LDP d10, d11, [sp, 16]
468 LDP d8, d9, [sp], 48
469 RET
470 6:
471 TBZ x1, 3, 7f
472 STR d3, [x7], 8
473 STR d2, [x17], 8
474 DUP d3, v3.d[1]
475 DUP d2, v2.d[1]
476 STR d1, [x16], 8
477 STR d0, [x6], 8
478 DUP d1, v1.d[1]
479 DUP d0, v0.d[1]
480 7:
481 TBZ x1, 2, 8f
482 STR s3, [x7], 4
483 STR s2, [x17], 4
484 DUP s3, v3.s[1]
485 DUP s2, v2.s[1]
486 STR s1, [x16], 4
487 STR s0, [x6], 4
488 DUP s1, v1.s[1]
489 DUP s0, v0.s[1]
490 8:
491 TBZ x1, 1, 9f
492 STR h3, [x7], 2
493 STR h2, [x17], 2
494 DUP h3, v3.h[1]
495 DUP h2, v2.h[1]
496 STR h1, [x16], 2
497 STR h0, [x6], 2
498 DUP h1, v1.h[1]
499 DUP h0, v0.h[1]
500 9:
501 TBZ x1, 0, 10f
502 STR b3, [x7]
503 STR b2, [x17]
504 STR b1, [x16]
505 STR b0, [x6]
506 10:
507 RET
508
509 END_FUNCTION xnn_qd8_f32_qc8w_igemm_minmax_ukernel_4x16c4__asm_aarch64_neondot_ld128
510
511 #ifdef __ELF__
512 .section ".note.GNU-stack","",%progbits
513 #endif