Debian Codegraph @

src/qd8-f32-qc8w-gemm/gen/qd8-f32-qc8w-gemm-4x16c4-minmax-asm-aarch64-neondot-ld128.S

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-gemm/4x16c4-aarch64-neondot-ld128.S.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2020 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include "xnnpack/assembly.h"
12
13 # void xnn_qd8_f32_qc8w_gemm_minmax_ukernel_4x16c4__asm_aarch64_neondot_ld128(
14 # size_t mr, x0
15 # size_t nc, x1
16 # size_t kc, x2 / x0
17 # const int8_t* restrict a, x3
18 # size_t a_stride, x4
19 # const void* restrict w, x5
20 # int8_t* restrict c, x6
21 # size_t cm_stride, x7
22 # size_t cn_stride, [sp] -> x12
23 # const union xnn_f32_minmax_params *params, [sp + 8] -> x11
24 # const struct xnn_qd8_quantization_params *quantization_params) [sp + 16] -> x16
25
26 # params structure is 8 bytes
27 # struct {
28 # int32_t zero_point;
29 # float scale;
30 # } scalar;
31
32 # d8-d15, x19-x30 need to be preserved if used. x18 is reserved by the OS.
33
34 // Register usage
35 // A0 x3 v0
36 // A1 x15 v1
37 // A2 x13 v2
38 // A3 x4 v3
39 // B x5 v4 v5 v6 v7
40 // C0 x6 v16 v20 v24 v28
41 // C1 x8 v17 v21 v25 v29
42 // C2 x9 v18 v22 v26 v30
43 // C3 x7 v19 v23 v27 v31
44 // unused v14 v15
45
46 BEGIN_FUNCTION xnn_qd8_f32_qc8w_gemm_minmax_ukernel_4x16c4__asm_aarch64_neondot_ld128
47
48 # Clamp A and C pointers
49 CMP x0, 2 // if mr < 2
50 ADD x2, x2, 3 // kc = (kc + 3) & ~3
51 ADD x15, x3, x4 // a1 = a0 + a_stride
52 ADD x8, x6, x7 // c1 = c0 + cm_stride
53 CSEL x15, x3, x15, LO // a1 = a0
54 CSEL x8, x6, x8, LO // c1 = c0
55 BIC x2, x2, 3
56
57 ADD x13, x15, x4 // a2 = a1 + a_stride
58 ADD x9, x8, x7 // c2 = c1 + cm_stride
59 // if mr <= 2
60 CSEL x13, x15, x13, LS // a2 = a1
61 CSEL x9, x8, x9, LS // c2 = c1
62
63 LDP x12, x11, [sp] // cn_stride, params
64 LDR x16, [sp, 16] // &quantization_params[0].zero_point
65 STP d8, d9, [sp, -48]!
66 STP d10, d11, [sp, 16]
67 STP d12, d13, [sp, 32]
68 LDP q12, q13, [x16] // v12 & v13 interleaved zero_point & scale
69
70 CMP x0, 4 // if mr < 4
71 ADD x4, x13, x4 // a3 = a2 + a_stride
72 ADD x7, x9, x7 // c3 = c2 + cm_stride
73 CSEL x4, x13, x4, LO // a3 = a2
74 CSEL x7, x9, x7, LO // c3 = c2
75
76 .p2align 3
77 0:
78 # Load initial bias from w into accumulators
79 SUBS x0, x2, 16 // k = kc - 16
80 LDP q0, q1, [x5], 32
81 MUL v16.4s, v0.4s, v12.s[0]
82 MUL v17.4s, v0.4s, v12.s[2]
83 MUL v18.4s, v0.4s, v13.s[0]
84 LDP q2, q3, [x5], 32
85 MUL v19.4s, v0.4s, v13.s[2]
86 MUL v20.4s, v1.4s, v12.s[0]
87 MUL v21.4s, v1.4s, v12.s[2]
88 MUL v22.4s, v1.4s, v13.s[0]
89 MUL v23.4s, v1.4s, v13.s[2]
90 MUL v24.4s, v2.4s, v12.s[0]
91 MUL v25.4s, v2.4s, v12.s[2]
92 MUL v26.4s, v2.4s, v13.s[0]
93 MUL v27.4s, v2.4s, v13.s[2]
94 MUL v28.4s, v3.4s, v12.s[0]
95 MUL v29.4s, v3.4s, v12.s[2]
96 MUL v30.4s, v3.4s, v13.s[0]
97 MUL v31.4s, v3.4s, v13.s[2]
98
99 # Is there at least 16 bytes?
100 B.LO 3f
101
102 # Main loop - 16 bytes of A
103 .p2align 3
104 1:
105 LDR q0, [x3], 16
106 LDR q4, [x5], 16
107 LDR q1, [x15], 16
108 LDR q2, [x13], 16
109 LDR q3, [x4], 16
110 LDR q5, [x5], 16
111 SDOT v16.4s, v4.16b, v0.4b[0]
112 SDOT v17.4s, v4.16b, v1.4b[0]
113 LDP q6, q7, [x5], 32
114 SDOT v18.4s, v4.16b, v2.4b[0]
115 SDOT v19.4s, v4.16b, v3.4b[0]
116 SDOT v20.4s, v5.16b, v0.4b[0]
117 SDOT v21.4s, v5.16b, v1.4b[0]
118 SDOT v22.4s, v5.16b, v2.4b[0]
119 SDOT v23.4s, v5.16b, v3.4b[0]
120 SDOT v24.4s, v6.16b, v0.4b[0]
121 SDOT v25.4s, v6.16b, v1.4b[0]
122 LDP q4, q5, [x5], 32
123 SDOT v26.4s, v6.16b, v2.4b[0]
124 SDOT v27.4s, v6.16b, v3.4b[0]
125 SDOT v28.4s, v7.16b, v0.4b[0]
126 SDOT v29.4s, v7.16b, v1.4b[0]
127 SDOT v30.4s, v7.16b, v2.4b[0]
128 SDOT v31.4s, v7.16b, v3.4b[0]
129
130 SDOT v16.4s, v4.16b, v0.4b[1]
131 SDOT v17.4s, v4.16b, v1.4b[1]
132 LDP q6, q7, [x5], 32
133 SDOT v18.4s, v4.16b, v2.4b[1]
134 SDOT v19.4s, v4.16b, v3.4b[1]
135 SDOT v20.4s, v5.16b, v0.4b[1]
136 SDOT v21.4s, v5.16b, v1.4b[1]
137 SDOT v22.4s, v5.16b, v2.4b[1]
138 SDOT v23.4s, v5.16b, v3.4b[1]
139 SDOT v24.4s, v6.16b, v0.4b[1]
140 SDOT v25.4s, v6.16b, v1.4b[1]
141 LDP q4, q5, [x5], 32
142 SDOT v26.4s, v6.16b, v2.4b[1]
143 SDOT v27.4s, v6.16b, v3.4b[1]
144 SDOT v28.4s, v7.16b, v0.4b[1]
145 SDOT v29.4s, v7.16b, v1.4b[1]
146 SDOT v30.4s, v7.16b, v2.4b[1]
147 SDOT v31.4s, v7.16b, v3.4b[1]
148
149 SDOT v16.4s, v4.16b, v0.4b[2]
150 SDOT v17.4s, v4.16b, v1.4b[2]
151 LDP q6, q7, [x5], 32
152 SDOT v18.4s, v4.16b, v2.4b[2]
153 SDOT v19.4s, v4.16b, v3.4b[2]
154 SDOT v20.4s, v5.16b, v0.4b[2]
155 SDOT v21.4s, v5.16b, v1.4b[2]
156 SDOT v22.4s, v5.16b, v2.4b[2]
157 SDOT v23.4s, v5.16b, v3.4b[2]
158 SDOT v24.4s, v6.16b, v0.4b[2]
159 SDOT v25.4s, v6.16b, v1.4b[2]
160 LDP q4, q5, [x5], 32
161 SDOT v26.4s, v6.16b, v2.4b[2]
162 SDOT v27.4s, v6.16b, v3.4b[2]
163 SDOT v28.4s, v7.16b, v0.4b[2]
164 SDOT v29.4s, v7.16b, v1.4b[2]
165 SDOT v30.4s, v7.16b, v2.4b[2]
166 SDOT v31.4s, v7.16b, v3.4b[2]
167
168 SDOT v16.4s, v4.16b, v0.4b[3]
169 SDOT v17.4s, v4.16b, v1.4b[3]
170 LDP q6, q7, [x5], 32
171 SDOT v18.4s, v4.16b, v2.4b[3]
172 SDOT v19.4s, v4.16b, v3.4b[3]
173 SDOT v20.4s, v5.16b, v0.4b[3]
174 SDOT v21.4s, v5.16b, v1.4b[3]
175 SDOT v22.4s, v5.16b, v2.4b[3]
176 SDOT v23.4s, v5.16b, v3.4b[3]
177 SDOT v24.4s, v6.16b, v0.4b[3]
178 SDOT v25.4s, v6.16b, v1.4b[3]
179 SDOT v26.4s, v6.16b, v2.4b[3]
180 SDOT v27.4s, v6.16b, v3.4b[3]
181 SUBS x0, x0, 16
182 SDOT v28.4s, v7.16b, v0.4b[3]
183 SDOT v29.4s, v7.16b, v1.4b[3]
184 SDOT v30.4s, v7.16b, v2.4b[3]
185 SDOT v31.4s, v7.16b, v3.4b[3]
186 B.HS 1b
187
188 # Is there a remainder?- 4 to 12 bytes of A
189 TST x0, 15
190 B.NE 3f
191
192 2:
193 LDP q0, q1, [x5], 32 // kernel_scale
194 SCVTF v19.4s, v19.4s
195 SCVTF v23.4s, v23.4s
196 SCVTF v27.4s, v27.4s
197 SCVTF v31.4s, v31.4s
198 SCVTF v18.4s, v18.4s
199 SCVTF v22.4s, v22.4s
200 SCVTF v26.4s, v26.4s
201 LDP q2, q3, [x5], 32
202 SCVTF v30.4s, v30.4s
203 SCVTF v17.4s, v17.4s
204 SCVTF v21.4s, v21.4s
205 SCVTF v25.4s, v25.4s
206 SCVTF v29.4s, v29.4s
207 SCVTF v16.4s, v16.4s
208 SCVTF v20.4s, v20.4s
209 SCVTF v24.4s, v24.4s
210 SCVTF v28.4s, v28.4s
211 FMUL v8.4s, v0.4s, v13.s[3] // kernel_scale * scale
212 FMUL v9.4s, v1.4s, v13.s[3]
213 FMUL v10.4s, v2.4s, v13.s[3]
214 FMUL v11.4s, v3.4s, v13.s[3]
215 FMUL v4.4s, v0.4s, v13.s[1]
216 FMUL v5.4s, v1.4s, v13.s[1]
217 FMUL v6.4s, v2.4s, v13.s[1]
218 FMUL v7.4s, v3.4s, v13.s[1]
219 FMUL v19.4s, v19.4s, v8.4s
220 FMUL v8.4s, v0.4s, v12.s[3]
221 FMUL v23.4s, v23.4s, v9.4s
222 FMUL v9.4s, v1.4s, v12.s[3]
223 FMUL v27.4s, v27.4s, v10.4s
224 FMUL v10.4s, v2.4s, v12.s[3]
225 FMUL v31.4s, v31.4s, v11.4s
226 FMUL v11.4s, v3.4s, v12.s[3]
227 FMUL v18.4s, v18.4s, v4.4s
228 FMUL v4.4s, v0.4s, v12.s[1]
229 FMUL v22.4s, v22.4s, v5.4s
230 FMUL v5.4s, v1.4s, v12.s[1]
231 LDP q0, q1, [x5], 32 // bias
232 FMUL v26.4s, v26.4s, v6.4s
233 FMUL v6.4s, v2.4s, v12.s[1]
234 FMUL v30.4s, v30.4s, v7.4s
235 FMUL v7.4s, v3.4s, v12.s[1]
236 FMUL v17.4s, v17.4s, v8.4s
237 FMUL v21.4s, v21.4s, v9.4s
238 FMUL v25.4s, v25.4s, v10.4s
239 FMUL v29.4s, v29.4s, v11.4s
240 LDP q2, q3, [x5], 32
241 FMUL v16.4s, v16.4s, v4.4s
242 FMUL v20.4s, v20.4s, v5.4s
243 FMUL v24.4s, v24.4s, v6.4s
244 FMUL v28.4s, v28.4s, v7.4s
245 LD2R {v4.4s, v5.4s}, [x11] // min max
246 FADD v19.4s, v19.4s, v0.4s
247 FADD v23.4s, v23.4s, v1.4s
248 FADD v27.4s, v27.4s, v2.4s
249 FADD v31.4s, v31.4s, v3.4s
250 FADD v18.4s, v18.4s, v0.4s
251 FADD v22.4s, v22.4s, v1.4s
252 FADD v26.4s, v26.4s, v2.4s
253 FADD v30.4s, v30.4s, v3.4s
254 FADD v17.4s, v17.4s, v0.4s
255 FADD v21.4s, v21.4s, v1.4s
256 FADD v25.4s, v25.4s, v2.4s
257 FADD v29.4s, v29.4s, v3.4s
258 FADD v16.4s, v16.4s, v0.4s
259 FADD v20.4s, v20.4s, v1.4s
260 FADD v24.4s, v24.4s, v2.4s
261 FADD v28.4s, v28.4s, v3.4s
262 FMAX v19.4s, v19.4s, v4.4s
263 FMAX v23.4s, v23.4s, v4.4s
264 FMAX v27.4s, v27.4s, v4.4s
265 FMAX v31.4s, v31.4s, v4.4s
266 FMAX v18.4s, v18.4s, v4.4s
267 FMAX v22.4s, v22.4s, v4.4s
268 FMAX v26.4s, v26.4s, v4.4s
269 FMAX v30.4s, v30.4s, v4.4s
270 FMAX v17.4s, v17.4s, v4.4s
271 FMAX v21.4s, v21.4s, v4.4s
272 FMAX v25.4s, v25.4s, v4.4s
273 FMAX v29.4s, v29.4s, v4.4s
274 FMAX v16.4s, v16.4s, v4.4s
275 FMAX v20.4s, v20.4s, v4.4s
276 FMAX v24.4s, v24.4s, v4.4s
277 FMAX v28.4s, v28.4s, v4.4s
278 FMIN v19.4s, v19.4s, v5.4s
279 FMIN v23.4s, v23.4s, v5.4s
280 FMIN v27.4s, v27.4s, v5.4s
281 FMIN v31.4s, v31.4s, v5.4s
282 FMIN v18.4s, v18.4s, v5.4s
283 FMIN v22.4s, v22.4s, v5.4s
284 FMIN v26.4s, v26.4s, v5.4s
285 FMIN v30.4s, v30.4s, v5.4s
286 FMIN v17.4s, v17.4s, v5.4s
287 FMIN v21.4s, v21.4s, v5.4s
288 FMIN v25.4s, v25.4s, v5.4s
289 FMIN v29.4s, v29.4s, v5.4s
290 FMIN v16.4s, v16.4s, v5.4s
291 FMIN v20.4s, v20.4s, v5.4s
292 FMIN v24.4s, v24.4s, v5.4s
293 FMIN v28.4s, v28.4s, v5.4s
294
295 SUBS x1, x1, 16
296 B.LO 5f
297 STP q19, q23, [x7]
298 STP q27, q31, [x7, #32]
299 ADD x7, x7, x12
300 STP q18, q22, [x9]
301 STP q26, q30, [x9, #32]
302 ADD x9, x9, x12
303 STP q17, q21, [x8]
304 STP q25, q29, [x8, #32]
305 ADD x8, x8, x12
306 STP q16, q20, [x6]
307 STP q24, q28, [x6, #32]
308 ADD x6, x6, x12
309 SUB x3, x3, x2 // a0 -= kc
310 SUB x15, x15, x2 // a1 -= kc
311 SUB x13, x13, x2 // a2 -= kc
312 SUB x4, x4, x2 // a3 -= kc
313 B.NE 0b
314
315 # Restore d8-d13 from stack
316 LDP d12, d13, [sp, 32]
317 LDP d10, d11, [sp, 16]
318 LDP d8, d9, [sp], 48
319 RET
320
321
322 # Remainder- 8 bytes of A
323 .p2align 3
324 3:
325 # Is there a remainder?- 8 bytes of A
326 TBZ x0, 3, 4f
327
328 LDR d0, [x3], 8
329 LDR q4, [x5], 16
330 LDR d1, [x15], 8
331 LDR d2, [x13], 8
332 LDR d3, [x4], 8
333 LDR q5, [x5], 16
334 SDOT v16.4s, v4.16b, v0.4b[0]
335 SDOT v17.4s, v4.16b, v1.4b[0]
336 LDP q6, q7, [x5], 32
337 SDOT v18.4s, v4.16b, v2.4b[0]
338 SDOT v19.4s, v4.16b, v3.4b[0]
339 SDOT v20.4s, v5.16b, v0.4b[0]
340 SDOT v21.4s, v5.16b, v1.4b[0]
341 SDOT v22.4s, v5.16b, v2.4b[0]
342 SDOT v23.4s, v5.16b, v3.4b[0]
343 SDOT v24.4s, v6.16b, v0.4b[0]
344 SDOT v25.4s, v6.16b, v1.4b[0]
345 LDP q4, q5, [x5], 32
346 SDOT v26.4s, v6.16b, v2.4b[0]
347 SDOT v27.4s, v6.16b, v3.4b[0]
348 SDOT v28.4s, v7.16b, v0.4b[0]
349 SDOT v29.4s, v7.16b, v1.4b[0]
350 SDOT v30.4s, v7.16b, v2.4b[0]
351 SDOT v31.4s, v7.16b, v3.4b[0]
352 SDOT v16.4s, v4.16b, v0.4b[1]
353 SDOT v17.4s, v4.16b, v1.4b[1]
354 LDP q6, q7, [x5], 32
355 SDOT v18.4s, v4.16b, v2.4b[1]
356 SDOT v19.4s, v4.16b, v3.4b[1]
357 SDOT v20.4s, v5.16b, v0.4b[1]
358 SDOT v21.4s, v5.16b, v1.4b[1]
359 SDOT v22.4s, v5.16b, v2.4b[1]
360 SDOT v23.4s, v5.16b, v3.4b[1]
361 SDOT v24.4s, v6.16b, v0.4b[1]
362 SDOT v25.4s, v6.16b, v1.4b[1]
363 SDOT v26.4s, v6.16b, v2.4b[1]
364 SDOT v27.4s, v6.16b, v3.4b[1]
365 SDOT v28.4s, v7.16b, v0.4b[1]
366 SDOT v29.4s, v7.16b, v1.4b[1]
367 SDOT v30.4s, v7.16b, v2.4b[1]
368 SDOT v31.4s, v7.16b, v3.4b[1]
369 # Is there a remainder?- 4 bytes of A
370 TBZ x0, 2, 2b
371
372 # Remainder- 4 bytes of A
373 4:
374 LDR s0, [x3], 4
375 LDR q4, [x5], 16
376 LDR s1, [x15], 4
377 LDR s2, [x13], 4
378 LDR s3, [x4], 4
379 SDOT v16.4s, v4.16b, v0.4b[0]
380 LDR q5, [x5], 16
381 SDOT v17.4s, v4.16b, v1.4b[0]
382 SDOT v18.4s, v4.16b, v2.4b[0]
383 SDOT v19.4s, v4.16b, v3.4b[0]
384 SDOT v20.4s, v5.16b, v0.4b[0]
385 LDP q6, q7, [x5], 32
386 SDOT v21.4s, v5.16b, v1.4b[0]
387 SDOT v22.4s, v5.16b, v2.4b[0]
388 SDOT v23.4s, v5.16b, v3.4b[0]
389 SDOT v24.4s, v6.16b, v0.4b[0]
390 SDOT v25.4s, v6.16b, v1.4b[0]
391 SDOT v26.4s, v6.16b, v2.4b[0]
392 SDOT v27.4s, v6.16b, v3.4b[0]
393 SDOT v28.4s, v7.16b, v0.4b[0]
394 SDOT v29.4s, v7.16b, v1.4b[0]
395 SDOT v30.4s, v7.16b, v2.4b[0]
396 SDOT v31.4s, v7.16b, v3.4b[0]
397 B 2b
398
399 # Store odd width
400 .p2align 3
401 5:
402 TBZ x1, 3, 6f
403 STP q19, q23, [x7]
404 STP q18, q22, [x9]
405 MOV v19.16b, v27.16b
406 MOV v23.16b, v31.16b
407 MOV v18.16b, v26.16b
408 MOV v22.16b, v30.16b
409 STP q17, q21, [x8]
410 STP q16, q20, [x6]
411 MOV v17.16b, v25.16b
412 MOV v21.16b, v29.16b
413 MOV v16.16b, v24.16b
414 MOV v20.16b, v28.16b
415 ADD x6, x6, #32
416 ADD x7, x7, #32
417 ADD x8, x8, #32
418 ADD x9, x9, #32
419 6:
420 TBZ x1, 2, 7f
421 STR q19, [x7]
422 STR q18, [x9]
423 MOV v19.16b, v23.16b
424 MOV v18.16b, v22.16b
425 STR q17, [x8]
426 STR q16, [x6]
427 MOV v17.16b, v21.16b
428 MOV v16.16b, v20.16b
429 ADD x6, x6, #16
430 ADD x7, x7, #16
431 ADD x8, x8, #16
432 ADD x9, x9, #16
433 7:
434 TBZ x1, 1, 8f
435 STR d19, [x7], 8
436 STR d18, [x9], 8
437 DUP d19, v19.d[1]
438 DUP d18, v18.d[1]
439 STR d17, [x8], 8
440 STR d16, [x6], 8
441 DUP d17, v17.d[1]
442 DUP d16, v16.d[1]
443 8:
444 TBZ x1, 0, 9f
445 STR s19, [x7]
446 STR s18, [x9]
447 STR s17, [x8]
448 STR s16, [x6]
449 9:
450 # Restore d8-d13 from stack
451 LDP d12, d13, [sp, 32]
452 LDP d10, d11, [sp, 16]
453 LDP d8, d9, [sp], 48
454 RET
455
456 END_FUNCTION xnn_qd8_f32_qc8w_gemm_minmax_ukernel_4x16c4__asm_aarch64_neondot_ld128
457
458 #ifdef __ELF__
459 .section ".note.GNU-stack","",%progbits
460 #endif