Debian Codegraph @

src/qd8-f32-qc8w-gemm/gen/qd8-f32-qc8w-gemm-4x16c4-minmax-asm-aarch64-neondot-ld64.S

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-gemm/4x16c4-aarch64-neondot-ld64.S.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2020 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include "xnnpack/assembly.h"
12
13 # void xnn_qd8_f32_qc8w_gemm_minmax__ukernel_4x16c4__asm_aarch64_neondot_ld64(
14 # size_t mr, x0
15 # size_t nc, x1
16 # size_t kc, x2 / x0
17 # const int8_t* restrict a, x3
18 # size_t a_stride, x4
19 # const void* restrict w, x5
20 # int8_t* restrict c, x6
21 # size_t cm_stride, x7
22 # size_t cn_stride, [sp] -> x12
23 # const union xnn_f32_minmax_params *params, [sp + 8] -> x11
24 # const struct xnn_qd8_quantization_params *quantization_params) [sp + 16] -> x16
25
26 # d8-d15, x19-x30 need to be preserved if used. x18 is reserved by the OS.
27
28 // Register usage
29 // A0 x3 v0
30 // A1 x15 v1
31 // A2 x13 v2
32 // A3 x4 v3
33 // B x5 v4 v5 v6 v7
34 // C0 x6 v16 v20 v24 v28
35 // C1 x8 v17 v21 v25 v29
36 // C2 x9 v18 v22 v26 v30
37 // C3 x7 v19 v23 v27 v31
38 // unused v14 v15
39
40 BEGIN_FUNCTION xnn_qd8_f32_qc8w_gemm_minmax_ukernel_4x16c4__asm_aarch64_neondot_ld64
41
42 # Clamp A and C pointers
43 CMP x0, 2 // if mr < 2
44 ADD x2, x2, 3 // kc = (kc + 3) & ~3
45 ADD x15, x3, x4 // a1 = a0 + a_stride
46 ADD x8, x6, x7 // c1 = c0 + cm_stride
47 CSEL x15, x3, x15, LO // a1 = a0
48 CSEL x8, x6, x8, LO // c1 = c0
49 BIC x2, x2, 3
50
51 ADD x13, x15, x4 // a2 = a1 + a_stride
52 ADD x9, x8, x7 // c2 = c1 + cm_stride
53 // if mr <= 2
54 CSEL x13, x15, x13, LS // a2 = a1
55 CSEL x9, x8, x9, LS // c2 = c1
56
57 LDP x12, x11, [sp] // cn_stride, params
58
59 LDR x16, [sp, 16] // &quantization_params[0].zero_point
60 STP d8, d9, [sp, -48]!
61 STP d10, d11, [sp, 16]
62 STP d12, d13, [sp, 32]
63 LDP q12, q13, [x16] // v12 & v13 interleaved zero_point & scale
64
65 CMP x0, 4 // if mr < 4
66 ADD x4, x13, x4 // a3 = a2 + a_stride
67 ADD x7, x9, x7 // c3 = c2 + cm_stride
68 CSEL x4, x13, x4, LO // a3 = a2
69 CSEL x7, x9, x7, LO // c3 = c2
70
71 .p2align 3
72 0:
73 # Load initial bias from w into accumulators
74 SUBS x0, x2, 8 // k = kc - 8
75 LDP q0, q1, [x5], 32
76 MUL v16.4s, v0.4s, v12.s[0]
77 MUL v17.4s, v0.4s, v12.s[2]
78 MUL v18.4s, v0.4s, v13.s[0]
79 LDP q2, q3, [x5], 32
80 MUL v19.4s, v0.4s, v13.s[2]
81 MUL v20.4s, v1.4s, v12.s[0]
82 MUL v21.4s, v1.4s, v12.s[2]
83 MUL v22.4s, v1.4s, v13.s[0]
84 MUL v23.4s, v1.4s, v13.s[2]
85 MUL v24.4s, v2.4s, v12.s[0]
86 MUL v25.4s, v2.4s, v12.s[2]
87 MUL v26.4s, v2.4s, v13.s[0]
88 MUL v27.4s, v2.4s, v13.s[2]
89 MUL v28.4s, v3.4s, v12.s[0]
90 MUL v29.4s, v3.4s, v12.s[2]
91 MUL v30.4s, v3.4s, v13.s[0]
92 MUL v31.4s, v3.4s, v13.s[2]
93
94 # Is there at least 8 bytes?
95 B.LO 3f
96
97 # Main loop - 8 bytes of A
98 .p2align 3
99 1:
100 LDR d0, [x3], 8
101 LDR q4, [x5], 16
102 LDR d1, [x15], 8
103 LDR d2, [x13], 8
104 LDR d3, [x4], 8
105 LDR q5, [x5], 16
106 SDOT v16.4s, v4.16b, v0.4b[0]
107 SDOT v17.4s, v4.16b, v1.4b[0]
108 LDP q6, q7, [x5], 32
109 SDOT v18.4s, v4.16b, v2.4b[0]
110 SDOT v19.4s, v4.16b, v3.4b[0]
111 SDOT v20.4s, v5.16b, v0.4b[0]
112 SDOT v21.4s, v5.16b, v1.4b[0]
113 SDOT v22.4s, v5.16b, v2.4b[0]
114 SDOT v23.4s, v5.16b, v3.4b[0]
115 SDOT v24.4s, v6.16b, v0.4b[0]
116 SDOT v25.4s, v6.16b, v1.4b[0]
117 LDP q4, q5, [x5], 32
118 SDOT v26.4s, v6.16b, v2.4b[0]
119 SDOT v27.4s, v6.16b, v3.4b[0]
120 SDOT v28.4s, v7.16b, v0.4b[0]
121 SDOT v29.4s, v7.16b, v1.4b[0]
122 SDOT v30.4s, v7.16b, v2.4b[0]
123 SDOT v31.4s, v7.16b, v3.4b[0]
124 SDOT v16.4s, v4.16b, v0.4b[1]
125 SDOT v17.4s, v4.16b, v1.4b[1]
126 LDP q6, q7, [x5], 32
127 SDOT v18.4s, v4.16b, v2.4b[1]
128 SDOT v19.4s, v4.16b, v3.4b[1]
129 SDOT v20.4s, v5.16b, v0.4b[1]
130 SDOT v21.4s, v5.16b, v1.4b[1]
131 SDOT v22.4s, v5.16b, v2.4b[1]
132 SDOT v23.4s, v5.16b, v3.4b[1]
133 SDOT v24.4s, v6.16b, v0.4b[1]
134 SDOT v25.4s, v6.16b, v1.4b[1]
135 SDOT v26.4s, v6.16b, v2.4b[1]
136 SDOT v27.4s, v6.16b, v3.4b[1]
137 SDOT v28.4s, v7.16b, v0.4b[1]
138 SDOT v29.4s, v7.16b, v1.4b[1]
139 SDOT v30.4s, v7.16b, v2.4b[1]
140 SUBS x0, x0, 8
141 SDOT v31.4s, v7.16b, v3.4b[1]
142 B.HS 1b
143
144 # Is there a remainder?- 4 bytes of A
145 TBNZ x0, 2, 3f
146
147 2:
148 LDP q0, q1, [x5], 32 // kernel_scale
149 LDP q2, q3, [x5], 32
150 SCVTF v19.4s, v19.4s
151 SCVTF v23.4s, v23.4s
152 SCVTF v27.4s, v27.4s
153 SCVTF v31.4s, v31.4s
154 SCVTF v18.4s, v18.4s
155 SCVTF v22.4s, v22.4s
156 SCVTF v26.4s, v26.4s
157 FMUL v8.4s, v0.4s, v13.s[3] // kernel_scale * scale
158 FMUL v9.4s, v1.4s, v13.s[3]
159 FMUL v10.4s, v2.4s, v13.s[3]
160 FMUL v11.4s, v3.4s, v13.s[3]
161 FMUL v4.4s, v0.4s, v13.s[1]
162 FMUL v5.4s, v1.4s, v13.s[1]
163 FMUL v6.4s, v2.4s, v13.s[1]
164 FMUL v7.4s, v3.4s, v13.s[1]
165 SCVTF v30.4s, v30.4s
166 SCVTF v17.4s, v17.4s
167 SCVTF v21.4s, v21.4s
168 SCVTF v25.4s, v25.4s
169 SCVTF v29.4s, v29.4s
170 SCVTF v16.4s, v16.4s
171 SCVTF v20.4s, v20.4s
172 SCVTF v24.4s, v24.4s
173 SCVTF v28.4s, v28.4s
174 FMUL v19.4s, v19.4s, v8.4s
175 FMUL v8.4s, v0.4s, v12.s[3]
176 FMUL v23.4s, v23.4s, v9.4s
177 FMUL v9.4s, v1.4s, v12.s[3]
178 FMUL v27.4s, v27.4s, v10.4s
179 FMUL v10.4s, v2.4s, v12.s[3]
180 FMUL v31.4s, v31.4s, v11.4s
181 FMUL v11.4s, v3.4s, v12.s[3]
182 FMUL v18.4s, v18.4s, v4.4s
183 FMUL v4.4s, v0.4s, v12.s[1]
184 FMUL v22.4s, v22.4s, v5.4s
185 FMUL v5.4s, v1.4s, v12.s[1]
186 FMUL v26.4s, v26.4s, v6.4s
187 FMUL v6.4s, v2.4s, v12.s[1]
188 FMUL v30.4s, v30.4s, v7.4s
189 FMUL v7.4s, v3.4s, v12.s[1]
190 LDP q0, q1, [x5], 32 // bias
191 LDP q2, q3, [x5], 32
192 FMUL v17.4s, v17.4s, v8.4s
193 FMUL v21.4s, v21.4s, v9.4s
194 FMUL v25.4s, v25.4s, v10.4s
195 FMUL v29.4s, v29.4s, v11.4s
196 FMUL v16.4s, v16.4s, v4.4s
197 FMUL v20.4s, v20.4s, v5.4s
198 FMUL v24.4s, v24.4s, v6.4s
199 FMUL v28.4s, v28.4s, v7.4s
200 LD2R {v4.4s, v5.4s}, [x11] // min max
201 FADD v19.4s, v19.4s, v0.4s
202 FADD v23.4s, v23.4s, v1.4s
203 FADD v27.4s, v27.4s, v2.4s
204 FADD v31.4s, v31.4s, v3.4s
205 FADD v18.4s, v18.4s, v0.4s
206 FADD v22.4s, v22.4s, v1.4s
207 FADD v26.4s, v26.4s, v2.4s
208 FADD v30.4s, v30.4s, v3.4s
209 FADD v17.4s, v17.4s, v0.4s
210 FADD v21.4s, v21.4s, v1.4s
211 FADD v25.4s, v25.4s, v2.4s
212 FADD v29.4s, v29.4s, v3.4s
213 FADD v16.4s, v16.4s, v0.4s
214 FADD v20.4s, v20.4s, v1.4s
215 FADD v24.4s, v24.4s, v2.4s
216 FADD v28.4s, v28.4s, v3.4s
217 FMAX v19.4s, v19.4s, v4.4s
218 FMAX v23.4s, v23.4s, v4.4s
219 FMAX v27.4s, v27.4s, v4.4s
220 FMAX v31.4s, v31.4s, v4.4s
221 FMAX v18.4s, v18.4s, v4.4s
222 FMAX v22.4s, v22.4s, v4.4s
223 FMAX v26.4s, v26.4s, v4.4s
224 FMAX v30.4s, v30.4s, v4.4s
225 FMAX v17.4s, v17.4s, v4.4s
226 FMAX v21.4s, v21.4s, v4.4s
227 FMAX v25.4s, v25.4s, v4.4s
228 FMAX v29.4s, v29.4s, v4.4s
229 FMAX v16.4s, v16.4s, v4.4s
230 FMAX v20.4s, v20.4s, v4.4s
231 FMAX v24.4s, v24.4s, v4.4s
232 FMAX v28.4s, v28.4s, v4.4s
233 FMIN v19.4s, v19.4s, v5.4s
234 FMIN v23.4s, v23.4s, v5.4s
235 FMIN v27.4s, v27.4s, v5.4s
236 FMIN v31.4s, v31.4s, v5.4s
237 FMIN v18.4s, v18.4s, v5.4s
238 FMIN v22.4s, v22.4s, v5.4s
239 FMIN v26.4s, v26.4s, v5.4s
240 FMIN v30.4s, v30.4s, v5.4s
241 FMIN v17.4s, v17.4s, v5.4s
242 FMIN v21.4s, v21.4s, v5.4s
243 FMIN v25.4s, v25.4s, v5.4s
244 FMIN v29.4s, v29.4s, v5.4s
245 FMIN v16.4s, v16.4s, v5.4s
246 FMIN v20.4s, v20.4s, v5.4s
247 FMIN v24.4s, v24.4s, v5.4s
248 FMIN v28.4s, v28.4s, v5.4s
249 SUBS x1, x1, 16
250 B.LO 4f
251 STP q19, q23, [x7]
252 STP q27, q31, [x7, #32]
253 ADD x7, x7, x12
254 STP q18, q22, [x9]
255 STP q26, q30, [x9, #32]
256 ADD x9, x9, x12
257 STP q17, q21, [x8]
258 STP q25, q29, [x8, #32]
259 ADD x8, x8, x12
260 STP q16, q20, [x6]
261 STP q24, q28, [x6, #32]
262 ADD x6, x6, x12
263 SUB x3, x3, x2 // a0 -= kc
264 SUB x15, x15, x2 // a1 -= kc
265 SUB x13, x13, x2 // a2 -= kc
266 SUB x4, x4, x2 // a3 -= kc
267 B.NE 0b
268
269 # Restore d8-d13 from stack
270 LDP d12, d13, [sp, 32]
271 LDP d10, d11, [sp, 16]
272 LDP d8, d9, [sp], 48
273 RET
274
275 # Remainder- 4 bytes of A
276 .p2align 3
277 3:
278 LDR s0, [x3], 4
279 LDR q4, [x5], 16
280 LDR s1, [x15], 4
281 LDR s2, [x13], 4
282 LDR s3, [x4], 4
283 SDOT v16.4s, v4.16b, v0.4b[0]
284 LDR q5, [x5], 16
285 SDOT v17.4s, v4.16b, v1.4b[0]
286 SDOT v18.4s, v4.16b, v2.4b[0]
287 SDOT v19.4s, v4.16b, v3.4b[0]
288 SDOT v20.4s, v5.16b, v0.4b[0]
289 LDP q6, q7, [x5], 32
290 SDOT v21.4s, v5.16b, v1.4b[0]
291 SDOT v22.4s, v5.16b, v2.4b[0]
292 SDOT v23.4s, v5.16b, v3.4b[0]
293 SDOT v24.4s, v6.16b, v0.4b[0]
294 SDOT v25.4s, v6.16b, v1.4b[0]
295 SDOT v26.4s, v6.16b, v2.4b[0]
296 SDOT v27.4s, v6.16b, v3.4b[0]
297 SDOT v28.4s, v7.16b, v0.4b[0]
298 SDOT v29.4s, v7.16b, v1.4b[0]
299 SDOT v30.4s, v7.16b, v2.4b[0]
300 SDOT v31.4s, v7.16b, v3.4b[0]
301 B 2b
302
303 # Store odd width
304 .p2align 3
305 4:
306 TBZ x1, 3, 5f
307 STP q19, q23, [x7]
308 STP q18, q22, [x9]
309 MOV v19.16b, v27.16b
310 MOV v23.16b, v31.16b
311 MOV v18.16b, v26.16b
312 MOV v22.16b, v30.16b
313 STP q17, q21, [x8]
314 STP q16, q20, [x6]
315 MOV v17.16b, v25.16b
316 MOV v21.16b, v29.16b
317 MOV v16.16b, v24.16b
318 MOV v20.16b, v28.16b
319 ADD x6, x6, #32
320 ADD x7, x7, #32
321 ADD x8, x8, #32
322 ADD x9, x9, #32
323 5:
324 TBZ x1, 2, 6f
325 STR q19, [x7]
326 STR q18, [x9]
327 MOV v19.16b, v23.16b
328 MOV v18.16b, v22.16b
329 STR q17, [x8]
330 STR q16, [x6]
331 MOV v17.16b, v21.16b
332 MOV v16.16b, v20.16b
333 ADD x6, x6, #16
334 ADD x7, x7, #16
335 ADD x8, x8, #16
336 ADD x9, x9, #16
337 6:
338 TBZ x1, 1, 7f
339 STR d19, [x7], 8
340 STR d18, [x9], 8
341 DUP d19, v19.d[1]
342 DUP d18, v18.d[1]
343 STR d17, [x8], 8
344 STR d16, [x6], 8
345 DUP d17, v17.d[1]
346 DUP d16, v16.d[1]
347 7:
348 TBZ x1, 0, 8f
349 STR s19, [x7]
350 STR s18, [x9]
351 STR s17, [x8]
352 STR s16, [x6]
353 8:
354 # Restore d8-d13 from stack
355 LDP d12, d13, [sp, 32]
356 LDP d10, d11, [sp, 16]
357 LDP d8, d9, [sp], 48
358 RET
359
360 END_FUNCTION xnn_qd8_f32_qc8w_gemm_minmax_ukernel_4x16c4__asm_aarch64_neondot_ld64
361
362 #ifdef __ELF__
363 .section ".note.GNU-stack","",%progbits
364 #endif