Debian Codegraph @

src/qs8-qc8w-igemm/gen/qs8-qc8w-igemm-1x8c8-minmax-fp32-asm-aarch64-neon-mlal-cortex-a53.S

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-igemm/1x8c8-aarch64-neon-mlal-cortex-a53.S.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2021 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include "xnnpack/assembly.h"
12
13 # void xnn_qs8_qc8w_igemm_minmax_fp32_ukernel_1x8c8__asm_aarch64_neon_mlal_cortex_a53(
14 # size_t mr, x0
15 # size_t nc, x1
16 # size_t kc, x2 / x0
17 # size_t ks, x3 / x9
18 # const int8_t** restrict a, x4
19 # const int8_t* restrict w, x5
20 # int8_t* restrict c, x6
21 # size_t cm_stride, (x7)
22 # size_t cn_stride, [sp] -> x10
23 # size_t a_offset, [sp + 8] -> x8
24 # const int8_t* zero, [sp + 16] -> x12
25 # const union xnn_qs8_qc8w_conv_minmax_params params [sp + 24] -> x11
26
27 # d8-d15, x19-x30 need to be preserved if used. x18 is reserved by the OS.
28
29 // Register usage
30 // A0 x13 v0 v6
31 // B x5 v4 v5 v2 v3
32 // C0 x6 v16 v18 v20 v22 v24 v26 v28 v30
33 // temp0 v17 v19 v21 v23
34 // x16, x17, x7 tenporary a53 gpr load data
35
36 BEGIN_FUNCTION xnn_qs8_qc8w_igemm_minmax_fp32_ukernel_1x8c8__asm_aarch64_neon_mlal_cortex_a53
37
38 # Clamp C pointers
39 LDP x10, x8, [sp] // Load cn_stride, a_offset
40 ADD x2, x2, 7 // kc = (kc + 7) & ~7
41 LDP x12, x11, [sp, 16] // Load zero, params pointer
42 BIC x2, x2, 7
43
44 .p2align 3
45 0:
46 # Load initial bias from w into accumulators
47 LDP s16, s18, [x5], 8
48 LDP s20, s22, [x5], 8
49 LDP s24, s26, [x5], 8
50 LDP s28, s30, [x5], 8
51 MOV x9, x3 // p = ks
52
53 .p2align 3
54 1:
55 # Load next A pointer
56 LDR x13, [x4], 8
57 CMP x13, x12 // if a0 == zero
58 ADD x13, x13, x8 // a0 += a_offset
59 CSEL x13, x12, x13, EQ // a0 = zero, else += a0 + a_offset
60
61 # Is there at least 16 bytes for epilogue?
62 SUBS x0, x2, 16 // k = kc - 16
63 B.LO 5f
64
65 # Prologue: load A0 and 4 B's
66 LDP d0, d6, [x13], 16 // Read A0
67 LDP d4, d5, [x5] // Read B
68 LDP d2, d3, [x5, 64] // Read B
69 LDR x16, [x5, 16] // Read B
70
71 # Is there at least 16 bytes for main loop?
72 SUBS x0, x0, 16 // k = k - 16
73 B.LO 3f
74
75 # Main loop - 16 bytes of A
76 # 4 groups of 2 mul/mla/adap + 2 load = 10 cycles.
77 # 1 load for A0 = +1 cycle. Total 41 cycles.
78
79 .p2align 3
80 2:
81 # BLOCK 0 - 6 cycles
82 SMULL v17.8h, v4.8b, v0.8b
83 LDR x17, [x5, 80]
84 SMULL v19.8h, v5.8b, v0.8b
85 LDR d5, [x5, 24]
86 INS v4.d[0], x16
87 SMLAL v17.8h, v2.8b, v6.8b
88 LDR x16, [x5, 32]
89 SMLAL v19.8h, v3.8b, v6.8b
90 LDR d3, [x5, 88]
91 INS v2.d[0], x17
92
93 # BLOCK 1 - 10 cycles
94 SMULL v21.8h, v4.8b, v0.8b
95 LDR x17, [x5, 96]
96 SMULL v23.8h, v5.8b, v0.8b
97 SADALP v16.4s, v17.8h
98 SADALP v18.4s, v19.8h
99 LDR d5, [x5, 40]
100 INS v4.d[0], x16
101 SMLAL v21.8h, v2.8b, v6.8b
102 LDR x16, [x5, 48]
103 SMLAL v23.8h, v3.8b, v6.8b
104 LDR d3, [x5, 104]
105 INS v2.d[0], x17
106
107 # BLOCK 2 - 10 cycles
108 SMULL v17.8h, v4.8b, v0.8b
109 LDR x17, [x5, 112]
110 SMULL v19.8h, v5.8b, v0.8b
111 SADALP v20.4s, v21.8h
112 SADALP v22.4s, v23.8h
113 LDR d5, [x5, 56]
114 INS v4.d[0], x16
115 SMLAL v17.8h, v2.8b, v6.8b
116 LDR x16, [x5, 128]
117 SMLAL v19.8h, v3.8b, v6.8b
118 LDR d3, [x5, 120]
119 INS v2.d[0], x17
120
121 # BLOCK 3 - 15 cycles
122 SMULL v21.8h, v4.8b, v0.8b
123 LDR x7, [x13], 8 // Read A0
124 SMULL v23.8h, v5.8b, v0.8b
125 LDR x17, [x5, 192] // Read B
126 SADALP v24.4s, v17.8h
127 SUBS x0, x0, 16
128 SADALP v26.4s, v19.8h
129 LDR d5, [x5, 136] // Read B
130 INS v4.d[0], x16
131 SMLAL v21.8h, v2.8b, v6.8b
132 LDR x16, [x5, 144]
133 SMLAL v23.8h, v3.8b, v6.8b
134 LDR d6, [x13], 8 // Read A0
135 INS v0.d[0], x7
136 LDR d3, [x5, 200] // Read B
137 INS v2.d[0], x17
138 SADALP v28.4s, v21.8h
139 ADD x5, x5, 128
140 SADALP v30.4s, v23.8h
141 B.HS 2b
142
143 # Epilogue
144 # Same as main loop except no loads at end of loop
145
146 .p2align 3
147 3:
148 # BLOCK 0 - 6 cycles
149 SMULL v17.8h, v4.8b, v0.8b
150 LDR x17, [x5, 80]
151 SMULL v19.8h, v5.8b, v0.8b
152 LDR d5, [x5, 24]
153 INS v4.d[0], x16
154 SMLAL v17.8h, v2.8b, v6.8b
155 LDR x16, [x5, 32]
156 SMLAL v19.8h, v3.8b, v6.8b
157 LDR d3, [x5, 88]
158 INS v2.d[0], x17
159
160 # BLOCK 1 - 10 cycles
161 SMULL v21.8h, v4.8b, v0.8b
162 LDR x17, [x5, 96]
163 SMULL v23.8h, v5.8b, v0.8b
164 SADALP v16.4s, v17.8h
165 SADALP v18.4s, v19.8h
166 LDR d5, [x5, 40]
167 INS v4.d[0], x16
168 SMLAL v21.8h, v2.8b, v6.8b
169 LDR x16, [x5, 48]
170 SMLAL v23.8h, v3.8b, v6.8b
171 LDR d3, [x5, 104]
172 INS v2.d[0], x17
173
174 # BLOCK 2 - 10 cycles
175 SMULL v17.8h, v4.8b, v0.8b
176 LDR x17, [x5, 112]
177 SMULL v19.8h, v5.8b, v0.8b
178 SADALP v20.4s, v21.8h
179 SADALP v22.4s, v23.8h
180 LDR d5, [x5, 56]
181 INS v4.d[0], x16
182 SMLAL v17.8h, v2.8b, v6.8b
183 SMLAL v19.8h, v3.8b, v6.8b
184 LDR d3, [x5, 120]
185 INS v2.d[0], x17
186
187 # BLOCK 3 - 12 cycles
188 SMULL v21.8h, v4.8b, v0.8b
189 SMULL v23.8h, v5.8b, v0.8b
190 SADALP v24.4s, v17.8h
191 SADALP v26.4s, v19.8h
192 SMLAL v21.8h, v2.8b, v6.8b
193 SMLAL v23.8h, v3.8b, v6.8b
194 SADALP v28.4s, v21.8h
195 ADD x5, x5, 128
196 SADALP v30.4s, v23.8h
197
198 # Is there a remainder?- 8 bytes of A
199 TBNZ x0, 3, 5f
200
201 # ks loop
202 SUBS x9, x9, 8 // ks -= MR * sizeof(int8_t*)
203 B.HI 1b
204
205 4:
206 # Add columns
207 ADDP v16.4s, v16.4s, v18.4s
208 ADDP v20.4s, v20.4s, v22.4s
209 ADDP v24.4s, v24.4s, v26.4s
210 ADDP v28.4s, v28.4s, v30.4s
211 ADDP v0.4s, v16.4s, v20.4s
212 ADDP v1.4s, v24.4s, v28.4s
213
214 # Load per channel scale values from weights
215 SCVTF v0.4s, v0.4s
216 LDR q4, [x5], 16
217 SCVTF v1.4s, v1.4s
218 LDR q5, [x5], 16
219 FMUL v0.4s, v0.4s, v4.4s
220 FMUL v1.4s, v1.4s, v5.4s
221
222 FCVTNS v0.4s, v0.4s
223 FCVTNS v1.4s, v1.4s
224
225 LD1R {v5.8h}, [x11], 2
226 SQXTN v0.4h, v0.4s
227 SQXTN2 v0.8h, v1.4s
228 SUBS x1, x1, 8
229 SQADD v0.8h, v0.8h, v5.8h
230 LD1R {v1.16b}, [x11], 1
231 SQXTN v0.8b, v0.8h
232 LD1R {v17.16b}, [x11]
233 SMAX v0.8b, v0.8b, v1.8b
234 SUB x11, x11, 3 // rewind params pointer
235
236 SMIN v0.8b, v0.8b, v17.8b
237 B.LO 6f
238
239 # Store full 1 x 8
240 ST1 {v0.8b}, [x6], x10
241 SUB x4, x4, x3 // a -= ks
242 B.HI 0b
243 RET
244
245 # Remainder - 8 bytes of A
246 .p2align 3
247 5:
248 LDR d0, [x13], 8
249 LDP d4, d5, [x5]
250 LDP d6, d7, [x5, 16]
251 SMULL v17.8h, v4.8b, v0.8b
252 SMULL v19.8h, v5.8b, v0.8b
253 SMULL v21.8h, v6.8b, v0.8b
254 SMULL v23.8h, v7.8b, v0.8b
255 LDP d4, d5, [x5, 32]
256 LDP d6, d7, [x5, 48]
257 SADALP v16.4s, v17.8h
258 SADALP v18.4s, v19.8h
259 SADALP v20.4s, v21.8h
260 SADALP v22.4s, v23.8h
261 SMULL v17.8h, v4.8b, v0.8b
262 SMULL v19.8h, v5.8b, v0.8b
263 SMULL v21.8h, v6.8b, v0.8b
264 SMULL v23.8h, v7.8b, v0.8b
265 ADD x5, x5, 64
266 SADALP v24.4s, v17.8h
267 SADALP v26.4s, v19.8h
268 SADALP v28.4s, v21.8h
269 SADALP v30.4s, v23.8h
270
271 # ks loop
272 SUBS x9, x9, 8 // ks -= MR * sizeof(int8_t*)
273 B.HI 1b
274 B 4b
275
276 # Store odd width
277 .p2align 3
278 6:
279 TBZ x1, 2, 7f
280 STR s0, [x6], 4
281 EXT v0.16b, v0.16b, v0.16b, 4
282
283 7:
284 TBZ x1, 1, 8f
285 STR h0, [x6], 2
286 EXT v0.16b, v0.16b, v0.16b, 2
287 8:
288 TBZ x1, 0, 9f
289 STR b0, [x6]
290 9:
291 RET
292
293 END_FUNCTION xnn_qs8_qc8w_igemm_minmax_fp32_ukernel_1x8c8__asm_aarch64_neon_mlal_cortex_a53
294
295 #ifdef __ELF__
296 .section ".note.GNU-stack","",%progbits
297 #endif