Debian Codegraph @

src/qs8-qc8w-igemm/gen/qs8-qc8w-igemm-1x8c8-minmax-fp32-asm-aarch64-neon-mlal-cortex-a53-prfm.S

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-igemm/1x8c8-aarch64-neon-mlal-cortex-a53.S.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2021 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include "xnnpack/assembly.h"
12
13 # void xnn_qs8_qc8w_igemm_minmax_fp32_ukernel_1x8c8__asm_aarch64_neon_mlal_cortex_a53_prfm(
14 # size_t mr, x0
15 # size_t nc, x1
16 # size_t kc, x2 / x0
17 # size_t ks, x3 / x9
18 # const int8_t** restrict a, x4
19 # const int8_t* restrict w, x5
20 # int8_t* restrict c, x6
21 # size_t cm_stride, (x7)
22 # size_t cn_stride, [sp] -> x10
23 # size_t a_offset, [sp + 8] -> x8
24 # const int8_t* zero, [sp + 16] -> x12
25 # const union xnn_qs8_qc8w_conv_minmax_params params [sp + 24] -> x11
26
27 # d8-d15, x19-x30 need to be preserved if used. x18 is reserved by the OS.
28
29 // Register usage
30 // A0 x13 v0 v6
31 // B x5 v4 v5 v2 v3
32 // C0 x6 v16 v18 v20 v22 v24 v26 v28 v30
33 // temp0 v17 v19 v21 v23
34 // x16, x17, x7 tenporary a53 gpr load data
35
36 BEGIN_FUNCTION xnn_qs8_qc8w_igemm_minmax_fp32_ukernel_1x8c8__asm_aarch64_neon_mlal_cortex_a53_prfm
37
38 # Clamp C pointers
39 LDP x10, x8, [sp] // Load cn_stride, a_offset
40 ADD x2, x2, 7 // kc = (kc + 7) & ~7
41 LDP x12, x11, [sp, 16] // Load zero, params pointer
42 BIC x2, x2, 7
43
44 .p2align 3
45 0:
46 # Load initial bias from w into accumulators
47 LDP s16, s18, [x5], 8
48 LDP s20, s22, [x5], 8
49 LDP s24, s26, [x5], 8
50 LDP s28, s30, [x5], 8
51 MOV x9, x3 // p = ks
52
53 .p2align 3
54 1:
55 # Load next A pointer
56 LDR x13, [x4], 8
57 CMP x13, x12 // if a0 == zero
58 ADD x13, x13, x8 // a0 += a_offset
59 CSEL x13, x12, x13, EQ // a0 = zero, else += a0 + a_offset
60
61 # Is there at least 16 bytes for epilogue?
62 SUBS x0, x2, 16 // k = kc - 16
63 B.LO 5f
64
65 # Prologue: load A0 and 4 B's
66 LDP d0, d6, [x13], 16 // Read A0
67 LDP d4, d5, [x5] // Read B
68 LDP d2, d3, [x5, 64] // Read B
69 LDR x16, [x5, 16] // Read B
70
71 # Is there at least 16 bytes for main loop?
72 SUBS x0, x0, 16 // k = k - 16
73 B.LO 3f
74
75 # Main loop - 16 bytes of A
76 # 4 groups of 2 mul/mla/adap + 2 load = 10 cycles.
77 # 1 load for A0 = +1 cycle. Total 41 cycles.
78
79 .p2align 3
80 2:
81 # BLOCK 0 - 6 cycles
82 SMULL v17.8h, v4.8b, v0.8b
83 LDR x17, [x5, 80]
84 SMULL v19.8h, v5.8b, v0.8b
85 LDR d5, [x5, 24]
86 INS v4.d[0], x16
87 SMLAL v17.8h, v2.8b, v6.8b
88 LDR x16, [x5, 32]
89 SMLAL v19.8h, v3.8b, v6.8b
90 LDR d3, [x5, 88]
91 INS v2.d[0], x17
92
93 # BLOCK 1 - 10 cycles
94 SMULL v21.8h, v4.8b, v0.8b
95 LDR x17, [x5, 96]
96 SMULL v23.8h, v5.8b, v0.8b
97 SADALP v16.4s, v17.8h
98 PRFM PLDL1KEEP, [x5, 448]
99 SADALP v18.4s, v19.8h
100 PRFM PLDL1KEEP, [x5, 512]
101 LDR d5, [x5, 40]
102 INS v4.d[0], x16
103 SMLAL v21.8h, v2.8b, v6.8b
104 LDR x16, [x5, 48]
105 SMLAL v23.8h, v3.8b, v6.8b
106 LDR d3, [x5, 104]
107 INS v2.d[0], x17
108
109 # BLOCK 2 - 10 cycles
110 SMULL v17.8h, v4.8b, v0.8b
111 LDR x17, [x5, 112]
112 SMULL v19.8h, v5.8b, v0.8b
113 SADALP v20.4s, v21.8h
114 PRFM PLDL1KEEP, [x13, 128]
115 SADALP v22.4s, v23.8h
116 LDR d5, [x5, 56]
117 INS v4.d[0], x16
118 SMLAL v17.8h, v2.8b, v6.8b
119 LDR x16, [x5, 128]
120 SMLAL v19.8h, v3.8b, v6.8b
121 LDR d3, [x5, 120]
122 INS v2.d[0], x17
123
124 # BLOCK 3 - 15 cycles
125 SMULL v21.8h, v4.8b, v0.8b
126 LDR x7, [x13], 8 // Read A0
127 SMULL v23.8h, v5.8b, v0.8b
128 LDR x17, [x5, 192] // Read B
129 SADALP v24.4s, v17.8h
130 SUBS x0, x0, 16
131 SADALP v26.4s, v19.8h
132 LDR d5, [x5, 136] // Read B
133 INS v4.d[0], x16
134 SMLAL v21.8h, v2.8b, v6.8b
135 LDR x16, [x5, 144]
136 SMLAL v23.8h, v3.8b, v6.8b
137 LDR d6, [x13], 8 // Read A0
138 INS v0.d[0], x7
139 LDR d3, [x5, 200] // Read B
140 INS v2.d[0], x17
141 SADALP v28.4s, v21.8h
142 ADD x5, x5, 128
143 SADALP v30.4s, v23.8h
144 B.HS 2b
145
146 # Epilogue
147 # Same as main loop except no loads at end of loop
148
149 .p2align 3
150 3:
151 # BLOCK 0 - 6 cycles
152 SMULL v17.8h, v4.8b, v0.8b
153 LDR x17, [x5, 80]
154 SMULL v19.8h, v5.8b, v0.8b
155 LDR d5, [x5, 24]
156 INS v4.d[0], x16
157 SMLAL v17.8h, v2.8b, v6.8b
158 LDR x16, [x5, 32]
159 SMLAL v19.8h, v3.8b, v6.8b
160 LDR d3, [x5, 88]
161 INS v2.d[0], x17
162
163 # BLOCK 1 - 10 cycles
164 SMULL v21.8h, v4.8b, v0.8b
165 LDR x17, [x5, 96]
166 SMULL v23.8h, v5.8b, v0.8b
167 SADALP v16.4s, v17.8h
168 SADALP v18.4s, v19.8h
169 LDR d5, [x5, 40]
170 INS v4.d[0], x16
171 SMLAL v21.8h, v2.8b, v6.8b
172 LDR x16, [x5, 48]
173 SMLAL v23.8h, v3.8b, v6.8b
174 LDR d3, [x5, 104]
175 INS v2.d[0], x17
176
177 # BLOCK 2 - 10 cycles
178 SMULL v17.8h, v4.8b, v0.8b
179 LDR x17, [x5, 112]
180 SMULL v19.8h, v5.8b, v0.8b
181 SADALP v20.4s, v21.8h
182 SADALP v22.4s, v23.8h
183 LDR d5, [x5, 56]
184 INS v4.d[0], x16
185 SMLAL v17.8h, v2.8b, v6.8b
186 SMLAL v19.8h, v3.8b, v6.8b
187 LDR d3, [x5, 120]
188 INS v2.d[0], x17
189
190 # BLOCK 3 - 12 cycles
191 SMULL v21.8h, v4.8b, v0.8b
192 SMULL v23.8h, v5.8b, v0.8b
193 SADALP v24.4s, v17.8h
194 SADALP v26.4s, v19.8h
195 SMLAL v21.8h, v2.8b, v6.8b
196 SMLAL v23.8h, v3.8b, v6.8b
197 SADALP v28.4s, v21.8h
198 ADD x5, x5, 128
199 SADALP v30.4s, v23.8h
200
201 # Is there a remainder?- 8 bytes of A
202 TBNZ x0, 3, 5f
203
204 # ks loop
205 SUBS x9, x9, 8 // ks -= MR * sizeof(int8_t*)
206 B.HI 1b
207
208 4:
209 # Add columns
210 ADDP v16.4s, v16.4s, v18.4s
211 ADDP v20.4s, v20.4s, v22.4s
212 ADDP v24.4s, v24.4s, v26.4s
213 ADDP v28.4s, v28.4s, v30.4s
214 ADDP v0.4s, v16.4s, v20.4s
215 ADDP v1.4s, v24.4s, v28.4s
216
217 # Load per channel scale values from weights
218 SCVTF v0.4s, v0.4s
219 LDR q4, [x5], 16
220 SCVTF v1.4s, v1.4s
221 LDR q5, [x5], 16
222 FMUL v0.4s, v0.4s, v4.4s
223 FMUL v1.4s, v1.4s, v5.4s
224
225 FCVTNS v0.4s, v0.4s
226 FCVTNS v1.4s, v1.4s
227
228 LD1R {v5.8h}, [x11], 2
229 SQXTN v0.4h, v0.4s
230 SQXTN2 v0.8h, v1.4s
231 SUBS x1, x1, 8
232 SQADD v0.8h, v0.8h, v5.8h
233 LD1R {v1.16b}, [x11], 1
234 SQXTN v0.8b, v0.8h
235 LD1R {v17.16b}, [x11]
236 SMAX v0.8b, v0.8b, v1.8b
237 SUB x11, x11, 3 // rewind params pointer
238
239 SMIN v0.8b, v0.8b, v17.8b
240 B.LO 6f
241
242 # Store full 1 x 8
243 ST1 {v0.8b}, [x6], x10
244 SUB x4, x4, x3 // a -= ks
245 B.HI 0b
246 RET
247
248 # Remainder - 8 bytes of A
249 .p2align 3
250 5:
251 LDR d0, [x13], 8
252 LDP d4, d5, [x5]
253 LDP d6, d7, [x5, 16]
254 SMULL v17.8h, v4.8b, v0.8b
255 SMULL v19.8h, v5.8b, v0.8b
256 SMULL v21.8h, v6.8b, v0.8b
257 SMULL v23.8h, v7.8b, v0.8b
258 LDP d4, d5, [x5, 32]
259 LDP d6, d7, [x5, 48]
260 SADALP v16.4s, v17.8h
261 SADALP v18.4s, v19.8h
262 SADALP v20.4s, v21.8h
263 SADALP v22.4s, v23.8h
264 SMULL v17.8h, v4.8b, v0.8b
265 SMULL v19.8h, v5.8b, v0.8b
266 SMULL v21.8h, v6.8b, v0.8b
267 SMULL v23.8h, v7.8b, v0.8b
268 ADD x5, x5, 64
269 SADALP v24.4s, v17.8h
270 SADALP v26.4s, v19.8h
271 SADALP v28.4s, v21.8h
272 SADALP v30.4s, v23.8h
273
274 # ks loop
275 SUBS x9, x9, 8 // ks -= MR * sizeof(int8_t*)
276 B.HI 1b
277 B 4b
278
279 # Store odd width
280 .p2align 3
281 6:
282 TBZ x1, 2, 7f
283 STR s0, [x6], 4
284 EXT v0.16b, v0.16b, v0.16b, 4
285
286 7:
287 TBZ x1, 1, 8f
288 STR h0, [x6], 2
289 EXT v0.16b, v0.16b, v0.16b, 2
290 8:
291 TBZ x1, 0, 9f
292 STR b0, [x6]
293 9:
294 RET
295
296 END_FUNCTION xnn_qs8_qc8w_igemm_minmax_fp32_ukernel_1x8c8__asm_aarch64_neon_mlal_cortex_a53_prfm
297
298 #ifdef __ELF__
299 .section ".note.GNU-stack","",%progbits
300 #endif