Debian Codegraph @

src/qs8-qc8w-igemm/gen/qs8-qc8w-igemm-1x8c8-minmax-fp32-asm-aarch64-neon-mlal.S

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-igemm/1x8c8-aarch64-neon-mlal.S.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2021 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include "xnnpack/assembly.h"
12
13 # void xnn_qs8_qc8w_igemm_minmax_fp32_ukernel_1x8c8__asm_aarch64_neon_mlal(
14 # size_t mr, x0
15 # size_t nc, x1
16 # size_t kc, x2 / x0
17 # size_t ks, x3 / x9
18 # const int8_t** restrict a, x4
19 # const int8_t* restrict w, x5
20 # int8_t* restrict c, x6
21 # size_t cm_stride, (x7)
22 # size_t cn_stride, [sp] -> x10
23 # size_t a_offset, [sp + 8] -> x8
24 # const int8_t* zero, [sp + 16] -> x12
25 # const union xnn_qs8_qc8w_conv_minmax_params params [sp + 24] -> x11
26
27 # d8-d15, x19-x30 need to be preserved if used. x18 is reserved by the OS.
28
29 // Register usage
30 // A0 x13 v0 v6
31 // B x5 v4 v5 v2 v3
32 // C0 x6 v16 v18 v20 v22 v24 v26 v28 v30
33 // temp0 v17 v19 v21 v23
34
35 BEGIN_FUNCTION xnn_qs8_qc8w_igemm_minmax_fp32_ukernel_1x8c8__asm_aarch64_neon_mlal
36 # Clamp C pointers
37 LDP x10, x8, [sp] // Load cn_stride, a_offset
38 ADD x2, x2, 7 // kc = (kc + 7) & ~7
39 LDP x12, x11, [sp, 16] // Load zero, params pointer
40 BIC x2, x2, 7
41
42 .p2align 3
43 0:
44 # Load initial bias from w into accumulators
45 LDP s16, s18, [x5], 8
46 LDP s20, s22, [x5], 8
47 LDP s24, s26, [x5], 8
48 LDP s28, s30, [x5], 8
49 MOV x9, x3 // p = ks
50
51 .p2align 3
52 1:
53 # Load next A pointer
54 LDR x13, [x4], 8
55 CMP x13, x12 // if a0 == zero
56 ADD x13, x13, x8 // a0 += a_offset
57 CSEL x13, x12, x13, EQ // a0 = zero, else += a0 + a_offset
58
59 # Is there at least 16 bytes for epilogue?
60 SUBS x0, x2, 16 // k = kc - 16
61 B.LO 5f
62
63 # Prologue: load A0 and 4 B's
64 LDP d0, d6, [x13], 16 // Read A0
65 LDP d4, d5, [x5] // Read B
66 LDP d2, d3, [x5, 64] // Read B
67
68 # Is there at least 16 bytes for main loop?
69 SUBS x0, x0, 16 // k = k - 16
70 B.LO 3f
71
72 # Main loop - 16 bytes of A
73 # 4 groups of 2 mul/mla/adap = 6 cycles.
74 # 2 load for A0, A1 = +4 cycle. Total 36 cycles.
75
76 .p2align 3
77 2:
78 # BLOCK 0 - 4 cycles
79 SMULL v17.8h, v4.8b, v0.8b
80 SMULL v19.8h, v5.8b, v0.8b
81 LDP d4, d5, [x5, 16]
82 SMLAL v17.8h, v2.8b, v6.8b
83 SMLAL v19.8h, v3.8b, v6.8b
84 LDP d2, d3, [x5, 80]
85
86 # BLOCK 1 - 6 cycles
87 SMULL v21.8h, v4.8b, v0.8b
88 SMULL v23.8h, v5.8b, v0.8b
89 SADALP v16.4s, v17.8h
90 SADALP v18.4s, v19.8h
91 LDP d4, d5, [x5, 32]
92 SMLAL v21.8h, v2.8b, v6.8b
93 SMLAL v23.8h, v3.8b, v6.8b
94 LDP d2, d3, [x5, 96]
95
96 # BLOCK 2 - 6 cycles
97 SMULL v17.8h, v4.8b, v0.8b
98 SMULL v19.8h, v5.8b, v0.8b
99 SADALP v20.4s, v21.8h
100 SADALP v22.4s, v23.8h
101 LDP d4, d5, [x5, 48]
102 SMLAL v17.8h, v2.8b, v6.8b
103 SMLAL v19.8h, v3.8b, v6.8b
104 LDP d2, d3, [x5, 112]
105
106 # BLOCK 3 - 14 cycles
107 SMULL v21.8h, v4.8b, v0.8b
108 ADD x5, x5, 128
109 SMULL v23.8h, v5.8b, v0.8b
110 SADALP v24.4s, v17.8h
111 SUBS x0, x0, 16
112 SADALP v26.4s, v19.8h
113 LDP d4, d5, [x5] // Read B
114 SMLAL v21.8h, v2.8b, v6.8b
115 SMLAL v23.8h, v3.8b, v6.8b
116 LDP d0, d6, [x13], 16 // Read A0
117 SADALP v28.4s, v21.8h
118 LDP d2, d3, [x5, 64] // Read B
119 SADALP v30.4s, v23.8h
120 B.HS 2b
121
122 # Epilogue
123 # Same as main loop except no loads at end of loop
124
125 .p2align 3
126 3:
127 # BLOCK 0 - 4 cycles
128 SMULL v17.8h, v4.8b, v0.8b
129 SMULL v19.8h, v5.8b, v0.8b
130 LDP d4, d5, [x5, 16]
131 SMLAL v17.8h, v2.8b, v6.8b
132 SMLAL v19.8h, v3.8b, v6.8b
133 LDP d2, d3, [x5, 80]
134
135 # BLOCK 1 - 6 cycles
136 SMULL v21.8h, v4.8b, v0.8b
137 SMULL v23.8h, v5.8b, v0.8b
138 SADALP v16.4s, v17.8h
139 SADALP v18.4s, v19.8h
140 LDP d4, d5, [x5, 32]
141 SMLAL v21.8h, v2.8b, v6.8b
142 SMLAL v23.8h, v3.8b, v6.8b
143 LDP d2, d3, [x5, 96]
144
145 # BLOCK 2 - 6 cycles
146 SMULL v17.8h, v4.8b, v0.8b
147 SMULL v19.8h, v5.8b, v0.8b
148 SADALP v20.4s, v21.8h
149 SADALP v22.4s, v23.8h
150 LDP d4, d5, [x5, 48]
151 SMLAL v17.8h, v2.8b, v6.8b
152 SMLAL v19.8h, v3.8b, v6.8b
153 LDP d2, d3, [x5, 112]
154
155 # BLOCK 3 - 8 cycles
156 SMULL v21.8h, v4.8b, v0.8b
157 ADD x5, x5, 128
158 SMULL v23.8h, v5.8b, v0.8b
159 SADALP v24.4s, v17.8h
160 SUBS x0, x0, 16
161 SADALP v26.4s, v19.8h
162 SMLAL v21.8h, v2.8b, v6.8b
163 SMLAL v23.8h, v3.8b, v6.8b
164 SADALP v28.4s, v21.8h
165 SADALP v30.4s, v23.8h
166
167 # Is there a remainder?- 8 bytes of A
168 TBNZ x0, 3, 5f
169
170 # ks loop
171 SUBS x9, x9, 8 // ks -= MR * sizeof(int8_t*)
172 B.HI 1b
173
174 4:
175 # Add columns
176 ADDP v16.4s, v16.4s, v18.4s
177 ADDP v20.4s, v20.4s, v22.4s
178 ADDP v24.4s, v24.4s, v26.4s
179 ADDP v28.4s, v28.4s, v30.4s
180 ADDP v0.4s, v16.4s, v20.4s
181 ADDP v1.4s, v24.4s, v28.4s
182
183 # Load per channel scale values from weights
184 SCVTF v0.4s, v0.4s
185 LDR q4, [x5], 16
186 SCVTF v1.4s, v1.4s
187 LDR q5, [x5], 16
188 FMUL v0.4s, v0.4s, v4.4s
189 FMUL v1.4s, v1.4s, v5.4s
190
191 FCVTNS v0.4s, v0.4s
192 FCVTNS v1.4s, v1.4s
193
194 LD1R {v5.8h}, [x11], 2
195 SQXTN v0.4h, v0.4s
196 SQXTN2 v0.8h, v1.4s
197 SUBS x1, x1, 8
198 SQADD v0.8h, v0.8h, v5.8h
199 LD1R {v1.16b}, [x11], 1
200 SQXTN v0.8b, v0.8h
201 LD1R {v17.16b}, [x11]
202 SMAX v0.8b, v0.8b, v1.8b
203 SUB x11, x11, 3 // rewind params pointer
204 SMIN v0.8b, v0.8b, v17.8b
205 B.LO 6f
206
207 # Store full 1 x 8
208 ST1 {v0.8b}, [x6], x10
209 SUB x4, x4, x3 // a -= ks
210 B.HI 0b
211 RET
212
213 # Remainder - 8 bytes of A
214 .p2align 3
215 5:
216 LDR d0, [x13], 8
217 LDP d4, d5, [x5]
218 LDP d6, d7, [x5, 16]
219 SMULL v17.8h, v4.8b, v0.8b
220 SMULL v19.8h, v5.8b, v0.8b
221 SMULL v21.8h, v6.8b, v0.8b
222 SMULL v23.8h, v7.8b, v0.8b
223 LDP d4, d5, [x5, 32]
224 LDP d6, d7, [x5, 48]
225 SADALP v16.4s, v17.8h
226 SADALP v18.4s, v19.8h
227 SADALP v20.4s, v21.8h
228 SADALP v22.4s, v23.8h
229 SMULL v17.8h, v4.8b, v0.8b
230 SMULL v19.8h, v5.8b, v0.8b
231 SMULL v21.8h, v6.8b, v0.8b
232 SMULL v23.8h, v7.8b, v0.8b
233 ADD x5, x5, 64
234 SADALP v24.4s, v17.8h
235 SADALP v26.4s, v19.8h
236 SADALP v28.4s, v21.8h
237 SADALP v30.4s, v23.8h
238
239 # ks loop
240 SUBS x9, x9, 8 // ks -= MR * sizeof(int8_t*)
241 B.HI 1b
242 B 4b
243
244 # Store odd width
245 .p2align 3
246 6:
247 TBZ x1, 2, 7f
248 STR s0, [x6], 4
249 EXT v0.16b, v0.16b, v0.16b, 4
250
251 7:
252 TBZ x1, 1, 8f
253 STR h0, [x6], 2
254 EXT v0.16b, v0.16b, v0.16b, 2
255 8:
256 TBZ x1, 0, 9f
257 STR b0, [x6]
258 9:
259 RET
260
261 END_FUNCTION xnn_qs8_qc8w_igemm_minmax_fp32_ukernel_1x8c8__asm_aarch64_neon_mlal
262
263 #ifdef __ELF__
264 .section ".note.GNU-stack","",%progbits
265 #endif