Debian Codegraph @

src/qs8-qc8w-igemm/gen/qs8-qc8w-igemm-1x8c8-minmax-fp32-asm-aarch64-neon-mlal-prfm.S

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-igemm/1x8c8-aarch64-neon-mlal.S.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2021 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include "xnnpack/assembly.h"
12
13 # void xnn_qs8_qc8w_igemm_minmax_fp32_ukernel_1x8c8__asm_aarch64_neon_mlal_prfm(
14 # size_t mr, x0
15 # size_t nc, x1
16 # size_t kc, x2 / x0
17 # size_t ks, x3 / x9
18 # const int8_t** restrict a, x4
19 # const int8_t* restrict w, x5
20 # int8_t* restrict c, x6
21 # size_t cm_stride, (x7)
22 # size_t cn_stride, [sp] -> x10
23 # size_t a_offset, [sp + 8] -> x8
24 # const int8_t* zero, [sp + 16] -> x12
25 # const union xnn_qs8_qc8w_conv_minmax_params params [sp + 24] -> x11
26
27 # d8-d15, x19-x30 need to be preserved if used. x18 is reserved by the OS.
28
29 // Register usage
30 // A0 x13 v0 v6
31 // B x5 v4 v5 v2 v3
32 // C0 x6 v16 v18 v20 v22 v24 v26 v28 v30
33 // temp0 v17 v19 v21 v23
34
35 BEGIN_FUNCTION xnn_qs8_qc8w_igemm_minmax_fp32_ukernel_1x8c8__asm_aarch64_neon_mlal_prfm
36 # Clamp C pointers
37 LDP x10, x8, [sp] // Load cn_stride, a_offset
38 ADD x2, x2, 7 // kc = (kc + 7) & ~7
39 LDP x12, x11, [sp, 16] // Load zero, params pointer
40 BIC x2, x2, 7
41
42 .p2align 3
43 0:
44 # Load initial bias from w into accumulators
45 LDP s16, s18, [x5], 8
46 LDP s20, s22, [x5], 8
47 LDP s24, s26, [x5], 8
48 LDP s28, s30, [x5], 8
49 MOV x9, x3 // p = ks
50
51 .p2align 3
52 1:
53 # Load next A pointer
54 LDR x13, [x4], 8
55 CMP x13, x12 // if a0 == zero
56 ADD x13, x13, x8 // a0 += a_offset
57 CSEL x13, x12, x13, EQ // a0 = zero, else += a0 + a_offset
58
59 # Is there at least 16 bytes for epilogue?
60 SUBS x0, x2, 16 // k = kc - 16
61 B.LO 5f
62
63 # Prologue: load A0 and 4 B's
64 LDP d0, d6, [x13], 16 // Read A0
65 LDP d4, d5, [x5] // Read B
66 LDP d2, d3, [x5, 64] // Read B
67
68 # Is there at least 16 bytes for main loop?
69 SUBS x0, x0, 16 // k = k - 16
70 B.LO 3f
71
72 # Main loop - 16 bytes of A
73 # 4 groups of 2 mul/mla/adap = 6 cycles.
74 # 2 load for A0, A1 = +4 cycle. Total 36 cycles.
75
76 .p2align 3
77 2:
78 # BLOCK 0 - 4 cycles
79 SMULL v17.8h, v4.8b, v0.8b
80 SMULL v19.8h, v5.8b, v0.8b
81 LDP d4, d5, [x5, 16]
82 SMLAL v17.8h, v2.8b, v6.8b
83 SMLAL v19.8h, v3.8b, v6.8b
84 LDP d2, d3, [x5, 80]
85
86 # BLOCK 1 - 6 cycles
87 SMULL v21.8h, v4.8b, v0.8b
88 SMULL v23.8h, v5.8b, v0.8b
89 PRFM PLDL1KEEP, [x5, 448]
90 SADALP v16.4s, v17.8h
91 PRFM PLDL1KEEP, [x5, 512]
92 SADALP v18.4s, v19.8h
93 LDP d4, d5, [x5, 32]
94 SMLAL v21.8h, v2.8b, v6.8b
95 SMLAL v23.8h, v3.8b, v6.8b
96 LDP d2, d3, [x5, 96]
97
98 # BLOCK 2 - 6 cycles
99 SMULL v17.8h, v4.8b, v0.8b
100 SMULL v19.8h, v5.8b, v0.8b
101 PRFM PLDL1KEEP, [x13, 128]
102 SADALP v20.4s, v21.8h
103 SADALP v22.4s, v23.8h
104 LDP d4, d5, [x5, 48]
105 SMLAL v17.8h, v2.8b, v6.8b
106 SMLAL v19.8h, v3.8b, v6.8b
107 LDP d2, d3, [x5, 112]
108
109 # BLOCK 3 - 14 cycles
110 SMULL v21.8h, v4.8b, v0.8b
111 ADD x5, x5, 128
112 SMULL v23.8h, v5.8b, v0.8b
113 SADALP v24.4s, v17.8h
114 SUBS x0, x0, 16
115 SADALP v26.4s, v19.8h
116 LDP d4, d5, [x5] // Read B
117 SMLAL v21.8h, v2.8b, v6.8b
118 SMLAL v23.8h, v3.8b, v6.8b
119 LDP d0, d6, [x13], 16 // Read A0
120 SADALP v28.4s, v21.8h
121 LDP d2, d3, [x5, 64] // Read B
122 SADALP v30.4s, v23.8h
123 B.HS 2b
124
125 # Epilogue
126 # Same as main loop except no loads at end of loop
127
128 .p2align 3
129 3:
130 # BLOCK 0 - 4 cycles
131 SMULL v17.8h, v4.8b, v0.8b
132 SMULL v19.8h, v5.8b, v0.8b
133 LDP d4, d5, [x5, 16]
134 SMLAL v17.8h, v2.8b, v6.8b
135 SMLAL v19.8h, v3.8b, v6.8b
136 LDP d2, d3, [x5, 80]
137
138 # BLOCK 1 - 6 cycles
139 SMULL v21.8h, v4.8b, v0.8b
140 SMULL v23.8h, v5.8b, v0.8b
141 PRFM PLDL1KEEP, [x5, 448]
142 SADALP v16.4s, v17.8h
143 PRFM PLDL1KEEP, [x5, 512]
144 SADALP v18.4s, v19.8h
145 LDP d4, d5, [x5, 32]
146 SMLAL v21.8h, v2.8b, v6.8b
147 SMLAL v23.8h, v3.8b, v6.8b
148 LDP d2, d3, [x5, 96]
149
150 # BLOCK 2 - 6 cycles
151 SMULL v17.8h, v4.8b, v0.8b
152 SMULL v19.8h, v5.8b, v0.8b
153 PRFM PLDL1KEEP, [x13, 128]
154 SADALP v20.4s, v21.8h
155 SADALP v22.4s, v23.8h
156 LDP d4, d5, [x5, 48]
157 SMLAL v17.8h, v2.8b, v6.8b
158 SMLAL v19.8h, v3.8b, v6.8b
159 LDP d2, d3, [x5, 112]
160
161 # BLOCK 3 - 8 cycles
162 SMULL v21.8h, v4.8b, v0.8b
163 ADD x5, x5, 128
164 SMULL v23.8h, v5.8b, v0.8b
165 SADALP v24.4s, v17.8h
166 SUBS x0, x0, 16
167 SADALP v26.4s, v19.8h
168 SMLAL v21.8h, v2.8b, v6.8b
169 SMLAL v23.8h, v3.8b, v6.8b
170 SADALP v28.4s, v21.8h
171 SADALP v30.4s, v23.8h
172
173 # Is there a remainder?- 8 bytes of A
174 TBNZ x0, 3, 5f
175
176 # ks loop
177 SUBS x9, x9, 8 // ks -= MR * sizeof(int8_t*)
178 B.HI 1b
179
180 4:
181 # Add columns
182 ADDP v16.4s, v16.4s, v18.4s
183 ADDP v20.4s, v20.4s, v22.4s
184 ADDP v24.4s, v24.4s, v26.4s
185 ADDP v28.4s, v28.4s, v30.4s
186 ADDP v0.4s, v16.4s, v20.4s
187 ADDP v1.4s, v24.4s, v28.4s
188
189 # Load per channel scale values from weights
190 SCVTF v0.4s, v0.4s
191 LDR q4, [x5], 16
192 SCVTF v1.4s, v1.4s
193 LDR q5, [x5], 16
194 FMUL v0.4s, v0.4s, v4.4s
195 FMUL v1.4s, v1.4s, v5.4s
196
197 FCVTNS v0.4s, v0.4s
198 FCVTNS v1.4s, v1.4s
199
200 LD1R {v5.8h}, [x11], 2
201 SQXTN v0.4h, v0.4s
202 SQXTN2 v0.8h, v1.4s
203 SUBS x1, x1, 8
204 SQADD v0.8h, v0.8h, v5.8h
205 LD1R {v1.16b}, [x11], 1
206 SQXTN v0.8b, v0.8h
207 LD1R {v17.16b}, [x11]
208 SMAX v0.8b, v0.8b, v1.8b
209 SUB x11, x11, 3 // rewind params pointer
210 SMIN v0.8b, v0.8b, v17.8b
211 B.LO 6f
212
213 # Store full 1 x 8
214 ST1 {v0.8b}, [x6], x10
215 SUB x4, x4, x3 // a -= ks
216 B.HI 0b
217 RET
218
219 # Remainder - 8 bytes of A
220 .p2align 3
221 5:
222 LDR d0, [x13], 8
223 LDP d4, d5, [x5]
224 LDP d6, d7, [x5, 16]
225 SMULL v17.8h, v4.8b, v0.8b
226 SMULL v19.8h, v5.8b, v0.8b
227 SMULL v21.8h, v6.8b, v0.8b
228 SMULL v23.8h, v7.8b, v0.8b
229 LDP d4, d5, [x5, 32]
230 LDP d6, d7, [x5, 48]
231 SADALP v16.4s, v17.8h
232 SADALP v18.4s, v19.8h
233 SADALP v20.4s, v21.8h
234 SADALP v22.4s, v23.8h
235 SMULL v17.8h, v4.8b, v0.8b
236 SMULL v19.8h, v5.8b, v0.8b
237 SMULL v21.8h, v6.8b, v0.8b
238 SMULL v23.8h, v7.8b, v0.8b
239 ADD x5, x5, 64
240 SADALP v24.4s, v17.8h
241 SADALP v26.4s, v19.8h
242 SADALP v28.4s, v21.8h
243 SADALP v30.4s, v23.8h
244
245 # ks loop
246 SUBS x9, x9, 8 // ks -= MR * sizeof(int8_t*)
247 B.HI 1b
248 B 4b
249
250 # Store odd width
251 .p2align 3
252 6:
253 TBZ x1, 2, 7f
254 STR s0, [x6], 4
255 EXT v0.16b, v0.16b, v0.16b, 4
256
257 7:
258 TBZ x1, 1, 8f
259 STR h0, [x6], 2
260 EXT v0.16b, v0.16b, v0.16b, 2
261 8:
262 TBZ x1, 0, 9f
263 STR b0, [x6]
264 9:
265 RET
266
267 END_FUNCTION xnn_qs8_qc8w_igemm_minmax_fp32_ukernel_1x8c8__asm_aarch64_neon_mlal_prfm
268
269 #ifdef __ELF__
270 .section ".note.GNU-stack","",%progbits
271 #endif