Debian Codegraph @

src/qs8-qc8w-igemm/gen/qs8-qc8w-igemm-1x8-minmax-fp32-asm-aarch32-neonv8-mlal-lane-cortex-a35.S

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-igemm/1x8-aarch32-neon-mlal-lane-cortex-a7.S.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2021 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include "xnnpack/assembly.h"
12
13 .syntax unified
14
15 // void xnn_qs8_qc8w_igemm_minmax_fp32_ukernel_1x8__asm_aarch32_neonv8_mlal_lane_cortex_a35
16 // size_t mr, (r0)
17 // size_t nc, r1
18 // size_t kc, (r2) -> sp + 56 -> r5
19 // size_t ks, (r3) -> sp + 60 -> r14
20 // const int8_t** restrict a, sp + 88 -> r2
21 // const void* restrict w, sp + 92 -> r9
22 // int8_t* restrict c, sp + 96 -> r11
23 // size_t cm_stride, sp + 100 -> r6
24 // size_t cn_stride, sp + 104 -> r12
25 // size_t a_offset, sp + 108 -> (r5)
26 // const int8_t* zero, sp + 112 -> r7
27 // xnn_qs8_qc8w_conv_minmax_params*params); sp + 116 -> (r5)
28
29 // d8-d15, r4-r11,r14(lr) need to be preserved if used. r13(sp),r15(pc) are reserved.
30
31 // Based on cortex_a53 microkernel but with Neon loads
32
33 // Register usage
34 // A0 r3 d0-d1 q0
35 // B r9 d8-d9 q4 q5
36 // C0 r11 d16-d17 q8 d18-d19 q9
37 // q2, q3 acc2
38 // unused r4, r8, r10, d15, q10-q15, q1-q3
39 // params structure is 4 bytes
40 // struct {
41 // int16_t output_zero_point; d13[2]
42 // int8_t output_min; d13[6]
43 // int8_t output_max; d13[7]
44 // } xnn_qs8_minmax_params.neonv8;
45
46 BEGIN_FUNCTION xnn_qs8_qc8w_igemm_minmax_fp32_ukernel_1x8__asm_aarch32_neonv8_mlal_lane_cortex_a35
47 # Push 88 bytes
48 # r2, r3 will be reloaded in outer loop.
49 PUSH {r2, r3, r5, r6, r7, r9, r11, lr} // +32
50 SUB sp, sp, 8 // +8
51 VPUSH {d8-d13} // +48 = 88
52
53 LDR r2, [sp, 88] // a
54 LDR r9, [sp, 92] // w
55 LDR r11, [sp, 96] // c
56 LDR r6, [sp, 100] // cm_stride
57 LDR r12, [sp, 104] // cn_stride
58 LDR r7, [sp, 112] // zero
59 LDR r5, [sp, 116] // params
60 MOV r14, r3 // p = ks
61
62 # Load params values
63 VLD1.32 {d13[]}, [r5] // QC8 neonv8 params
64
65
66 .p2align 3
67 0:
68 # Load initial bias from w into accumulators
69 VLDM r9!, {d16-d19} // Bias
70 VMOV.I32 q2, 0 // second set of C for pipelining FMLA
71 VMOV.I32 q3, 0
72
73 .p2align 3
74 1:
75 # Load next A pointer
76 LDR r3, [r2, 0]
77
78 # Add a_offset
79 LDR r5, [sp, 108] // a_offset
80 ADD r2, r2, 4
81 CMP r3, r7 // if a0 == zero
82 ADD r3, r3, r5 // a0 += a_offset
83 MOVEQ r3, r7 // a0 = zero, else += a0 + a_offset
84
85 LDR r5, [sp, 56] // kc
86 SUBS r5, r5, 8 // kc - 8
87 BLO 5f // less than 8 channels?
88
89 // Prologue - load A0 and B0
90 VLD1.8 {d0}, [r3]! // A0
91 SUBS r5, r5, 8 // k = k - 8
92 VLD1.8 {d8}, [r9]! // B0
93 BLO 3f // less than 8 channels?
94
95 // Main loop - 8 bytes
96 // 64 bytes for weights.
97
98 .p2align 3
99 2:
100 // Extend
101 VMOVL.S8 q0, d0
102 VMOVL.S8 q4, d8
103
104 // BLOCK 0
105 VLD1.8 {d10}, [r9]! // B1
106 VMLAL.S16 q8, d8, d0[0]
107 VMLAL.S16 q9, d9, d0[0]
108 VMOVL.S8 q5, d10
109
110 // BLOCK 1
111 VLD1.8 {d8}, [r9]! // B2
112 VMLAL.S16 q2, d10, d0[1]
113 VMLAL.S16 q3, d11, d0[1]
114 VMOVL.S8 q4, d8
115
116 // BLOCK 2
117 VLD1.8 {d10}, [r9]! // B3
118 VMLAL.S16 q8, d8, d0[2]
119 VMLAL.S16 q9, d9, d0[2]
120 VMOVL.S8 q5, d10
121
122 // BLOCK 3
123 VLD1.8 {d8}, [r9]! // B4
124 VMLAL.S16 q2, d10, d0[3]
125 VMLAL.S16 q3, d11, d0[3]
126 VLD1.8 {d0}, [r3]! // A0
127 VMOVL.S8 q4, d8
128
129 // BLOCK 4
130 VLD1.8 {d10}, [r9]! // B5
131 VMLAL.S16 q8, d8, d1[0]
132 VMLAL.S16 q9, d9, d1[0]
133 VMOVL.S8 q5, d10
134
135 // BLOCK 5
136 VLD1.8 {d8}, [r9]! // B6
137 VMLAL.S16 q2, d10, d1[1]
138 VMLAL.S16 q3, d11, d1[1]
139 VMOVL.S8 q4, d8
140
141 // BLOCK 6
142 VLD1.8 {d10}, [r9]! // B7
143 VMLAL.S16 q8, d8, d1[2]
144 VMLAL.S16 q9, d9, d1[2]
145 VMOVL.S8 q5, d10
146 SUBS r5, r5, 8
147
148 // BLOCK 7
149 VLD1.8 {d8}, [r9]! // B0
150 VMLAL.S16 q2, d10, d1[3]
151 VMLAL.S16 q3, d11, d1[3]
152 BHS 2b
153
154 // Epilogue
155
156 .p2align 3
157 3:
158 // Extend
159 VMOVL.S8 q0, d0
160 VMOVL.S8 q4, d8
161
162 // BLOCK 0
163 VLD1.8 {d10}, [r9]! // B1
164 VMLAL.S16 q8, d8, d0[0]
165 VMLAL.S16 q9, d9, d0[0]
166 VMOVL.S8 q5, d10
167
168 // BLOCK 1
169 VLD1.8 {d8}, [r9]! // B2
170 VMLAL.S16 q2, d10, d0[1]
171 VMLAL.S16 q3, d11, d0[1]
172 VMOVL.S8 q4, d8
173
174 // BLOCK 2
175 VLD1.8 {d10}, [r9]! // B3
176 VMLAL.S16 q8, d8, d0[2]
177 VMLAL.S16 q9, d9, d0[2]
178 VMOVL.S8 q5, d10
179
180 // BLOCK 3
181 VLD1.8 {d8}, [r9]! // B4
182 VMLAL.S16 q2, d10, d0[3]
183 VMLAL.S16 q3, d11, d0[3]
184 VMOVL.S8 q4, d8
185
186 // BLOCK 4
187 VLD1.8 {d10}, [r9]! // B5
188 VMLAL.S16 q8, d8, d1[0]
189 VMLAL.S16 q9, d9, d1[0]
190 VMOVL.S8 q5, d10
191
192 // BLOCK 5
193 VLD1.8 {d8}, [r9]! // B6
194 VMLAL.S16 q2, d10, d1[1]
195 VMLAL.S16 q3, d11, d1[1]
196 VMOVL.S8 q4, d8
197
198 // BLOCK 6
199 VLD1.8 {d10}, [r9]! // B7
200 VMLAL.S16 q8, d8, d1[2]
201 VMLAL.S16 q9, d9, d1[2]
202 VMOVL.S8 q5, d10
203 ADDS r5, r5, 8
204
205 VMLAL.S16 q2, d10, d1[3]
206 VMLAL.S16 q3, d11, d1[3]
207
208 # Is there a remainder?- 1-7 bytes of A
209 BNE 6f
210
211 4:
212 # ks loop
213 SUBS r14, r14, 4 // ks -= MR * sizeof(void*)
214 BHI 1b
215
216 LDR r14, [sp, 60] // p = ks
217
218 VADD.S32 q8, q8, q2
219 VADD.S32 q9, q9, q3
220
221 # QC8 FP32 quantization
222 VLD1.8 {q0-q1}, [r9]!
223
224 VCVT.F32.S32 q8, q8
225 VCVT.F32.S32 q9, q9
226
227 VMUL.F32 q8, q8, q0 // multiplier
228 VMUL.F32 q9, q9, q1
229
230 VCVTN.S32.F32 q8, q8
231 VCVTN.S32.F32 q9, q9
232
233 VDUP.16 q0, d13[2] // output_zero_point
234
235 VQMOVN.S32 d16, q8
236 VQMOVN.S32 d17, q9
237
238 VQADD.S16 q8, q8, q0
239
240 VDUP.8 d24, d13[6] // output_min
241
242 VQMOVN.S16 d0, q8
243
244 VDUP.8 d25, d13[7] // output_max
245
246 VMAX.S8 d0, d0, d24
247
248 SUBS r1, r1, 8
249
250 VMIN.S8 d0, d0, d25
251
252 # Store full 1 x 8
253 BLO 7f
254 VST1.8 {d0}, [r11], r12
255 SUB r2, r2, r14 // a -= ks
256 BHI 0b
257
258 VPOP {d8-d13}
259 ADD sp, sp, 16 // skip pad of 8, r2, r3
260 POP {r5, r6, r7, r9, r11, pc}
261
262 # Remainder- 1 to 7 bytes of A
263 .p2align 3
264 5:
265 AND r5, r5, 7 // kc remainder 1 to 7
266 6:
267 VLD1.8 {d0}, [r3]
268 VLD1.8 {d8}, [r9]!
269
270 VMOVL.S8 q0, d0
271 VMOVL.S8 q4, d8
272 VMLAL.S16 q8, d8, d0[0]
273 VMLAL.S16 q9, d9, d0[0]
274 CMP r5, 2
275 BLO 4b
276
277 VLD1.8 {d8}, [r9]!
278 VMOVL.S8 q4, d8
279 VMLAL.S16 q8, d8, d0[1]
280 VMLAL.S16 q9, d9, d0[1]
281 BEQ 4b
282
283 VLD1.8 {d8}, [r9]!
284 VMOVL.S8 q4, d8
285 VMLAL.S16 q8, d8, d0[2]
286 VMLAL.S16 q9, d9, d0[2]
287 CMP r5, 4
288 BLO 4b
289
290 VLD1.8 {d8}, [r9]!
291 VMOVL.S8 q4, d8
292 VMLAL.S16 q8, d8, d0[3]
293 VMLAL.S16 q9, d9, d0[3]
294 BEQ 4b
295
296 VLD1.8 {d8}, [r9]!
297 VMOVL.S8 q4, d8
298 VMLAL.S16 q8, d8, d1[0]
299 VMLAL.S16 q9, d9, d1[0]
300 CMP r5, 6
301 BLO 4b
302
303 VLD1.8 {d8}, [r9]!
304 VMOVL.S8 q4, d8
305 VMLAL.S16 q8, d8, d1[1]
306 VMLAL.S16 q9, d9, d1[1]
307 BEQ 4b
308
309 VLD1.8 {d8}, [r9]!
310 VMOVL.S8 q4, d8
311 VMLAL.S16 q8, d8, d1[2]
312 VMLAL.S16 q9, d9, d1[2]
313 B 4b
314
315 # Store odd width
316 .p2align 3
317 7:
318 TST r1, 4
319 BEQ 8f
320 VST1.32 {d0[0]}, [r11]!
321 VEXT.8 q0, q0, q0, 4
322 8:
323 TST r1, 2
324 BEQ 9f
325 VST1.16 {d0[0]}, [r11]!
326 VEXT.8 q0, q0, q0, 2
327
328 9:
329 TST r1, 1
330 BEQ 10f
331 VST1.8 {d0[0]}, [r11]
332
333 10:
334 VPOP {d8-d13}
335 ADD sp, sp, 16 // skip pad of 8, r2, r3
336 POP {r5, r6, r7, r9, r11, pc}
337
338 END_FUNCTION xnn_qs8_qc8w_igemm_minmax_fp32_ukernel_1x8__asm_aarch32_neonv8_mlal_lane_cortex_a35
339
340 #ifdef __ELF__
341 .section ".note.GNU-stack","",%progbits
342 #endif