Debian Codegraph @

src/qs8-qc8w-igemm/gen/qs8-qc8w-igemm-1x8-minmax-fp32-asm-aarch32-neonv8-mlal-lane-cortex-a35-prfm.S

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-igemm/1x8-aarch32-neon-mlal-lane-cortex-a7.S.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2021 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include "xnnpack/assembly.h"
12
13 .syntax unified
14
15 // void xnn_qs8_qc8w_igemm_minmax_fp32_ukernel_1x8__asm_aarch32_neonv8_mlal_lane_cortex_a35_prfm
16 // size_t mr, (r0)
17 // size_t nc, r1
18 // size_t kc, (r2) -> sp + 56 -> r5
19 // size_t ks, (r3) -> sp + 60 -> r14
20 // const int8_t** restrict a, sp + 88 -> r2
21 // const void* restrict w, sp + 92 -> r9
22 // int8_t* restrict c, sp + 96 -> r11
23 // size_t cm_stride, sp + 100 -> r6
24 // size_t cn_stride, sp + 104 -> r12
25 // size_t a_offset, sp + 108 -> (r5)
26 // const int8_t* zero, sp + 112 -> r7
27 // xnn_qs8_qc8w_conv_minmax_params*params); sp + 116 -> (r5)
28
29 // d8-d15, r4-r11,r14(lr) need to be preserved if used. r13(sp),r15(pc) are reserved.
30
31 // Based on cortex_a53 microkernel but with Neon loads
32
33 // Register usage
34 // A0 r3 d0-d1 q0
35 // B r9 d8-d9 q4 q5
36 // C0 r11 d16-d17 q8 d18-d19 q9
37 // q2, q3 acc2
38 // unused r4, r8, r10, d15, q10-q15, q1-q3
39 // params structure is 4 bytes
40 // struct {
41 // int16_t output_zero_point; d13[2]
42 // int8_t output_min; d13[6]
43 // int8_t output_max; d13[7]
44 // } xnn_qs8_minmax_params.neonv8;
45
46 BEGIN_FUNCTION xnn_qs8_qc8w_igemm_minmax_fp32_ukernel_1x8__asm_aarch32_neonv8_mlal_lane_cortex_a35_prfm
47 # Push 88 bytes
48 # r2, r3 will be reloaded in outer loop.
49 PUSH {r2, r3, r5, r6, r7, r9, r11, lr} // +32
50 SUB sp, sp, 8 // +8
51 VPUSH {d8-d13} // +48 = 88
52
53 LDR r2, [sp, 88] // a
54 LDR r9, [sp, 92] // w
55 LDR r11, [sp, 96] // c
56 LDR r6, [sp, 100] // cm_stride
57 LDR r12, [sp, 104] // cn_stride
58 LDR r7, [sp, 112] // zero
59 LDR r5, [sp, 116] // params
60 MOV r14, r3 // p = ks
61
62 # Load params values
63 VLD1.32 {d13[]}, [r5] // QC8 neonv8 params
64
65 PLD [r9, 64] // Prefetch B
66 PLD [r9, 112]
67 PLD [r9, 192]
68 PLD [r9, 256]
69 PLD [r9, 320]
70 PLD [r9, 384]
71
72 .p2align 3
73 0:
74 # Load initial bias from w into accumulators
75 VLDM r9!, {d16-d19} // Bias
76 VMOV.I32 q2, 0 // second set of C for pipelining FMLA
77 VMOV.I32 q3, 0
78
79 .p2align 3
80 1:
81 # Load next A pointer
82 LDR r3, [r2, 0]
83
84 # Add a_offset
85 LDR r5, [sp, 108] // a_offset
86 ADD r2, r2, 4
87 CMP r3, r7 // if a0 == zero
88 ADD r3, r3, r5 // a0 += a_offset
89 MOVEQ r3, r7 // a0 = zero, else += a0 + a_offset
90
91 LDR r5, [sp, 56] // kc
92 SUBS r5, r5, 8 // kc - 8
93 BLO 5f // less than 8 channels?
94
95 // Prologue - load A0 and B0
96 VLD1.8 {d0}, [r3]! // A0
97 SUBS r5, r5, 8 // k = k - 8
98 VLD1.8 {d8}, [r9]! // B0
99 BLO 3f // less than 8 channels?
100
101 // Main loop - 8 bytes
102 // 64 bytes for weights.
103
104 .p2align 3
105 2:
106 // Extend
107 VMOVL.S8 q0, d0
108 VMOVL.S8 q4, d8
109 PLD [r9, 448]
110
111 // BLOCK 0
112 VLD1.8 {d10}, [r9]! // B1
113 VMLAL.S16 q8, d8, d0[0]
114 VMLAL.S16 q9, d9, d0[0]
115 VMOVL.S8 q5, d10
116
117 // BLOCK 1
118 VLD1.8 {d8}, [r9]! // B2
119 VMLAL.S16 q2, d10, d0[1]
120 VMLAL.S16 q3, d11, d0[1]
121 VMOVL.S8 q4, d8
122
123 // BLOCK 2
124 VLD1.8 {d10}, [r9]! // B3
125 VMLAL.S16 q8, d8, d0[2]
126 VMLAL.S16 q9, d9, d0[2]
127 VMOVL.S8 q5, d10
128
129 // BLOCK 3
130 VLD1.8 {d8}, [r9]! // B4
131 VMLAL.S16 q2, d10, d0[3]
132 VMLAL.S16 q3, d11, d0[3]
133 VLD1.8 {d0}, [r3]! // A0
134 VMOVL.S8 q4, d8
135
136 // BLOCK 4
137 VLD1.8 {d10}, [r9]! // B5
138 VMLAL.S16 q8, d8, d1[0]
139 VMLAL.S16 q9, d9, d1[0]
140 VMOVL.S8 q5, d10
141
142 // BLOCK 5
143 VLD1.8 {d8}, [r9]! // B6
144 VMLAL.S16 q2, d10, d1[1]
145 VMLAL.S16 q3, d11, d1[1]
146 VMOVL.S8 q4, d8
147
148 // BLOCK 6
149 VLD1.8 {d10}, [r9]! // B7
150 VMLAL.S16 q8, d8, d1[2]
151 VMLAL.S16 q9, d9, d1[2]
152 VMOVL.S8 q5, d10
153 SUBS r5, r5, 8
154
155 // BLOCK 7
156 VLD1.8 {d8}, [r9]! // B0
157 VMLAL.S16 q2, d10, d1[3]
158 VMLAL.S16 q3, d11, d1[3]
159 BHS 2b
160
161 // Epilogue
162
163 .p2align 3
164 3:
165 // Extend
166 VMOVL.S8 q0, d0
167 VMOVL.S8 q4, d8
168 PLD [r9, 448]
169
170 // BLOCK 0
171 VLD1.8 {d10}, [r9]! // B1
172 VMLAL.S16 q8, d8, d0[0]
173 VMLAL.S16 q9, d9, d0[0]
174 VMOVL.S8 q5, d10
175
176 // BLOCK 1
177 VLD1.8 {d8}, [r9]! // B2
178 VMLAL.S16 q2, d10, d0[1]
179 VMLAL.S16 q3, d11, d0[1]
180 VMOVL.S8 q4, d8
181
182 // BLOCK 2
183 VLD1.8 {d10}, [r9]! // B3
184 VMLAL.S16 q8, d8, d0[2]
185 VMLAL.S16 q9, d9, d0[2]
186 VMOVL.S8 q5, d10
187
188 // BLOCK 3
189 VLD1.8 {d8}, [r9]! // B4
190 VMLAL.S16 q2, d10, d0[3]
191 VMLAL.S16 q3, d11, d0[3]
192 VMOVL.S8 q4, d8
193
194 // BLOCK 4
195 VLD1.8 {d10}, [r9]! // B5
196 VMLAL.S16 q8, d8, d1[0]
197 VMLAL.S16 q9, d9, d1[0]
198 VMOVL.S8 q5, d10
199
200 // BLOCK 5
201 VLD1.8 {d8}, [r9]! // B6
202 VMLAL.S16 q2, d10, d1[1]
203 VMLAL.S16 q3, d11, d1[1]
204 VMOVL.S8 q4, d8
205
206 // BLOCK 6
207 VLD1.8 {d10}, [r9]! // B7
208 VMLAL.S16 q8, d8, d1[2]
209 VMLAL.S16 q9, d9, d1[2]
210 VMOVL.S8 q5, d10
211 ADDS r5, r5, 8
212
213 VMLAL.S16 q2, d10, d1[3]
214 VMLAL.S16 q3, d11, d1[3]
215
216 # Is there a remainder?- 1-7 bytes of A
217 BNE 6f
218
219 4:
220 # ks loop
221 SUBS r14, r14, 4 // ks -= MR * sizeof(void*)
222 BHI 1b
223
224 LDR r14, [sp, 60] // p = ks
225
226 VADD.S32 q8, q8, q2
227 VADD.S32 q9, q9, q3
228
229 # QC8 FP32 quantization
230 VLD1.8 {q0-q1}, [r9]!
231
232 VCVT.F32.S32 q8, q8
233 VCVT.F32.S32 q9, q9
234
235 VMUL.F32 q8, q8, q0 // multiplier
236 VMUL.F32 q9, q9, q1
237
238 VCVTN.S32.F32 q8, q8
239 VCVTN.S32.F32 q9, q9
240
241 VDUP.16 q0, d13[2] // output_zero_point
242
243 VQMOVN.S32 d16, q8
244 VQMOVN.S32 d17, q9
245
246 VQADD.S16 q8, q8, q0
247
248 VDUP.8 d24, d13[6] // output_min
249
250 VQMOVN.S16 d0, q8
251
252 VDUP.8 d25, d13[7] // output_max
253
254 VMAX.S8 d0, d0, d24
255
256 SUBS r1, r1, 8
257
258 VMIN.S8 d0, d0, d25
259
260 # Store full 1 x 8
261 BLO 7f
262 VST1.8 {d0}, [r11], r12
263 SUB r2, r2, r14 // a -= ks
264 BHI 0b
265
266 VPOP {d8-d13}
267 ADD sp, sp, 16 // skip pad of 8, r2, r3
268 POP {r5, r6, r7, r9, r11, pc}
269
270 # Remainder- 1 to 7 bytes of A
271 .p2align 3
272 5:
273 AND r5, r5, 7 // kc remainder 1 to 7
274 6:
275 VLD1.8 {d0}, [r3]
276 VLD1.8 {d8}, [r9]!
277
278 VMOVL.S8 q0, d0
279 VMOVL.S8 q4, d8
280 VMLAL.S16 q8, d8, d0[0]
281 VMLAL.S16 q9, d9, d0[0]
282 CMP r5, 2
283 BLO 4b
284
285 VLD1.8 {d8}, [r9]!
286 VMOVL.S8 q4, d8
287 VMLAL.S16 q8, d8, d0[1]
288 VMLAL.S16 q9, d9, d0[1]
289 BEQ 4b
290
291 VLD1.8 {d8}, [r9]!
292 VMOVL.S8 q4, d8
293 VMLAL.S16 q8, d8, d0[2]
294 VMLAL.S16 q9, d9, d0[2]
295 CMP r5, 4
296 BLO 4b
297
298 VLD1.8 {d8}, [r9]!
299 VMOVL.S8 q4, d8
300 VMLAL.S16 q8, d8, d0[3]
301 VMLAL.S16 q9, d9, d0[3]
302 BEQ 4b
303
304 VLD1.8 {d8}, [r9]!
305 VMOVL.S8 q4, d8
306 VMLAL.S16 q8, d8, d1[0]
307 VMLAL.S16 q9, d9, d1[0]
308 CMP r5, 6
309 BLO 4b
310
311 VLD1.8 {d8}, [r9]!
312 VMOVL.S8 q4, d8
313 VMLAL.S16 q8, d8, d1[1]
314 VMLAL.S16 q9, d9, d1[1]
315 BEQ 4b
316
317 VLD1.8 {d8}, [r9]!
318 VMOVL.S8 q4, d8
319 VMLAL.S16 q8, d8, d1[2]
320 VMLAL.S16 q9, d9, d1[2]
321 B 4b
322
323 # Store odd width
324 .p2align 3
325 7:
326 TST r1, 4
327 BEQ 8f
328 VST1.32 {d0[0]}, [r11]!
329 VEXT.8 q0, q0, q0, 4
330 8:
331 TST r1, 2
332 BEQ 9f
333 VST1.16 {d0[0]}, [r11]!
334 VEXT.8 q0, q0, q0, 2
335
336 9:
337 TST r1, 1
338 BEQ 10f
339 VST1.8 {d0[0]}, [r11]
340
341 10:
342 VPOP {d8-d13}
343 ADD sp, sp, 16 // skip pad of 8, r2, r3
344 POP {r5, r6, r7, r9, r11, pc}
345
346 END_FUNCTION xnn_qs8_qc8w_igemm_minmax_fp32_ukernel_1x8__asm_aarch32_neonv8_mlal_lane_cortex_a35_prfm
347
348 #ifdef __ELF__
349 .section ".note.GNU-stack","",%progbits
350 #endif