Debian Codegraph @

src/qs8-qc8w-igemm/gen/qs8-qc8w-igemm-1x8-minmax-fp32-asm-aarch32-neon-mlal-lane-cortex-a7-prfm.S

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-igemm/1x8-aarch32-neon-mlal-lane-cortex-a7.S.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2021 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include "xnnpack/assembly.h"
12
13 .syntax unified
14
15 // void xnn_qs8_qc8w_igemm_minmax_fp32_ukernel_1x8__asm_aarch32_neon_mlal_lane_cortex_a7_prfm
16 // size_t mr, (r0)
17 // size_t nc, r1
18 // size_t kc, (r2) -> sp + 56 -> r5
19 // size_t ks, (r3) -> sp + 60 -> r14
20 // const int8_t** restrict a, sp + 88 -> r2
21 // const void* restrict w, sp + 92 -> r9
22 // int8_t* restrict c, sp + 96 -> r11
23 // size_t cm_stride, sp + 100 -> r6
24 // size_t cn_stride, sp + 104 -> r12
25 // size_t a_offset, sp + 108 -> (r5)
26 // const int8_t* zero, sp + 112 -> r7
27 // xnn_qs8_qc8w_conv_minmax_params*params); sp + 116 -> (r5)
28
29 // d8-d15, r4-r11,r14(lr) need to be preserved if used. r13(sp),r15(pc) are reserved.
30
31 // Based on cortex_a53 microkernel but with Neon loads
32
33 // Register usage
34 // A0 r3 d0-d1 q0
35 // B r9 d8-d9 q4 q5
36 // C0 r11 d16-d17 q8 d18-d19 q9
37 // q2, q3 acc2
38 // unused r4, r8, r10, d15, q10-q15, q1-q3
39 // params structure is 10 bytes
40 // struct {
41 // float magic_bias; d12[0]
42 // int32_t magic_bias_less_output_zero_point; d12[1]
43 // int8_t output_min; d13[6]
44 // int8_t output_max; d13[7]
45 // } xnn_qs8_minmax_params.neon;
46
47 BEGIN_FUNCTION xnn_qs8_qc8w_igemm_minmax_fp32_ukernel_1x8__asm_aarch32_neon_mlal_lane_cortex_a7_prfm
48 # Push 88 bytes
49 # r2, r3 will be reloaded in outer loop.
50 PUSH {r2, r3, r5, r6, r7, r9, r11, lr} // +32
51 SUB sp, sp, 8 // +8
52 VPUSH {d8-d13} // +48 = 88
53
54 LDR r2, [sp, 88] // a
55 LDR r9, [sp, 92] // w
56 LDR r11, [sp, 96] // c
57 LDR r6, [sp, 100] // cm_stride
58 LDR r12, [sp, 104] // cn_stride
59 LDR r7, [sp, 112] // zero
60 LDR r5, [sp, 116] // params
61 MOV r14, r3 // p = ks
62
63 # Load params values
64 VLDM r5!, {d12} // QC8 neon params
65 VLD1.16 {d13[]}, [r5]
66
67 PLD [r9, 64] // Prefetch B
68 PLD [r9, 112]
69 PLD [r9, 192]
70 PLD [r9, 256]
71 PLD [r9, 320]
72 PLD [r9, 384]
73
74 .p2align 3
75 0:
76 # Load initial bias from w into accumulators
77 VLDM r9!, {d16-d19} // Bias
78 VMOV.I32 q2, 0 // second set of C for pipelining FMLA
79 VMOV.I32 q3, 0
80
81 .p2align 3
82 1:
83 # Load next A pointer
84 LDR r3, [r2, 0]
85
86 # Add a_offset
87 LDR r5, [sp, 108] // a_offset
88 ADD r2, r2, 4
89 CMP r3, r7 // if a0 == zero
90 ADD r3, r3, r5 // a0 += a_offset
91 MOVEQ r3, r7 // a0 = zero, else += a0 + a_offset
92
93 LDR r5, [sp, 56] // kc
94 SUBS r5, r5, 8 // kc - 8
95 BLO 5f // less than 8 channels?
96
97 // Prologue - load A0 and B0
98 VLD1.8 {d0}, [r3]! // A0
99 SUBS r5, r5, 8 // k = k - 8
100 VLD1.8 {d8}, [r9]! // B0
101 BLO 3f // less than 8 channels?
102
103 // Main loop - 8 bytes
104 // 64 bytes for weights.
105
106 .p2align 3
107 2:
108 // Extend
109 VMOVL.S8 q0, d0
110 VMOVL.S8 q4, d8
111 PLD [r9, 448]
112
113 // BLOCK 0
114 VLD1.8 {d10}, [r9]! // B1
115 VMLAL.S16 q8, d8, d0[0]
116 VMLAL.S16 q9, d9, d0[0]
117 VMOVL.S8 q5, d10
118
119 // BLOCK 1
120 VLD1.8 {d8}, [r9]! // B2
121 VMLAL.S16 q2, d10, d0[1]
122 VMLAL.S16 q3, d11, d0[1]
123 VMOVL.S8 q4, d8
124
125 // BLOCK 2
126 VLD1.8 {d10}, [r9]! // B3
127 VMLAL.S16 q8, d8, d0[2]
128 VMLAL.S16 q9, d9, d0[2]
129 VMOVL.S8 q5, d10
130
131 // BLOCK 3
132 VLD1.8 {d8}, [r9]! // B4
133 VMLAL.S16 q2, d10, d0[3]
134 VMLAL.S16 q3, d11, d0[3]
135 VLD1.8 {d0}, [r3]! // A0
136 VMOVL.S8 q4, d8
137
138 // BLOCK 4
139 VLD1.8 {d10}, [r9]! // B5
140 VMLAL.S16 q8, d8, d1[0]
141 VMLAL.S16 q9, d9, d1[0]
142 VMOVL.S8 q5, d10
143
144 // BLOCK 5
145 VLD1.8 {d8}, [r9]! // B6
146 VMLAL.S16 q2, d10, d1[1]
147 VMLAL.S16 q3, d11, d1[1]
148 VMOVL.S8 q4, d8
149
150 // BLOCK 6
151 VLD1.8 {d10}, [r9]! // B7
152 VMLAL.S16 q8, d8, d1[2]
153 VMLAL.S16 q9, d9, d1[2]
154 VMOVL.S8 q5, d10
155 SUBS r5, r5, 8
156
157 // BLOCK 7
158 VLD1.8 {d8}, [r9]! // B0
159 VMLAL.S16 q2, d10, d1[3]
160 VMLAL.S16 q3, d11, d1[3]
161 BHS 2b
162
163 // Epilogue
164
165 .p2align 3
166 3:
167 // Extend
168 VMOVL.S8 q0, d0
169 VMOVL.S8 q4, d8
170 PLD [r9, 448]
171
172 // BLOCK 0
173 VLD1.8 {d10}, [r9]! // B1
174 VMLAL.S16 q8, d8, d0[0]
175 VMLAL.S16 q9, d9, d0[0]
176 VMOVL.S8 q5, d10
177
178 // BLOCK 1
179 VLD1.8 {d8}, [r9]! // B2
180 VMLAL.S16 q2, d10, d0[1]
181 VMLAL.S16 q3, d11, d0[1]
182 VMOVL.S8 q4, d8
183
184 // BLOCK 2
185 VLD1.8 {d10}, [r9]! // B3
186 VMLAL.S16 q8, d8, d0[2]
187 VMLAL.S16 q9, d9, d0[2]
188 VMOVL.S8 q5, d10
189
190 // BLOCK 3
191 VLD1.8 {d8}, [r9]! // B4
192 VMLAL.S16 q2, d10, d0[3]
193 VMLAL.S16 q3, d11, d0[3]
194 VMOVL.S8 q4, d8
195
196 // BLOCK 4
197 VLD1.8 {d10}, [r9]! // B5
198 VMLAL.S16 q8, d8, d1[0]
199 VMLAL.S16 q9, d9, d1[0]
200 VMOVL.S8 q5, d10
201
202 // BLOCK 5
203 VLD1.8 {d8}, [r9]! // B6
204 VMLAL.S16 q2, d10, d1[1]
205 VMLAL.S16 q3, d11, d1[1]
206 VMOVL.S8 q4, d8
207
208 // BLOCK 6
209 VLD1.8 {d10}, [r9]! // B7
210 VMLAL.S16 q8, d8, d1[2]
211 VMLAL.S16 q9, d9, d1[2]
212 VMOVL.S8 q5, d10
213 ADDS r5, r5, 8
214
215 VMLAL.S16 q2, d10, d1[3]
216 VMLAL.S16 q3, d11, d1[3]
217
218 # Is there a remainder?- 1-7 bytes of A
219 BNE 6f
220
221 4:
222 # ks loop
223 SUBS r14, r14, 4 // ks -= MR * sizeof(void*)
224 BHI 1b
225
226 LDR r14, [sp, 60] // p = ks
227
228 VADD.S32 q8, q8, q2
229 VADD.S32 q9, q9, q3
230
231 # QC8 FP32 quantization
232 VLD1.8 {q0-q1}, [r9]!
233
234 VDUP.32 q2, d12[0] // magic_bias
235 VDUP.32 q3, d12[1] // magic_bias_less_output_zero_point
236
237 VCVT.F32.S32 q8, q8
238 VCVT.F32.S32 q9, q9
239
240 VMUL.F32 q8, q8, q0 // multiplier
241 VMUL.F32 q9, q9, q1
242
243 VADD.F32 q8, q8, q2 // magic_bias
244 VADD.F32 q9, q9, q2
245
246 VQSUB.S32 q8, q8, q3 // magic_bias_less_output_zero_point
247 VQSUB.S32 q9, q9, q3
248
249
250 VQMOVN.S32 d16, q8
251 VQMOVN.S32 d17, q9
252
253
254 VDUP.8 d24, d13[6] // output_min
255
256 VQMOVN.S16 d0, q8
257
258 VDUP.8 d25, d13[7] // output_max
259
260 VMAX.S8 d0, d0, d24
261
262 SUBS r1, r1, 8
263
264 VMIN.S8 d0, d0, d25
265
266 # Store full 1 x 8
267 BLO 7f
268 VST1.8 {d0}, [r11], r12
269 SUB r2, r2, r14 // a -= ks
270 BHI 0b
271
272 VPOP {d8-d13}
273 ADD sp, sp, 16 // skip pad of 8, r2, r3
274 POP {r5, r6, r7, r9, r11, pc}
275
276 # Remainder- 1 to 7 bytes of A
277 .p2align 3
278 5:
279 AND r5, r5, 7 // kc remainder 1 to 7
280 6:
281 VLD1.8 {d0}, [r3]
282 VLD1.8 {d8}, [r9]!
283
284 VMOVL.S8 q0, d0
285 VMOVL.S8 q4, d8
286 VMLAL.S16 q8, d8, d0[0]
287 VMLAL.S16 q9, d9, d0[0]
288 CMP r5, 2
289 BLO 4b
290
291 VLD1.8 {d8}, [r9]!
292 VMOVL.S8 q4, d8
293 VMLAL.S16 q8, d8, d0[1]
294 VMLAL.S16 q9, d9, d0[1]
295 BEQ 4b
296
297 VLD1.8 {d8}, [r9]!
298 VMOVL.S8 q4, d8
299 VMLAL.S16 q8, d8, d0[2]
300 VMLAL.S16 q9, d9, d0[2]
301 CMP r5, 4
302 BLO 4b
303
304 VLD1.8 {d8}, [r9]!
305 VMOVL.S8 q4, d8
306 VMLAL.S16 q8, d8, d0[3]
307 VMLAL.S16 q9, d9, d0[3]
308 BEQ 4b
309
310 VLD1.8 {d8}, [r9]!
311 VMOVL.S8 q4, d8
312 VMLAL.S16 q8, d8, d1[0]
313 VMLAL.S16 q9, d9, d1[0]
314 CMP r5, 6
315 BLO 4b
316
317 VLD1.8 {d8}, [r9]!
318 VMOVL.S8 q4, d8
319 VMLAL.S16 q8, d8, d1[1]
320 VMLAL.S16 q9, d9, d1[1]
321 BEQ 4b
322
323 VLD1.8 {d8}, [r9]!
324 VMOVL.S8 q4, d8
325 VMLAL.S16 q8, d8, d1[2]
326 VMLAL.S16 q9, d9, d1[2]
327 B 4b
328
329 # Store odd width
330 .p2align 3
331 7:
332 TST r1, 4
333 BEQ 8f
334 VST1.32 {d0[0]}, [r11]!
335 VEXT.8 q0, q0, q0, 4
336 8:
337 TST r1, 2
338 BEQ 9f
339 VST1.16 {d0[0]}, [r11]!
340 VEXT.8 q0, q0, q0, 2
341
342 9:
343 TST r1, 1
344 BEQ 10f
345 VST1.8 {d0[0]}, [r11]
346
347 10:
348 VPOP {d8-d13}
349 ADD sp, sp, 16 // skip pad of 8, r2, r3
350 POP {r5, r6, r7, r9, r11, pc}
351
352 END_FUNCTION xnn_qs8_qc8w_igemm_minmax_fp32_ukernel_1x8__asm_aarch32_neon_mlal_lane_cortex_a7_prfm
353
354 #ifdef __ELF__
355 .section ".note.GNU-stack","",%progbits
356 #endif