Debian Codegraph @

src/qs8-qc8w-igemm/gen/qs8-qc8w-igemm-1x8-minmax-fp32-asm-aarch32-neon-mlal-lane-cortex-a7.S

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-igemm/1x8-aarch32-neon-mlal-lane-cortex-a7.S.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2021 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include "xnnpack/assembly.h"
12
13 .syntax unified
14
15 // void xnn_qs8_qc8w_igemm_minmax_fp32_ukernel_1x8__asm_aarch32_neon_mlal_lane_cortex_a7
16 // size_t mr, (r0)
17 // size_t nc, r1
18 // size_t kc, (r2) -> sp + 56 -> r5
19 // size_t ks, (r3) -> sp + 60 -> r14
20 // const int8_t** restrict a, sp + 88 -> r2
21 // const void* restrict w, sp + 92 -> r9
22 // int8_t* restrict c, sp + 96 -> r11
23 // size_t cm_stride, sp + 100 -> r6
24 // size_t cn_stride, sp + 104 -> r12
25 // size_t a_offset, sp + 108 -> (r5)
26 // const int8_t* zero, sp + 112 -> r7
27 // xnn_qs8_qc8w_conv_minmax_params*params); sp + 116 -> (r5)
28
29 // d8-d15, r4-r11,r14(lr) need to be preserved if used. r13(sp),r15(pc) are reserved.
30
31 // Based on cortex_a53 microkernel but with Neon loads
32
33 // Register usage
34 // A0 r3 d0-d1 q0
35 // B r9 d8-d9 q4 q5
36 // C0 r11 d16-d17 q8 d18-d19 q9
37 // q2, q3 acc2
38 // unused r4, r8, r10, d15, q10-q15, q1-q3
39 // params structure is 10 bytes
40 // struct {
41 // float magic_bias; d12[0]
42 // int32_t magic_bias_less_output_zero_point; d12[1]
43 // int8_t output_min; d13[6]
44 // int8_t output_max; d13[7]
45 // } xnn_qs8_minmax_params.neon;
46
47 BEGIN_FUNCTION xnn_qs8_qc8w_igemm_minmax_fp32_ukernel_1x8__asm_aarch32_neon_mlal_lane_cortex_a7
48 # Push 88 bytes
49 # r2, r3 will be reloaded in outer loop.
50 PUSH {r2, r3, r5, r6, r7, r9, r11, lr} // +32
51 SUB sp, sp, 8 // +8
52 VPUSH {d8-d13} // +48 = 88
53
54 LDR r2, [sp, 88] // a
55 LDR r9, [sp, 92] // w
56 LDR r11, [sp, 96] // c
57 LDR r6, [sp, 100] // cm_stride
58 LDR r12, [sp, 104] // cn_stride
59 LDR r7, [sp, 112] // zero
60 LDR r5, [sp, 116] // params
61 MOV r14, r3 // p = ks
62
63 # Load params values
64 VLDM r5!, {d12} // QC8 neon params
65 VLD1.16 {d13[]}, [r5]
66
67
68 .p2align 3
69 0:
70 # Load initial bias from w into accumulators
71 VLDM r9!, {d16-d19} // Bias
72 VMOV.I32 q2, 0 // second set of C for pipelining FMLA
73 VMOV.I32 q3, 0
74
75 .p2align 3
76 1:
77 # Load next A pointer
78 LDR r3, [r2, 0]
79
80 # Add a_offset
81 LDR r5, [sp, 108] // a_offset
82 ADD r2, r2, 4
83 CMP r3, r7 // if a0 == zero
84 ADD r3, r3, r5 // a0 += a_offset
85 MOVEQ r3, r7 // a0 = zero, else += a0 + a_offset
86
87 LDR r5, [sp, 56] // kc
88 SUBS r5, r5, 8 // kc - 8
89 BLO 5f // less than 8 channels?
90
91 // Prologue - load A0 and B0
92 VLD1.8 {d0}, [r3]! // A0
93 SUBS r5, r5, 8 // k = k - 8
94 VLD1.8 {d8}, [r9]! // B0
95 BLO 3f // less than 8 channels?
96
97 // Main loop - 8 bytes
98 // 64 bytes for weights.
99
100 .p2align 3
101 2:
102 // Extend
103 VMOVL.S8 q0, d0
104 VMOVL.S8 q4, d8
105
106 // BLOCK 0
107 VLD1.8 {d10}, [r9]! // B1
108 VMLAL.S16 q8, d8, d0[0]
109 VMLAL.S16 q9, d9, d0[0]
110 VMOVL.S8 q5, d10
111
112 // BLOCK 1
113 VLD1.8 {d8}, [r9]! // B2
114 VMLAL.S16 q2, d10, d0[1]
115 VMLAL.S16 q3, d11, d0[1]
116 VMOVL.S8 q4, d8
117
118 // BLOCK 2
119 VLD1.8 {d10}, [r9]! // B3
120 VMLAL.S16 q8, d8, d0[2]
121 VMLAL.S16 q9, d9, d0[2]
122 VMOVL.S8 q5, d10
123
124 // BLOCK 3
125 VLD1.8 {d8}, [r9]! // B4
126 VMLAL.S16 q2, d10, d0[3]
127 VMLAL.S16 q3, d11, d0[3]
128 VLD1.8 {d0}, [r3]! // A0
129 VMOVL.S8 q4, d8
130
131 // BLOCK 4
132 VLD1.8 {d10}, [r9]! // B5
133 VMLAL.S16 q8, d8, d1[0]
134 VMLAL.S16 q9, d9, d1[0]
135 VMOVL.S8 q5, d10
136
137 // BLOCK 5
138 VLD1.8 {d8}, [r9]! // B6
139 VMLAL.S16 q2, d10, d1[1]
140 VMLAL.S16 q3, d11, d1[1]
141 VMOVL.S8 q4, d8
142
143 // BLOCK 6
144 VLD1.8 {d10}, [r9]! // B7
145 VMLAL.S16 q8, d8, d1[2]
146 VMLAL.S16 q9, d9, d1[2]
147 VMOVL.S8 q5, d10
148 SUBS r5, r5, 8
149
150 // BLOCK 7
151 VLD1.8 {d8}, [r9]! // B0
152 VMLAL.S16 q2, d10, d1[3]
153 VMLAL.S16 q3, d11, d1[3]
154 BHS 2b
155
156 // Epilogue
157
158 .p2align 3
159 3:
160 // Extend
161 VMOVL.S8 q0, d0
162 VMOVL.S8 q4, d8
163
164 // BLOCK 0
165 VLD1.8 {d10}, [r9]! // B1
166 VMLAL.S16 q8, d8, d0[0]
167 VMLAL.S16 q9, d9, d0[0]
168 VMOVL.S8 q5, d10
169
170 // BLOCK 1
171 VLD1.8 {d8}, [r9]! // B2
172 VMLAL.S16 q2, d10, d0[1]
173 VMLAL.S16 q3, d11, d0[1]
174 VMOVL.S8 q4, d8
175
176 // BLOCK 2
177 VLD1.8 {d10}, [r9]! // B3
178 VMLAL.S16 q8, d8, d0[2]
179 VMLAL.S16 q9, d9, d0[2]
180 VMOVL.S8 q5, d10
181
182 // BLOCK 3
183 VLD1.8 {d8}, [r9]! // B4
184 VMLAL.S16 q2, d10, d0[3]
185 VMLAL.S16 q3, d11, d0[3]
186 VMOVL.S8 q4, d8
187
188 // BLOCK 4
189 VLD1.8 {d10}, [r9]! // B5
190 VMLAL.S16 q8, d8, d1[0]
191 VMLAL.S16 q9, d9, d1[0]
192 VMOVL.S8 q5, d10
193
194 // BLOCK 5
195 VLD1.8 {d8}, [r9]! // B6
196 VMLAL.S16 q2, d10, d1[1]
197 VMLAL.S16 q3, d11, d1[1]
198 VMOVL.S8 q4, d8
199
200 // BLOCK 6
201 VLD1.8 {d10}, [r9]! // B7
202 VMLAL.S16 q8, d8, d1[2]
203 VMLAL.S16 q9, d9, d1[2]
204 VMOVL.S8 q5, d10
205 ADDS r5, r5, 8
206
207 VMLAL.S16 q2, d10, d1[3]
208 VMLAL.S16 q3, d11, d1[3]
209
210 # Is there a remainder?- 1-7 bytes of A
211 BNE 6f
212
213 4:
214 # ks loop
215 SUBS r14, r14, 4 // ks -= MR * sizeof(void*)
216 BHI 1b
217
218 LDR r14, [sp, 60] // p = ks
219
220 VADD.S32 q8, q8, q2
221 VADD.S32 q9, q9, q3
222
223 # QC8 FP32 quantization
224 VLD1.8 {q0-q1}, [r9]!
225
226 VDUP.32 q2, d12[0] // magic_bias
227 VDUP.32 q3, d12[1] // magic_bias_less_output_zero_point
228
229 VCVT.F32.S32 q8, q8
230 VCVT.F32.S32 q9, q9
231
232 VMUL.F32 q8, q8, q0 // multiplier
233 VMUL.F32 q9, q9, q1
234
235 VADD.F32 q8, q8, q2 // magic_bias
236 VADD.F32 q9, q9, q2
237
238 VQSUB.S32 q8, q8, q3 // magic_bias_less_output_zero_point
239 VQSUB.S32 q9, q9, q3
240
241
242 VQMOVN.S32 d16, q8
243 VQMOVN.S32 d17, q9
244
245
246 VDUP.8 d24, d13[6] // output_min
247
248 VQMOVN.S16 d0, q8
249
250 VDUP.8 d25, d13[7] // output_max
251
252 VMAX.S8 d0, d0, d24
253
254 SUBS r1, r1, 8
255
256 VMIN.S8 d0, d0, d25
257
258 # Store full 1 x 8
259 BLO 7f
260 VST1.8 {d0}, [r11], r12
261 SUB r2, r2, r14 // a -= ks
262 BHI 0b
263
264 VPOP {d8-d13}
265 ADD sp, sp, 16 // skip pad of 8, r2, r3
266 POP {r5, r6, r7, r9, r11, pc}
267
268 # Remainder- 1 to 7 bytes of A
269 .p2align 3
270 5:
271 AND r5, r5, 7 // kc remainder 1 to 7
272 6:
273 VLD1.8 {d0}, [r3]
274 VLD1.8 {d8}, [r9]!
275
276 VMOVL.S8 q0, d0
277 VMOVL.S8 q4, d8
278 VMLAL.S16 q8, d8, d0[0]
279 VMLAL.S16 q9, d9, d0[0]
280 CMP r5, 2
281 BLO 4b
282
283 VLD1.8 {d8}, [r9]!
284 VMOVL.S8 q4, d8
285 VMLAL.S16 q8, d8, d0[1]
286 VMLAL.S16 q9, d9, d0[1]
287 BEQ 4b
288
289 VLD1.8 {d8}, [r9]!
290 VMOVL.S8 q4, d8
291 VMLAL.S16 q8, d8, d0[2]
292 VMLAL.S16 q9, d9, d0[2]
293 CMP r5, 4
294 BLO 4b
295
296 VLD1.8 {d8}, [r9]!
297 VMOVL.S8 q4, d8
298 VMLAL.S16 q8, d8, d0[3]
299 VMLAL.S16 q9, d9, d0[3]
300 BEQ 4b
301
302 VLD1.8 {d8}, [r9]!
303 VMOVL.S8 q4, d8
304 VMLAL.S16 q8, d8, d1[0]
305 VMLAL.S16 q9, d9, d1[0]
306 CMP r5, 6
307 BLO 4b
308
309 VLD1.8 {d8}, [r9]!
310 VMOVL.S8 q4, d8
311 VMLAL.S16 q8, d8, d1[1]
312 VMLAL.S16 q9, d9, d1[1]
313 BEQ 4b
314
315 VLD1.8 {d8}, [r9]!
316 VMOVL.S8 q4, d8
317 VMLAL.S16 q8, d8, d1[2]
318 VMLAL.S16 q9, d9, d1[2]
319 B 4b
320
321 # Store odd width
322 .p2align 3
323 7:
324 TST r1, 4
325 BEQ 8f
326 VST1.32 {d0[0]}, [r11]!
327 VEXT.8 q0, q0, q0, 4
328 8:
329 TST r1, 2
330 BEQ 9f
331 VST1.16 {d0[0]}, [r11]!
332 VEXT.8 q0, q0, q0, 2
333
334 9:
335 TST r1, 1
336 BEQ 10f
337 VST1.8 {d0[0]}, [r11]
338
339 10:
340 VPOP {d8-d13}
341 ADD sp, sp, 16 // skip pad of 8, r2, r3
342 POP {r5, r6, r7, r9, r11, pc}
343
344 END_FUNCTION xnn_qs8_qc8w_igemm_minmax_fp32_ukernel_1x8__asm_aarch32_neon_mlal_lane_cortex_a7
345
346 #ifdef __ELF__
347 .section ".note.GNU-stack","",%progbits
348 #endif