Debian Codegraph @

src/x32-packw/gen/x32-packw-x8s4-gemm-goi-neon-ld4lane-u4-prfm.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/neon.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include <arm_neon.h>
16
17 #include "xnnpack/packw.h"
18 #include "xnnpack/prefetch.h"
19
20
21 void xnn_x32_packw_gemm_goi_ukernel_x8s4__neon_ld4lane_u4_prfm(
22 size_t g,
23 size_t nc,
24 size_t kc,
25 size_t nr,
26 size_t kr,
27 size_t sr,
28 const uint32_t* weights,
29 const uint32_t* bias,
30 const void* scale,
31 uint32_t* packed_weights,
32 size_t extra_bytes,
33 const void* params)
34 {
35 assert(g != 0);
36 assert(nc != 0);
37 assert(kc != 0);
38 assert(nr == 8);
39 assert(kr == 1);
40 assert(sr == 4);
41 assert(weights != NULL);
42 assert(packed_weights != NULL);
43 uint32x4x4_t vtmp0123x0123;
44 vtmp0123x0123.val[0] = vdupq_n_u32(0);
45 vtmp0123x0123.val[1] = vdupq_n_u32(0);
46 vtmp0123x0123.val[2] = vdupq_n_u32(0);
47 vtmp0123x0123.val[3] = vdupq_n_u32(0);
48 uint32x4x4_t vtmp0123x4567;
49 vtmp0123x4567.val[0] = vdupq_n_u32(0);
50 vtmp0123x4567.val[1] = vdupq_n_u32(0);
51 vtmp0123x4567.val[2] = vdupq_n_u32(0);
52 vtmp0123x4567.val[3] = vdupq_n_u32(0);
53
54 do {
55 // NC main loop multiple of 8
56 const uint32_t* w0 = weights;
57 size_t n = nc;
58
59 for (; n >= 8; n -= 8) {
60 if XNN_LIKELY(bias != NULL) {
61 uint32x4_t vb0 = vld1q_u32(bias); bias += 4;
62 uint32x4_t vb4 = vld1q_u32(bias); bias += 4;
63 vst1q_u32(packed_weights, vb0); packed_weights += 4;
64 vst1q_u32(packed_weights, vb4); packed_weights += 4;
65 } else {
66 const uint32x4_t vzero = vmovq_n_u32(0);
67 vst1q_u32(packed_weights, vzero); packed_weights += 4;
68 vst1q_u32(packed_weights, vzero); packed_weights += 4;
69 }
70
71 const uint32_t* w1 = w0 + kc;
72 const uint32_t* w2 = w1 + kc;
73 const uint32_t* w3 = w2 + kc;
74 const uint32_t* w4 = w3 + kc;
75 const uint32_t* w5 = w4 + kc;
76 const uint32_t* w6 = w5 + kc;
77 const uint32_t* w7 = w6 + kc;
78 xnn_prefetch_to_l1((const int8_t*) w0);
79 xnn_prefetch_to_l1((const int8_t*) w0 + 64);
80 xnn_prefetch_to_l1((const int8_t*) w1);
81 xnn_prefetch_to_l1((const int8_t*) w1 + 64);
82 xnn_prefetch_to_l1((const int8_t*) w2);
83 xnn_prefetch_to_l1((const int8_t*) w2 + 64);
84 xnn_prefetch_to_l1((const int8_t*) w3);
85 xnn_prefetch_to_l1((const int8_t*) w3 + 64);
86 xnn_prefetch_to_l1((const int8_t*) w4);
87 xnn_prefetch_to_l1((const int8_t*) w4 + 64);
88 xnn_prefetch_to_l1((const int8_t*) w5);
89 xnn_prefetch_to_l1((const int8_t*) w5 + 64);
90 xnn_prefetch_to_l1((const int8_t*) w6);
91 xnn_prefetch_to_l1((const int8_t*) w6 + 64);
92 xnn_prefetch_to_l1((const int8_t*) w7);
93 xnn_prefetch_to_l1((const int8_t*) w7 + 64);
94
95 // KC main loop multiple of 4
96 size_t k = kc;
97 for (; k >= 4; k -= 4) {
98 uint32x4_t vsrtmp;
99 vtmp0123x0123 = vld4q_lane_u32(w0, vtmp0123x0123, 0); w0 += 4;
100 vsrtmp = vtmp0123x0123.val[3];
101 vtmp0123x0123.val[3] = vtmp0123x0123.val[2];
102 vtmp0123x0123.val[2] = vtmp0123x0123.val[1];
103 vtmp0123x0123.val[1] = vtmp0123x0123.val[0];
104 vtmp0123x0123.val[0] = vsrtmp;
105 vtmp0123x0123 = vld4q_lane_u32(w1, vtmp0123x0123, 1); w1 += 4;
106 vsrtmp = vtmp0123x0123.val[3];
107 vtmp0123x0123.val[3] = vtmp0123x0123.val[2];
108 vtmp0123x0123.val[2] = vtmp0123x0123.val[1];
109 vtmp0123x0123.val[1] = vtmp0123x0123.val[0];
110 vtmp0123x0123.val[0] = vsrtmp;
111 vtmp0123x0123 = vld4q_lane_u32(w2, vtmp0123x0123, 2); w2 += 4;
112 vsrtmp = vtmp0123x0123.val[3];
113 vtmp0123x0123.val[3] = vtmp0123x0123.val[2];
114 vtmp0123x0123.val[2] = vtmp0123x0123.val[1];
115 vtmp0123x0123.val[1] = vtmp0123x0123.val[0];
116 vtmp0123x0123.val[0] = vsrtmp;
117 vtmp0123x0123 = vld4q_lane_u32(w3, vtmp0123x0123, 3); w3 += 4;
118 vsrtmp = vtmp0123x0123.val[3];
119 vtmp0123x0123.val[3] = vtmp0123x0123.val[2];
120 vtmp0123x0123.val[2] = vtmp0123x0123.val[1];
121 vtmp0123x0123.val[1] = vtmp0123x0123.val[0];
122 vtmp0123x0123.val[0] = vsrtmp;
123 vtmp0123x4567 = vld4q_lane_u32(w4, vtmp0123x4567, 0); w4 += 4;
124 vsrtmp = vtmp0123x4567.val[3];
125 vtmp0123x4567.val[3] = vtmp0123x4567.val[2];
126 vtmp0123x4567.val[2] = vtmp0123x4567.val[1];
127 vtmp0123x4567.val[1] = vtmp0123x4567.val[0];
128 vtmp0123x4567.val[0] = vsrtmp;
129 vtmp0123x4567 = vld4q_lane_u32(w5, vtmp0123x4567, 1); w5 += 4;
130 vsrtmp = vtmp0123x4567.val[3];
131 vtmp0123x4567.val[3] = vtmp0123x4567.val[2];
132 vtmp0123x4567.val[2] = vtmp0123x4567.val[1];
133 vtmp0123x4567.val[1] = vtmp0123x4567.val[0];
134 vtmp0123x4567.val[0] = vsrtmp;
135 vtmp0123x4567 = vld4q_lane_u32(w6, vtmp0123x4567, 2); w6 += 4;
136 vsrtmp = vtmp0123x4567.val[3];
137 vtmp0123x4567.val[3] = vtmp0123x4567.val[2];
138 vtmp0123x4567.val[2] = vtmp0123x4567.val[1];
139 vtmp0123x4567.val[1] = vtmp0123x4567.val[0];
140 vtmp0123x4567.val[0] = vsrtmp;
141 vtmp0123x4567 = vld4q_lane_u32(w7, vtmp0123x4567, 3); w7 += 4;
142 vsrtmp = vtmp0123x4567.val[3];
143 vtmp0123x4567.val[3] = vtmp0123x4567.val[2];
144 vtmp0123x4567.val[2] = vtmp0123x4567.val[1];
145 vtmp0123x4567.val[1] = vtmp0123x4567.val[0];
146 vtmp0123x4567.val[0] = vsrtmp;
147 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
148 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
149 xnn_prefetch_to_l1((const int8_t*) w2 + 128);
150 xnn_prefetch_to_l1((const int8_t*) w3 + 128);
151 xnn_prefetch_to_l1((const int8_t*) w4 + 128);
152 xnn_prefetch_to_l1((const int8_t*) w5 + 128);
153 xnn_prefetch_to_l1((const int8_t*) w6 + 128);
154 xnn_prefetch_to_l1((const int8_t*) w7 + 128);
155 vst1q_u32(packed_weights, vtmp0123x0123.val[0]); packed_weights += 4;
156 vst1q_u32(packed_weights, vtmp0123x4567.val[0]); packed_weights += 4;
157 vst1q_u32(packed_weights, vtmp0123x0123.val[1]); packed_weights += 4;
158 vst1q_u32(packed_weights, vtmp0123x4567.val[1]); packed_weights += 4;
159 vst1q_u32(packed_weights, vtmp0123x0123.val[2]); packed_weights += 4;
160 vst1q_u32(packed_weights, vtmp0123x4567.val[2]); packed_weights += 4;
161 vst1q_u32(packed_weights, vtmp0123x0123.val[3]); packed_weights += 4;
162 vst1q_u32(packed_weights, vtmp0123x4567.val[3]); packed_weights += 4;
163 }
164
165 // KC remainder of 1..3
166 // Same as main loop but ld1, ld2 or ld3
167 if XNN_UNLIKELY(k != 0) {
168 assert(k >= 1);
169 assert(k <= 3);
170 switch (k) {
171 // KC remainder of 1
172 case 1:
173 {
174 uint32x4_t vtmp0x0123 = vdupq_n_u32(0);
175 uint32x4_t vtmp0x4567 = vdupq_n_u32(0);
176 uint32x4_t vsrtmp;
177 uint32x4_t vsrtmp1x0123 = vdupq_n_u32(0);
178 uint32x4_t vsrtmp2x0123 = vdupq_n_u32(0);
179 uint32x4_t vsrtmp3x0123 = vdupq_n_u32(0);
180 uint32x4_t vsrtmp1x4567 = vdupq_n_u32(0);
181 uint32x4_t vsrtmp2x4567 = vdupq_n_u32(0);
182 uint32x4_t vsrtmp3x4567 = vdupq_n_u32(0);
183 vtmp0x0123 = vld1q_lane_u32(w0, vtmp0x0123, 0); w0 += 1;
184 vsrtmp = vsrtmp3x0123;
185 vsrtmp3x0123 = vsrtmp2x0123;
186 vsrtmp2x0123 = vsrtmp1x0123;
187 vsrtmp1x0123 = vtmp0x0123;
188 vtmp0x0123 = vsrtmp;
189 vtmp0x0123 = vld1q_lane_u32(w1, vtmp0x0123, 1); w1 += 1;
190 vsrtmp = vsrtmp3x0123;
191 vsrtmp3x0123 = vsrtmp2x0123;
192 vsrtmp2x0123 = vsrtmp1x0123;
193 vsrtmp1x0123 = vtmp0x0123;
194 vtmp0x0123 = vsrtmp;
195 vtmp0x0123 = vld1q_lane_u32(w2, vtmp0x0123, 2); w2 += 1;
196 vsrtmp = vsrtmp3x0123;
197 vsrtmp3x0123 = vsrtmp2x0123;
198 vsrtmp2x0123 = vsrtmp1x0123;
199 vsrtmp1x0123 = vtmp0x0123;
200 vtmp0x0123 = vsrtmp;
201 vtmp0x0123 = vld1q_lane_u32(w3, vtmp0x0123, 3); w3 += 1;
202 vsrtmp = vsrtmp3x0123;
203 vsrtmp3x0123 = vsrtmp2x0123;
204 vsrtmp2x0123 = vsrtmp1x0123;
205 vsrtmp1x0123 = vtmp0x0123;
206 vtmp0x0123 = vsrtmp;
207 vtmp0x4567 = vld1q_lane_u32(w4, vtmp0x4567, 0); w4 += 1;
208 vsrtmp = vsrtmp3x4567;
209 vsrtmp3x4567 = vsrtmp2x4567;
210 vsrtmp2x4567 = vsrtmp1x4567;
211 vsrtmp1x4567 = vtmp0x4567;
212 vtmp0x4567 = vsrtmp;
213 vtmp0x4567 = vld1q_lane_u32(w5, vtmp0x4567, 1); w5 += 1;
214 vsrtmp = vsrtmp3x4567;
215 vsrtmp3x4567 = vsrtmp2x4567;
216 vsrtmp2x4567 = vsrtmp1x4567;
217 vsrtmp1x4567 = vtmp0x4567;
218 vtmp0x4567 = vsrtmp;
219 vtmp0x4567 = vld1q_lane_u32(w6, vtmp0x4567, 2); w6 += 1;
220 vsrtmp = vsrtmp3x4567;
221 vsrtmp3x4567 = vsrtmp2x4567;
222 vsrtmp2x4567 = vsrtmp1x4567;
223 vsrtmp1x4567 = vtmp0x4567;
224 vtmp0x4567 = vsrtmp;
225 vtmp0x4567 = vld1q_lane_u32(w7, vtmp0x4567, 3); w7 += 1;
226 vsrtmp = vsrtmp3x4567;
227 vsrtmp3x4567 = vsrtmp2x4567;
228 vsrtmp2x4567 = vsrtmp1x4567;
229 vsrtmp1x4567 = vtmp0x4567;
230 vtmp0x4567 = vsrtmp;
231 vst1q_u32(packed_weights, vtmp0x0123); packed_weights += 4;
232 vst1q_u32(packed_weights, vtmp0x4567); packed_weights += 4;
233 vst1q_u32(packed_weights, vsrtmp1x0123); packed_weights += 4;
234 vst1q_u32(packed_weights, vsrtmp1x4567); packed_weights += 4;
235 vst1q_u32(packed_weights, vsrtmp2x0123); packed_weights += 4;
236 vst1q_u32(packed_weights, vsrtmp2x4567); packed_weights += 4;
237 vst1q_u32(packed_weights, vsrtmp3x0123); packed_weights += 4;
238 vst1q_u32(packed_weights, vsrtmp3x4567); packed_weights += 4;
239 break;
240 }
241 // KC remainder of 2
242 case 2:
243 {
244 uint32x4x2_t vtmp01x0123;
245 vtmp01x0123.val[0] = vdupq_n_u32(0);
246 vtmp01x0123.val[1] = vdupq_n_u32(0);
247 uint32x4x2_t vtmp01x4567;
248 vtmp01x4567.val[0] = vdupq_n_u32(0);
249 vtmp01x4567.val[1] = vdupq_n_u32(0);
250 uint32x4_t vsrtmp;
251 uint32x4_t vsrtmp2x0123 = vdupq_n_u32(0);
252 uint32x4_t vsrtmp3x0123 = vdupq_n_u32(0);
253 uint32x4_t vsrtmp2x4567 = vdupq_n_u32(0);
254 uint32x4_t vsrtmp3x4567 = vdupq_n_u32(0);
255 vtmp01x0123 = vld2q_lane_u32(w0, vtmp01x0123, 0); w0 += 2;
256 vsrtmp = vsrtmp3x0123;
257 vsrtmp3x0123 = vsrtmp2x0123;
258 vsrtmp2x0123 = vtmp01x0123.val[1];
259 vtmp01x0123.val[1] = vtmp01x0123.val[0];
260 vtmp01x0123.val[0] = vsrtmp;
261 vtmp01x0123 = vld2q_lane_u32(w1, vtmp01x0123, 1); w1 += 2;
262 vsrtmp = vsrtmp3x0123;
263 vsrtmp3x0123 = vsrtmp2x0123;
264 vsrtmp2x0123 = vtmp01x0123.val[1];
265 vtmp01x0123.val[1] = vtmp01x0123.val[0];
266 vtmp01x0123.val[0] = vsrtmp;
267 vtmp01x0123 = vld2q_lane_u32(w2, vtmp01x0123, 2); w2 += 2;
268 vsrtmp = vsrtmp3x0123;
269 vsrtmp3x0123 = vsrtmp2x0123;
270 vsrtmp2x0123 = vtmp01x0123.val[1];
271 vtmp01x0123.val[1] = vtmp01x0123.val[0];
272 vtmp01x0123.val[0] = vsrtmp;
273 vtmp01x0123 = vld2q_lane_u32(w3, vtmp01x0123, 3); w3 += 2;
274 vsrtmp = vsrtmp3x0123;
275 vsrtmp3x0123 = vsrtmp2x0123;
276 vsrtmp2x0123 = vtmp01x0123.val[1];
277 vtmp01x0123.val[1] = vtmp01x0123.val[0];
278 vtmp01x0123.val[0] = vsrtmp;
279 vtmp01x4567 = vld2q_lane_u32(w4, vtmp01x4567, 0); w4 += 2;
280 vsrtmp = vsrtmp3x4567;
281 vsrtmp3x4567 = vsrtmp2x4567;
282 vsrtmp2x4567 = vtmp01x4567.val[1];
283 vtmp01x4567.val[1] = vtmp01x4567.val[0];
284 vtmp01x4567.val[0] = vsrtmp;
285 vtmp01x4567 = vld2q_lane_u32(w5, vtmp01x4567, 1); w5 += 2;
286 vsrtmp = vsrtmp3x4567;
287 vsrtmp3x4567 = vsrtmp2x4567;
288 vsrtmp2x4567 = vtmp01x4567.val[1];
289 vtmp01x4567.val[1] = vtmp01x4567.val[0];
290 vtmp01x4567.val[0] = vsrtmp;
291 vtmp01x4567 = vld2q_lane_u32(w6, vtmp01x4567, 2); w6 += 2;
292 vsrtmp = vsrtmp3x4567;
293 vsrtmp3x4567 = vsrtmp2x4567;
294 vsrtmp2x4567 = vtmp01x4567.val[1];
295 vtmp01x4567.val[1] = vtmp01x4567.val[0];
296 vtmp01x4567.val[0] = vsrtmp;
297 vtmp01x4567 = vld2q_lane_u32(w7, vtmp01x4567, 3); w7 += 2;
298 vsrtmp = vsrtmp3x4567;
299 vsrtmp3x4567 = vsrtmp2x4567;
300 vsrtmp2x4567 = vtmp01x4567.val[1];
301 vtmp01x4567.val[1] = vtmp01x4567.val[0];
302 vtmp01x4567.val[0] = vsrtmp;
303 vst1q_u32(packed_weights, vtmp01x0123.val[0]); packed_weights += 4;
304 vst1q_u32(packed_weights, vtmp01x4567.val[0]); packed_weights += 4;
305 vst1q_u32(packed_weights, vtmp01x0123.val[1]); packed_weights += 4;
306 vst1q_u32(packed_weights, vtmp01x4567.val[1]); packed_weights += 4;
307 vst1q_u32(packed_weights, vsrtmp2x0123); packed_weights += 4;
308 vst1q_u32(packed_weights, vsrtmp2x4567); packed_weights += 4;
309 vst1q_u32(packed_weights, vsrtmp3x0123); packed_weights += 4;
310 vst1q_u32(packed_weights, vsrtmp3x4567); packed_weights += 4;
311 break;
312 }
313 // KC remainder of 3
314 case 3:
315 {
316 uint32x4x3_t vtmp012x0123;
317 vtmp012x0123.val[0] = vdupq_n_u32(0);
318 vtmp012x0123.val[1] = vdupq_n_u32(0);
319 vtmp012x0123.val[2] = vdupq_n_u32(0);
320 uint32x4x3_t vtmp012x4567;
321 vtmp012x4567.val[0] = vdupq_n_u32(0);
322 vtmp012x4567.val[1] = vdupq_n_u32(0);
323 vtmp012x4567.val[2] = vdupq_n_u32(0);
324 uint32x4_t vsrtmp;
325 uint32x4_t vsrtmp3x0123 = vdupq_n_u32(0);
326 uint32x4_t vsrtmp3x4567 = vdupq_n_u32(0);
327 vtmp012x0123 = vld3q_lane_u32(w0, vtmp012x0123, 0); w0 += 3;
328 vsrtmp = vsrtmp3x0123;
329 vsrtmp3x0123 = vtmp012x0123.val[2];
330 vtmp012x0123.val[2] = vtmp012x0123.val[1];
331 vtmp012x0123.val[1] = vtmp012x0123.val[0];
332 vtmp012x0123.val[0] = vsrtmp;
333 vtmp012x0123 = vld3q_lane_u32(w1, vtmp012x0123, 1); w1 += 3;
334 vsrtmp = vsrtmp3x0123;
335 vsrtmp3x0123 = vtmp012x0123.val[2];
336 vtmp012x0123.val[2] = vtmp012x0123.val[1];
337 vtmp012x0123.val[1] = vtmp012x0123.val[0];
338 vtmp012x0123.val[0] = vsrtmp;
339 vtmp012x0123 = vld3q_lane_u32(w2, vtmp012x0123, 2); w2 += 3;
340 vsrtmp = vsrtmp3x0123;
341 vsrtmp3x0123 = vtmp012x0123.val[2];
342 vtmp012x0123.val[2] = vtmp012x0123.val[1];
343 vtmp012x0123.val[1] = vtmp012x0123.val[0];
344 vtmp012x0123.val[0] = vsrtmp;
345 vtmp012x0123 = vld3q_lane_u32(w3, vtmp012x0123, 3); w3 += 3;
346 vsrtmp = vsrtmp3x0123;
347 vsrtmp3x0123 = vtmp012x0123.val[2];
348 vtmp012x0123.val[2] = vtmp012x0123.val[1];
349 vtmp012x0123.val[1] = vtmp012x0123.val[0];
350 vtmp012x0123.val[0] = vsrtmp;
351 vtmp012x4567 = vld3q_lane_u32(w4, vtmp012x4567, 0); w4 += 3;
352 vsrtmp = vsrtmp3x4567;
353 vsrtmp3x4567 = vtmp012x4567.val[2];
354 vtmp012x4567.val[2] = vtmp012x4567.val[1];
355 vtmp012x4567.val[1] = vtmp012x4567.val[0];
356 vtmp012x4567.val[0] = vsrtmp;
357 vtmp012x4567 = vld3q_lane_u32(w5, vtmp012x4567, 1); w5 += 3;
358 vsrtmp = vsrtmp3x4567;
359 vsrtmp3x4567 = vtmp012x4567.val[2];
360 vtmp012x4567.val[2] = vtmp012x4567.val[1];
361 vtmp012x4567.val[1] = vtmp012x4567.val[0];
362 vtmp012x4567.val[0] = vsrtmp;
363 vtmp012x4567 = vld3q_lane_u32(w6, vtmp012x4567, 2); w6 += 3;
364 vsrtmp = vsrtmp3x4567;
365 vsrtmp3x4567 = vtmp012x4567.val[2];
366 vtmp012x4567.val[2] = vtmp012x4567.val[1];
367 vtmp012x4567.val[1] = vtmp012x4567.val[0];
368 vtmp012x4567.val[0] = vsrtmp;
369 vtmp012x4567 = vld3q_lane_u32(w7, vtmp012x4567, 3); w7 += 3;
370 vsrtmp = vsrtmp3x4567;
371 vsrtmp3x4567 = vtmp012x4567.val[2];
372 vtmp012x4567.val[2] = vtmp012x4567.val[1];
373 vtmp012x4567.val[1] = vtmp012x4567.val[0];
374 vtmp012x4567.val[0] = vsrtmp;
375 vst1q_u32(packed_weights, vtmp012x0123.val[0]); packed_weights += 4;
376 vst1q_u32(packed_weights, vtmp012x4567.val[0]); packed_weights += 4;
377 vst1q_u32(packed_weights, vtmp012x0123.val[1]); packed_weights += 4;
378 vst1q_u32(packed_weights, vtmp012x4567.val[1]); packed_weights += 4;
379 vst1q_u32(packed_weights, vtmp012x0123.val[2]); packed_weights += 4;
380 vst1q_u32(packed_weights, vtmp012x4567.val[2]); packed_weights += 4;
381 vst1q_u32(packed_weights, vsrtmp3x0123); packed_weights += 4;
382 vst1q_u32(packed_weights, vsrtmp3x4567); packed_weights += 4;
383 break;
384 }
385 default:
386 XNN_UNREACHABLE;
387 }
388 }
389 packed_weights = (uint32_t*) ((uintptr_t) packed_weights + extra_bytes);
390 w0 = w7;
391 }
392
393 // NC remainder (1..7)
394 if XNN_UNLIKELY(n != 0) {
395 assert(n >= 1);
396 assert(n <= 7);
397 if XNN_LIKELY(bias != NULL) {
398 size_t nb = n;
399 do {
400 *packed_weights++ = *bias++;
401 } while (--nb != 0);
402 packed_weights += (8 - n);
403 } else {
404 const uint32x4_t vzero = vmovq_n_u32(0);
405 vst1q_u32(packed_weights, vzero); packed_weights += 4;
406 vst1q_u32(packed_weights, vzero); packed_weights += 4;
407 }
408
409 // NR remainder has less than 8 rows so last row is not loaded
410 const uint32_t* w1 = w0 + kc;
411 if XNN_UNPREDICTABLE(n < 2) {
412 w1 = w0;
413 }
414 const uint32_t* w2 = w1 + kc;
415 if XNN_UNPREDICTABLE(n <= 2) {
416 w2 = w1;
417 }
418 const uint32_t* w3 = w2 + kc;
419 if XNN_UNPREDICTABLE(n < 4) {
420 w3 = w2;
421 }
422 const uint32_t* w4 = w3 + kc;
423 if XNN_UNPREDICTABLE(n <= 4) {
424 w4 = w3;
425 }
426 const uint32_t* w5 = w4 + kc;
427 if XNN_UNPREDICTABLE(n < 6) {
428 w5 = w4;
429 }
430 const uint32_t* w6 = w5 + kc;
431 if XNN_UNPREDICTABLE(n <= 6) {
432 w6 = w5;
433 }
434
435 // KC main loop multiple of 4
436 size_t k = kc;
437 for (; k >= 4; k -= 4) {
438 uint32x4_t vsrtmp;
439 vtmp0123x0123 = vld4q_lane_u32(w0, vtmp0123x0123, 0); w0 += 4;
440 vsrtmp = vtmp0123x0123.val[3];
441 vtmp0123x0123.val[3] = vtmp0123x0123.val[2];
442 vtmp0123x0123.val[2] = vtmp0123x0123.val[1];
443 vtmp0123x0123.val[1] = vtmp0123x0123.val[0];
444 vtmp0123x0123.val[0] = vsrtmp;
445 vtmp0123x0123 = vld4q_lane_u32(w1, vtmp0123x0123, 1); w1 += 4;
446 vsrtmp = vtmp0123x0123.val[3];
447 vtmp0123x0123.val[3] = vtmp0123x0123.val[2];
448 vtmp0123x0123.val[2] = vtmp0123x0123.val[1];
449 vtmp0123x0123.val[1] = vtmp0123x0123.val[0];
450 vtmp0123x0123.val[0] = vsrtmp;
451 vtmp0123x0123 = vld4q_lane_u32(w2, vtmp0123x0123, 2); w2 += 4;
452 vsrtmp = vtmp0123x0123.val[3];
453 vtmp0123x0123.val[3] = vtmp0123x0123.val[2];
454 vtmp0123x0123.val[2] = vtmp0123x0123.val[1];
455 vtmp0123x0123.val[1] = vtmp0123x0123.val[0];
456 vtmp0123x0123.val[0] = vsrtmp;
457 vtmp0123x0123 = vld4q_lane_u32(w3, vtmp0123x0123, 3); w3 += 4;
458 vsrtmp = vtmp0123x0123.val[3];
459 vtmp0123x0123.val[3] = vtmp0123x0123.val[2];
460 vtmp0123x0123.val[2] = vtmp0123x0123.val[1];
461 vtmp0123x0123.val[1] = vtmp0123x0123.val[0];
462 vtmp0123x0123.val[0] = vsrtmp;
463 vtmp0123x4567 = vld4q_lane_u32(w4, vtmp0123x4567, 0); w4 += 4;
464 vsrtmp = vtmp0123x4567.val[3];
465 vtmp0123x4567.val[3] = vtmp0123x4567.val[2];
466 vtmp0123x4567.val[2] = vtmp0123x4567.val[1];
467 vtmp0123x4567.val[1] = vtmp0123x4567.val[0];
468 vtmp0123x4567.val[0] = vsrtmp;
469 vtmp0123x4567 = vld4q_lane_u32(w5, vtmp0123x4567, 1); w5 += 4;
470 vsrtmp = vtmp0123x4567.val[3];
471 vtmp0123x4567.val[3] = vtmp0123x4567.val[2];
472 vtmp0123x4567.val[2] = vtmp0123x4567.val[1];
473 vtmp0123x4567.val[1] = vtmp0123x4567.val[0];
474 vtmp0123x4567.val[0] = vsrtmp;
475 vtmp0123x4567 = vld4q_lane_u32(w6, vtmp0123x4567, 2); w6 += 4;
476 vsrtmp = vtmp0123x4567.val[3];
477 vtmp0123x4567.val[3] = vtmp0123x4567.val[2];
478 vtmp0123x4567.val[2] = vtmp0123x4567.val[1];
479 vtmp0123x4567.val[1] = vtmp0123x4567.val[0];
480 vtmp0123x4567.val[0] = vsrtmp;
481 vsrtmp = vtmp0123x4567.val[3];
482 vtmp0123x4567.val[3] = vtmp0123x4567.val[2];
483 vtmp0123x4567.val[2] = vtmp0123x4567.val[1];
484 vtmp0123x4567.val[1] = vtmp0123x4567.val[0];
485 vtmp0123x4567.val[0] = vsrtmp;
486 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
487 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
488 xnn_prefetch_to_l1((const int8_t*) w2 + 128);
489 xnn_prefetch_to_l1((const int8_t*) w3 + 128);
490 xnn_prefetch_to_l1((const int8_t*) w4 + 128);
491 xnn_prefetch_to_l1((const int8_t*) w5 + 128);
492 xnn_prefetch_to_l1((const int8_t*) w6 + 128);
493 vst1q_u32(packed_weights, vtmp0123x0123.val[0]); packed_weights += 4;
494 vst1q_u32(packed_weights, vtmp0123x4567.val[0]); packed_weights += 4;
495 vst1q_u32(packed_weights, vtmp0123x0123.val[1]); packed_weights += 4;
496 vst1q_u32(packed_weights, vtmp0123x4567.val[1]); packed_weights += 4;
497 vst1q_u32(packed_weights, vtmp0123x0123.val[2]); packed_weights += 4;
498 vst1q_u32(packed_weights, vtmp0123x4567.val[2]); packed_weights += 4;
499 vst1q_u32(packed_weights, vtmp0123x0123.val[3]); packed_weights += 4;
500 vst1q_u32(packed_weights, vtmp0123x4567.val[3]); packed_weights += 4;
501 }
502
503 // KC remainder of 1..3
504 // Same as main loop but ld1, ld2 or ld3
505 if XNN_UNLIKELY(k != 0) {
506 assert(k >= 1);
507 assert(k <= 3);
508 switch (k) {
509 // KC remainder of 1
510 case 1:
511 {
512 uint32x4_t vtmp0x0123 = vdupq_n_u32(0);
513 uint32x4_t vtmp0x4567 = vdupq_n_u32(0);
514 uint32x4_t vsrtmp;
515 uint32x4_t vsrtmp1x0123 = vdupq_n_u32(0);
516 uint32x4_t vsrtmp2x0123 = vdupq_n_u32(0);
517 uint32x4_t vsrtmp3x0123 = vdupq_n_u32(0);
518 uint32x4_t vsrtmp1x4567 = vdupq_n_u32(0);
519 uint32x4_t vsrtmp2x4567 = vdupq_n_u32(0);
520 uint32x4_t vsrtmp3x4567 = vdupq_n_u32(0);
521 vtmp0x0123 = vld1q_lane_u32(w0, vtmp0x0123, 0);
522 vsrtmp = vsrtmp3x0123;
523 vsrtmp3x0123 = vsrtmp2x0123;
524 vsrtmp2x0123 = vsrtmp1x0123;
525 vsrtmp1x0123 = vtmp0x0123;
526 vtmp0x0123 = vsrtmp;
527 vtmp0x0123 = vld1q_lane_u32(w1, vtmp0x0123, 1);
528 vsrtmp = vsrtmp3x0123;
529 vsrtmp3x0123 = vsrtmp2x0123;
530 vsrtmp2x0123 = vsrtmp1x0123;
531 vsrtmp1x0123 = vtmp0x0123;
532 vtmp0x0123 = vsrtmp;
533 vtmp0x0123 = vld1q_lane_u32(w2, vtmp0x0123, 2);
534 vsrtmp = vsrtmp3x0123;
535 vsrtmp3x0123 = vsrtmp2x0123;
536 vsrtmp2x0123 = vsrtmp1x0123;
537 vsrtmp1x0123 = vtmp0x0123;
538 vtmp0x0123 = vsrtmp;
539 vtmp0x0123 = vld1q_lane_u32(w3, vtmp0x0123, 3);
540 vsrtmp = vsrtmp3x0123;
541 vsrtmp3x0123 = vsrtmp2x0123;
542 vsrtmp2x0123 = vsrtmp1x0123;
543 vsrtmp1x0123 = vtmp0x0123;
544 vtmp0x0123 = vsrtmp;
545 vtmp0x4567 = vld1q_lane_u32(w4, vtmp0x4567, 0);
546 vsrtmp = vsrtmp3x4567;
547 vsrtmp3x4567 = vsrtmp2x4567;
548 vsrtmp2x4567 = vsrtmp1x4567;
549 vsrtmp1x4567 = vtmp0x4567;
550 vtmp0x4567 = vsrtmp;
551 vtmp0x4567 = vld1q_lane_u32(w5, vtmp0x4567, 1);
552 vsrtmp = vsrtmp3x4567;
553 vsrtmp3x4567 = vsrtmp2x4567;
554 vsrtmp2x4567 = vsrtmp1x4567;
555 vsrtmp1x4567 = vtmp0x4567;
556 vtmp0x4567 = vsrtmp;
557 vtmp0x4567 = vld1q_lane_u32(w6, vtmp0x4567, 2);
558 vsrtmp = vsrtmp3x4567;
559 vsrtmp3x4567 = vsrtmp2x4567;
560 vsrtmp2x4567 = vsrtmp1x4567;
561 vsrtmp1x4567 = vtmp0x4567;
562 vtmp0x4567 = vsrtmp;
563 vsrtmp = vsrtmp3x4567;
564 vsrtmp3x4567 = vsrtmp2x4567;
565 vsrtmp2x4567 = vsrtmp1x4567;
566 vsrtmp1x4567 = vtmp0x4567;
567 vtmp0x4567 = vsrtmp;
568 vst1q_u32(packed_weights, vtmp0x0123); packed_weights += 4;
569 vst1q_u32(packed_weights, vtmp0x4567); packed_weights += 4;
570 vst1q_u32(packed_weights, vsrtmp1x0123); packed_weights += 4;
571 vst1q_u32(packed_weights, vsrtmp1x4567); packed_weights += 4;
572 vst1q_u32(packed_weights, vsrtmp2x0123); packed_weights += 4;
573 vst1q_u32(packed_weights, vsrtmp2x4567); packed_weights += 4;
574 vst1q_u32(packed_weights, vsrtmp3x0123); packed_weights += 4;
575 vst1q_u32(packed_weights, vsrtmp3x4567); packed_weights += 4;
576 break;
577 }
578 // KC remainder of 2
579 case 2:
580 {
581 uint32x4x2_t vtmp01x0123;
582 vtmp01x0123.val[0] = vdupq_n_u32(0);
583 vtmp01x0123.val[1] = vdupq_n_u32(0);
584 uint32x4x2_t vtmp01x4567;
585 vtmp01x4567.val[0] = vdupq_n_u32(0);
586 vtmp01x4567.val[1] = vdupq_n_u32(0);
587 uint32x4_t vsrtmp;
588 uint32x4_t vsrtmp2x0123 = vdupq_n_u32(0);
589 uint32x4_t vsrtmp3x0123 = vdupq_n_u32(0);
590 uint32x4_t vsrtmp2x4567 = vdupq_n_u32(0);
591 uint32x4_t vsrtmp3x4567 = vdupq_n_u32(0);
592 vtmp01x0123 = vld2q_lane_u32(w0, vtmp01x0123, 0);
593 vsrtmp = vsrtmp3x0123;
594 vsrtmp3x0123 = vsrtmp2x0123;
595 vsrtmp2x0123 = vtmp01x0123.val[1];
596 vtmp01x0123.val[1] = vtmp01x0123.val[0];
597 vtmp01x0123.val[0] = vsrtmp;
598 vtmp01x0123 = vld2q_lane_u32(w1, vtmp01x0123, 1);
599 vsrtmp = vsrtmp3x0123;
600 vsrtmp3x0123 = vsrtmp2x0123;
601 vsrtmp2x0123 = vtmp01x0123.val[1];
602 vtmp01x0123.val[1] = vtmp01x0123.val[0];
603 vtmp01x0123.val[0] = vsrtmp;
604 vtmp01x0123 = vld2q_lane_u32(w2, vtmp01x0123, 2);
605 vsrtmp = vsrtmp3x0123;
606 vsrtmp3x0123 = vsrtmp2x0123;
607 vsrtmp2x0123 = vtmp01x0123.val[1];
608 vtmp01x0123.val[1] = vtmp01x0123.val[0];
609 vtmp01x0123.val[0] = vsrtmp;
610 vtmp01x0123 = vld2q_lane_u32(w3, vtmp01x0123, 3);
611 vsrtmp = vsrtmp3x0123;
612 vsrtmp3x0123 = vsrtmp2x0123;
613 vsrtmp2x0123 = vtmp01x0123.val[1];
614 vtmp01x0123.val[1] = vtmp01x0123.val[0];
615 vtmp01x0123.val[0] = vsrtmp;
616 vtmp01x4567 = vld2q_lane_u32(w4, vtmp01x4567, 0);
617 vsrtmp = vsrtmp3x4567;
618 vsrtmp3x4567 = vsrtmp2x4567;
619 vsrtmp2x4567 = vtmp01x4567.val[1];
620 vtmp01x4567.val[1] = vtmp01x4567.val[0];
621 vtmp01x4567.val[0] = vsrtmp;
622 vtmp01x4567 = vld2q_lane_u32(w5, vtmp01x4567, 1);
623 vsrtmp = vsrtmp3x4567;
624 vsrtmp3x4567 = vsrtmp2x4567;
625 vsrtmp2x4567 = vtmp01x4567.val[1];
626 vtmp01x4567.val[1] = vtmp01x4567.val[0];
627 vtmp01x4567.val[0] = vsrtmp;
628 vtmp01x4567 = vld2q_lane_u32(w6, vtmp01x4567, 2);
629 vsrtmp = vsrtmp3x4567;
630 vsrtmp3x4567 = vsrtmp2x4567;
631 vsrtmp2x4567 = vtmp01x4567.val[1];
632 vtmp01x4567.val[1] = vtmp01x4567.val[0];
633 vtmp01x4567.val[0] = vsrtmp;
634 vsrtmp = vsrtmp3x4567;
635 vsrtmp3x4567 = vsrtmp2x4567;
636 vsrtmp2x4567 = vtmp01x4567.val[1];
637 vtmp01x4567.val[1] = vtmp01x4567.val[0];
638 vtmp01x4567.val[0] = vsrtmp;
639 vst1q_u32(packed_weights, vtmp01x0123.val[0]); packed_weights += 4;
640 vst1q_u32(packed_weights, vtmp01x4567.val[0]); packed_weights += 4;
641 vst1q_u32(packed_weights, vtmp01x0123.val[1]); packed_weights += 4;
642 vst1q_u32(packed_weights, vtmp01x4567.val[1]); packed_weights += 4;
643 vst1q_u32(packed_weights, vsrtmp2x0123); packed_weights += 4;
644 vst1q_u32(packed_weights, vsrtmp2x4567); packed_weights += 4;
645 vst1q_u32(packed_weights, vsrtmp3x0123); packed_weights += 4;
646 vst1q_u32(packed_weights, vsrtmp3x4567); packed_weights += 4;
647 break;
648 }
649 // KC remainder of 3
650 case 3:
651 {
652 uint32x4x3_t vtmp012x0123;
653 vtmp012x0123.val[0] = vdupq_n_u32(0);
654 vtmp012x0123.val[1] = vdupq_n_u32(0);
655 vtmp012x0123.val[2] = vdupq_n_u32(0);
656 uint32x4x3_t vtmp012x4567;
657 vtmp012x4567.val[0] = vdupq_n_u32(0);
658 vtmp012x4567.val[1] = vdupq_n_u32(0);
659 vtmp012x4567.val[2] = vdupq_n_u32(0);
660 uint32x4_t vsrtmp;
661 uint32x4_t vsrtmp3x0123 = vdupq_n_u32(0);
662 uint32x4_t vsrtmp3x4567 = vdupq_n_u32(0);
663 vtmp012x0123 = vld3q_lane_u32(w0, vtmp012x0123, 0);
664 vsrtmp = vsrtmp3x0123;
665 vsrtmp3x0123 = vtmp012x0123.val[2];
666 vtmp012x0123.val[2] = vtmp012x0123.val[1];
667 vtmp012x0123.val[1] = vtmp012x0123.val[0];
668 vtmp012x0123.val[0] = vsrtmp;
669 vtmp012x0123 = vld3q_lane_u32(w1, vtmp012x0123, 1);
670 vsrtmp = vsrtmp3x0123;
671 vsrtmp3x0123 = vtmp012x0123.val[2];
672 vtmp012x0123.val[2] = vtmp012x0123.val[1];
673 vtmp012x0123.val[1] = vtmp012x0123.val[0];
674 vtmp012x0123.val[0] = vsrtmp;
675 vtmp012x0123 = vld3q_lane_u32(w2, vtmp012x0123, 2);
676 vsrtmp = vsrtmp3x0123;
677 vsrtmp3x0123 = vtmp012x0123.val[2];
678 vtmp012x0123.val[2] = vtmp012x0123.val[1];
679 vtmp012x0123.val[1] = vtmp012x0123.val[0];
680 vtmp012x0123.val[0] = vsrtmp;
681 vtmp012x0123 = vld3q_lane_u32(w3, vtmp012x0123, 3);
682 vsrtmp = vsrtmp3x0123;
683 vsrtmp3x0123 = vtmp012x0123.val[2];
684 vtmp012x0123.val[2] = vtmp012x0123.val[1];
685 vtmp012x0123.val[1] = vtmp012x0123.val[0];
686 vtmp012x0123.val[0] = vsrtmp;
687 vtmp012x4567 = vld3q_lane_u32(w4, vtmp012x4567, 0);
688 vsrtmp = vsrtmp3x4567;
689 vsrtmp3x4567 = vtmp012x4567.val[2];
690 vtmp012x4567.val[2] = vtmp012x4567.val[1];
691 vtmp012x4567.val[1] = vtmp012x4567.val[0];
692 vtmp012x4567.val[0] = vsrtmp;
693 vtmp012x4567 = vld3q_lane_u32(w5, vtmp012x4567, 1);
694 vsrtmp = vsrtmp3x4567;
695 vsrtmp3x4567 = vtmp012x4567.val[2];
696 vtmp012x4567.val[2] = vtmp012x4567.val[1];
697 vtmp012x4567.val[1] = vtmp012x4567.val[0];
698 vtmp012x4567.val[0] = vsrtmp;
699 vtmp012x4567 = vld3q_lane_u32(w6, vtmp012x4567, 2);
700 vsrtmp = vsrtmp3x4567;
701 vsrtmp3x4567 = vtmp012x4567.val[2];
702 vtmp012x4567.val[2] = vtmp012x4567.val[1];
703 vtmp012x4567.val[1] = vtmp012x4567.val[0];
704 vtmp012x4567.val[0] = vsrtmp;
705 vsrtmp = vsrtmp3x4567;
706 vsrtmp3x4567 = vtmp012x4567.val[2];
707 vtmp012x4567.val[2] = vtmp012x4567.val[1];
708 vtmp012x4567.val[1] = vtmp012x4567.val[0];
709 vtmp012x4567.val[0] = vsrtmp;
710 vst1q_u32(packed_weights, vtmp012x0123.val[0]); packed_weights += 4;
711 vst1q_u32(packed_weights, vtmp012x4567.val[0]); packed_weights += 4;
712 vst1q_u32(packed_weights, vtmp012x0123.val[1]); packed_weights += 4;
713 vst1q_u32(packed_weights, vtmp012x4567.val[1]); packed_weights += 4;
714 vst1q_u32(packed_weights, vtmp012x0123.val[2]); packed_weights += 4;
715 vst1q_u32(packed_weights, vtmp012x4567.val[2]); packed_weights += 4;
716 vst1q_u32(packed_weights, vsrtmp3x0123); packed_weights += 4;
717 vst1q_u32(packed_weights, vsrtmp3x4567); packed_weights += 4;
718 break;
719 }
720 default:
721 XNN_UNREACHABLE;
722 }
723 }
724 packed_weights = (uint32_t*) ((uintptr_t) packed_weights + extra_bytes);
725 }
726
727 weights += nc * kc;
728 } while (--g != 0);
729 }