Debian Codegraph @

src/x32-packw/gen/x32-packw-x8-gemm-goi-neon-ld4lane-u8-prfm.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/neon.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include <arm_neon.h>
16
17 #include "xnnpack/packw.h"
18 #include "xnnpack/prefetch.h"
19
20
21 void xnn_x32_packw_gemm_goi_ukernel_x8__neon_ld4lane_u8_prfm(
22 size_t g,
23 size_t nc,
24 size_t kc,
25 size_t nr,
26 size_t kr,
27 size_t sr,
28 const uint32_t* weights,
29 const uint32_t* bias,
30 const void* scale,
31 uint32_t* packed_weights,
32 size_t extra_bytes,
33 const void* params)
34 {
35 assert(g != 0);
36 assert(nc != 0);
37 assert(kc != 0);
38 assert(nr == 8);
39 assert(kr == 1);
40 assert(sr == 1);
41 assert(weights != NULL);
42 assert(packed_weights != NULL);
43 uint32x4x4_t vtmp0123x0123;
44 vtmp0123x0123.val[0] = vdupq_n_u32(0);
45 vtmp0123x0123.val[1] = vdupq_n_u32(0);
46 vtmp0123x0123.val[2] = vdupq_n_u32(0);
47 vtmp0123x0123.val[3] = vdupq_n_u32(0);
48 uint32x4x4_t vtmp4567x0123;
49 vtmp4567x0123.val[0] = vdupq_n_u32(0);
50 vtmp4567x0123.val[1] = vdupq_n_u32(0);
51 vtmp4567x0123.val[2] = vdupq_n_u32(0);
52 vtmp4567x0123.val[3] = vdupq_n_u32(0);
53 uint32x4x4_t vtmp0123x4567;
54 vtmp0123x4567.val[0] = vdupq_n_u32(0);
55 vtmp0123x4567.val[1] = vdupq_n_u32(0);
56 vtmp0123x4567.val[2] = vdupq_n_u32(0);
57 vtmp0123x4567.val[3] = vdupq_n_u32(0);
58 uint32x4x4_t vtmp4567x4567;
59 vtmp4567x4567.val[0] = vdupq_n_u32(0);
60 vtmp4567x4567.val[1] = vdupq_n_u32(0);
61 vtmp4567x4567.val[2] = vdupq_n_u32(0);
62 vtmp4567x4567.val[3] = vdupq_n_u32(0);
63
64 do {
65 // NC main loop multiple of 8
66 const uint32_t* w0 = weights;
67 size_t n = nc;
68
69 for (; n >= 8; n -= 8) {
70 if XNN_LIKELY(bias != NULL) {
71 uint32x4_t vb0 = vld1q_u32(bias); bias += 4;
72 uint32x4_t vb4 = vld1q_u32(bias); bias += 4;
73 vst1q_u32(packed_weights, vb0); packed_weights += 4;
74 vst1q_u32(packed_weights, vb4); packed_weights += 4;
75 } else {
76 const uint32x4_t vzero = vmovq_n_u32(0);
77 vst1q_u32(packed_weights, vzero); packed_weights += 4;
78 vst1q_u32(packed_weights, vzero); packed_weights += 4;
79 }
80
81 const uint32_t* w1 = w0 + kc;
82 const uint32_t* w2 = w1 + kc;
83 const uint32_t* w3 = w2 + kc;
84 const uint32_t* w4 = w3 + kc;
85 const uint32_t* w5 = w4 + kc;
86 const uint32_t* w6 = w5 + kc;
87 const uint32_t* w7 = w6 + kc;
88 xnn_prefetch_to_l1((const int8_t*) w0);
89 xnn_prefetch_to_l1((const int8_t*) w0 + 64);
90 xnn_prefetch_to_l1((const int8_t*) w1);
91 xnn_prefetch_to_l1((const int8_t*) w1 + 64);
92 xnn_prefetch_to_l1((const int8_t*) w2);
93 xnn_prefetch_to_l1((const int8_t*) w2 + 64);
94 xnn_prefetch_to_l1((const int8_t*) w3);
95 xnn_prefetch_to_l1((const int8_t*) w3 + 64);
96 xnn_prefetch_to_l1((const int8_t*) w4);
97 xnn_prefetch_to_l1((const int8_t*) w4 + 64);
98 xnn_prefetch_to_l1((const int8_t*) w5);
99 xnn_prefetch_to_l1((const int8_t*) w5 + 64);
100 xnn_prefetch_to_l1((const int8_t*) w6);
101 xnn_prefetch_to_l1((const int8_t*) w6 + 64);
102 xnn_prefetch_to_l1((const int8_t*) w7);
103 xnn_prefetch_to_l1((const int8_t*) w7 + 64);
104
105 // KC main loop multiple of 8
106 size_t k = kc;
107 for (; k >= 8; k -= 8) {
108 vtmp0123x0123 = vld4q_lane_u32(w0, vtmp0123x0123, 0); w0 += 4;
109 vtmp0123x0123 = vld4q_lane_u32(w1, vtmp0123x0123, 1); w1 += 4;
110 vtmp0123x0123 = vld4q_lane_u32(w2, vtmp0123x0123, 2); w2 += 4;
111 vtmp0123x0123 = vld4q_lane_u32(w3, vtmp0123x0123, 3); w3 += 4;
112 vtmp0123x4567 = vld4q_lane_u32(w4, vtmp0123x4567, 0); w4 += 4;
113 vtmp0123x4567 = vld4q_lane_u32(w5, vtmp0123x4567, 1); w5 += 4;
114 vtmp0123x4567 = vld4q_lane_u32(w6, vtmp0123x4567, 2); w6 += 4;
115 vtmp0123x4567 = vld4q_lane_u32(w7, vtmp0123x4567, 3); w7 += 4;
116 vtmp4567x0123 = vld4q_lane_u32(w0, vtmp4567x0123, 0); w0 += 4;
117 vtmp4567x0123 = vld4q_lane_u32(w1, vtmp4567x0123, 1); w1 += 4;
118 vtmp4567x0123 = vld4q_lane_u32(w2, vtmp4567x0123, 2); w2 += 4;
119 vtmp4567x0123 = vld4q_lane_u32(w3, vtmp4567x0123, 3); w3 += 4;
120 vtmp4567x4567 = vld4q_lane_u32(w4, vtmp4567x4567, 0); w4 += 4;
121 vtmp4567x4567 = vld4q_lane_u32(w5, vtmp4567x4567, 1); w5 += 4;
122 vtmp4567x4567 = vld4q_lane_u32(w6, vtmp4567x4567, 2); w6 += 4;
123 vtmp4567x4567 = vld4q_lane_u32(w7, vtmp4567x4567, 3); w7 += 4;
124 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
125 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
126 xnn_prefetch_to_l1((const int8_t*) w2 + 128);
127 xnn_prefetch_to_l1((const int8_t*) w3 + 128);
128 xnn_prefetch_to_l1((const int8_t*) w4 + 128);
129 xnn_prefetch_to_l1((const int8_t*) w5 + 128);
130 xnn_prefetch_to_l1((const int8_t*) w6 + 128);
131 xnn_prefetch_to_l1((const int8_t*) w7 + 128);
132 vst1q_u32(packed_weights, vtmp0123x0123.val[0]); packed_weights += 4;
133 vst1q_u32(packed_weights, vtmp0123x4567.val[0]); packed_weights += 4;
134 vst1q_u32(packed_weights, vtmp0123x0123.val[1]); packed_weights += 4;
135 vst1q_u32(packed_weights, vtmp0123x4567.val[1]); packed_weights += 4;
136 vst1q_u32(packed_weights, vtmp0123x0123.val[2]); packed_weights += 4;
137 vst1q_u32(packed_weights, vtmp0123x4567.val[2]); packed_weights += 4;
138 vst1q_u32(packed_weights, vtmp0123x0123.val[3]); packed_weights += 4;
139 vst1q_u32(packed_weights, vtmp0123x4567.val[3]); packed_weights += 4;
140 vst1q_u32(packed_weights, vtmp4567x0123.val[0]); packed_weights += 4;
141 vst1q_u32(packed_weights, vtmp4567x4567.val[0]); packed_weights += 4;
142 vst1q_u32(packed_weights, vtmp4567x0123.val[1]); packed_weights += 4;
143 vst1q_u32(packed_weights, vtmp4567x4567.val[1]); packed_weights += 4;
144 vst1q_u32(packed_weights, vtmp4567x0123.val[2]); packed_weights += 4;
145 vst1q_u32(packed_weights, vtmp4567x4567.val[2]); packed_weights += 4;
146 vst1q_u32(packed_weights, vtmp4567x0123.val[3]); packed_weights += 4;
147 vst1q_u32(packed_weights, vtmp4567x4567.val[3]); packed_weights += 4;
148 }
149 // KC remainder multiple of 4
150 if (k >= 4) {
151 vtmp0123x0123 = vld4q_lane_u32(w0, vtmp0123x0123, 0); w0 += 4;
152 vtmp0123x0123 = vld4q_lane_u32(w1, vtmp0123x0123, 1); w1 += 4;
153 vtmp0123x0123 = vld4q_lane_u32(w2, vtmp0123x0123, 2); w2 += 4;
154 vtmp0123x0123 = vld4q_lane_u32(w3, vtmp0123x0123, 3); w3 += 4;
155 vtmp0123x4567 = vld4q_lane_u32(w4, vtmp0123x4567, 0); w4 += 4;
156 vtmp0123x4567 = vld4q_lane_u32(w5, vtmp0123x4567, 1); w5 += 4;
157 vtmp0123x4567 = vld4q_lane_u32(w6, vtmp0123x4567, 2); w6 += 4;
158 vtmp0123x4567 = vld4q_lane_u32(w7, vtmp0123x4567, 3); w7 += 4;
159 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
160 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
161 xnn_prefetch_to_l1((const int8_t*) w2 + 128);
162 xnn_prefetch_to_l1((const int8_t*) w3 + 128);
163 xnn_prefetch_to_l1((const int8_t*) w4 + 128);
164 xnn_prefetch_to_l1((const int8_t*) w5 + 128);
165 xnn_prefetch_to_l1((const int8_t*) w6 + 128);
166 xnn_prefetch_to_l1((const int8_t*) w7 + 128);
167 vst1q_u32(packed_weights, vtmp0123x0123.val[0]); packed_weights += 4;
168 vst1q_u32(packed_weights, vtmp0123x4567.val[0]); packed_weights += 4;
169 vst1q_u32(packed_weights, vtmp0123x0123.val[1]); packed_weights += 4;
170 vst1q_u32(packed_weights, vtmp0123x4567.val[1]); packed_weights += 4;
171 vst1q_u32(packed_weights, vtmp0123x0123.val[2]); packed_weights += 4;
172 vst1q_u32(packed_weights, vtmp0123x4567.val[2]); packed_weights += 4;
173 vst1q_u32(packed_weights, vtmp0123x0123.val[3]); packed_weights += 4;
174 vst1q_u32(packed_weights, vtmp0123x4567.val[3]); packed_weights += 4;
175 k -= 4;
176 }
177
178 // KC remainder of 1..3
179 // Same as main loop but ld1, ld2 or ld3
180 if XNN_UNLIKELY(k != 0) {
181 assert(k >= 1);
182 assert(k <= 3);
183 switch (k) {
184 // KC remainder of 1
185 case 1:
186 {
187 uint32x4_t vtmp0x0123 = vdupq_n_u32(0);
188 uint32x4_t vtmp0x4567 = vdupq_n_u32(0);
189 vtmp0x0123 = vld1q_lane_u32(w0, vtmp0x0123, 0); w0 += 1;
190 vtmp0x0123 = vld1q_lane_u32(w1, vtmp0x0123, 1); w1 += 1;
191 vtmp0x0123 = vld1q_lane_u32(w2, vtmp0x0123, 2); w2 += 1;
192 vtmp0x0123 = vld1q_lane_u32(w3, vtmp0x0123, 3); w3 += 1;
193 vtmp0x4567 = vld1q_lane_u32(w4, vtmp0x4567, 0); w4 += 1;
194 vtmp0x4567 = vld1q_lane_u32(w5, vtmp0x4567, 1); w5 += 1;
195 vtmp0x4567 = vld1q_lane_u32(w6, vtmp0x4567, 2); w6 += 1;
196 vtmp0x4567 = vld1q_lane_u32(w7, vtmp0x4567, 3); w7 += 1;
197 vst1q_u32(packed_weights, vtmp0x0123); packed_weights += 4;
198 vst1q_u32(packed_weights, vtmp0x4567); packed_weights += 4;
199 break;
200 }
201 // KC remainder of 2
202 case 2:
203 {
204 uint32x4x2_t vtmp01x0123;
205 vtmp01x0123.val[0] = vdupq_n_u32(0);
206 vtmp01x0123.val[1] = vdupq_n_u32(0);
207 uint32x4x2_t vtmp01x4567;
208 vtmp01x4567.val[0] = vdupq_n_u32(0);
209 vtmp01x4567.val[1] = vdupq_n_u32(0);
210 vtmp01x0123 = vld2q_lane_u32(w0, vtmp01x0123, 0); w0 += 2;
211 vtmp01x0123 = vld2q_lane_u32(w1, vtmp01x0123, 1); w1 += 2;
212 vtmp01x0123 = vld2q_lane_u32(w2, vtmp01x0123, 2); w2 += 2;
213 vtmp01x0123 = vld2q_lane_u32(w3, vtmp01x0123, 3); w3 += 2;
214 vtmp01x4567 = vld2q_lane_u32(w4, vtmp01x4567, 0); w4 += 2;
215 vtmp01x4567 = vld2q_lane_u32(w5, vtmp01x4567, 1); w5 += 2;
216 vtmp01x4567 = vld2q_lane_u32(w6, vtmp01x4567, 2); w6 += 2;
217 vtmp01x4567 = vld2q_lane_u32(w7, vtmp01x4567, 3); w7 += 2;
218 vst1q_u32(packed_weights, vtmp01x0123.val[0]); packed_weights += 4;
219 vst1q_u32(packed_weights, vtmp01x4567.val[0]); packed_weights += 4;
220 vst1q_u32(packed_weights, vtmp01x0123.val[1]); packed_weights += 4;
221 vst1q_u32(packed_weights, vtmp01x4567.val[1]); packed_weights += 4;
222 break;
223 }
224 // KC remainder of 3
225 case 3:
226 {
227 uint32x4x3_t vtmp012x0123;
228 vtmp012x0123.val[0] = vdupq_n_u32(0);
229 vtmp012x0123.val[1] = vdupq_n_u32(0);
230 vtmp012x0123.val[2] = vdupq_n_u32(0);
231 uint32x4x3_t vtmp012x4567;
232 vtmp012x4567.val[0] = vdupq_n_u32(0);
233 vtmp012x4567.val[1] = vdupq_n_u32(0);
234 vtmp012x4567.val[2] = vdupq_n_u32(0);
235 vtmp012x0123 = vld3q_lane_u32(w0, vtmp012x0123, 0); w0 += 3;
236 vtmp012x0123 = vld3q_lane_u32(w1, vtmp012x0123, 1); w1 += 3;
237 vtmp012x0123 = vld3q_lane_u32(w2, vtmp012x0123, 2); w2 += 3;
238 vtmp012x0123 = vld3q_lane_u32(w3, vtmp012x0123, 3); w3 += 3;
239 vtmp012x4567 = vld3q_lane_u32(w4, vtmp012x4567, 0); w4 += 3;
240 vtmp012x4567 = vld3q_lane_u32(w5, vtmp012x4567, 1); w5 += 3;
241 vtmp012x4567 = vld3q_lane_u32(w6, vtmp012x4567, 2); w6 += 3;
242 vtmp012x4567 = vld3q_lane_u32(w7, vtmp012x4567, 3); w7 += 3;
243 vst1q_u32(packed_weights, vtmp012x0123.val[0]); packed_weights += 4;
244 vst1q_u32(packed_weights, vtmp012x4567.val[0]); packed_weights += 4;
245 vst1q_u32(packed_weights, vtmp012x0123.val[1]); packed_weights += 4;
246 vst1q_u32(packed_weights, vtmp012x4567.val[1]); packed_weights += 4;
247 vst1q_u32(packed_weights, vtmp012x0123.val[2]); packed_weights += 4;
248 vst1q_u32(packed_weights, vtmp012x4567.val[2]); packed_weights += 4;
249 break;
250 }
251 default:
252 XNN_UNREACHABLE;
253 }
254 }
255 packed_weights = (uint32_t*) ((uintptr_t) packed_weights + extra_bytes);
256 w0 = w7;
257 }
258
259 // NC remainder (1..7)
260 if XNN_UNLIKELY(n != 0) {
261 assert(n >= 1);
262 assert(n <= 7);
263 if XNN_LIKELY(bias != NULL) {
264 size_t nb = n;
265 do {
266 *packed_weights++ = *bias++;
267 } while (--nb != 0);
268 packed_weights += (8 - n);
269 } else {
270 const uint32x4_t vzero = vmovq_n_u32(0);
271 vst1q_u32(packed_weights, vzero); packed_weights += 4;
272 vst1q_u32(packed_weights, vzero); packed_weights += 4;
273 }
274
275 // NR remainder has less than 8 rows so last row is not loaded
276 const uint32_t* w1 = w0 + kc;
277 if XNN_UNPREDICTABLE(n < 2) {
278 w1 = w0;
279 }
280 const uint32_t* w2 = w1 + kc;
281 if XNN_UNPREDICTABLE(n <= 2) {
282 w2 = w1;
283 }
284 const uint32_t* w3 = w2 + kc;
285 if XNN_UNPREDICTABLE(n < 4) {
286 w3 = w2;
287 }
288 const uint32_t* w4 = w3 + kc;
289 if XNN_UNPREDICTABLE(n <= 4) {
290 w4 = w3;
291 }
292 const uint32_t* w5 = w4 + kc;
293 if XNN_UNPREDICTABLE(n < 6) {
294 w5 = w4;
295 }
296 const uint32_t* w6 = w5 + kc;
297 if XNN_UNPREDICTABLE(n <= 6) {
298 w6 = w5;
299 }
300
301 // KC main loop multiple of 8
302 size_t k = kc;
303 for (; k >= 8; k -= 8) {
304 vtmp0123x0123 = vld4q_lane_u32(w0, vtmp0123x0123, 0); w0 += 4;
305 vtmp0123x0123 = vld4q_lane_u32(w1, vtmp0123x0123, 1); w1 += 4;
306 vtmp0123x0123 = vld4q_lane_u32(w2, vtmp0123x0123, 2); w2 += 4;
307 vtmp0123x0123 = vld4q_lane_u32(w3, vtmp0123x0123, 3); w3 += 4;
308 vtmp0123x4567 = vld4q_lane_u32(w4, vtmp0123x4567, 0); w4 += 4;
309 vtmp0123x4567 = vld4q_lane_u32(w5, vtmp0123x4567, 1); w5 += 4;
310 vtmp0123x4567 = vld4q_lane_u32(w6, vtmp0123x4567, 2); w6 += 4;
311 vtmp4567x0123 = vld4q_lane_u32(w0, vtmp4567x0123, 0); w0 += 4;
312 vtmp4567x0123 = vld4q_lane_u32(w1, vtmp4567x0123, 1); w1 += 4;
313 vtmp4567x0123 = vld4q_lane_u32(w2, vtmp4567x0123, 2); w2 += 4;
314 vtmp4567x0123 = vld4q_lane_u32(w3, vtmp4567x0123, 3); w3 += 4;
315 vtmp4567x4567 = vld4q_lane_u32(w4, vtmp4567x4567, 0); w4 += 4;
316 vtmp4567x4567 = vld4q_lane_u32(w5, vtmp4567x4567, 1); w5 += 4;
317 vtmp4567x4567 = vld4q_lane_u32(w6, vtmp4567x4567, 2); w6 += 4;
318 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
319 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
320 xnn_prefetch_to_l1((const int8_t*) w2 + 128);
321 xnn_prefetch_to_l1((const int8_t*) w3 + 128);
322 xnn_prefetch_to_l1((const int8_t*) w4 + 128);
323 xnn_prefetch_to_l1((const int8_t*) w5 + 128);
324 xnn_prefetch_to_l1((const int8_t*) w6 + 128);
325 vst1q_u32(packed_weights, vtmp0123x0123.val[0]); packed_weights += 4;
326 vst1q_u32(packed_weights, vtmp0123x4567.val[0]); packed_weights += 4;
327 vst1q_u32(packed_weights, vtmp0123x0123.val[1]); packed_weights += 4;
328 vst1q_u32(packed_weights, vtmp0123x4567.val[1]); packed_weights += 4;
329 vst1q_u32(packed_weights, vtmp0123x0123.val[2]); packed_weights += 4;
330 vst1q_u32(packed_weights, vtmp0123x4567.val[2]); packed_weights += 4;
331 vst1q_u32(packed_weights, vtmp0123x0123.val[3]); packed_weights += 4;
332 vst1q_u32(packed_weights, vtmp0123x4567.val[3]); packed_weights += 4;
333 vst1q_u32(packed_weights, vtmp4567x0123.val[0]); packed_weights += 4;
334 vst1q_u32(packed_weights, vtmp4567x4567.val[0]); packed_weights += 4;
335 vst1q_u32(packed_weights, vtmp4567x0123.val[1]); packed_weights += 4;
336 vst1q_u32(packed_weights, vtmp4567x4567.val[1]); packed_weights += 4;
337 vst1q_u32(packed_weights, vtmp4567x0123.val[2]); packed_weights += 4;
338 vst1q_u32(packed_weights, vtmp4567x4567.val[2]); packed_weights += 4;
339 vst1q_u32(packed_weights, vtmp4567x0123.val[3]); packed_weights += 4;
340 vst1q_u32(packed_weights, vtmp4567x4567.val[3]); packed_weights += 4;
341 }
342 // KC remainder multiple of 4
343 if (k >= 4) {
344 vtmp0123x0123 = vld4q_lane_u32(w0, vtmp0123x0123, 0); w0 += 4;
345 vtmp0123x0123 = vld4q_lane_u32(w1, vtmp0123x0123, 1); w1 += 4;
346 vtmp0123x0123 = vld4q_lane_u32(w2, vtmp0123x0123, 2); w2 += 4;
347 vtmp0123x0123 = vld4q_lane_u32(w3, vtmp0123x0123, 3); w3 += 4;
348 vtmp0123x4567 = vld4q_lane_u32(w4, vtmp0123x4567, 0); w4 += 4;
349 vtmp0123x4567 = vld4q_lane_u32(w5, vtmp0123x4567, 1); w5 += 4;
350 vtmp0123x4567 = vld4q_lane_u32(w6, vtmp0123x4567, 2); w6 += 4;
351 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
352 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
353 xnn_prefetch_to_l1((const int8_t*) w2 + 128);
354 xnn_prefetch_to_l1((const int8_t*) w3 + 128);
355 xnn_prefetch_to_l1((const int8_t*) w4 + 128);
356 xnn_prefetch_to_l1((const int8_t*) w5 + 128);
357 xnn_prefetch_to_l1((const int8_t*) w6 + 128);
358 vst1q_u32(packed_weights, vtmp0123x0123.val[0]); packed_weights += 4;
359 vst1q_u32(packed_weights, vtmp0123x4567.val[0]); packed_weights += 4;
360 vst1q_u32(packed_weights, vtmp0123x0123.val[1]); packed_weights += 4;
361 vst1q_u32(packed_weights, vtmp0123x4567.val[1]); packed_weights += 4;
362 vst1q_u32(packed_weights, vtmp0123x0123.val[2]); packed_weights += 4;
363 vst1q_u32(packed_weights, vtmp0123x4567.val[2]); packed_weights += 4;
364 vst1q_u32(packed_weights, vtmp0123x0123.val[3]); packed_weights += 4;
365 vst1q_u32(packed_weights, vtmp0123x4567.val[3]); packed_weights += 4;
366 k -= 4;
367 }
368
369 // KC remainder of 1..3
370 // Same as main loop but ld1, ld2 or ld3
371 if XNN_UNLIKELY(k != 0) {
372 assert(k >= 1);
373 assert(k <= 3);
374 switch (k) {
375 // KC remainder of 1
376 case 1:
377 {
378 uint32x4_t vtmp0x0123 = vdupq_n_u32(0);
379 uint32x4_t vtmp0x4567 = vdupq_n_u32(0);
380 vtmp0x0123 = vld1q_lane_u32(w0, vtmp0x0123, 0);
381 vtmp0x0123 = vld1q_lane_u32(w1, vtmp0x0123, 1);
382 vtmp0x0123 = vld1q_lane_u32(w2, vtmp0x0123, 2);
383 vtmp0x0123 = vld1q_lane_u32(w3, vtmp0x0123, 3);
384 vtmp0x4567 = vld1q_lane_u32(w4, vtmp0x4567, 0);
385 vtmp0x4567 = vld1q_lane_u32(w5, vtmp0x4567, 1);
386 vtmp0x4567 = vld1q_lane_u32(w6, vtmp0x4567, 2);
387 vst1q_u32(packed_weights, vtmp0x0123); packed_weights += 4;
388 vst1q_u32(packed_weights, vtmp0x4567); packed_weights += 4;
389 break;
390 }
391 // KC remainder of 2
392 case 2:
393 {
394 uint32x4x2_t vtmp01x0123;
395 vtmp01x0123.val[0] = vdupq_n_u32(0);
396 vtmp01x0123.val[1] = vdupq_n_u32(0);
397 uint32x4x2_t vtmp01x4567;
398 vtmp01x4567.val[0] = vdupq_n_u32(0);
399 vtmp01x4567.val[1] = vdupq_n_u32(0);
400 vtmp01x0123 = vld2q_lane_u32(w0, vtmp01x0123, 0);
401 vtmp01x0123 = vld2q_lane_u32(w1, vtmp01x0123, 1);
402 vtmp01x0123 = vld2q_lane_u32(w2, vtmp01x0123, 2);
403 vtmp01x0123 = vld2q_lane_u32(w3, vtmp01x0123, 3);
404 vtmp01x4567 = vld2q_lane_u32(w4, vtmp01x4567, 0);
405 vtmp01x4567 = vld2q_lane_u32(w5, vtmp01x4567, 1);
406 vtmp01x4567 = vld2q_lane_u32(w6, vtmp01x4567, 2);
407 vst1q_u32(packed_weights, vtmp01x0123.val[0]); packed_weights += 4;
408 vst1q_u32(packed_weights, vtmp01x4567.val[0]); packed_weights += 4;
409 vst1q_u32(packed_weights, vtmp01x0123.val[1]); packed_weights += 4;
410 vst1q_u32(packed_weights, vtmp01x4567.val[1]); packed_weights += 4;
411 break;
412 }
413 // KC remainder of 3
414 case 3:
415 {
416 uint32x4x3_t vtmp012x0123;
417 vtmp012x0123.val[0] = vdupq_n_u32(0);
418 vtmp012x0123.val[1] = vdupq_n_u32(0);
419 vtmp012x0123.val[2] = vdupq_n_u32(0);
420 uint32x4x3_t vtmp012x4567;
421 vtmp012x4567.val[0] = vdupq_n_u32(0);
422 vtmp012x4567.val[1] = vdupq_n_u32(0);
423 vtmp012x4567.val[2] = vdupq_n_u32(0);
424 vtmp012x0123 = vld3q_lane_u32(w0, vtmp012x0123, 0);
425 vtmp012x0123 = vld3q_lane_u32(w1, vtmp012x0123, 1);
426 vtmp012x0123 = vld3q_lane_u32(w2, vtmp012x0123, 2);
427 vtmp012x0123 = vld3q_lane_u32(w3, vtmp012x0123, 3);
428 vtmp012x4567 = vld3q_lane_u32(w4, vtmp012x4567, 0);
429 vtmp012x4567 = vld3q_lane_u32(w5, vtmp012x4567, 1);
430 vtmp012x4567 = vld3q_lane_u32(w6, vtmp012x4567, 2);
431 vst1q_u32(packed_weights, vtmp012x0123.val[0]); packed_weights += 4;
432 vst1q_u32(packed_weights, vtmp012x4567.val[0]); packed_weights += 4;
433 vst1q_u32(packed_weights, vtmp012x0123.val[1]); packed_weights += 4;
434 vst1q_u32(packed_weights, vtmp012x4567.val[1]); packed_weights += 4;
435 vst1q_u32(packed_weights, vtmp012x0123.val[2]); packed_weights += 4;
436 vst1q_u32(packed_weights, vtmp012x4567.val[2]); packed_weights += 4;
437 break;
438 }
439 default:
440 XNN_UNREACHABLE;
441 }
442 }
443 packed_weights = (uint32_t*) ((uintptr_t) packed_weights + extra_bytes);
444 }
445
446 weights += nc * kc;
447 } while (--g != 0);
448 }