Debian Codegraph @

src/x32-packw/gen/x32-packw-x16-gemm-goi-neon-ld4lane-u8.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/neon.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include <arm_neon.h>
16
17 #include "xnnpack/packw.h"
18
19
20 void xnn_x32_packw_gemm_goi_ukernel_x16__neon_ld4lane_u8(
21 size_t g,
22 size_t nc,
23 size_t kc,
24 size_t nr,
25 size_t kr,
26 size_t sr,
27 const uint32_t* weights,
28 const uint32_t* bias,
29 const void* scale,
30 uint32_t* packed_weights,
31 size_t extra_bytes,
32 const void* params)
33 {
34 assert(g != 0);
35 assert(nc != 0);
36 assert(kc != 0);
37 assert(nr == 16);
38 assert(kr == 1);
39 assert(sr == 1);
40 assert(weights != NULL);
41 assert(packed_weights != NULL);
42 uint32x4x4_t vtmp0123x0123;
43 vtmp0123x0123.val[0] = vdupq_n_u32(0);
44 vtmp0123x0123.val[1] = vdupq_n_u32(0);
45 vtmp0123x0123.val[2] = vdupq_n_u32(0);
46 vtmp0123x0123.val[3] = vdupq_n_u32(0);
47 uint32x4x4_t vtmp4567x0123;
48 vtmp4567x0123.val[0] = vdupq_n_u32(0);
49 vtmp4567x0123.val[1] = vdupq_n_u32(0);
50 vtmp4567x0123.val[2] = vdupq_n_u32(0);
51 vtmp4567x0123.val[3] = vdupq_n_u32(0);
52 uint32x4x4_t vtmp0123x4567;
53 vtmp0123x4567.val[0] = vdupq_n_u32(0);
54 vtmp0123x4567.val[1] = vdupq_n_u32(0);
55 vtmp0123x4567.val[2] = vdupq_n_u32(0);
56 vtmp0123x4567.val[3] = vdupq_n_u32(0);
57 uint32x4x4_t vtmp4567x4567;
58 vtmp4567x4567.val[0] = vdupq_n_u32(0);
59 vtmp4567x4567.val[1] = vdupq_n_u32(0);
60 vtmp4567x4567.val[2] = vdupq_n_u32(0);
61 vtmp4567x4567.val[3] = vdupq_n_u32(0);
62 uint32x4x4_t vtmp0123x89AB;
63 vtmp0123x89AB.val[0] = vdupq_n_u32(0);
64 vtmp0123x89AB.val[1] = vdupq_n_u32(0);
65 vtmp0123x89AB.val[2] = vdupq_n_u32(0);
66 vtmp0123x89AB.val[3] = vdupq_n_u32(0);
67 uint32x4x4_t vtmp4567x89AB;
68 vtmp4567x89AB.val[0] = vdupq_n_u32(0);
69 vtmp4567x89AB.val[1] = vdupq_n_u32(0);
70 vtmp4567x89AB.val[2] = vdupq_n_u32(0);
71 vtmp4567x89AB.val[3] = vdupq_n_u32(0);
72 uint32x4x4_t vtmp0123xCDEF;
73 vtmp0123xCDEF.val[0] = vdupq_n_u32(0);
74 vtmp0123xCDEF.val[1] = vdupq_n_u32(0);
75 vtmp0123xCDEF.val[2] = vdupq_n_u32(0);
76 vtmp0123xCDEF.val[3] = vdupq_n_u32(0);
77 uint32x4x4_t vtmp4567xCDEF;
78 vtmp4567xCDEF.val[0] = vdupq_n_u32(0);
79 vtmp4567xCDEF.val[1] = vdupq_n_u32(0);
80 vtmp4567xCDEF.val[2] = vdupq_n_u32(0);
81 vtmp4567xCDEF.val[3] = vdupq_n_u32(0);
82
83 do {
84 // NC main loop multiple of 16
85 const uint32_t* w0 = weights;
86 size_t n = nc;
87
88 for (; n >= 16; n -= 16) {
89 if XNN_LIKELY(bias != NULL) {
90 uint32x4_t vb0 = vld1q_u32(bias); bias += 4;
91 uint32x4_t vb4 = vld1q_u32(bias); bias += 4;
92 uint32x4_t vb8 = vld1q_u32(bias); bias += 4;
93 uint32x4_t vb12 = vld1q_u32(bias); bias += 4;
94 vst1q_u32(packed_weights, vb0); packed_weights += 4;
95 vst1q_u32(packed_weights, vb4); packed_weights += 4;
96 vst1q_u32(packed_weights, vb8); packed_weights += 4;
97 vst1q_u32(packed_weights, vb12); packed_weights += 4;
98 } else {
99 const uint32x4_t vzero = vmovq_n_u32(0);
100 vst1q_u32(packed_weights, vzero); packed_weights += 4;
101 vst1q_u32(packed_weights, vzero); packed_weights += 4;
102 vst1q_u32(packed_weights, vzero); packed_weights += 4;
103 vst1q_u32(packed_weights, vzero); packed_weights += 4;
104 }
105
106 const uint32_t* w1 = w0 + kc;
107 const uint32_t* w2 = w1 + kc;
108 const uint32_t* w3 = w2 + kc;
109 const uint32_t* w4 = w3 + kc;
110 const uint32_t* w5 = w4 + kc;
111 const uint32_t* w6 = w5 + kc;
112 const uint32_t* w7 = w6 + kc;
113 const uint32_t* w8 = w7 + kc;
114 const uint32_t* w9 = w8 + kc;
115 const uint32_t* w10 = w9 + kc;
116 const uint32_t* w11 = w10 + kc;
117 const uint32_t* w12 = w11 + kc;
118 const uint32_t* w13 = w12 + kc;
119 const uint32_t* w14 = w13 + kc;
120 const uint32_t* w15 = w14 + kc;
121
122 // KC main loop multiple of 8
123 size_t k = kc;
124 for (; k >= 8; k -= 8) {
125 vtmp0123x0123 = vld4q_lane_u32(w0, vtmp0123x0123, 0); w0 += 4;
126 vtmp0123x0123 = vld4q_lane_u32(w1, vtmp0123x0123, 1); w1 += 4;
127 vtmp0123x0123 = vld4q_lane_u32(w2, vtmp0123x0123, 2); w2 += 4;
128 vtmp0123x0123 = vld4q_lane_u32(w3, vtmp0123x0123, 3); w3 += 4;
129 vtmp0123x4567 = vld4q_lane_u32(w4, vtmp0123x4567, 0); w4 += 4;
130 vtmp0123x4567 = vld4q_lane_u32(w5, vtmp0123x4567, 1); w5 += 4;
131 vtmp0123x4567 = vld4q_lane_u32(w6, vtmp0123x4567, 2); w6 += 4;
132 vtmp0123x4567 = vld4q_lane_u32(w7, vtmp0123x4567, 3); w7 += 4;
133 vtmp0123x89AB = vld4q_lane_u32(w8, vtmp0123x89AB, 0); w8 += 4;
134 vtmp0123x89AB = vld4q_lane_u32(w9, vtmp0123x89AB, 1); w9 += 4;
135 vtmp0123x89AB = vld4q_lane_u32(w10, vtmp0123x89AB, 2); w10 += 4;
136 vtmp0123x89AB = vld4q_lane_u32(w11, vtmp0123x89AB, 3); w11 += 4;
137 vtmp0123xCDEF = vld4q_lane_u32(w12, vtmp0123xCDEF, 0); w12 += 4;
138 vtmp0123xCDEF = vld4q_lane_u32(w13, vtmp0123xCDEF, 1); w13 += 4;
139 vtmp0123xCDEF = vld4q_lane_u32(w14, vtmp0123xCDEF, 2); w14 += 4;
140 vtmp0123xCDEF = vld4q_lane_u32(w15, vtmp0123xCDEF, 3); w15 += 4;
141 vtmp4567x0123 = vld4q_lane_u32(w0, vtmp4567x0123, 0); w0 += 4;
142 vtmp4567x0123 = vld4q_lane_u32(w1, vtmp4567x0123, 1); w1 += 4;
143 vtmp4567x0123 = vld4q_lane_u32(w2, vtmp4567x0123, 2); w2 += 4;
144 vtmp4567x0123 = vld4q_lane_u32(w3, vtmp4567x0123, 3); w3 += 4;
145 vtmp4567x4567 = vld4q_lane_u32(w4, vtmp4567x4567, 0); w4 += 4;
146 vtmp4567x4567 = vld4q_lane_u32(w5, vtmp4567x4567, 1); w5 += 4;
147 vtmp4567x4567 = vld4q_lane_u32(w6, vtmp4567x4567, 2); w6 += 4;
148 vtmp4567x4567 = vld4q_lane_u32(w7, vtmp4567x4567, 3); w7 += 4;
149 vtmp4567x89AB = vld4q_lane_u32(w8, vtmp4567x89AB, 0); w8 += 4;
150 vtmp4567x89AB = vld4q_lane_u32(w9, vtmp4567x89AB, 1); w9 += 4;
151 vtmp4567x89AB = vld4q_lane_u32(w10, vtmp4567x89AB, 2); w10 += 4;
152 vtmp4567x89AB = vld4q_lane_u32(w11, vtmp4567x89AB, 3); w11 += 4;
153 vtmp4567xCDEF = vld4q_lane_u32(w12, vtmp4567xCDEF, 0); w12 += 4;
154 vtmp4567xCDEF = vld4q_lane_u32(w13, vtmp4567xCDEF, 1); w13 += 4;
155 vtmp4567xCDEF = vld4q_lane_u32(w14, vtmp4567xCDEF, 2); w14 += 4;
156 vtmp4567xCDEF = vld4q_lane_u32(w15, vtmp4567xCDEF, 3); w15 += 4;
157 vst1q_u32(packed_weights, vtmp0123x0123.val[0]); packed_weights += 4;
158 vst1q_u32(packed_weights, vtmp0123x4567.val[0]); packed_weights += 4;
159 vst1q_u32(packed_weights, vtmp0123x89AB.val[0]); packed_weights += 4;
160 vst1q_u32(packed_weights, vtmp0123xCDEF.val[0]); packed_weights += 4;
161 vst1q_u32(packed_weights, vtmp0123x0123.val[1]); packed_weights += 4;
162 vst1q_u32(packed_weights, vtmp0123x4567.val[1]); packed_weights += 4;
163 vst1q_u32(packed_weights, vtmp0123x89AB.val[1]); packed_weights += 4;
164 vst1q_u32(packed_weights, vtmp0123xCDEF.val[1]); packed_weights += 4;
165 vst1q_u32(packed_weights, vtmp0123x0123.val[2]); packed_weights += 4;
166 vst1q_u32(packed_weights, vtmp0123x4567.val[2]); packed_weights += 4;
167 vst1q_u32(packed_weights, vtmp0123x89AB.val[2]); packed_weights += 4;
168 vst1q_u32(packed_weights, vtmp0123xCDEF.val[2]); packed_weights += 4;
169 vst1q_u32(packed_weights, vtmp0123x0123.val[3]); packed_weights += 4;
170 vst1q_u32(packed_weights, vtmp0123x4567.val[3]); packed_weights += 4;
171 vst1q_u32(packed_weights, vtmp0123x89AB.val[3]); packed_weights += 4;
172 vst1q_u32(packed_weights, vtmp0123xCDEF.val[3]); packed_weights += 4;
173 vst1q_u32(packed_weights, vtmp4567x0123.val[0]); packed_weights += 4;
174 vst1q_u32(packed_weights, vtmp4567x4567.val[0]); packed_weights += 4;
175 vst1q_u32(packed_weights, vtmp4567x89AB.val[0]); packed_weights += 4;
176 vst1q_u32(packed_weights, vtmp4567xCDEF.val[0]); packed_weights += 4;
177 vst1q_u32(packed_weights, vtmp4567x0123.val[1]); packed_weights += 4;
178 vst1q_u32(packed_weights, vtmp4567x4567.val[1]); packed_weights += 4;
179 vst1q_u32(packed_weights, vtmp4567x89AB.val[1]); packed_weights += 4;
180 vst1q_u32(packed_weights, vtmp4567xCDEF.val[1]); packed_weights += 4;
181 vst1q_u32(packed_weights, vtmp4567x0123.val[2]); packed_weights += 4;
182 vst1q_u32(packed_weights, vtmp4567x4567.val[2]); packed_weights += 4;
183 vst1q_u32(packed_weights, vtmp4567x89AB.val[2]); packed_weights += 4;
184 vst1q_u32(packed_weights, vtmp4567xCDEF.val[2]); packed_weights += 4;
185 vst1q_u32(packed_weights, vtmp4567x0123.val[3]); packed_weights += 4;
186 vst1q_u32(packed_weights, vtmp4567x4567.val[3]); packed_weights += 4;
187 vst1q_u32(packed_weights, vtmp4567x89AB.val[3]); packed_weights += 4;
188 vst1q_u32(packed_weights, vtmp4567xCDEF.val[3]); packed_weights += 4;
189 }
190 // KC remainder multiple of 4
191 if (k >= 4) {
192 vtmp0123x0123 = vld4q_lane_u32(w0, vtmp0123x0123, 0); w0 += 4;
193 vtmp0123x0123 = vld4q_lane_u32(w1, vtmp0123x0123, 1); w1 += 4;
194 vtmp0123x0123 = vld4q_lane_u32(w2, vtmp0123x0123, 2); w2 += 4;
195 vtmp0123x0123 = vld4q_lane_u32(w3, vtmp0123x0123, 3); w3 += 4;
196 vtmp0123x4567 = vld4q_lane_u32(w4, vtmp0123x4567, 0); w4 += 4;
197 vtmp0123x4567 = vld4q_lane_u32(w5, vtmp0123x4567, 1); w5 += 4;
198 vtmp0123x4567 = vld4q_lane_u32(w6, vtmp0123x4567, 2); w6 += 4;
199 vtmp0123x4567 = vld4q_lane_u32(w7, vtmp0123x4567, 3); w7 += 4;
200 vtmp0123x89AB = vld4q_lane_u32(w8, vtmp0123x89AB, 0); w8 += 4;
201 vtmp0123x89AB = vld4q_lane_u32(w9, vtmp0123x89AB, 1); w9 += 4;
202 vtmp0123x89AB = vld4q_lane_u32(w10, vtmp0123x89AB, 2); w10 += 4;
203 vtmp0123x89AB = vld4q_lane_u32(w11, vtmp0123x89AB, 3); w11 += 4;
204 vtmp0123xCDEF = vld4q_lane_u32(w12, vtmp0123xCDEF, 0); w12 += 4;
205 vtmp0123xCDEF = vld4q_lane_u32(w13, vtmp0123xCDEF, 1); w13 += 4;
206 vtmp0123xCDEF = vld4q_lane_u32(w14, vtmp0123xCDEF, 2); w14 += 4;
207 vtmp0123xCDEF = vld4q_lane_u32(w15, vtmp0123xCDEF, 3); w15 += 4;
208 vst1q_u32(packed_weights, vtmp0123x0123.val[0]); packed_weights += 4;
209 vst1q_u32(packed_weights, vtmp0123x4567.val[0]); packed_weights += 4;
210 vst1q_u32(packed_weights, vtmp0123x89AB.val[0]); packed_weights += 4;
211 vst1q_u32(packed_weights, vtmp0123xCDEF.val[0]); packed_weights += 4;
212 vst1q_u32(packed_weights, vtmp0123x0123.val[1]); packed_weights += 4;
213 vst1q_u32(packed_weights, vtmp0123x4567.val[1]); packed_weights += 4;
214 vst1q_u32(packed_weights, vtmp0123x89AB.val[1]); packed_weights += 4;
215 vst1q_u32(packed_weights, vtmp0123xCDEF.val[1]); packed_weights += 4;
216 vst1q_u32(packed_weights, vtmp0123x0123.val[2]); packed_weights += 4;
217 vst1q_u32(packed_weights, vtmp0123x4567.val[2]); packed_weights += 4;
218 vst1q_u32(packed_weights, vtmp0123x89AB.val[2]); packed_weights += 4;
219 vst1q_u32(packed_weights, vtmp0123xCDEF.val[2]); packed_weights += 4;
220 vst1q_u32(packed_weights, vtmp0123x0123.val[3]); packed_weights += 4;
221 vst1q_u32(packed_weights, vtmp0123x4567.val[3]); packed_weights += 4;
222 vst1q_u32(packed_weights, vtmp0123x89AB.val[3]); packed_weights += 4;
223 vst1q_u32(packed_weights, vtmp0123xCDEF.val[3]); packed_weights += 4;
224 k -= 4;
225 }
226
227 // KC remainder of 1..3
228 // Same as main loop but ld1, ld2 or ld3
229 if XNN_UNLIKELY(k != 0) {
230 assert(k >= 1);
231 assert(k <= 3);
232 switch (k) {
233 // KC remainder of 1
234 case 1:
235 {
236 uint32x4_t vtmp0x0123 = vdupq_n_u32(0);
237 uint32x4_t vtmp0x4567 = vdupq_n_u32(0);
238 uint32x4_t vtmp0x89AB = vdupq_n_u32(0);
239 uint32x4_t vtmp0xCDEF = vdupq_n_u32(0);
240 vtmp0x0123 = vld1q_lane_u32(w0, vtmp0x0123, 0); w0 += 1;
241 vtmp0x0123 = vld1q_lane_u32(w1, vtmp0x0123, 1); w1 += 1;
242 vtmp0x0123 = vld1q_lane_u32(w2, vtmp0x0123, 2); w2 += 1;
243 vtmp0x0123 = vld1q_lane_u32(w3, vtmp0x0123, 3); w3 += 1;
244 vtmp0x4567 = vld1q_lane_u32(w4, vtmp0x4567, 0); w4 += 1;
245 vtmp0x4567 = vld1q_lane_u32(w5, vtmp0x4567, 1); w5 += 1;
246 vtmp0x4567 = vld1q_lane_u32(w6, vtmp0x4567, 2); w6 += 1;
247 vtmp0x4567 = vld1q_lane_u32(w7, vtmp0x4567, 3); w7 += 1;
248 vtmp0x89AB = vld1q_lane_u32(w8, vtmp0x89AB, 0); w8 += 1;
249 vtmp0x89AB = vld1q_lane_u32(w9, vtmp0x89AB, 1); w9 += 1;
250 vtmp0x89AB = vld1q_lane_u32(w10, vtmp0x89AB, 2); w10 += 1;
251 vtmp0x89AB = vld1q_lane_u32(w11, vtmp0x89AB, 3); w11 += 1;
252 vtmp0xCDEF = vld1q_lane_u32(w12, vtmp0xCDEF, 0); w12 += 1;
253 vtmp0xCDEF = vld1q_lane_u32(w13, vtmp0xCDEF, 1); w13 += 1;
254 vtmp0xCDEF = vld1q_lane_u32(w14, vtmp0xCDEF, 2); w14 += 1;
255 vtmp0xCDEF = vld1q_lane_u32(w15, vtmp0xCDEF, 3); w15 += 1;
256 vst1q_u32(packed_weights, vtmp0x0123); packed_weights += 4;
257 vst1q_u32(packed_weights, vtmp0x4567); packed_weights += 4;
258 vst1q_u32(packed_weights, vtmp0x89AB); packed_weights += 4;
259 vst1q_u32(packed_weights, vtmp0xCDEF); packed_weights += 4;
260 break;
261 }
262 // KC remainder of 2
263 case 2:
264 {
265 uint32x4x2_t vtmp01x0123;
266 vtmp01x0123.val[0] = vdupq_n_u32(0);
267 vtmp01x0123.val[1] = vdupq_n_u32(0);
268 uint32x4x2_t vtmp01x4567;
269 vtmp01x4567.val[0] = vdupq_n_u32(0);
270 vtmp01x4567.val[1] = vdupq_n_u32(0);
271 uint32x4x2_t vtmp01x89AB;
272 vtmp01x89AB.val[0] = vdupq_n_u32(0);
273 vtmp01x89AB.val[1] = vdupq_n_u32(0);
274 uint32x4x2_t vtmp01xCDEF;
275 vtmp01xCDEF.val[0] = vdupq_n_u32(0);
276 vtmp01xCDEF.val[1] = vdupq_n_u32(0);
277 vtmp01x0123 = vld2q_lane_u32(w0, vtmp01x0123, 0); w0 += 2;
278 vtmp01x0123 = vld2q_lane_u32(w1, vtmp01x0123, 1); w1 += 2;
279 vtmp01x0123 = vld2q_lane_u32(w2, vtmp01x0123, 2); w2 += 2;
280 vtmp01x0123 = vld2q_lane_u32(w3, vtmp01x0123, 3); w3 += 2;
281 vtmp01x4567 = vld2q_lane_u32(w4, vtmp01x4567, 0); w4 += 2;
282 vtmp01x4567 = vld2q_lane_u32(w5, vtmp01x4567, 1); w5 += 2;
283 vtmp01x4567 = vld2q_lane_u32(w6, vtmp01x4567, 2); w6 += 2;
284 vtmp01x4567 = vld2q_lane_u32(w7, vtmp01x4567, 3); w7 += 2;
285 vtmp01x89AB = vld2q_lane_u32(w8, vtmp01x89AB, 0); w8 += 2;
286 vtmp01x89AB = vld2q_lane_u32(w9, vtmp01x89AB, 1); w9 += 2;
287 vtmp01x89AB = vld2q_lane_u32(w10, vtmp01x89AB, 2); w10 += 2;
288 vtmp01x89AB = vld2q_lane_u32(w11, vtmp01x89AB, 3); w11 += 2;
289 vtmp01xCDEF = vld2q_lane_u32(w12, vtmp01xCDEF, 0); w12 += 2;
290 vtmp01xCDEF = vld2q_lane_u32(w13, vtmp01xCDEF, 1); w13 += 2;
291 vtmp01xCDEF = vld2q_lane_u32(w14, vtmp01xCDEF, 2); w14 += 2;
292 vtmp01xCDEF = vld2q_lane_u32(w15, vtmp01xCDEF, 3); w15 += 2;
293 vst1q_u32(packed_weights, vtmp01x0123.val[0]); packed_weights += 4;
294 vst1q_u32(packed_weights, vtmp01x4567.val[0]); packed_weights += 4;
295 vst1q_u32(packed_weights, vtmp01x89AB.val[0]); packed_weights += 4;
296 vst1q_u32(packed_weights, vtmp01xCDEF.val[0]); packed_weights += 4;
297 vst1q_u32(packed_weights, vtmp01x0123.val[1]); packed_weights += 4;
298 vst1q_u32(packed_weights, vtmp01x4567.val[1]); packed_weights += 4;
299 vst1q_u32(packed_weights, vtmp01x89AB.val[1]); packed_weights += 4;
300 vst1q_u32(packed_weights, vtmp01xCDEF.val[1]); packed_weights += 4;
301 break;
302 }
303 // KC remainder of 3
304 case 3:
305 {
306 uint32x4x3_t vtmp012x0123;
307 vtmp012x0123.val[0] = vdupq_n_u32(0);
308 vtmp012x0123.val[1] = vdupq_n_u32(0);
309 vtmp012x0123.val[2] = vdupq_n_u32(0);
310 uint32x4x3_t vtmp012x4567;
311 vtmp012x4567.val[0] = vdupq_n_u32(0);
312 vtmp012x4567.val[1] = vdupq_n_u32(0);
313 vtmp012x4567.val[2] = vdupq_n_u32(0);
314 uint32x4x3_t vtmp012x89AB;
315 vtmp012x89AB.val[0] = vdupq_n_u32(0);
316 vtmp012x89AB.val[1] = vdupq_n_u32(0);
317 vtmp012x89AB.val[2] = vdupq_n_u32(0);
318 uint32x4x3_t vtmp012xCDEF;
319 vtmp012xCDEF.val[0] = vdupq_n_u32(0);
320 vtmp012xCDEF.val[1] = vdupq_n_u32(0);
321 vtmp012xCDEF.val[2] = vdupq_n_u32(0);
322 vtmp012x0123 = vld3q_lane_u32(w0, vtmp012x0123, 0); w0 += 3;
323 vtmp012x0123 = vld3q_lane_u32(w1, vtmp012x0123, 1); w1 += 3;
324 vtmp012x0123 = vld3q_lane_u32(w2, vtmp012x0123, 2); w2 += 3;
325 vtmp012x0123 = vld3q_lane_u32(w3, vtmp012x0123, 3); w3 += 3;
326 vtmp012x4567 = vld3q_lane_u32(w4, vtmp012x4567, 0); w4 += 3;
327 vtmp012x4567 = vld3q_lane_u32(w5, vtmp012x4567, 1); w5 += 3;
328 vtmp012x4567 = vld3q_lane_u32(w6, vtmp012x4567, 2); w6 += 3;
329 vtmp012x4567 = vld3q_lane_u32(w7, vtmp012x4567, 3); w7 += 3;
330 vtmp012x89AB = vld3q_lane_u32(w8, vtmp012x89AB, 0); w8 += 3;
331 vtmp012x89AB = vld3q_lane_u32(w9, vtmp012x89AB, 1); w9 += 3;
332 vtmp012x89AB = vld3q_lane_u32(w10, vtmp012x89AB, 2); w10 += 3;
333 vtmp012x89AB = vld3q_lane_u32(w11, vtmp012x89AB, 3); w11 += 3;
334 vtmp012xCDEF = vld3q_lane_u32(w12, vtmp012xCDEF, 0); w12 += 3;
335 vtmp012xCDEF = vld3q_lane_u32(w13, vtmp012xCDEF, 1); w13 += 3;
336 vtmp012xCDEF = vld3q_lane_u32(w14, vtmp012xCDEF, 2); w14 += 3;
337 vtmp012xCDEF = vld3q_lane_u32(w15, vtmp012xCDEF, 3); w15 += 3;
338 vst1q_u32(packed_weights, vtmp012x0123.val[0]); packed_weights += 4;
339 vst1q_u32(packed_weights, vtmp012x4567.val[0]); packed_weights += 4;
340 vst1q_u32(packed_weights, vtmp012x89AB.val[0]); packed_weights += 4;
341 vst1q_u32(packed_weights, vtmp012xCDEF.val[0]); packed_weights += 4;
342 vst1q_u32(packed_weights, vtmp012x0123.val[1]); packed_weights += 4;
343 vst1q_u32(packed_weights, vtmp012x4567.val[1]); packed_weights += 4;
344 vst1q_u32(packed_weights, vtmp012x89AB.val[1]); packed_weights += 4;
345 vst1q_u32(packed_weights, vtmp012xCDEF.val[1]); packed_weights += 4;
346 vst1q_u32(packed_weights, vtmp012x0123.val[2]); packed_weights += 4;
347 vst1q_u32(packed_weights, vtmp012x4567.val[2]); packed_weights += 4;
348 vst1q_u32(packed_weights, vtmp012x89AB.val[2]); packed_weights += 4;
349 vst1q_u32(packed_weights, vtmp012xCDEF.val[2]); packed_weights += 4;
350 break;
351 }
352 default:
353 XNN_UNREACHABLE;
354 }
355 }
356 packed_weights = (uint32_t*) ((uintptr_t) packed_weights + extra_bytes);
357 w0 = w15;
358 }
359
360 // NC remainder (1..15)
361 if XNN_UNLIKELY(n != 0) {
362 assert(n >= 1);
363 assert(n <= 15);
364 if XNN_LIKELY(bias != NULL) {
365 size_t nb = n;
366 do {
367 *packed_weights++ = *bias++;
368 } while (--nb != 0);
369 packed_weights += (16 - n);
370 } else {
371 const uint32x4_t vzero = vmovq_n_u32(0);
372 vst1q_u32(packed_weights, vzero); packed_weights += 4;
373 vst1q_u32(packed_weights, vzero); packed_weights += 4;
374 vst1q_u32(packed_weights, vzero); packed_weights += 4;
375 vst1q_u32(packed_weights, vzero); packed_weights += 4;
376 }
377
378 // NR remainder has less than 16 rows so last row is not loaded
379 const uint32_t* w1 = w0 + kc;
380 if XNN_UNPREDICTABLE(n < 2) {
381 w1 = w0;
382 }
383 const uint32_t* w2 = w1 + kc;
384 if XNN_UNPREDICTABLE(n <= 2) {
385 w2 = w1;
386 }
387 const uint32_t* w3 = w2 + kc;
388 if XNN_UNPREDICTABLE(n < 4) {
389 w3 = w2;
390 }
391 const uint32_t* w4 = w3 + kc;
392 if XNN_UNPREDICTABLE(n <= 4) {
393 w4 = w3;
394 }
395 const uint32_t* w5 = w4 + kc;
396 if XNN_UNPREDICTABLE(n < 6) {
397 w5 = w4;
398 }
399 const uint32_t* w6 = w5 + kc;
400 if XNN_UNPREDICTABLE(n <= 6) {
401 w6 = w5;
402 }
403 const uint32_t* w7 = w6 + kc;
404 if XNN_UNPREDICTABLE(n < 8) {
405 w7 = w6;
406 }
407 const uint32_t* w8 = w7 + kc;
408 if XNN_UNPREDICTABLE(n <= 8) {
409 w8 = w7;
410 }
411 const uint32_t* w9 = w8 + kc;
412 if XNN_UNPREDICTABLE(n < 10) {
413 w9 = w8;
414 }
415 const uint32_t* w10 = w9 + kc;
416 if XNN_UNPREDICTABLE(n <= 10) {
417 w10 = w9;
418 }
419 const uint32_t* w11 = w10 + kc;
420 if XNN_UNPREDICTABLE(n < 12) {
421 w11 = w10;
422 }
423 const uint32_t* w12 = w11 + kc;
424 if XNN_UNPREDICTABLE(n <= 12) {
425 w12 = w11;
426 }
427 const uint32_t* w13 = w12 + kc;
428 if XNN_UNPREDICTABLE(n < 14) {
429 w13 = w12;
430 }
431 const uint32_t* w14 = w13 + kc;
432 if XNN_UNPREDICTABLE(n <= 14) {
433 w14 = w13;
434 }
435
436 // KC main loop multiple of 8
437 size_t k = kc;
438 for (; k >= 8; k -= 8) {
439 vtmp0123x0123 = vld4q_lane_u32(w0, vtmp0123x0123, 0); w0 += 4;
440 vtmp0123x0123 = vld4q_lane_u32(w1, vtmp0123x0123, 1); w1 += 4;
441 vtmp0123x0123 = vld4q_lane_u32(w2, vtmp0123x0123, 2); w2 += 4;
442 vtmp0123x0123 = vld4q_lane_u32(w3, vtmp0123x0123, 3); w3 += 4;
443 vtmp0123x4567 = vld4q_lane_u32(w4, vtmp0123x4567, 0); w4 += 4;
444 vtmp0123x4567 = vld4q_lane_u32(w5, vtmp0123x4567, 1); w5 += 4;
445 vtmp0123x4567 = vld4q_lane_u32(w6, vtmp0123x4567, 2); w6 += 4;
446 vtmp0123x4567 = vld4q_lane_u32(w7, vtmp0123x4567, 3); w7 += 4;
447 vtmp0123x89AB = vld4q_lane_u32(w8, vtmp0123x89AB, 0); w8 += 4;
448 vtmp0123x89AB = vld4q_lane_u32(w9, vtmp0123x89AB, 1); w9 += 4;
449 vtmp0123x89AB = vld4q_lane_u32(w10, vtmp0123x89AB, 2); w10 += 4;
450 vtmp0123x89AB = vld4q_lane_u32(w11, vtmp0123x89AB, 3); w11 += 4;
451 vtmp0123xCDEF = vld4q_lane_u32(w12, vtmp0123xCDEF, 0); w12 += 4;
452 vtmp0123xCDEF = vld4q_lane_u32(w13, vtmp0123xCDEF, 1); w13 += 4;
453 vtmp0123xCDEF = vld4q_lane_u32(w14, vtmp0123xCDEF, 2); w14 += 4;
454 vtmp4567x0123 = vld4q_lane_u32(w0, vtmp4567x0123, 0); w0 += 4;
455 vtmp4567x0123 = vld4q_lane_u32(w1, vtmp4567x0123, 1); w1 += 4;
456 vtmp4567x0123 = vld4q_lane_u32(w2, vtmp4567x0123, 2); w2 += 4;
457 vtmp4567x0123 = vld4q_lane_u32(w3, vtmp4567x0123, 3); w3 += 4;
458 vtmp4567x4567 = vld4q_lane_u32(w4, vtmp4567x4567, 0); w4 += 4;
459 vtmp4567x4567 = vld4q_lane_u32(w5, vtmp4567x4567, 1); w5 += 4;
460 vtmp4567x4567 = vld4q_lane_u32(w6, vtmp4567x4567, 2); w6 += 4;
461 vtmp4567x4567 = vld4q_lane_u32(w7, vtmp4567x4567, 3); w7 += 4;
462 vtmp4567x89AB = vld4q_lane_u32(w8, vtmp4567x89AB, 0); w8 += 4;
463 vtmp4567x89AB = vld4q_lane_u32(w9, vtmp4567x89AB, 1); w9 += 4;
464 vtmp4567x89AB = vld4q_lane_u32(w10, vtmp4567x89AB, 2); w10 += 4;
465 vtmp4567x89AB = vld4q_lane_u32(w11, vtmp4567x89AB, 3); w11 += 4;
466 vtmp4567xCDEF = vld4q_lane_u32(w12, vtmp4567xCDEF, 0); w12 += 4;
467 vtmp4567xCDEF = vld4q_lane_u32(w13, vtmp4567xCDEF, 1); w13 += 4;
468 vtmp4567xCDEF = vld4q_lane_u32(w14, vtmp4567xCDEF, 2); w14 += 4;
469 vst1q_u32(packed_weights, vtmp0123x0123.val[0]); packed_weights += 4;
470 vst1q_u32(packed_weights, vtmp0123x4567.val[0]); packed_weights += 4;
471 vst1q_u32(packed_weights, vtmp0123x89AB.val[0]); packed_weights += 4;
472 vst1q_u32(packed_weights, vtmp0123xCDEF.val[0]); packed_weights += 4;
473 vst1q_u32(packed_weights, vtmp0123x0123.val[1]); packed_weights += 4;
474 vst1q_u32(packed_weights, vtmp0123x4567.val[1]); packed_weights += 4;
475 vst1q_u32(packed_weights, vtmp0123x89AB.val[1]); packed_weights += 4;
476 vst1q_u32(packed_weights, vtmp0123xCDEF.val[1]); packed_weights += 4;
477 vst1q_u32(packed_weights, vtmp0123x0123.val[2]); packed_weights += 4;
478 vst1q_u32(packed_weights, vtmp0123x4567.val[2]); packed_weights += 4;
479 vst1q_u32(packed_weights, vtmp0123x89AB.val[2]); packed_weights += 4;
480 vst1q_u32(packed_weights, vtmp0123xCDEF.val[2]); packed_weights += 4;
481 vst1q_u32(packed_weights, vtmp0123x0123.val[3]); packed_weights += 4;
482 vst1q_u32(packed_weights, vtmp0123x4567.val[3]); packed_weights += 4;
483 vst1q_u32(packed_weights, vtmp0123x89AB.val[3]); packed_weights += 4;
484 vst1q_u32(packed_weights, vtmp0123xCDEF.val[3]); packed_weights += 4;
485 vst1q_u32(packed_weights, vtmp4567x0123.val[0]); packed_weights += 4;
486 vst1q_u32(packed_weights, vtmp4567x4567.val[0]); packed_weights += 4;
487 vst1q_u32(packed_weights, vtmp4567x89AB.val[0]); packed_weights += 4;
488 vst1q_u32(packed_weights, vtmp4567xCDEF.val[0]); packed_weights += 4;
489 vst1q_u32(packed_weights, vtmp4567x0123.val[1]); packed_weights += 4;
490 vst1q_u32(packed_weights, vtmp4567x4567.val[1]); packed_weights += 4;
491 vst1q_u32(packed_weights, vtmp4567x89AB.val[1]); packed_weights += 4;
492 vst1q_u32(packed_weights, vtmp4567xCDEF.val[1]); packed_weights += 4;
493 vst1q_u32(packed_weights, vtmp4567x0123.val[2]); packed_weights += 4;
494 vst1q_u32(packed_weights, vtmp4567x4567.val[2]); packed_weights += 4;
495 vst1q_u32(packed_weights, vtmp4567x89AB.val[2]); packed_weights += 4;
496 vst1q_u32(packed_weights, vtmp4567xCDEF.val[2]); packed_weights += 4;
497 vst1q_u32(packed_weights, vtmp4567x0123.val[3]); packed_weights += 4;
498 vst1q_u32(packed_weights, vtmp4567x4567.val[3]); packed_weights += 4;
499 vst1q_u32(packed_weights, vtmp4567x89AB.val[3]); packed_weights += 4;
500 vst1q_u32(packed_weights, vtmp4567xCDEF.val[3]); packed_weights += 4;
501 }
502 // KC remainder multiple of 4
503 if (k >= 4) {
504 vtmp0123x0123 = vld4q_lane_u32(w0, vtmp0123x0123, 0); w0 += 4;
505 vtmp0123x0123 = vld4q_lane_u32(w1, vtmp0123x0123, 1); w1 += 4;
506 vtmp0123x0123 = vld4q_lane_u32(w2, vtmp0123x0123, 2); w2 += 4;
507 vtmp0123x0123 = vld4q_lane_u32(w3, vtmp0123x0123, 3); w3 += 4;
508 vtmp0123x4567 = vld4q_lane_u32(w4, vtmp0123x4567, 0); w4 += 4;
509 vtmp0123x4567 = vld4q_lane_u32(w5, vtmp0123x4567, 1); w5 += 4;
510 vtmp0123x4567 = vld4q_lane_u32(w6, vtmp0123x4567, 2); w6 += 4;
511 vtmp0123x4567 = vld4q_lane_u32(w7, vtmp0123x4567, 3); w7 += 4;
512 vtmp0123x89AB = vld4q_lane_u32(w8, vtmp0123x89AB, 0); w8 += 4;
513 vtmp0123x89AB = vld4q_lane_u32(w9, vtmp0123x89AB, 1); w9 += 4;
514 vtmp0123x89AB = vld4q_lane_u32(w10, vtmp0123x89AB, 2); w10 += 4;
515 vtmp0123x89AB = vld4q_lane_u32(w11, vtmp0123x89AB, 3); w11 += 4;
516 vtmp0123xCDEF = vld4q_lane_u32(w12, vtmp0123xCDEF, 0); w12 += 4;
517 vtmp0123xCDEF = vld4q_lane_u32(w13, vtmp0123xCDEF, 1); w13 += 4;
518 vtmp0123xCDEF = vld4q_lane_u32(w14, vtmp0123xCDEF, 2); w14 += 4;
519 vst1q_u32(packed_weights, vtmp0123x0123.val[0]); packed_weights += 4;
520 vst1q_u32(packed_weights, vtmp0123x4567.val[0]); packed_weights += 4;
521 vst1q_u32(packed_weights, vtmp0123x89AB.val[0]); packed_weights += 4;
522 vst1q_u32(packed_weights, vtmp0123xCDEF.val[0]); packed_weights += 4;
523 vst1q_u32(packed_weights, vtmp0123x0123.val[1]); packed_weights += 4;
524 vst1q_u32(packed_weights, vtmp0123x4567.val[1]); packed_weights += 4;
525 vst1q_u32(packed_weights, vtmp0123x89AB.val[1]); packed_weights += 4;
526 vst1q_u32(packed_weights, vtmp0123xCDEF.val[1]); packed_weights += 4;
527 vst1q_u32(packed_weights, vtmp0123x0123.val[2]); packed_weights += 4;
528 vst1q_u32(packed_weights, vtmp0123x4567.val[2]); packed_weights += 4;
529 vst1q_u32(packed_weights, vtmp0123x89AB.val[2]); packed_weights += 4;
530 vst1q_u32(packed_weights, vtmp0123xCDEF.val[2]); packed_weights += 4;
531 vst1q_u32(packed_weights, vtmp0123x0123.val[3]); packed_weights += 4;
532 vst1q_u32(packed_weights, vtmp0123x4567.val[3]); packed_weights += 4;
533 vst1q_u32(packed_weights, vtmp0123x89AB.val[3]); packed_weights += 4;
534 vst1q_u32(packed_weights, vtmp0123xCDEF.val[3]); packed_weights += 4;
535 k -= 4;
536 }
537
538 // KC remainder of 1..3
539 // Same as main loop but ld1, ld2 or ld3
540 if XNN_UNLIKELY(k != 0) {
541 assert(k >= 1);
542 assert(k <= 3);
543 switch (k) {
544 // KC remainder of 1
545 case 1:
546 {
547 uint32x4_t vtmp0x0123 = vdupq_n_u32(0);
548 uint32x4_t vtmp0x4567 = vdupq_n_u32(0);
549 uint32x4_t vtmp0x89AB = vdupq_n_u32(0);
550 uint32x4_t vtmp0xCDEF = vdupq_n_u32(0);
551 vtmp0x0123 = vld1q_lane_u32(w0, vtmp0x0123, 0);
552 vtmp0x0123 = vld1q_lane_u32(w1, vtmp0x0123, 1);
553 vtmp0x0123 = vld1q_lane_u32(w2, vtmp0x0123, 2);
554 vtmp0x0123 = vld1q_lane_u32(w3, vtmp0x0123, 3);
555 vtmp0x4567 = vld1q_lane_u32(w4, vtmp0x4567, 0);
556 vtmp0x4567 = vld1q_lane_u32(w5, vtmp0x4567, 1);
557 vtmp0x4567 = vld1q_lane_u32(w6, vtmp0x4567, 2);
558 vtmp0x4567 = vld1q_lane_u32(w7, vtmp0x4567, 3);
559 vtmp0x89AB = vld1q_lane_u32(w8, vtmp0x89AB, 0);
560 vtmp0x89AB = vld1q_lane_u32(w9, vtmp0x89AB, 1);
561 vtmp0x89AB = vld1q_lane_u32(w10, vtmp0x89AB, 2);
562 vtmp0x89AB = vld1q_lane_u32(w11, vtmp0x89AB, 3);
563 vtmp0xCDEF = vld1q_lane_u32(w12, vtmp0xCDEF, 0);
564 vtmp0xCDEF = vld1q_lane_u32(w13, vtmp0xCDEF, 1);
565 vtmp0xCDEF = vld1q_lane_u32(w14, vtmp0xCDEF, 2);
566 vst1q_u32(packed_weights, vtmp0x0123); packed_weights += 4;
567 vst1q_u32(packed_weights, vtmp0x4567); packed_weights += 4;
568 vst1q_u32(packed_weights, vtmp0x89AB); packed_weights += 4;
569 vst1q_u32(packed_weights, vtmp0xCDEF); packed_weights += 4;
570 break;
571 }
572 // KC remainder of 2
573 case 2:
574 {
575 uint32x4x2_t vtmp01x0123;
576 vtmp01x0123.val[0] = vdupq_n_u32(0);
577 vtmp01x0123.val[1] = vdupq_n_u32(0);
578 uint32x4x2_t vtmp01x4567;
579 vtmp01x4567.val[0] = vdupq_n_u32(0);
580 vtmp01x4567.val[1] = vdupq_n_u32(0);
581 uint32x4x2_t vtmp01x89AB;
582 vtmp01x89AB.val[0] = vdupq_n_u32(0);
583 vtmp01x89AB.val[1] = vdupq_n_u32(0);
584 uint32x4x2_t vtmp01xCDEF;
585 vtmp01xCDEF.val[0] = vdupq_n_u32(0);
586 vtmp01xCDEF.val[1] = vdupq_n_u32(0);
587 vtmp01x0123 = vld2q_lane_u32(w0, vtmp01x0123, 0);
588 vtmp01x0123 = vld2q_lane_u32(w1, vtmp01x0123, 1);
589 vtmp01x0123 = vld2q_lane_u32(w2, vtmp01x0123, 2);
590 vtmp01x0123 = vld2q_lane_u32(w3, vtmp01x0123, 3);
591 vtmp01x4567 = vld2q_lane_u32(w4, vtmp01x4567, 0);
592 vtmp01x4567 = vld2q_lane_u32(w5, vtmp01x4567, 1);
593 vtmp01x4567 = vld2q_lane_u32(w6, vtmp01x4567, 2);
594 vtmp01x4567 = vld2q_lane_u32(w7, vtmp01x4567, 3);
595 vtmp01x89AB = vld2q_lane_u32(w8, vtmp01x89AB, 0);
596 vtmp01x89AB = vld2q_lane_u32(w9, vtmp01x89AB, 1);
597 vtmp01x89AB = vld2q_lane_u32(w10, vtmp01x89AB, 2);
598 vtmp01x89AB = vld2q_lane_u32(w11, vtmp01x89AB, 3);
599 vtmp01xCDEF = vld2q_lane_u32(w12, vtmp01xCDEF, 0);
600 vtmp01xCDEF = vld2q_lane_u32(w13, vtmp01xCDEF, 1);
601 vtmp01xCDEF = vld2q_lane_u32(w14, vtmp01xCDEF, 2);
602 vst1q_u32(packed_weights, vtmp01x0123.val[0]); packed_weights += 4;
603 vst1q_u32(packed_weights, vtmp01x4567.val[0]); packed_weights += 4;
604 vst1q_u32(packed_weights, vtmp01x89AB.val[0]); packed_weights += 4;
605 vst1q_u32(packed_weights, vtmp01xCDEF.val[0]); packed_weights += 4;
606 vst1q_u32(packed_weights, vtmp01x0123.val[1]); packed_weights += 4;
607 vst1q_u32(packed_weights, vtmp01x4567.val[1]); packed_weights += 4;
608 vst1q_u32(packed_weights, vtmp01x89AB.val[1]); packed_weights += 4;
609 vst1q_u32(packed_weights, vtmp01xCDEF.val[1]); packed_weights += 4;
610 break;
611 }
612 // KC remainder of 3
613 case 3:
614 {
615 uint32x4x3_t vtmp012x0123;
616 vtmp012x0123.val[0] = vdupq_n_u32(0);
617 vtmp012x0123.val[1] = vdupq_n_u32(0);
618 vtmp012x0123.val[2] = vdupq_n_u32(0);
619 uint32x4x3_t vtmp012x4567;
620 vtmp012x4567.val[0] = vdupq_n_u32(0);
621 vtmp012x4567.val[1] = vdupq_n_u32(0);
622 vtmp012x4567.val[2] = vdupq_n_u32(0);
623 uint32x4x3_t vtmp012x89AB;
624 vtmp012x89AB.val[0] = vdupq_n_u32(0);
625 vtmp012x89AB.val[1] = vdupq_n_u32(0);
626 vtmp012x89AB.val[2] = vdupq_n_u32(0);
627 uint32x4x3_t vtmp012xCDEF;
628 vtmp012xCDEF.val[0] = vdupq_n_u32(0);
629 vtmp012xCDEF.val[1] = vdupq_n_u32(0);
630 vtmp012xCDEF.val[2] = vdupq_n_u32(0);
631 vtmp012x0123 = vld3q_lane_u32(w0, vtmp012x0123, 0);
632 vtmp012x0123 = vld3q_lane_u32(w1, vtmp012x0123, 1);
633 vtmp012x0123 = vld3q_lane_u32(w2, vtmp012x0123, 2);
634 vtmp012x0123 = vld3q_lane_u32(w3, vtmp012x0123, 3);
635 vtmp012x4567 = vld3q_lane_u32(w4, vtmp012x4567, 0);
636 vtmp012x4567 = vld3q_lane_u32(w5, vtmp012x4567, 1);
637 vtmp012x4567 = vld3q_lane_u32(w6, vtmp012x4567, 2);
638 vtmp012x4567 = vld3q_lane_u32(w7, vtmp012x4567, 3);
639 vtmp012x89AB = vld3q_lane_u32(w8, vtmp012x89AB, 0);
640 vtmp012x89AB = vld3q_lane_u32(w9, vtmp012x89AB, 1);
641 vtmp012x89AB = vld3q_lane_u32(w10, vtmp012x89AB, 2);
642 vtmp012x89AB = vld3q_lane_u32(w11, vtmp012x89AB, 3);
643 vtmp012xCDEF = vld3q_lane_u32(w12, vtmp012xCDEF, 0);
644 vtmp012xCDEF = vld3q_lane_u32(w13, vtmp012xCDEF, 1);
645 vtmp012xCDEF = vld3q_lane_u32(w14, vtmp012xCDEF, 2);
646 vst1q_u32(packed_weights, vtmp012x0123.val[0]); packed_weights += 4;
647 vst1q_u32(packed_weights, vtmp012x4567.val[0]); packed_weights += 4;
648 vst1q_u32(packed_weights, vtmp012x89AB.val[0]); packed_weights += 4;
649 vst1q_u32(packed_weights, vtmp012xCDEF.val[0]); packed_weights += 4;
650 vst1q_u32(packed_weights, vtmp012x0123.val[1]); packed_weights += 4;
651 vst1q_u32(packed_weights, vtmp012x4567.val[1]); packed_weights += 4;
652 vst1q_u32(packed_weights, vtmp012x89AB.val[1]); packed_weights += 4;
653 vst1q_u32(packed_weights, vtmp012xCDEF.val[1]); packed_weights += 4;
654 vst1q_u32(packed_weights, vtmp012x0123.val[2]); packed_weights += 4;
655 vst1q_u32(packed_weights, vtmp012x4567.val[2]); packed_weights += 4;
656 vst1q_u32(packed_weights, vtmp012x89AB.val[2]); packed_weights += 4;
657 vst1q_u32(packed_weights, vtmp012xCDEF.val[2]); packed_weights += 4;
658 break;
659 }
660 default:
661 XNN_UNREACHABLE;
662 }
663 }
664 packed_weights = (uint32_t*) ((uintptr_t) packed_weights + extra_bytes);
665 }
666
667 weights += nc * kc;
668 } while (--g != 0);
669 }