Debian Codegraph @

src/x32-packw/gen/x32-packw-x12-gemm-goi-neon-ld4lane-u4.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/neon.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include <arm_neon.h>
16
17 #include "xnnpack/packw.h"
18
19
20 void xnn_x32_packw_gemm_goi_ukernel_x12__neon_ld4lane_u4(
21 size_t g,
22 size_t nc,
23 size_t kc,
24 size_t nr,
25 size_t kr,
26 size_t sr,
27 const uint32_t* weights,
28 const uint32_t* bias,
29 const void* scale,
30 uint32_t* packed_weights,
31 size_t extra_bytes,
32 const void* params)
33 {
34 assert(g != 0);
35 assert(nc != 0);
36 assert(kc != 0);
37 assert(nr == 12);
38 assert(kr == 1);
39 assert(sr == 1);
40 assert(weights != NULL);
41 assert(packed_weights != NULL);
42 uint32x4x4_t vtmp0123x0123;
43 vtmp0123x0123.val[0] = vdupq_n_u32(0);
44 vtmp0123x0123.val[1] = vdupq_n_u32(0);
45 vtmp0123x0123.val[2] = vdupq_n_u32(0);
46 vtmp0123x0123.val[3] = vdupq_n_u32(0);
47 uint32x4x4_t vtmp0123x4567;
48 vtmp0123x4567.val[0] = vdupq_n_u32(0);
49 vtmp0123x4567.val[1] = vdupq_n_u32(0);
50 vtmp0123x4567.val[2] = vdupq_n_u32(0);
51 vtmp0123x4567.val[3] = vdupq_n_u32(0);
52 uint32x4x4_t vtmp0123x89AB;
53 vtmp0123x89AB.val[0] = vdupq_n_u32(0);
54 vtmp0123x89AB.val[1] = vdupq_n_u32(0);
55 vtmp0123x89AB.val[2] = vdupq_n_u32(0);
56 vtmp0123x89AB.val[3] = vdupq_n_u32(0);
57
58 do {
59 // NC main loop multiple of 12
60 const uint32_t* w0 = weights;
61 size_t n = nc;
62
63 for (; n >= 12; n -= 12) {
64 if XNN_LIKELY(bias != NULL) {
65 uint32x4_t vb0 = vld1q_u32(bias); bias += 4;
66 uint32x4_t vb4 = vld1q_u32(bias); bias += 4;
67 uint32x4_t vb8 = vld1q_u32(bias); bias += 4;
68 vst1q_u32(packed_weights, vb0); packed_weights += 4;
69 vst1q_u32(packed_weights, vb4); packed_weights += 4;
70 vst1q_u32(packed_weights, vb8); packed_weights += 4;
71 } else {
72 const uint32x4_t vzero = vmovq_n_u32(0);
73 vst1q_u32(packed_weights, vzero); packed_weights += 4;
74 vst1q_u32(packed_weights, vzero); packed_weights += 4;
75 vst1q_u32(packed_weights, vzero); packed_weights += 4;
76 }
77
78 const uint32_t* w1 = w0 + kc;
79 const uint32_t* w2 = w1 + kc;
80 const uint32_t* w3 = w2 + kc;
81 const uint32_t* w4 = w3 + kc;
82 const uint32_t* w5 = w4 + kc;
83 const uint32_t* w6 = w5 + kc;
84 const uint32_t* w7 = w6 + kc;
85 const uint32_t* w8 = w7 + kc;
86 const uint32_t* w9 = w8 + kc;
87 const uint32_t* w10 = w9 + kc;
88 const uint32_t* w11 = w10 + kc;
89
90 // KC main loop multiple of 4
91 size_t k = kc;
92 for (; k >= 4; k -= 4) {
93 vtmp0123x0123 = vld4q_lane_u32(w0, vtmp0123x0123, 0); w0 += 4;
94 vtmp0123x0123 = vld4q_lane_u32(w1, vtmp0123x0123, 1); w1 += 4;
95 vtmp0123x0123 = vld4q_lane_u32(w2, vtmp0123x0123, 2); w2 += 4;
96 vtmp0123x0123 = vld4q_lane_u32(w3, vtmp0123x0123, 3); w3 += 4;
97 vtmp0123x4567 = vld4q_lane_u32(w4, vtmp0123x4567, 0); w4 += 4;
98 vtmp0123x4567 = vld4q_lane_u32(w5, vtmp0123x4567, 1); w5 += 4;
99 vtmp0123x4567 = vld4q_lane_u32(w6, vtmp0123x4567, 2); w6 += 4;
100 vtmp0123x4567 = vld4q_lane_u32(w7, vtmp0123x4567, 3); w7 += 4;
101 vtmp0123x89AB = vld4q_lane_u32(w8, vtmp0123x89AB, 0); w8 += 4;
102 vtmp0123x89AB = vld4q_lane_u32(w9, vtmp0123x89AB, 1); w9 += 4;
103 vtmp0123x89AB = vld4q_lane_u32(w10, vtmp0123x89AB, 2); w10 += 4;
104 vtmp0123x89AB = vld4q_lane_u32(w11, vtmp0123x89AB, 3); w11 += 4;
105 vst1q_u32(packed_weights, vtmp0123x0123.val[0]); packed_weights += 4;
106 vst1q_u32(packed_weights, vtmp0123x4567.val[0]); packed_weights += 4;
107 vst1q_u32(packed_weights, vtmp0123x89AB.val[0]); packed_weights += 4;
108 vst1q_u32(packed_weights, vtmp0123x0123.val[1]); packed_weights += 4;
109 vst1q_u32(packed_weights, vtmp0123x4567.val[1]); packed_weights += 4;
110 vst1q_u32(packed_weights, vtmp0123x89AB.val[1]); packed_weights += 4;
111 vst1q_u32(packed_weights, vtmp0123x0123.val[2]); packed_weights += 4;
112 vst1q_u32(packed_weights, vtmp0123x4567.val[2]); packed_weights += 4;
113 vst1q_u32(packed_weights, vtmp0123x89AB.val[2]); packed_weights += 4;
114 vst1q_u32(packed_weights, vtmp0123x0123.val[3]); packed_weights += 4;
115 vst1q_u32(packed_weights, vtmp0123x4567.val[3]); packed_weights += 4;
116 vst1q_u32(packed_weights, vtmp0123x89AB.val[3]); packed_weights += 4;
117 }
118
119 // KC remainder of 1..3
120 // Same as main loop but ld1, ld2 or ld3
121 if XNN_UNLIKELY(k != 0) {
122 assert(k >= 1);
123 assert(k <= 3);
124 switch (k) {
125 // KC remainder of 1
126 case 1:
127 {
128 uint32x4_t vtmp0x0123 = vdupq_n_u32(0);
129 uint32x4_t vtmp0x4567 = vdupq_n_u32(0);
130 uint32x4_t vtmp0x89AB = vdupq_n_u32(0);
131 vtmp0x0123 = vld1q_lane_u32(w0, vtmp0x0123, 0); w0 += 1;
132 vtmp0x0123 = vld1q_lane_u32(w1, vtmp0x0123, 1); w1 += 1;
133 vtmp0x0123 = vld1q_lane_u32(w2, vtmp0x0123, 2); w2 += 1;
134 vtmp0x0123 = vld1q_lane_u32(w3, vtmp0x0123, 3); w3 += 1;
135 vtmp0x4567 = vld1q_lane_u32(w4, vtmp0x4567, 0); w4 += 1;
136 vtmp0x4567 = vld1q_lane_u32(w5, vtmp0x4567, 1); w5 += 1;
137 vtmp0x4567 = vld1q_lane_u32(w6, vtmp0x4567, 2); w6 += 1;
138 vtmp0x4567 = vld1q_lane_u32(w7, vtmp0x4567, 3); w7 += 1;
139 vtmp0x89AB = vld1q_lane_u32(w8, vtmp0x89AB, 0); w8 += 1;
140 vtmp0x89AB = vld1q_lane_u32(w9, vtmp0x89AB, 1); w9 += 1;
141 vtmp0x89AB = vld1q_lane_u32(w10, vtmp0x89AB, 2); w10 += 1;
142 vtmp0x89AB = vld1q_lane_u32(w11, vtmp0x89AB, 3); w11 += 1;
143 vst1q_u32(packed_weights, vtmp0x0123); packed_weights += 4;
144 vst1q_u32(packed_weights, vtmp0x4567); packed_weights += 4;
145 vst1q_u32(packed_weights, vtmp0x89AB); packed_weights += 4;
146 break;
147 }
148 // KC remainder of 2
149 case 2:
150 {
151 uint32x4x2_t vtmp01x0123;
152 vtmp01x0123.val[0] = vdupq_n_u32(0);
153 vtmp01x0123.val[1] = vdupq_n_u32(0);
154 uint32x4x2_t vtmp01x4567;
155 vtmp01x4567.val[0] = vdupq_n_u32(0);
156 vtmp01x4567.val[1] = vdupq_n_u32(0);
157 uint32x4x2_t vtmp01x89AB;
158 vtmp01x89AB.val[0] = vdupq_n_u32(0);
159 vtmp01x89AB.val[1] = vdupq_n_u32(0);
160 vtmp01x0123 = vld2q_lane_u32(w0, vtmp01x0123, 0); w0 += 2;
161 vtmp01x0123 = vld2q_lane_u32(w1, vtmp01x0123, 1); w1 += 2;
162 vtmp01x0123 = vld2q_lane_u32(w2, vtmp01x0123, 2); w2 += 2;
163 vtmp01x0123 = vld2q_lane_u32(w3, vtmp01x0123, 3); w3 += 2;
164 vtmp01x4567 = vld2q_lane_u32(w4, vtmp01x4567, 0); w4 += 2;
165 vtmp01x4567 = vld2q_lane_u32(w5, vtmp01x4567, 1); w5 += 2;
166 vtmp01x4567 = vld2q_lane_u32(w6, vtmp01x4567, 2); w6 += 2;
167 vtmp01x4567 = vld2q_lane_u32(w7, vtmp01x4567, 3); w7 += 2;
168 vtmp01x89AB = vld2q_lane_u32(w8, vtmp01x89AB, 0); w8 += 2;
169 vtmp01x89AB = vld2q_lane_u32(w9, vtmp01x89AB, 1); w9 += 2;
170 vtmp01x89AB = vld2q_lane_u32(w10, vtmp01x89AB, 2); w10 += 2;
171 vtmp01x89AB = vld2q_lane_u32(w11, vtmp01x89AB, 3); w11 += 2;
172 vst1q_u32(packed_weights, vtmp01x0123.val[0]); packed_weights += 4;
173 vst1q_u32(packed_weights, vtmp01x4567.val[0]); packed_weights += 4;
174 vst1q_u32(packed_weights, vtmp01x89AB.val[0]); packed_weights += 4;
175 vst1q_u32(packed_weights, vtmp01x0123.val[1]); packed_weights += 4;
176 vst1q_u32(packed_weights, vtmp01x4567.val[1]); packed_weights += 4;
177 vst1q_u32(packed_weights, vtmp01x89AB.val[1]); packed_weights += 4;
178 break;
179 }
180 // KC remainder of 3
181 case 3:
182 {
183 uint32x4x3_t vtmp012x0123;
184 vtmp012x0123.val[0] = vdupq_n_u32(0);
185 vtmp012x0123.val[1] = vdupq_n_u32(0);
186 vtmp012x0123.val[2] = vdupq_n_u32(0);
187 uint32x4x3_t vtmp012x4567;
188 vtmp012x4567.val[0] = vdupq_n_u32(0);
189 vtmp012x4567.val[1] = vdupq_n_u32(0);
190 vtmp012x4567.val[2] = vdupq_n_u32(0);
191 uint32x4x3_t vtmp012x89AB;
192 vtmp012x89AB.val[0] = vdupq_n_u32(0);
193 vtmp012x89AB.val[1] = vdupq_n_u32(0);
194 vtmp012x89AB.val[2] = vdupq_n_u32(0);
195 vtmp012x0123 = vld3q_lane_u32(w0, vtmp012x0123, 0); w0 += 3;
196 vtmp012x0123 = vld3q_lane_u32(w1, vtmp012x0123, 1); w1 += 3;
197 vtmp012x0123 = vld3q_lane_u32(w2, vtmp012x0123, 2); w2 += 3;
198 vtmp012x0123 = vld3q_lane_u32(w3, vtmp012x0123, 3); w3 += 3;
199 vtmp012x4567 = vld3q_lane_u32(w4, vtmp012x4567, 0); w4 += 3;
200 vtmp012x4567 = vld3q_lane_u32(w5, vtmp012x4567, 1); w5 += 3;
201 vtmp012x4567 = vld3q_lane_u32(w6, vtmp012x4567, 2); w6 += 3;
202 vtmp012x4567 = vld3q_lane_u32(w7, vtmp012x4567, 3); w7 += 3;
203 vtmp012x89AB = vld3q_lane_u32(w8, vtmp012x89AB, 0); w8 += 3;
204 vtmp012x89AB = vld3q_lane_u32(w9, vtmp012x89AB, 1); w9 += 3;
205 vtmp012x89AB = vld3q_lane_u32(w10, vtmp012x89AB, 2); w10 += 3;
206 vtmp012x89AB = vld3q_lane_u32(w11, vtmp012x89AB, 3); w11 += 3;
207 vst1q_u32(packed_weights, vtmp012x0123.val[0]); packed_weights += 4;
208 vst1q_u32(packed_weights, vtmp012x4567.val[0]); packed_weights += 4;
209 vst1q_u32(packed_weights, vtmp012x89AB.val[0]); packed_weights += 4;
210 vst1q_u32(packed_weights, vtmp012x0123.val[1]); packed_weights += 4;
211 vst1q_u32(packed_weights, vtmp012x4567.val[1]); packed_weights += 4;
212 vst1q_u32(packed_weights, vtmp012x89AB.val[1]); packed_weights += 4;
213 vst1q_u32(packed_weights, vtmp012x0123.val[2]); packed_weights += 4;
214 vst1q_u32(packed_weights, vtmp012x4567.val[2]); packed_weights += 4;
215 vst1q_u32(packed_weights, vtmp012x89AB.val[2]); packed_weights += 4;
216 break;
217 }
218 default:
219 XNN_UNREACHABLE;
220 }
221 }
222 packed_weights = (uint32_t*) ((uintptr_t) packed_weights + extra_bytes);
223 w0 = w11;
224 }
225
226 // NC remainder (1..11)
227 if XNN_UNLIKELY(n != 0) {
228 assert(n >= 1);
229 assert(n <= 11);
230 if XNN_LIKELY(bias != NULL) {
231 size_t nb = n;
232 do {
233 *packed_weights++ = *bias++;
234 } while (--nb != 0);
235 packed_weights += (12 - n);
236 } else {
237 const uint32x4_t vzero = vmovq_n_u32(0);
238 vst1q_u32(packed_weights, vzero); packed_weights += 4;
239 vst1q_u32(packed_weights, vzero); packed_weights += 4;
240 vst1q_u32(packed_weights, vzero); packed_weights += 4;
241 }
242
243 // NR remainder has less than 12 rows so last row is not loaded
244 const uint32_t* w1 = w0 + kc;
245 if XNN_UNPREDICTABLE(n < 2) {
246 w1 = w0;
247 }
248 const uint32_t* w2 = w1 + kc;
249 if XNN_UNPREDICTABLE(n <= 2) {
250 w2 = w1;
251 }
252 const uint32_t* w3 = w2 + kc;
253 if XNN_UNPREDICTABLE(n < 4) {
254 w3 = w2;
255 }
256 const uint32_t* w4 = w3 + kc;
257 if XNN_UNPREDICTABLE(n <= 4) {
258 w4 = w3;
259 }
260 const uint32_t* w5 = w4 + kc;
261 if XNN_UNPREDICTABLE(n < 6) {
262 w5 = w4;
263 }
264 const uint32_t* w6 = w5 + kc;
265 if XNN_UNPREDICTABLE(n <= 6) {
266 w6 = w5;
267 }
268 const uint32_t* w7 = w6 + kc;
269 if XNN_UNPREDICTABLE(n < 8) {
270 w7 = w6;
271 }
272 const uint32_t* w8 = w7 + kc;
273 if XNN_UNPREDICTABLE(n <= 8) {
274 w8 = w7;
275 }
276 const uint32_t* w9 = w8 + kc;
277 if XNN_UNPREDICTABLE(n < 10) {
278 w9 = w8;
279 }
280 const uint32_t* w10 = w9 + kc;
281 if XNN_UNPREDICTABLE(n <= 10) {
282 w10 = w9;
283 }
284
285 // KC main loop multiple of 4
286 size_t k = kc;
287 for (; k >= 4; k -= 4) {
288 vtmp0123x0123 = vld4q_lane_u32(w0, vtmp0123x0123, 0); w0 += 4;
289 vtmp0123x0123 = vld4q_lane_u32(w1, vtmp0123x0123, 1); w1 += 4;
290 vtmp0123x0123 = vld4q_lane_u32(w2, vtmp0123x0123, 2); w2 += 4;
291 vtmp0123x0123 = vld4q_lane_u32(w3, vtmp0123x0123, 3); w3 += 4;
292 vtmp0123x4567 = vld4q_lane_u32(w4, vtmp0123x4567, 0); w4 += 4;
293 vtmp0123x4567 = vld4q_lane_u32(w5, vtmp0123x4567, 1); w5 += 4;
294 vtmp0123x4567 = vld4q_lane_u32(w6, vtmp0123x4567, 2); w6 += 4;
295 vtmp0123x4567 = vld4q_lane_u32(w7, vtmp0123x4567, 3); w7 += 4;
296 vtmp0123x89AB = vld4q_lane_u32(w8, vtmp0123x89AB, 0); w8 += 4;
297 vtmp0123x89AB = vld4q_lane_u32(w9, vtmp0123x89AB, 1); w9 += 4;
298 vtmp0123x89AB = vld4q_lane_u32(w10, vtmp0123x89AB, 2); w10 += 4;
299 vst1q_u32(packed_weights, vtmp0123x0123.val[0]); packed_weights += 4;
300 vst1q_u32(packed_weights, vtmp0123x4567.val[0]); packed_weights += 4;
301 vst1q_u32(packed_weights, vtmp0123x89AB.val[0]); packed_weights += 4;
302 vst1q_u32(packed_weights, vtmp0123x0123.val[1]); packed_weights += 4;
303 vst1q_u32(packed_weights, vtmp0123x4567.val[1]); packed_weights += 4;
304 vst1q_u32(packed_weights, vtmp0123x89AB.val[1]); packed_weights += 4;
305 vst1q_u32(packed_weights, vtmp0123x0123.val[2]); packed_weights += 4;
306 vst1q_u32(packed_weights, vtmp0123x4567.val[2]); packed_weights += 4;
307 vst1q_u32(packed_weights, vtmp0123x89AB.val[2]); packed_weights += 4;
308 vst1q_u32(packed_weights, vtmp0123x0123.val[3]); packed_weights += 4;
309 vst1q_u32(packed_weights, vtmp0123x4567.val[3]); packed_weights += 4;
310 vst1q_u32(packed_weights, vtmp0123x89AB.val[3]); packed_weights += 4;
311 }
312
313 // KC remainder of 1..3
314 // Same as main loop but ld1, ld2 or ld3
315 if XNN_UNLIKELY(k != 0) {
316 assert(k >= 1);
317 assert(k <= 3);
318 switch (k) {
319 // KC remainder of 1
320 case 1:
321 {
322 uint32x4_t vtmp0x0123 = vdupq_n_u32(0);
323 uint32x4_t vtmp0x4567 = vdupq_n_u32(0);
324 uint32x4_t vtmp0x89AB = vdupq_n_u32(0);
325 vtmp0x0123 = vld1q_lane_u32(w0, vtmp0x0123, 0);
326 vtmp0x0123 = vld1q_lane_u32(w1, vtmp0x0123, 1);
327 vtmp0x0123 = vld1q_lane_u32(w2, vtmp0x0123, 2);
328 vtmp0x0123 = vld1q_lane_u32(w3, vtmp0x0123, 3);
329 vtmp0x4567 = vld1q_lane_u32(w4, vtmp0x4567, 0);
330 vtmp0x4567 = vld1q_lane_u32(w5, vtmp0x4567, 1);
331 vtmp0x4567 = vld1q_lane_u32(w6, vtmp0x4567, 2);
332 vtmp0x4567 = vld1q_lane_u32(w7, vtmp0x4567, 3);
333 vtmp0x89AB = vld1q_lane_u32(w8, vtmp0x89AB, 0);
334 vtmp0x89AB = vld1q_lane_u32(w9, vtmp0x89AB, 1);
335 vtmp0x89AB = vld1q_lane_u32(w10, vtmp0x89AB, 2);
336 vst1q_u32(packed_weights, vtmp0x0123); packed_weights += 4;
337 vst1q_u32(packed_weights, vtmp0x4567); packed_weights += 4;
338 vst1q_u32(packed_weights, vtmp0x89AB); packed_weights += 4;
339 break;
340 }
341 // KC remainder of 2
342 case 2:
343 {
344 uint32x4x2_t vtmp01x0123;
345 vtmp01x0123.val[0] = vdupq_n_u32(0);
346 vtmp01x0123.val[1] = vdupq_n_u32(0);
347 uint32x4x2_t vtmp01x4567;
348 vtmp01x4567.val[0] = vdupq_n_u32(0);
349 vtmp01x4567.val[1] = vdupq_n_u32(0);
350 uint32x4x2_t vtmp01x89AB;
351 vtmp01x89AB.val[0] = vdupq_n_u32(0);
352 vtmp01x89AB.val[1] = vdupq_n_u32(0);
353 vtmp01x0123 = vld2q_lane_u32(w0, vtmp01x0123, 0);
354 vtmp01x0123 = vld2q_lane_u32(w1, vtmp01x0123, 1);
355 vtmp01x0123 = vld2q_lane_u32(w2, vtmp01x0123, 2);
356 vtmp01x0123 = vld2q_lane_u32(w3, vtmp01x0123, 3);
357 vtmp01x4567 = vld2q_lane_u32(w4, vtmp01x4567, 0);
358 vtmp01x4567 = vld2q_lane_u32(w5, vtmp01x4567, 1);
359 vtmp01x4567 = vld2q_lane_u32(w6, vtmp01x4567, 2);
360 vtmp01x4567 = vld2q_lane_u32(w7, vtmp01x4567, 3);
361 vtmp01x89AB = vld2q_lane_u32(w8, vtmp01x89AB, 0);
362 vtmp01x89AB = vld2q_lane_u32(w9, vtmp01x89AB, 1);
363 vtmp01x89AB = vld2q_lane_u32(w10, vtmp01x89AB, 2);
364 vst1q_u32(packed_weights, vtmp01x0123.val[0]); packed_weights += 4;
365 vst1q_u32(packed_weights, vtmp01x4567.val[0]); packed_weights += 4;
366 vst1q_u32(packed_weights, vtmp01x89AB.val[0]); packed_weights += 4;
367 vst1q_u32(packed_weights, vtmp01x0123.val[1]); packed_weights += 4;
368 vst1q_u32(packed_weights, vtmp01x4567.val[1]); packed_weights += 4;
369 vst1q_u32(packed_weights, vtmp01x89AB.val[1]); packed_weights += 4;
370 break;
371 }
372 // KC remainder of 3
373 case 3:
374 {
375 uint32x4x3_t vtmp012x0123;
376 vtmp012x0123.val[0] = vdupq_n_u32(0);
377 vtmp012x0123.val[1] = vdupq_n_u32(0);
378 vtmp012x0123.val[2] = vdupq_n_u32(0);
379 uint32x4x3_t vtmp012x4567;
380 vtmp012x4567.val[0] = vdupq_n_u32(0);
381 vtmp012x4567.val[1] = vdupq_n_u32(0);
382 vtmp012x4567.val[2] = vdupq_n_u32(0);
383 uint32x4x3_t vtmp012x89AB;
384 vtmp012x89AB.val[0] = vdupq_n_u32(0);
385 vtmp012x89AB.val[1] = vdupq_n_u32(0);
386 vtmp012x89AB.val[2] = vdupq_n_u32(0);
387 vtmp012x0123 = vld3q_lane_u32(w0, vtmp012x0123, 0);
388 vtmp012x0123 = vld3q_lane_u32(w1, vtmp012x0123, 1);
389 vtmp012x0123 = vld3q_lane_u32(w2, vtmp012x0123, 2);
390 vtmp012x0123 = vld3q_lane_u32(w3, vtmp012x0123, 3);
391 vtmp012x4567 = vld3q_lane_u32(w4, vtmp012x4567, 0);
392 vtmp012x4567 = vld3q_lane_u32(w5, vtmp012x4567, 1);
393 vtmp012x4567 = vld3q_lane_u32(w6, vtmp012x4567, 2);
394 vtmp012x4567 = vld3q_lane_u32(w7, vtmp012x4567, 3);
395 vtmp012x89AB = vld3q_lane_u32(w8, vtmp012x89AB, 0);
396 vtmp012x89AB = vld3q_lane_u32(w9, vtmp012x89AB, 1);
397 vtmp012x89AB = vld3q_lane_u32(w10, vtmp012x89AB, 2);
398 vst1q_u32(packed_weights, vtmp012x0123.val[0]); packed_weights += 4;
399 vst1q_u32(packed_weights, vtmp012x4567.val[0]); packed_weights += 4;
400 vst1q_u32(packed_weights, vtmp012x89AB.val[0]); packed_weights += 4;
401 vst1q_u32(packed_weights, vtmp012x0123.val[1]); packed_weights += 4;
402 vst1q_u32(packed_weights, vtmp012x4567.val[1]); packed_weights += 4;
403 vst1q_u32(packed_weights, vtmp012x89AB.val[1]); packed_weights += 4;
404 vst1q_u32(packed_weights, vtmp012x0123.val[2]); packed_weights += 4;
405 vst1q_u32(packed_weights, vtmp012x4567.val[2]); packed_weights += 4;
406 vst1q_u32(packed_weights, vtmp012x89AB.val[2]); packed_weights += 4;
407 break;
408 }
409 default:
410 XNN_UNREACHABLE;
411 }
412 }
413 packed_weights = (uint32_t*) ((uintptr_t) packed_weights + extra_bytes);
414 }
415
416 weights += nc * kc;
417 } while (--g != 0);
418 }