Debian Codegraph @

src/x32-packw/gen/x32-packw-x8-gemm-goi-neon-ld4lane-u8.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/neon.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include <arm_neon.h>
16
17 #include "xnnpack/packw.h"
18
19
20 void xnn_x32_packw_gemm_goi_ukernel_x8__neon_ld4lane_u8(
21 size_t g,
22 size_t nc,
23 size_t kc,
24 size_t nr,
25 size_t kr,
26 size_t sr,
27 const uint32_t* weights,
28 const uint32_t* bias,
29 const void* scale,
30 uint32_t* packed_weights,
31 size_t extra_bytes,
32 const void* params)
33 {
34 assert(g != 0);
35 assert(nc != 0);
36 assert(kc != 0);
37 assert(nr == 8);
38 assert(kr == 1);
39 assert(sr == 1);
40 assert(weights != NULL);
41 assert(packed_weights != NULL);
42 uint32x4x4_t vtmp0123x0123;
43 vtmp0123x0123.val[0] = vdupq_n_u32(0);
44 vtmp0123x0123.val[1] = vdupq_n_u32(0);
45 vtmp0123x0123.val[2] = vdupq_n_u32(0);
46 vtmp0123x0123.val[3] = vdupq_n_u32(0);
47 uint32x4x4_t vtmp4567x0123;
48 vtmp4567x0123.val[0] = vdupq_n_u32(0);
49 vtmp4567x0123.val[1] = vdupq_n_u32(0);
50 vtmp4567x0123.val[2] = vdupq_n_u32(0);
51 vtmp4567x0123.val[3] = vdupq_n_u32(0);
52 uint32x4x4_t vtmp0123x4567;
53 vtmp0123x4567.val[0] = vdupq_n_u32(0);
54 vtmp0123x4567.val[1] = vdupq_n_u32(0);
55 vtmp0123x4567.val[2] = vdupq_n_u32(0);
56 vtmp0123x4567.val[3] = vdupq_n_u32(0);
57 uint32x4x4_t vtmp4567x4567;
58 vtmp4567x4567.val[0] = vdupq_n_u32(0);
59 vtmp4567x4567.val[1] = vdupq_n_u32(0);
60 vtmp4567x4567.val[2] = vdupq_n_u32(0);
61 vtmp4567x4567.val[3] = vdupq_n_u32(0);
62
63 do {
64 // NC main loop multiple of 8
65 const uint32_t* w0 = weights;
66 size_t n = nc;
67
68 for (; n >= 8; n -= 8) {
69 if XNN_LIKELY(bias != NULL) {
70 uint32x4_t vb0 = vld1q_u32(bias); bias += 4;
71 uint32x4_t vb4 = vld1q_u32(bias); bias += 4;
72 vst1q_u32(packed_weights, vb0); packed_weights += 4;
73 vst1q_u32(packed_weights, vb4); packed_weights += 4;
74 } else {
75 const uint32x4_t vzero = vmovq_n_u32(0);
76 vst1q_u32(packed_weights, vzero); packed_weights += 4;
77 vst1q_u32(packed_weights, vzero); packed_weights += 4;
78 }
79
80 const uint32_t* w1 = w0 + kc;
81 const uint32_t* w2 = w1 + kc;
82 const uint32_t* w3 = w2 + kc;
83 const uint32_t* w4 = w3 + kc;
84 const uint32_t* w5 = w4 + kc;
85 const uint32_t* w6 = w5 + kc;
86 const uint32_t* w7 = w6 + kc;
87
88 // KC main loop multiple of 8
89 size_t k = kc;
90 for (; k >= 8; k -= 8) {
91 vtmp0123x0123 = vld4q_lane_u32(w0, vtmp0123x0123, 0); w0 += 4;
92 vtmp0123x0123 = vld4q_lane_u32(w1, vtmp0123x0123, 1); w1 += 4;
93 vtmp0123x0123 = vld4q_lane_u32(w2, vtmp0123x0123, 2); w2 += 4;
94 vtmp0123x0123 = vld4q_lane_u32(w3, vtmp0123x0123, 3); w3 += 4;
95 vtmp0123x4567 = vld4q_lane_u32(w4, vtmp0123x4567, 0); w4 += 4;
96 vtmp0123x4567 = vld4q_lane_u32(w5, vtmp0123x4567, 1); w5 += 4;
97 vtmp0123x4567 = vld4q_lane_u32(w6, vtmp0123x4567, 2); w6 += 4;
98 vtmp0123x4567 = vld4q_lane_u32(w7, vtmp0123x4567, 3); w7 += 4;
99 vtmp4567x0123 = vld4q_lane_u32(w0, vtmp4567x0123, 0); w0 += 4;
100 vtmp4567x0123 = vld4q_lane_u32(w1, vtmp4567x0123, 1); w1 += 4;
101 vtmp4567x0123 = vld4q_lane_u32(w2, vtmp4567x0123, 2); w2 += 4;
102 vtmp4567x0123 = vld4q_lane_u32(w3, vtmp4567x0123, 3); w3 += 4;
103 vtmp4567x4567 = vld4q_lane_u32(w4, vtmp4567x4567, 0); w4 += 4;
104 vtmp4567x4567 = vld4q_lane_u32(w5, vtmp4567x4567, 1); w5 += 4;
105 vtmp4567x4567 = vld4q_lane_u32(w6, vtmp4567x4567, 2); w6 += 4;
106 vtmp4567x4567 = vld4q_lane_u32(w7, vtmp4567x4567, 3); w7 += 4;
107 vst1q_u32(packed_weights, vtmp0123x0123.val[0]); packed_weights += 4;
108 vst1q_u32(packed_weights, vtmp0123x4567.val[0]); packed_weights += 4;
109 vst1q_u32(packed_weights, vtmp0123x0123.val[1]); packed_weights += 4;
110 vst1q_u32(packed_weights, vtmp0123x4567.val[1]); packed_weights += 4;
111 vst1q_u32(packed_weights, vtmp0123x0123.val[2]); packed_weights += 4;
112 vst1q_u32(packed_weights, vtmp0123x4567.val[2]); packed_weights += 4;
113 vst1q_u32(packed_weights, vtmp0123x0123.val[3]); packed_weights += 4;
114 vst1q_u32(packed_weights, vtmp0123x4567.val[3]); packed_weights += 4;
115 vst1q_u32(packed_weights, vtmp4567x0123.val[0]); packed_weights += 4;
116 vst1q_u32(packed_weights, vtmp4567x4567.val[0]); packed_weights += 4;
117 vst1q_u32(packed_weights, vtmp4567x0123.val[1]); packed_weights += 4;
118 vst1q_u32(packed_weights, vtmp4567x4567.val[1]); packed_weights += 4;
119 vst1q_u32(packed_weights, vtmp4567x0123.val[2]); packed_weights += 4;
120 vst1q_u32(packed_weights, vtmp4567x4567.val[2]); packed_weights += 4;
121 vst1q_u32(packed_weights, vtmp4567x0123.val[3]); packed_weights += 4;
122 vst1q_u32(packed_weights, vtmp4567x4567.val[3]); packed_weights += 4;
123 }
124 // KC remainder multiple of 4
125 if (k >= 4) {
126 vtmp0123x0123 = vld4q_lane_u32(w0, vtmp0123x0123, 0); w0 += 4;
127 vtmp0123x0123 = vld4q_lane_u32(w1, vtmp0123x0123, 1); w1 += 4;
128 vtmp0123x0123 = vld4q_lane_u32(w2, vtmp0123x0123, 2); w2 += 4;
129 vtmp0123x0123 = vld4q_lane_u32(w3, vtmp0123x0123, 3); w3 += 4;
130 vtmp0123x4567 = vld4q_lane_u32(w4, vtmp0123x4567, 0); w4 += 4;
131 vtmp0123x4567 = vld4q_lane_u32(w5, vtmp0123x4567, 1); w5 += 4;
132 vtmp0123x4567 = vld4q_lane_u32(w6, vtmp0123x4567, 2); w6 += 4;
133 vtmp0123x4567 = vld4q_lane_u32(w7, vtmp0123x4567, 3); w7 += 4;
134 vst1q_u32(packed_weights, vtmp0123x0123.val[0]); packed_weights += 4;
135 vst1q_u32(packed_weights, vtmp0123x4567.val[0]); packed_weights += 4;
136 vst1q_u32(packed_weights, vtmp0123x0123.val[1]); packed_weights += 4;
137 vst1q_u32(packed_weights, vtmp0123x4567.val[1]); packed_weights += 4;
138 vst1q_u32(packed_weights, vtmp0123x0123.val[2]); packed_weights += 4;
139 vst1q_u32(packed_weights, vtmp0123x4567.val[2]); packed_weights += 4;
140 vst1q_u32(packed_weights, vtmp0123x0123.val[3]); packed_weights += 4;
141 vst1q_u32(packed_weights, vtmp0123x4567.val[3]); packed_weights += 4;
142 k -= 4;
143 }
144
145 // KC remainder of 1..3
146 // Same as main loop but ld1, ld2 or ld3
147 if XNN_UNLIKELY(k != 0) {
148 assert(k >= 1);
149 assert(k <= 3);
150 switch (k) {
151 // KC remainder of 1
152 case 1:
153 {
154 uint32x4_t vtmp0x0123 = vdupq_n_u32(0);
155 uint32x4_t vtmp0x4567 = vdupq_n_u32(0);
156 vtmp0x0123 = vld1q_lane_u32(w0, vtmp0x0123, 0); w0 += 1;
157 vtmp0x0123 = vld1q_lane_u32(w1, vtmp0x0123, 1); w1 += 1;
158 vtmp0x0123 = vld1q_lane_u32(w2, vtmp0x0123, 2); w2 += 1;
159 vtmp0x0123 = vld1q_lane_u32(w3, vtmp0x0123, 3); w3 += 1;
160 vtmp0x4567 = vld1q_lane_u32(w4, vtmp0x4567, 0); w4 += 1;
161 vtmp0x4567 = vld1q_lane_u32(w5, vtmp0x4567, 1); w5 += 1;
162 vtmp0x4567 = vld1q_lane_u32(w6, vtmp0x4567, 2); w6 += 1;
163 vtmp0x4567 = vld1q_lane_u32(w7, vtmp0x4567, 3); w7 += 1;
164 vst1q_u32(packed_weights, vtmp0x0123); packed_weights += 4;
165 vst1q_u32(packed_weights, vtmp0x4567); packed_weights += 4;
166 break;
167 }
168 // KC remainder of 2
169 case 2:
170 {
171 uint32x4x2_t vtmp01x0123;
172 vtmp01x0123.val[0] = vdupq_n_u32(0);
173 vtmp01x0123.val[1] = vdupq_n_u32(0);
174 uint32x4x2_t vtmp01x4567;
175 vtmp01x4567.val[0] = vdupq_n_u32(0);
176 vtmp01x4567.val[1] = vdupq_n_u32(0);
177 vtmp01x0123 = vld2q_lane_u32(w0, vtmp01x0123, 0); w0 += 2;
178 vtmp01x0123 = vld2q_lane_u32(w1, vtmp01x0123, 1); w1 += 2;
179 vtmp01x0123 = vld2q_lane_u32(w2, vtmp01x0123, 2); w2 += 2;
180 vtmp01x0123 = vld2q_lane_u32(w3, vtmp01x0123, 3); w3 += 2;
181 vtmp01x4567 = vld2q_lane_u32(w4, vtmp01x4567, 0); w4 += 2;
182 vtmp01x4567 = vld2q_lane_u32(w5, vtmp01x4567, 1); w5 += 2;
183 vtmp01x4567 = vld2q_lane_u32(w6, vtmp01x4567, 2); w6 += 2;
184 vtmp01x4567 = vld2q_lane_u32(w7, vtmp01x4567, 3); w7 += 2;
185 vst1q_u32(packed_weights, vtmp01x0123.val[0]); packed_weights += 4;
186 vst1q_u32(packed_weights, vtmp01x4567.val[0]); packed_weights += 4;
187 vst1q_u32(packed_weights, vtmp01x0123.val[1]); packed_weights += 4;
188 vst1q_u32(packed_weights, vtmp01x4567.val[1]); packed_weights += 4;
189 break;
190 }
191 // KC remainder of 3
192 case 3:
193 {
194 uint32x4x3_t vtmp012x0123;
195 vtmp012x0123.val[0] = vdupq_n_u32(0);
196 vtmp012x0123.val[1] = vdupq_n_u32(0);
197 vtmp012x0123.val[2] = vdupq_n_u32(0);
198 uint32x4x3_t vtmp012x4567;
199 vtmp012x4567.val[0] = vdupq_n_u32(0);
200 vtmp012x4567.val[1] = vdupq_n_u32(0);
201 vtmp012x4567.val[2] = vdupq_n_u32(0);
202 vtmp012x0123 = vld3q_lane_u32(w0, vtmp012x0123, 0); w0 += 3;
203 vtmp012x0123 = vld3q_lane_u32(w1, vtmp012x0123, 1); w1 += 3;
204 vtmp012x0123 = vld3q_lane_u32(w2, vtmp012x0123, 2); w2 += 3;
205 vtmp012x0123 = vld3q_lane_u32(w3, vtmp012x0123, 3); w3 += 3;
206 vtmp012x4567 = vld3q_lane_u32(w4, vtmp012x4567, 0); w4 += 3;
207 vtmp012x4567 = vld3q_lane_u32(w5, vtmp012x4567, 1); w5 += 3;
208 vtmp012x4567 = vld3q_lane_u32(w6, vtmp012x4567, 2); w6 += 3;
209 vtmp012x4567 = vld3q_lane_u32(w7, vtmp012x4567, 3); w7 += 3;
210 vst1q_u32(packed_weights, vtmp012x0123.val[0]); packed_weights += 4;
211 vst1q_u32(packed_weights, vtmp012x4567.val[0]); packed_weights += 4;
212 vst1q_u32(packed_weights, vtmp012x0123.val[1]); packed_weights += 4;
213 vst1q_u32(packed_weights, vtmp012x4567.val[1]); packed_weights += 4;
214 vst1q_u32(packed_weights, vtmp012x0123.val[2]); packed_weights += 4;
215 vst1q_u32(packed_weights, vtmp012x4567.val[2]); packed_weights += 4;
216 break;
217 }
218 default:
219 XNN_UNREACHABLE;
220 }
221 }
222 packed_weights = (uint32_t*) ((uintptr_t) packed_weights + extra_bytes);
223 w0 = w7;
224 }
225
226 // NC remainder (1..7)
227 if XNN_UNLIKELY(n != 0) {
228 assert(n >= 1);
229 assert(n <= 7);
230 if XNN_LIKELY(bias != NULL) {
231 size_t nb = n;
232 do {
233 *packed_weights++ = *bias++;
234 } while (--nb != 0);
235 packed_weights += (8 - n);
236 } else {
237 const uint32x4_t vzero = vmovq_n_u32(0);
238 vst1q_u32(packed_weights, vzero); packed_weights += 4;
239 vst1q_u32(packed_weights, vzero); packed_weights += 4;
240 }
241
242 // NR remainder has less than 8 rows so last row is not loaded
243 const uint32_t* w1 = w0 + kc;
244 if XNN_UNPREDICTABLE(n < 2) {
245 w1 = w0;
246 }
247 const uint32_t* w2 = w1 + kc;
248 if XNN_UNPREDICTABLE(n <= 2) {
249 w2 = w1;
250 }
251 const uint32_t* w3 = w2 + kc;
252 if XNN_UNPREDICTABLE(n < 4) {
253 w3 = w2;
254 }
255 const uint32_t* w4 = w3 + kc;
256 if XNN_UNPREDICTABLE(n <= 4) {
257 w4 = w3;
258 }
259 const uint32_t* w5 = w4 + kc;
260 if XNN_UNPREDICTABLE(n < 6) {
261 w5 = w4;
262 }
263 const uint32_t* w6 = w5 + kc;
264 if XNN_UNPREDICTABLE(n <= 6) {
265 w6 = w5;
266 }
267
268 // KC main loop multiple of 8
269 size_t k = kc;
270 for (; k >= 8; k -= 8) {
271 vtmp0123x0123 = vld4q_lane_u32(w0, vtmp0123x0123, 0); w0 += 4;
272 vtmp0123x0123 = vld4q_lane_u32(w1, vtmp0123x0123, 1); w1 += 4;
273 vtmp0123x0123 = vld4q_lane_u32(w2, vtmp0123x0123, 2); w2 += 4;
274 vtmp0123x0123 = vld4q_lane_u32(w3, vtmp0123x0123, 3); w3 += 4;
275 vtmp0123x4567 = vld4q_lane_u32(w4, vtmp0123x4567, 0); w4 += 4;
276 vtmp0123x4567 = vld4q_lane_u32(w5, vtmp0123x4567, 1); w5 += 4;
277 vtmp0123x4567 = vld4q_lane_u32(w6, vtmp0123x4567, 2); w6 += 4;
278 vtmp4567x0123 = vld4q_lane_u32(w0, vtmp4567x0123, 0); w0 += 4;
279 vtmp4567x0123 = vld4q_lane_u32(w1, vtmp4567x0123, 1); w1 += 4;
280 vtmp4567x0123 = vld4q_lane_u32(w2, vtmp4567x0123, 2); w2 += 4;
281 vtmp4567x0123 = vld4q_lane_u32(w3, vtmp4567x0123, 3); w3 += 4;
282 vtmp4567x4567 = vld4q_lane_u32(w4, vtmp4567x4567, 0); w4 += 4;
283 vtmp4567x4567 = vld4q_lane_u32(w5, vtmp4567x4567, 1); w5 += 4;
284 vtmp4567x4567 = vld4q_lane_u32(w6, vtmp4567x4567, 2); w6 += 4;
285 vst1q_u32(packed_weights, vtmp0123x0123.val[0]); packed_weights += 4;
286 vst1q_u32(packed_weights, vtmp0123x4567.val[0]); packed_weights += 4;
287 vst1q_u32(packed_weights, vtmp0123x0123.val[1]); packed_weights += 4;
288 vst1q_u32(packed_weights, vtmp0123x4567.val[1]); packed_weights += 4;
289 vst1q_u32(packed_weights, vtmp0123x0123.val[2]); packed_weights += 4;
290 vst1q_u32(packed_weights, vtmp0123x4567.val[2]); packed_weights += 4;
291 vst1q_u32(packed_weights, vtmp0123x0123.val[3]); packed_weights += 4;
292 vst1q_u32(packed_weights, vtmp0123x4567.val[3]); packed_weights += 4;
293 vst1q_u32(packed_weights, vtmp4567x0123.val[0]); packed_weights += 4;
294 vst1q_u32(packed_weights, vtmp4567x4567.val[0]); packed_weights += 4;
295 vst1q_u32(packed_weights, vtmp4567x0123.val[1]); packed_weights += 4;
296 vst1q_u32(packed_weights, vtmp4567x4567.val[1]); packed_weights += 4;
297 vst1q_u32(packed_weights, vtmp4567x0123.val[2]); packed_weights += 4;
298 vst1q_u32(packed_weights, vtmp4567x4567.val[2]); packed_weights += 4;
299 vst1q_u32(packed_weights, vtmp4567x0123.val[3]); packed_weights += 4;
300 vst1q_u32(packed_weights, vtmp4567x4567.val[3]); packed_weights += 4;
301 }
302 // KC remainder multiple of 4
303 if (k >= 4) {
304 vtmp0123x0123 = vld4q_lane_u32(w0, vtmp0123x0123, 0); w0 += 4;
305 vtmp0123x0123 = vld4q_lane_u32(w1, vtmp0123x0123, 1); w1 += 4;
306 vtmp0123x0123 = vld4q_lane_u32(w2, vtmp0123x0123, 2); w2 += 4;
307 vtmp0123x0123 = vld4q_lane_u32(w3, vtmp0123x0123, 3); w3 += 4;
308 vtmp0123x4567 = vld4q_lane_u32(w4, vtmp0123x4567, 0); w4 += 4;
309 vtmp0123x4567 = vld4q_lane_u32(w5, vtmp0123x4567, 1); w5 += 4;
310 vtmp0123x4567 = vld4q_lane_u32(w6, vtmp0123x4567, 2); w6 += 4;
311 vst1q_u32(packed_weights, vtmp0123x0123.val[0]); packed_weights += 4;
312 vst1q_u32(packed_weights, vtmp0123x4567.val[0]); packed_weights += 4;
313 vst1q_u32(packed_weights, vtmp0123x0123.val[1]); packed_weights += 4;
314 vst1q_u32(packed_weights, vtmp0123x4567.val[1]); packed_weights += 4;
315 vst1q_u32(packed_weights, vtmp0123x0123.val[2]); packed_weights += 4;
316 vst1q_u32(packed_weights, vtmp0123x4567.val[2]); packed_weights += 4;
317 vst1q_u32(packed_weights, vtmp0123x0123.val[3]); packed_weights += 4;
318 vst1q_u32(packed_weights, vtmp0123x4567.val[3]); packed_weights += 4;
319 k -= 4;
320 }
321
322 // KC remainder of 1..3
323 // Same as main loop but ld1, ld2 or ld3
324 if XNN_UNLIKELY(k != 0) {
325 assert(k >= 1);
326 assert(k <= 3);
327 switch (k) {
328 // KC remainder of 1
329 case 1:
330 {
331 uint32x4_t vtmp0x0123 = vdupq_n_u32(0);
332 uint32x4_t vtmp0x4567 = vdupq_n_u32(0);
333 vtmp0x0123 = vld1q_lane_u32(w0, vtmp0x0123, 0);
334 vtmp0x0123 = vld1q_lane_u32(w1, vtmp0x0123, 1);
335 vtmp0x0123 = vld1q_lane_u32(w2, vtmp0x0123, 2);
336 vtmp0x0123 = vld1q_lane_u32(w3, vtmp0x0123, 3);
337 vtmp0x4567 = vld1q_lane_u32(w4, vtmp0x4567, 0);
338 vtmp0x4567 = vld1q_lane_u32(w5, vtmp0x4567, 1);
339 vtmp0x4567 = vld1q_lane_u32(w6, vtmp0x4567, 2);
340 vst1q_u32(packed_weights, vtmp0x0123); packed_weights += 4;
341 vst1q_u32(packed_weights, vtmp0x4567); packed_weights += 4;
342 break;
343 }
344 // KC remainder of 2
345 case 2:
346 {
347 uint32x4x2_t vtmp01x0123;
348 vtmp01x0123.val[0] = vdupq_n_u32(0);
349 vtmp01x0123.val[1] = vdupq_n_u32(0);
350 uint32x4x2_t vtmp01x4567;
351 vtmp01x4567.val[0] = vdupq_n_u32(0);
352 vtmp01x4567.val[1] = vdupq_n_u32(0);
353 vtmp01x0123 = vld2q_lane_u32(w0, vtmp01x0123, 0);
354 vtmp01x0123 = vld2q_lane_u32(w1, vtmp01x0123, 1);
355 vtmp01x0123 = vld2q_lane_u32(w2, vtmp01x0123, 2);
356 vtmp01x0123 = vld2q_lane_u32(w3, vtmp01x0123, 3);
357 vtmp01x4567 = vld2q_lane_u32(w4, vtmp01x4567, 0);
358 vtmp01x4567 = vld2q_lane_u32(w5, vtmp01x4567, 1);
359 vtmp01x4567 = vld2q_lane_u32(w6, vtmp01x4567, 2);
360 vst1q_u32(packed_weights, vtmp01x0123.val[0]); packed_weights += 4;
361 vst1q_u32(packed_weights, vtmp01x4567.val[0]); packed_weights += 4;
362 vst1q_u32(packed_weights, vtmp01x0123.val[1]); packed_weights += 4;
363 vst1q_u32(packed_weights, vtmp01x4567.val[1]); packed_weights += 4;
364 break;
365 }
366 // KC remainder of 3
367 case 3:
368 {
369 uint32x4x3_t vtmp012x0123;
370 vtmp012x0123.val[0] = vdupq_n_u32(0);
371 vtmp012x0123.val[1] = vdupq_n_u32(0);
372 vtmp012x0123.val[2] = vdupq_n_u32(0);
373 uint32x4x3_t vtmp012x4567;
374 vtmp012x4567.val[0] = vdupq_n_u32(0);
375 vtmp012x4567.val[1] = vdupq_n_u32(0);
376 vtmp012x4567.val[2] = vdupq_n_u32(0);
377 vtmp012x0123 = vld3q_lane_u32(w0, vtmp012x0123, 0);
378 vtmp012x0123 = vld3q_lane_u32(w1, vtmp012x0123, 1);
379 vtmp012x0123 = vld3q_lane_u32(w2, vtmp012x0123, 2);
380 vtmp012x0123 = vld3q_lane_u32(w3, vtmp012x0123, 3);
381 vtmp012x4567 = vld3q_lane_u32(w4, vtmp012x4567, 0);
382 vtmp012x4567 = vld3q_lane_u32(w5, vtmp012x4567, 1);
383 vtmp012x4567 = vld3q_lane_u32(w6, vtmp012x4567, 2);
384 vst1q_u32(packed_weights, vtmp012x0123.val[0]); packed_weights += 4;
385 vst1q_u32(packed_weights, vtmp012x4567.val[0]); packed_weights += 4;
386 vst1q_u32(packed_weights, vtmp012x0123.val[1]); packed_weights += 4;
387 vst1q_u32(packed_weights, vtmp012x4567.val[1]); packed_weights += 4;
388 vst1q_u32(packed_weights, vtmp012x0123.val[2]); packed_weights += 4;
389 vst1q_u32(packed_weights, vtmp012x4567.val[2]); packed_weights += 4;
390 break;
391 }
392 default:
393 XNN_UNREACHABLE;
394 }
395 }
396 packed_weights = (uint32_t*) ((uintptr_t) packed_weights + extra_bytes);
397 }
398
399 weights += nc * kc;
400 } while (--g != 0);
401 }