Debian Codegraph @

src/x32-packw/gen/x32-packw-x12-gemm-goi-neon-ld4lane-u4-prfm.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/neon.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include <arm_neon.h>
16
17 #include "xnnpack/packw.h"
18 #include "xnnpack/prefetch.h"
19
20
21 void xnn_x32_packw_gemm_goi_ukernel_x12__neon_ld4lane_u4_prfm(
22 size_t g,
23 size_t nc,
24 size_t kc,
25 size_t nr,
26 size_t kr,
27 size_t sr,
28 const uint32_t* weights,
29 const uint32_t* bias,
30 const void* scale,
31 uint32_t* packed_weights,
32 size_t extra_bytes,
33 const void* params)
34 {
35 assert(g != 0);
36 assert(nc != 0);
37 assert(kc != 0);
38 assert(nr == 12);
39 assert(kr == 1);
40 assert(sr == 1);
41 assert(weights != NULL);
42 assert(packed_weights != NULL);
43 uint32x4x4_t vtmp0123x0123;
44 vtmp0123x0123.val[0] = vdupq_n_u32(0);
45 vtmp0123x0123.val[1] = vdupq_n_u32(0);
46 vtmp0123x0123.val[2] = vdupq_n_u32(0);
47 vtmp0123x0123.val[3] = vdupq_n_u32(0);
48 uint32x4x4_t vtmp0123x4567;
49 vtmp0123x4567.val[0] = vdupq_n_u32(0);
50 vtmp0123x4567.val[1] = vdupq_n_u32(0);
51 vtmp0123x4567.val[2] = vdupq_n_u32(0);
52 vtmp0123x4567.val[3] = vdupq_n_u32(0);
53 uint32x4x4_t vtmp0123x89AB;
54 vtmp0123x89AB.val[0] = vdupq_n_u32(0);
55 vtmp0123x89AB.val[1] = vdupq_n_u32(0);
56 vtmp0123x89AB.val[2] = vdupq_n_u32(0);
57 vtmp0123x89AB.val[3] = vdupq_n_u32(0);
58
59 do {
60 // NC main loop multiple of 12
61 const uint32_t* w0 = weights;
62 size_t n = nc;
63
64 for (; n >= 12; n -= 12) {
65 if XNN_LIKELY(bias != NULL) {
66 uint32x4_t vb0 = vld1q_u32(bias); bias += 4;
67 uint32x4_t vb4 = vld1q_u32(bias); bias += 4;
68 uint32x4_t vb8 = vld1q_u32(bias); bias += 4;
69 vst1q_u32(packed_weights, vb0); packed_weights += 4;
70 vst1q_u32(packed_weights, vb4); packed_weights += 4;
71 vst1q_u32(packed_weights, vb8); packed_weights += 4;
72 } else {
73 const uint32x4_t vzero = vmovq_n_u32(0);
74 vst1q_u32(packed_weights, vzero); packed_weights += 4;
75 vst1q_u32(packed_weights, vzero); packed_weights += 4;
76 vst1q_u32(packed_weights, vzero); packed_weights += 4;
77 }
78
79 const uint32_t* w1 = w0 + kc;
80 const uint32_t* w2 = w1 + kc;
81 const uint32_t* w3 = w2 + kc;
82 const uint32_t* w4 = w3 + kc;
83 const uint32_t* w5 = w4 + kc;
84 const uint32_t* w6 = w5 + kc;
85 const uint32_t* w7 = w6 + kc;
86 const uint32_t* w8 = w7 + kc;
87 const uint32_t* w9 = w8 + kc;
88 const uint32_t* w10 = w9 + kc;
89 const uint32_t* w11 = w10 + kc;
90 xnn_prefetch_to_l1((const int8_t*) w0);
91 xnn_prefetch_to_l1((const int8_t*) w0 + 64);
92 xnn_prefetch_to_l1((const int8_t*) w1);
93 xnn_prefetch_to_l1((const int8_t*) w1 + 64);
94 xnn_prefetch_to_l1((const int8_t*) w2);
95 xnn_prefetch_to_l1((const int8_t*) w2 + 64);
96 xnn_prefetch_to_l1((const int8_t*) w3);
97 xnn_prefetch_to_l1((const int8_t*) w3 + 64);
98 xnn_prefetch_to_l1((const int8_t*) w4);
99 xnn_prefetch_to_l1((const int8_t*) w4 + 64);
100 xnn_prefetch_to_l1((const int8_t*) w5);
101 xnn_prefetch_to_l1((const int8_t*) w5 + 64);
102 xnn_prefetch_to_l1((const int8_t*) w6);
103 xnn_prefetch_to_l1((const int8_t*) w6 + 64);
104 xnn_prefetch_to_l1((const int8_t*) w7);
105 xnn_prefetch_to_l1((const int8_t*) w7 + 64);
106 xnn_prefetch_to_l1((const int8_t*) w8);
107 xnn_prefetch_to_l1((const int8_t*) w8 + 64);
108 xnn_prefetch_to_l1((const int8_t*) w9);
109 xnn_prefetch_to_l1((const int8_t*) w9 + 64);
110 xnn_prefetch_to_l1((const int8_t*) w10);
111 xnn_prefetch_to_l1((const int8_t*) w10 + 64);
112 xnn_prefetch_to_l1((const int8_t*) w11);
113 xnn_prefetch_to_l1((const int8_t*) w11 + 64);
114
115 // KC main loop multiple of 4
116 size_t k = kc;
117 for (; k >= 4; k -= 4) {
118 vtmp0123x0123 = vld4q_lane_u32(w0, vtmp0123x0123, 0); w0 += 4;
119 vtmp0123x0123 = vld4q_lane_u32(w1, vtmp0123x0123, 1); w1 += 4;
120 vtmp0123x0123 = vld4q_lane_u32(w2, vtmp0123x0123, 2); w2 += 4;
121 vtmp0123x0123 = vld4q_lane_u32(w3, vtmp0123x0123, 3); w3 += 4;
122 vtmp0123x4567 = vld4q_lane_u32(w4, vtmp0123x4567, 0); w4 += 4;
123 vtmp0123x4567 = vld4q_lane_u32(w5, vtmp0123x4567, 1); w5 += 4;
124 vtmp0123x4567 = vld4q_lane_u32(w6, vtmp0123x4567, 2); w6 += 4;
125 vtmp0123x4567 = vld4q_lane_u32(w7, vtmp0123x4567, 3); w7 += 4;
126 vtmp0123x89AB = vld4q_lane_u32(w8, vtmp0123x89AB, 0); w8 += 4;
127 vtmp0123x89AB = vld4q_lane_u32(w9, vtmp0123x89AB, 1); w9 += 4;
128 vtmp0123x89AB = vld4q_lane_u32(w10, vtmp0123x89AB, 2); w10 += 4;
129 vtmp0123x89AB = vld4q_lane_u32(w11, vtmp0123x89AB, 3); w11 += 4;
130 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
131 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
132 xnn_prefetch_to_l1((const int8_t*) w2 + 128);
133 xnn_prefetch_to_l1((const int8_t*) w3 + 128);
134 xnn_prefetch_to_l1((const int8_t*) w4 + 128);
135 xnn_prefetch_to_l1((const int8_t*) w5 + 128);
136 xnn_prefetch_to_l1((const int8_t*) w6 + 128);
137 xnn_prefetch_to_l1((const int8_t*) w7 + 128);
138 xnn_prefetch_to_l1((const int8_t*) w8 + 128);
139 xnn_prefetch_to_l1((const int8_t*) w9 + 128);
140 xnn_prefetch_to_l1((const int8_t*) w10 + 128);
141 xnn_prefetch_to_l1((const int8_t*) w11 + 128);
142 vst1q_u32(packed_weights, vtmp0123x0123.val[0]); packed_weights += 4;
143 vst1q_u32(packed_weights, vtmp0123x4567.val[0]); packed_weights += 4;
144 vst1q_u32(packed_weights, vtmp0123x89AB.val[0]); packed_weights += 4;
145 vst1q_u32(packed_weights, vtmp0123x0123.val[1]); packed_weights += 4;
146 vst1q_u32(packed_weights, vtmp0123x4567.val[1]); packed_weights += 4;
147 vst1q_u32(packed_weights, vtmp0123x89AB.val[1]); packed_weights += 4;
148 vst1q_u32(packed_weights, vtmp0123x0123.val[2]); packed_weights += 4;
149 vst1q_u32(packed_weights, vtmp0123x4567.val[2]); packed_weights += 4;
150 vst1q_u32(packed_weights, vtmp0123x89AB.val[2]); packed_weights += 4;
151 vst1q_u32(packed_weights, vtmp0123x0123.val[3]); packed_weights += 4;
152 vst1q_u32(packed_weights, vtmp0123x4567.val[3]); packed_weights += 4;
153 vst1q_u32(packed_weights, vtmp0123x89AB.val[3]); packed_weights += 4;
154 }
155
156 // KC remainder of 1..3
157 // Same as main loop but ld1, ld2 or ld3
158 if XNN_UNLIKELY(k != 0) {
159 assert(k >= 1);
160 assert(k <= 3);
161 switch (k) {
162 // KC remainder of 1
163 case 1:
164 {
165 uint32x4_t vtmp0x0123 = vdupq_n_u32(0);
166 uint32x4_t vtmp0x4567 = vdupq_n_u32(0);
167 uint32x4_t vtmp0x89AB = vdupq_n_u32(0);
168 vtmp0x0123 = vld1q_lane_u32(w0, vtmp0x0123, 0); w0 += 1;
169 vtmp0x0123 = vld1q_lane_u32(w1, vtmp0x0123, 1); w1 += 1;
170 vtmp0x0123 = vld1q_lane_u32(w2, vtmp0x0123, 2); w2 += 1;
171 vtmp0x0123 = vld1q_lane_u32(w3, vtmp0x0123, 3); w3 += 1;
172 vtmp0x4567 = vld1q_lane_u32(w4, vtmp0x4567, 0); w4 += 1;
173 vtmp0x4567 = vld1q_lane_u32(w5, vtmp0x4567, 1); w5 += 1;
174 vtmp0x4567 = vld1q_lane_u32(w6, vtmp0x4567, 2); w6 += 1;
175 vtmp0x4567 = vld1q_lane_u32(w7, vtmp0x4567, 3); w7 += 1;
176 vtmp0x89AB = vld1q_lane_u32(w8, vtmp0x89AB, 0); w8 += 1;
177 vtmp0x89AB = vld1q_lane_u32(w9, vtmp0x89AB, 1); w9 += 1;
178 vtmp0x89AB = vld1q_lane_u32(w10, vtmp0x89AB, 2); w10 += 1;
179 vtmp0x89AB = vld1q_lane_u32(w11, vtmp0x89AB, 3); w11 += 1;
180 vst1q_u32(packed_weights, vtmp0x0123); packed_weights += 4;
181 vst1q_u32(packed_weights, vtmp0x4567); packed_weights += 4;
182 vst1q_u32(packed_weights, vtmp0x89AB); packed_weights += 4;
183 break;
184 }
185 // KC remainder of 2
186 case 2:
187 {
188 uint32x4x2_t vtmp01x0123;
189 vtmp01x0123.val[0] = vdupq_n_u32(0);
190 vtmp01x0123.val[1] = vdupq_n_u32(0);
191 uint32x4x2_t vtmp01x4567;
192 vtmp01x4567.val[0] = vdupq_n_u32(0);
193 vtmp01x4567.val[1] = vdupq_n_u32(0);
194 uint32x4x2_t vtmp01x89AB;
195 vtmp01x89AB.val[0] = vdupq_n_u32(0);
196 vtmp01x89AB.val[1] = vdupq_n_u32(0);
197 vtmp01x0123 = vld2q_lane_u32(w0, vtmp01x0123, 0); w0 += 2;
198 vtmp01x0123 = vld2q_lane_u32(w1, vtmp01x0123, 1); w1 += 2;
199 vtmp01x0123 = vld2q_lane_u32(w2, vtmp01x0123, 2); w2 += 2;
200 vtmp01x0123 = vld2q_lane_u32(w3, vtmp01x0123, 3); w3 += 2;
201 vtmp01x4567 = vld2q_lane_u32(w4, vtmp01x4567, 0); w4 += 2;
202 vtmp01x4567 = vld2q_lane_u32(w5, vtmp01x4567, 1); w5 += 2;
203 vtmp01x4567 = vld2q_lane_u32(w6, vtmp01x4567, 2); w6 += 2;
204 vtmp01x4567 = vld2q_lane_u32(w7, vtmp01x4567, 3); w7 += 2;
205 vtmp01x89AB = vld2q_lane_u32(w8, vtmp01x89AB, 0); w8 += 2;
206 vtmp01x89AB = vld2q_lane_u32(w9, vtmp01x89AB, 1); w9 += 2;
207 vtmp01x89AB = vld2q_lane_u32(w10, vtmp01x89AB, 2); w10 += 2;
208 vtmp01x89AB = vld2q_lane_u32(w11, vtmp01x89AB, 3); w11 += 2;
209 vst1q_u32(packed_weights, vtmp01x0123.val[0]); packed_weights += 4;
210 vst1q_u32(packed_weights, vtmp01x4567.val[0]); packed_weights += 4;
211 vst1q_u32(packed_weights, vtmp01x89AB.val[0]); packed_weights += 4;
212 vst1q_u32(packed_weights, vtmp01x0123.val[1]); packed_weights += 4;
213 vst1q_u32(packed_weights, vtmp01x4567.val[1]); packed_weights += 4;
214 vst1q_u32(packed_weights, vtmp01x89AB.val[1]); packed_weights += 4;
215 break;
216 }
217 // KC remainder of 3
218 case 3:
219 {
220 uint32x4x3_t vtmp012x0123;
221 vtmp012x0123.val[0] = vdupq_n_u32(0);
222 vtmp012x0123.val[1] = vdupq_n_u32(0);
223 vtmp012x0123.val[2] = vdupq_n_u32(0);
224 uint32x4x3_t vtmp012x4567;
225 vtmp012x4567.val[0] = vdupq_n_u32(0);
226 vtmp012x4567.val[1] = vdupq_n_u32(0);
227 vtmp012x4567.val[2] = vdupq_n_u32(0);
228 uint32x4x3_t vtmp012x89AB;
229 vtmp012x89AB.val[0] = vdupq_n_u32(0);
230 vtmp012x89AB.val[1] = vdupq_n_u32(0);
231 vtmp012x89AB.val[2] = vdupq_n_u32(0);
232 vtmp012x0123 = vld3q_lane_u32(w0, vtmp012x0123, 0); w0 += 3;
233 vtmp012x0123 = vld3q_lane_u32(w1, vtmp012x0123, 1); w1 += 3;
234 vtmp012x0123 = vld3q_lane_u32(w2, vtmp012x0123, 2); w2 += 3;
235 vtmp012x0123 = vld3q_lane_u32(w3, vtmp012x0123, 3); w3 += 3;
236 vtmp012x4567 = vld3q_lane_u32(w4, vtmp012x4567, 0); w4 += 3;
237 vtmp012x4567 = vld3q_lane_u32(w5, vtmp012x4567, 1); w5 += 3;
238 vtmp012x4567 = vld3q_lane_u32(w6, vtmp012x4567, 2); w6 += 3;
239 vtmp012x4567 = vld3q_lane_u32(w7, vtmp012x4567, 3); w7 += 3;
240 vtmp012x89AB = vld3q_lane_u32(w8, vtmp012x89AB, 0); w8 += 3;
241 vtmp012x89AB = vld3q_lane_u32(w9, vtmp012x89AB, 1); w9 += 3;
242 vtmp012x89AB = vld3q_lane_u32(w10, vtmp012x89AB, 2); w10 += 3;
243 vtmp012x89AB = vld3q_lane_u32(w11, vtmp012x89AB, 3); w11 += 3;
244 vst1q_u32(packed_weights, vtmp012x0123.val[0]); packed_weights += 4;
245 vst1q_u32(packed_weights, vtmp012x4567.val[0]); packed_weights += 4;
246 vst1q_u32(packed_weights, vtmp012x89AB.val[0]); packed_weights += 4;
247 vst1q_u32(packed_weights, vtmp012x0123.val[1]); packed_weights += 4;
248 vst1q_u32(packed_weights, vtmp012x4567.val[1]); packed_weights += 4;
249 vst1q_u32(packed_weights, vtmp012x89AB.val[1]); packed_weights += 4;
250 vst1q_u32(packed_weights, vtmp012x0123.val[2]); packed_weights += 4;
251 vst1q_u32(packed_weights, vtmp012x4567.val[2]); packed_weights += 4;
252 vst1q_u32(packed_weights, vtmp012x89AB.val[2]); packed_weights += 4;
253 break;
254 }
255 default:
256 XNN_UNREACHABLE;
257 }
258 }
259 packed_weights = (uint32_t*) ((uintptr_t) packed_weights + extra_bytes);
260 w0 = w11;
261 }
262
263 // NC remainder (1..11)
264 if XNN_UNLIKELY(n != 0) {
265 assert(n >= 1);
266 assert(n <= 11);
267 if XNN_LIKELY(bias != NULL) {
268 size_t nb = n;
269 do {
270 *packed_weights++ = *bias++;
271 } while (--nb != 0);
272 packed_weights += (12 - n);
273 } else {
274 const uint32x4_t vzero = vmovq_n_u32(0);
275 vst1q_u32(packed_weights, vzero); packed_weights += 4;
276 vst1q_u32(packed_weights, vzero); packed_weights += 4;
277 vst1q_u32(packed_weights, vzero); packed_weights += 4;
278 }
279
280 // NR remainder has less than 12 rows so last row is not loaded
281 const uint32_t* w1 = w0 + kc;
282 if XNN_UNPREDICTABLE(n < 2) {
283 w1 = w0;
284 }
285 const uint32_t* w2 = w1 + kc;
286 if XNN_UNPREDICTABLE(n <= 2) {
287 w2 = w1;
288 }
289 const uint32_t* w3 = w2 + kc;
290 if XNN_UNPREDICTABLE(n < 4) {
291 w3 = w2;
292 }
293 const uint32_t* w4 = w3 + kc;
294 if XNN_UNPREDICTABLE(n <= 4) {
295 w4 = w3;
296 }
297 const uint32_t* w5 = w4 + kc;
298 if XNN_UNPREDICTABLE(n < 6) {
299 w5 = w4;
300 }
301 const uint32_t* w6 = w5 + kc;
302 if XNN_UNPREDICTABLE(n <= 6) {
303 w6 = w5;
304 }
305 const uint32_t* w7 = w6 + kc;
306 if XNN_UNPREDICTABLE(n < 8) {
307 w7 = w6;
308 }
309 const uint32_t* w8 = w7 + kc;
310 if XNN_UNPREDICTABLE(n <= 8) {
311 w8 = w7;
312 }
313 const uint32_t* w9 = w8 + kc;
314 if XNN_UNPREDICTABLE(n < 10) {
315 w9 = w8;
316 }
317 const uint32_t* w10 = w9 + kc;
318 if XNN_UNPREDICTABLE(n <= 10) {
319 w10 = w9;
320 }
321
322 // KC main loop multiple of 4
323 size_t k = kc;
324 for (; k >= 4; k -= 4) {
325 vtmp0123x0123 = vld4q_lane_u32(w0, vtmp0123x0123, 0); w0 += 4;
326 vtmp0123x0123 = vld4q_lane_u32(w1, vtmp0123x0123, 1); w1 += 4;
327 vtmp0123x0123 = vld4q_lane_u32(w2, vtmp0123x0123, 2); w2 += 4;
328 vtmp0123x0123 = vld4q_lane_u32(w3, vtmp0123x0123, 3); w3 += 4;
329 vtmp0123x4567 = vld4q_lane_u32(w4, vtmp0123x4567, 0); w4 += 4;
330 vtmp0123x4567 = vld4q_lane_u32(w5, vtmp0123x4567, 1); w5 += 4;
331 vtmp0123x4567 = vld4q_lane_u32(w6, vtmp0123x4567, 2); w6 += 4;
332 vtmp0123x4567 = vld4q_lane_u32(w7, vtmp0123x4567, 3); w7 += 4;
333 vtmp0123x89AB = vld4q_lane_u32(w8, vtmp0123x89AB, 0); w8 += 4;
334 vtmp0123x89AB = vld4q_lane_u32(w9, vtmp0123x89AB, 1); w9 += 4;
335 vtmp0123x89AB = vld4q_lane_u32(w10, vtmp0123x89AB, 2); w10 += 4;
336 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
337 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
338 xnn_prefetch_to_l1((const int8_t*) w2 + 128);
339 xnn_prefetch_to_l1((const int8_t*) w3 + 128);
340 xnn_prefetch_to_l1((const int8_t*) w4 + 128);
341 xnn_prefetch_to_l1((const int8_t*) w5 + 128);
342 xnn_prefetch_to_l1((const int8_t*) w6 + 128);
343 xnn_prefetch_to_l1((const int8_t*) w7 + 128);
344 xnn_prefetch_to_l1((const int8_t*) w8 + 128);
345 xnn_prefetch_to_l1((const int8_t*) w9 + 128);
346 xnn_prefetch_to_l1((const int8_t*) w10 + 128);
347 vst1q_u32(packed_weights, vtmp0123x0123.val[0]); packed_weights += 4;
348 vst1q_u32(packed_weights, vtmp0123x4567.val[0]); packed_weights += 4;
349 vst1q_u32(packed_weights, vtmp0123x89AB.val[0]); packed_weights += 4;
350 vst1q_u32(packed_weights, vtmp0123x0123.val[1]); packed_weights += 4;
351 vst1q_u32(packed_weights, vtmp0123x4567.val[1]); packed_weights += 4;
352 vst1q_u32(packed_weights, vtmp0123x89AB.val[1]); packed_weights += 4;
353 vst1q_u32(packed_weights, vtmp0123x0123.val[2]); packed_weights += 4;
354 vst1q_u32(packed_weights, vtmp0123x4567.val[2]); packed_weights += 4;
355 vst1q_u32(packed_weights, vtmp0123x89AB.val[2]); packed_weights += 4;
356 vst1q_u32(packed_weights, vtmp0123x0123.val[3]); packed_weights += 4;
357 vst1q_u32(packed_weights, vtmp0123x4567.val[3]); packed_weights += 4;
358 vst1q_u32(packed_weights, vtmp0123x89AB.val[3]); packed_weights += 4;
359 }
360
361 // KC remainder of 1..3
362 // Same as main loop but ld1, ld2 or ld3
363 if XNN_UNLIKELY(k != 0) {
364 assert(k >= 1);
365 assert(k <= 3);
366 switch (k) {
367 // KC remainder of 1
368 case 1:
369 {
370 uint32x4_t vtmp0x0123 = vdupq_n_u32(0);
371 uint32x4_t vtmp0x4567 = vdupq_n_u32(0);
372 uint32x4_t vtmp0x89AB = vdupq_n_u32(0);
373 vtmp0x0123 = vld1q_lane_u32(w0, vtmp0x0123, 0);
374 vtmp0x0123 = vld1q_lane_u32(w1, vtmp0x0123, 1);
375 vtmp0x0123 = vld1q_lane_u32(w2, vtmp0x0123, 2);
376 vtmp0x0123 = vld1q_lane_u32(w3, vtmp0x0123, 3);
377 vtmp0x4567 = vld1q_lane_u32(w4, vtmp0x4567, 0);
378 vtmp0x4567 = vld1q_lane_u32(w5, vtmp0x4567, 1);
379 vtmp0x4567 = vld1q_lane_u32(w6, vtmp0x4567, 2);
380 vtmp0x4567 = vld1q_lane_u32(w7, vtmp0x4567, 3);
381 vtmp0x89AB = vld1q_lane_u32(w8, vtmp0x89AB, 0);
382 vtmp0x89AB = vld1q_lane_u32(w9, vtmp0x89AB, 1);
383 vtmp0x89AB = vld1q_lane_u32(w10, vtmp0x89AB, 2);
384 vst1q_u32(packed_weights, vtmp0x0123); packed_weights += 4;
385 vst1q_u32(packed_weights, vtmp0x4567); packed_weights += 4;
386 vst1q_u32(packed_weights, vtmp0x89AB); packed_weights += 4;
387 break;
388 }
389 // KC remainder of 2
390 case 2:
391 {
392 uint32x4x2_t vtmp01x0123;
393 vtmp01x0123.val[0] = vdupq_n_u32(0);
394 vtmp01x0123.val[1] = vdupq_n_u32(0);
395 uint32x4x2_t vtmp01x4567;
396 vtmp01x4567.val[0] = vdupq_n_u32(0);
397 vtmp01x4567.val[1] = vdupq_n_u32(0);
398 uint32x4x2_t vtmp01x89AB;
399 vtmp01x89AB.val[0] = vdupq_n_u32(0);
400 vtmp01x89AB.val[1] = vdupq_n_u32(0);
401 vtmp01x0123 = vld2q_lane_u32(w0, vtmp01x0123, 0);
402 vtmp01x0123 = vld2q_lane_u32(w1, vtmp01x0123, 1);
403 vtmp01x0123 = vld2q_lane_u32(w2, vtmp01x0123, 2);
404 vtmp01x0123 = vld2q_lane_u32(w3, vtmp01x0123, 3);
405 vtmp01x4567 = vld2q_lane_u32(w4, vtmp01x4567, 0);
406 vtmp01x4567 = vld2q_lane_u32(w5, vtmp01x4567, 1);
407 vtmp01x4567 = vld2q_lane_u32(w6, vtmp01x4567, 2);
408 vtmp01x4567 = vld2q_lane_u32(w7, vtmp01x4567, 3);
409 vtmp01x89AB = vld2q_lane_u32(w8, vtmp01x89AB, 0);
410 vtmp01x89AB = vld2q_lane_u32(w9, vtmp01x89AB, 1);
411 vtmp01x89AB = vld2q_lane_u32(w10, vtmp01x89AB, 2);
412 vst1q_u32(packed_weights, vtmp01x0123.val[0]); packed_weights += 4;
413 vst1q_u32(packed_weights, vtmp01x4567.val[0]); packed_weights += 4;
414 vst1q_u32(packed_weights, vtmp01x89AB.val[0]); packed_weights += 4;
415 vst1q_u32(packed_weights, vtmp01x0123.val[1]); packed_weights += 4;
416 vst1q_u32(packed_weights, vtmp01x4567.val[1]); packed_weights += 4;
417 vst1q_u32(packed_weights, vtmp01x89AB.val[1]); packed_weights += 4;
418 break;
419 }
420 // KC remainder of 3
421 case 3:
422 {
423 uint32x4x3_t vtmp012x0123;
424 vtmp012x0123.val[0] = vdupq_n_u32(0);
425 vtmp012x0123.val[1] = vdupq_n_u32(0);
426 vtmp012x0123.val[2] = vdupq_n_u32(0);
427 uint32x4x3_t vtmp012x4567;
428 vtmp012x4567.val[0] = vdupq_n_u32(0);
429 vtmp012x4567.val[1] = vdupq_n_u32(0);
430 vtmp012x4567.val[2] = vdupq_n_u32(0);
431 uint32x4x3_t vtmp012x89AB;
432 vtmp012x89AB.val[0] = vdupq_n_u32(0);
433 vtmp012x89AB.val[1] = vdupq_n_u32(0);
434 vtmp012x89AB.val[2] = vdupq_n_u32(0);
435 vtmp012x0123 = vld3q_lane_u32(w0, vtmp012x0123, 0);
436 vtmp012x0123 = vld3q_lane_u32(w1, vtmp012x0123, 1);
437 vtmp012x0123 = vld3q_lane_u32(w2, vtmp012x0123, 2);
438 vtmp012x0123 = vld3q_lane_u32(w3, vtmp012x0123, 3);
439 vtmp012x4567 = vld3q_lane_u32(w4, vtmp012x4567, 0);
440 vtmp012x4567 = vld3q_lane_u32(w5, vtmp012x4567, 1);
441 vtmp012x4567 = vld3q_lane_u32(w6, vtmp012x4567, 2);
442 vtmp012x4567 = vld3q_lane_u32(w7, vtmp012x4567, 3);
443 vtmp012x89AB = vld3q_lane_u32(w8, vtmp012x89AB, 0);
444 vtmp012x89AB = vld3q_lane_u32(w9, vtmp012x89AB, 1);
445 vtmp012x89AB = vld3q_lane_u32(w10, vtmp012x89AB, 2);
446 vst1q_u32(packed_weights, vtmp012x0123.val[0]); packed_weights += 4;
447 vst1q_u32(packed_weights, vtmp012x4567.val[0]); packed_weights += 4;
448 vst1q_u32(packed_weights, vtmp012x89AB.val[0]); packed_weights += 4;
449 vst1q_u32(packed_weights, vtmp012x0123.val[1]); packed_weights += 4;
450 vst1q_u32(packed_weights, vtmp012x4567.val[1]); packed_weights += 4;
451 vst1q_u32(packed_weights, vtmp012x89AB.val[1]); packed_weights += 4;
452 vst1q_u32(packed_weights, vtmp012x0123.val[2]); packed_weights += 4;
453 vst1q_u32(packed_weights, vtmp012x4567.val[2]); packed_weights += 4;
454 vst1q_u32(packed_weights, vtmp012x89AB.val[2]); packed_weights += 4;
455 break;
456 }
457 default:
458 XNN_UNREACHABLE;
459 }
460 }
461 packed_weights = (uint32_t*) ((uintptr_t) packed_weights + extra_bytes);
462 }
463
464 weights += nc * kc;
465 } while (--g != 0);
466 }