Debian Codegraph @

src/x32-packw/gen/x32-packw-x16-gemm-goi-neon-ld4lane-u4.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/neon.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include <arm_neon.h>
16
17 #include "xnnpack/packw.h"
18
19
20 void xnn_x32_packw_gemm_goi_ukernel_x16__neon_ld4lane_u4(
21 size_t g,
22 size_t nc,
23 size_t kc,
24 size_t nr,
25 size_t kr,
26 size_t sr,
27 const uint32_t* weights,
28 const uint32_t* bias,
29 const void* scale,
30 uint32_t* packed_weights,
31 size_t extra_bytes,
32 const void* params)
33 {
34 assert(g != 0);
35 assert(nc != 0);
36 assert(kc != 0);
37 assert(nr == 16);
38 assert(kr == 1);
39 assert(sr == 1);
40 assert(weights != NULL);
41 assert(packed_weights != NULL);
42 uint32x4x4_t vtmp0123x0123;
43 vtmp0123x0123.val[0] = vdupq_n_u32(0);
44 vtmp0123x0123.val[1] = vdupq_n_u32(0);
45 vtmp0123x0123.val[2] = vdupq_n_u32(0);
46 vtmp0123x0123.val[3] = vdupq_n_u32(0);
47 uint32x4x4_t vtmp0123x4567;
48 vtmp0123x4567.val[0] = vdupq_n_u32(0);
49 vtmp0123x4567.val[1] = vdupq_n_u32(0);
50 vtmp0123x4567.val[2] = vdupq_n_u32(0);
51 vtmp0123x4567.val[3] = vdupq_n_u32(0);
52 uint32x4x4_t vtmp0123x89AB;
53 vtmp0123x89AB.val[0] = vdupq_n_u32(0);
54 vtmp0123x89AB.val[1] = vdupq_n_u32(0);
55 vtmp0123x89AB.val[2] = vdupq_n_u32(0);
56 vtmp0123x89AB.val[3] = vdupq_n_u32(0);
57 uint32x4x4_t vtmp0123xCDEF;
58 vtmp0123xCDEF.val[0] = vdupq_n_u32(0);
59 vtmp0123xCDEF.val[1] = vdupq_n_u32(0);
60 vtmp0123xCDEF.val[2] = vdupq_n_u32(0);
61 vtmp0123xCDEF.val[3] = vdupq_n_u32(0);
62
63 do {
64 // NC main loop multiple of 16
65 const uint32_t* w0 = weights;
66 size_t n = nc;
67
68 for (; n >= 16; n -= 16) {
69 if XNN_LIKELY(bias != NULL) {
70 uint32x4_t vb0 = vld1q_u32(bias); bias += 4;
71 uint32x4_t vb4 = vld1q_u32(bias); bias += 4;
72 uint32x4_t vb8 = vld1q_u32(bias); bias += 4;
73 uint32x4_t vb12 = vld1q_u32(bias); bias += 4;
74 vst1q_u32(packed_weights, vb0); packed_weights += 4;
75 vst1q_u32(packed_weights, vb4); packed_weights += 4;
76 vst1q_u32(packed_weights, vb8); packed_weights += 4;
77 vst1q_u32(packed_weights, vb12); packed_weights += 4;
78 } else {
79 const uint32x4_t vzero = vmovq_n_u32(0);
80 vst1q_u32(packed_weights, vzero); packed_weights += 4;
81 vst1q_u32(packed_weights, vzero); packed_weights += 4;
82 vst1q_u32(packed_weights, vzero); packed_weights += 4;
83 vst1q_u32(packed_weights, vzero); packed_weights += 4;
84 }
85
86 const uint32_t* w1 = w0 + kc;
87 const uint32_t* w2 = w1 + kc;
88 const uint32_t* w3 = w2 + kc;
89 const uint32_t* w4 = w3 + kc;
90 const uint32_t* w5 = w4 + kc;
91 const uint32_t* w6 = w5 + kc;
92 const uint32_t* w7 = w6 + kc;
93 const uint32_t* w8 = w7 + kc;
94 const uint32_t* w9 = w8 + kc;
95 const uint32_t* w10 = w9 + kc;
96 const uint32_t* w11 = w10 + kc;
97 const uint32_t* w12 = w11 + kc;
98 const uint32_t* w13 = w12 + kc;
99 const uint32_t* w14 = w13 + kc;
100 const uint32_t* w15 = w14 + kc;
101
102 // KC main loop multiple of 4
103 size_t k = kc;
104 for (; k >= 4; k -= 4) {
105 vtmp0123x0123 = vld4q_lane_u32(w0, vtmp0123x0123, 0); w0 += 4;
106 vtmp0123x0123 = vld4q_lane_u32(w1, vtmp0123x0123, 1); w1 += 4;
107 vtmp0123x0123 = vld4q_lane_u32(w2, vtmp0123x0123, 2); w2 += 4;
108 vtmp0123x0123 = vld4q_lane_u32(w3, vtmp0123x0123, 3); w3 += 4;
109 vtmp0123x4567 = vld4q_lane_u32(w4, vtmp0123x4567, 0); w4 += 4;
110 vtmp0123x4567 = vld4q_lane_u32(w5, vtmp0123x4567, 1); w5 += 4;
111 vtmp0123x4567 = vld4q_lane_u32(w6, vtmp0123x4567, 2); w6 += 4;
112 vtmp0123x4567 = vld4q_lane_u32(w7, vtmp0123x4567, 3); w7 += 4;
113 vtmp0123x89AB = vld4q_lane_u32(w8, vtmp0123x89AB, 0); w8 += 4;
114 vtmp0123x89AB = vld4q_lane_u32(w9, vtmp0123x89AB, 1); w9 += 4;
115 vtmp0123x89AB = vld4q_lane_u32(w10, vtmp0123x89AB, 2); w10 += 4;
116 vtmp0123x89AB = vld4q_lane_u32(w11, vtmp0123x89AB, 3); w11 += 4;
117 vtmp0123xCDEF = vld4q_lane_u32(w12, vtmp0123xCDEF, 0); w12 += 4;
118 vtmp0123xCDEF = vld4q_lane_u32(w13, vtmp0123xCDEF, 1); w13 += 4;
119 vtmp0123xCDEF = vld4q_lane_u32(w14, vtmp0123xCDEF, 2); w14 += 4;
120 vtmp0123xCDEF = vld4q_lane_u32(w15, vtmp0123xCDEF, 3); w15 += 4;
121 vst1q_u32(packed_weights, vtmp0123x0123.val[0]); packed_weights += 4;
122 vst1q_u32(packed_weights, vtmp0123x4567.val[0]); packed_weights += 4;
123 vst1q_u32(packed_weights, vtmp0123x89AB.val[0]); packed_weights += 4;
124 vst1q_u32(packed_weights, vtmp0123xCDEF.val[0]); packed_weights += 4;
125 vst1q_u32(packed_weights, vtmp0123x0123.val[1]); packed_weights += 4;
126 vst1q_u32(packed_weights, vtmp0123x4567.val[1]); packed_weights += 4;
127 vst1q_u32(packed_weights, vtmp0123x89AB.val[1]); packed_weights += 4;
128 vst1q_u32(packed_weights, vtmp0123xCDEF.val[1]); packed_weights += 4;
129 vst1q_u32(packed_weights, vtmp0123x0123.val[2]); packed_weights += 4;
130 vst1q_u32(packed_weights, vtmp0123x4567.val[2]); packed_weights += 4;
131 vst1q_u32(packed_weights, vtmp0123x89AB.val[2]); packed_weights += 4;
132 vst1q_u32(packed_weights, vtmp0123xCDEF.val[2]); packed_weights += 4;
133 vst1q_u32(packed_weights, vtmp0123x0123.val[3]); packed_weights += 4;
134 vst1q_u32(packed_weights, vtmp0123x4567.val[3]); packed_weights += 4;
135 vst1q_u32(packed_weights, vtmp0123x89AB.val[3]); packed_weights += 4;
136 vst1q_u32(packed_weights, vtmp0123xCDEF.val[3]); packed_weights += 4;
137 }
138
139 // KC remainder of 1..3
140 // Same as main loop but ld1, ld2 or ld3
141 if XNN_UNLIKELY(k != 0) {
142 assert(k >= 1);
143 assert(k <= 3);
144 switch (k) {
145 // KC remainder of 1
146 case 1:
147 {
148 uint32x4_t vtmp0x0123 = vdupq_n_u32(0);
149 uint32x4_t vtmp0x4567 = vdupq_n_u32(0);
150 uint32x4_t vtmp0x89AB = vdupq_n_u32(0);
151 uint32x4_t vtmp0xCDEF = vdupq_n_u32(0);
152 vtmp0x0123 = vld1q_lane_u32(w0, vtmp0x0123, 0); w0 += 1;
153 vtmp0x0123 = vld1q_lane_u32(w1, vtmp0x0123, 1); w1 += 1;
154 vtmp0x0123 = vld1q_lane_u32(w2, vtmp0x0123, 2); w2 += 1;
155 vtmp0x0123 = vld1q_lane_u32(w3, vtmp0x0123, 3); w3 += 1;
156 vtmp0x4567 = vld1q_lane_u32(w4, vtmp0x4567, 0); w4 += 1;
157 vtmp0x4567 = vld1q_lane_u32(w5, vtmp0x4567, 1); w5 += 1;
158 vtmp0x4567 = vld1q_lane_u32(w6, vtmp0x4567, 2); w6 += 1;
159 vtmp0x4567 = vld1q_lane_u32(w7, vtmp0x4567, 3); w7 += 1;
160 vtmp0x89AB = vld1q_lane_u32(w8, vtmp0x89AB, 0); w8 += 1;
161 vtmp0x89AB = vld1q_lane_u32(w9, vtmp0x89AB, 1); w9 += 1;
162 vtmp0x89AB = vld1q_lane_u32(w10, vtmp0x89AB, 2); w10 += 1;
163 vtmp0x89AB = vld1q_lane_u32(w11, vtmp0x89AB, 3); w11 += 1;
164 vtmp0xCDEF = vld1q_lane_u32(w12, vtmp0xCDEF, 0); w12 += 1;
165 vtmp0xCDEF = vld1q_lane_u32(w13, vtmp0xCDEF, 1); w13 += 1;
166 vtmp0xCDEF = vld1q_lane_u32(w14, vtmp0xCDEF, 2); w14 += 1;
167 vtmp0xCDEF = vld1q_lane_u32(w15, vtmp0xCDEF, 3); w15 += 1;
168 vst1q_u32(packed_weights, vtmp0x0123); packed_weights += 4;
169 vst1q_u32(packed_weights, vtmp0x4567); packed_weights += 4;
170 vst1q_u32(packed_weights, vtmp0x89AB); packed_weights += 4;
171 vst1q_u32(packed_weights, vtmp0xCDEF); packed_weights += 4;
172 break;
173 }
174 // KC remainder of 2
175 case 2:
176 {
177 uint32x4x2_t vtmp01x0123;
178 vtmp01x0123.val[0] = vdupq_n_u32(0);
179 vtmp01x0123.val[1] = vdupq_n_u32(0);
180 uint32x4x2_t vtmp01x4567;
181 vtmp01x4567.val[0] = vdupq_n_u32(0);
182 vtmp01x4567.val[1] = vdupq_n_u32(0);
183 uint32x4x2_t vtmp01x89AB;
184 vtmp01x89AB.val[0] = vdupq_n_u32(0);
185 vtmp01x89AB.val[1] = vdupq_n_u32(0);
186 uint32x4x2_t vtmp01xCDEF;
187 vtmp01xCDEF.val[0] = vdupq_n_u32(0);
188 vtmp01xCDEF.val[1] = vdupq_n_u32(0);
189 vtmp01x0123 = vld2q_lane_u32(w0, vtmp01x0123, 0); w0 += 2;
190 vtmp01x0123 = vld2q_lane_u32(w1, vtmp01x0123, 1); w1 += 2;
191 vtmp01x0123 = vld2q_lane_u32(w2, vtmp01x0123, 2); w2 += 2;
192 vtmp01x0123 = vld2q_lane_u32(w3, vtmp01x0123, 3); w3 += 2;
193 vtmp01x4567 = vld2q_lane_u32(w4, vtmp01x4567, 0); w4 += 2;
194 vtmp01x4567 = vld2q_lane_u32(w5, vtmp01x4567, 1); w5 += 2;
195 vtmp01x4567 = vld2q_lane_u32(w6, vtmp01x4567, 2); w6 += 2;
196 vtmp01x4567 = vld2q_lane_u32(w7, vtmp01x4567, 3); w7 += 2;
197 vtmp01x89AB = vld2q_lane_u32(w8, vtmp01x89AB, 0); w8 += 2;
198 vtmp01x89AB = vld2q_lane_u32(w9, vtmp01x89AB, 1); w9 += 2;
199 vtmp01x89AB = vld2q_lane_u32(w10, vtmp01x89AB, 2); w10 += 2;
200 vtmp01x89AB = vld2q_lane_u32(w11, vtmp01x89AB, 3); w11 += 2;
201 vtmp01xCDEF = vld2q_lane_u32(w12, vtmp01xCDEF, 0); w12 += 2;
202 vtmp01xCDEF = vld2q_lane_u32(w13, vtmp01xCDEF, 1); w13 += 2;
203 vtmp01xCDEF = vld2q_lane_u32(w14, vtmp01xCDEF, 2); w14 += 2;
204 vtmp01xCDEF = vld2q_lane_u32(w15, vtmp01xCDEF, 3); w15 += 2;
205 vst1q_u32(packed_weights, vtmp01x0123.val[0]); packed_weights += 4;
206 vst1q_u32(packed_weights, vtmp01x4567.val[0]); packed_weights += 4;
207 vst1q_u32(packed_weights, vtmp01x89AB.val[0]); packed_weights += 4;
208 vst1q_u32(packed_weights, vtmp01xCDEF.val[0]); packed_weights += 4;
209 vst1q_u32(packed_weights, vtmp01x0123.val[1]); packed_weights += 4;
210 vst1q_u32(packed_weights, vtmp01x4567.val[1]); packed_weights += 4;
211 vst1q_u32(packed_weights, vtmp01x89AB.val[1]); packed_weights += 4;
212 vst1q_u32(packed_weights, vtmp01xCDEF.val[1]); packed_weights += 4;
213 break;
214 }
215 // KC remainder of 3
216 case 3:
217 {
218 uint32x4x3_t vtmp012x0123;
219 vtmp012x0123.val[0] = vdupq_n_u32(0);
220 vtmp012x0123.val[1] = vdupq_n_u32(0);
221 vtmp012x0123.val[2] = vdupq_n_u32(0);
222 uint32x4x3_t vtmp012x4567;
223 vtmp012x4567.val[0] = vdupq_n_u32(0);
224 vtmp012x4567.val[1] = vdupq_n_u32(0);
225 vtmp012x4567.val[2] = vdupq_n_u32(0);
226 uint32x4x3_t vtmp012x89AB;
227 vtmp012x89AB.val[0] = vdupq_n_u32(0);
228 vtmp012x89AB.val[1] = vdupq_n_u32(0);
229 vtmp012x89AB.val[2] = vdupq_n_u32(0);
230 uint32x4x3_t vtmp012xCDEF;
231 vtmp012xCDEF.val[0] = vdupq_n_u32(0);
232 vtmp012xCDEF.val[1] = vdupq_n_u32(0);
233 vtmp012xCDEF.val[2] = vdupq_n_u32(0);
234 vtmp012x0123 = vld3q_lane_u32(w0, vtmp012x0123, 0); w0 += 3;
235 vtmp012x0123 = vld3q_lane_u32(w1, vtmp012x0123, 1); w1 += 3;
236 vtmp012x0123 = vld3q_lane_u32(w2, vtmp012x0123, 2); w2 += 3;
237 vtmp012x0123 = vld3q_lane_u32(w3, vtmp012x0123, 3); w3 += 3;
238 vtmp012x4567 = vld3q_lane_u32(w4, vtmp012x4567, 0); w4 += 3;
239 vtmp012x4567 = vld3q_lane_u32(w5, vtmp012x4567, 1); w5 += 3;
240 vtmp012x4567 = vld3q_lane_u32(w6, vtmp012x4567, 2); w6 += 3;
241 vtmp012x4567 = vld3q_lane_u32(w7, vtmp012x4567, 3); w7 += 3;
242 vtmp012x89AB = vld3q_lane_u32(w8, vtmp012x89AB, 0); w8 += 3;
243 vtmp012x89AB = vld3q_lane_u32(w9, vtmp012x89AB, 1); w9 += 3;
244 vtmp012x89AB = vld3q_lane_u32(w10, vtmp012x89AB, 2); w10 += 3;
245 vtmp012x89AB = vld3q_lane_u32(w11, vtmp012x89AB, 3); w11 += 3;
246 vtmp012xCDEF = vld3q_lane_u32(w12, vtmp012xCDEF, 0); w12 += 3;
247 vtmp012xCDEF = vld3q_lane_u32(w13, vtmp012xCDEF, 1); w13 += 3;
248 vtmp012xCDEF = vld3q_lane_u32(w14, vtmp012xCDEF, 2); w14 += 3;
249 vtmp012xCDEF = vld3q_lane_u32(w15, vtmp012xCDEF, 3); w15 += 3;
250 vst1q_u32(packed_weights, vtmp012x0123.val[0]); packed_weights += 4;
251 vst1q_u32(packed_weights, vtmp012x4567.val[0]); packed_weights += 4;
252 vst1q_u32(packed_weights, vtmp012x89AB.val[0]); packed_weights += 4;
253 vst1q_u32(packed_weights, vtmp012xCDEF.val[0]); packed_weights += 4;
254 vst1q_u32(packed_weights, vtmp012x0123.val[1]); packed_weights += 4;
255 vst1q_u32(packed_weights, vtmp012x4567.val[1]); packed_weights += 4;
256 vst1q_u32(packed_weights, vtmp012x89AB.val[1]); packed_weights += 4;
257 vst1q_u32(packed_weights, vtmp012xCDEF.val[1]); packed_weights += 4;
258 vst1q_u32(packed_weights, vtmp012x0123.val[2]); packed_weights += 4;
259 vst1q_u32(packed_weights, vtmp012x4567.val[2]); packed_weights += 4;
260 vst1q_u32(packed_weights, vtmp012x89AB.val[2]); packed_weights += 4;
261 vst1q_u32(packed_weights, vtmp012xCDEF.val[2]); packed_weights += 4;
262 break;
263 }
264 default:
265 XNN_UNREACHABLE;
266 }
267 }
268 packed_weights = (uint32_t*) ((uintptr_t) packed_weights + extra_bytes);
269 w0 = w15;
270 }
271
272 // NC remainder (1..15)
273 if XNN_UNLIKELY(n != 0) {
274 assert(n >= 1);
275 assert(n <= 15);
276 if XNN_LIKELY(bias != NULL) {
277 size_t nb = n;
278 do {
279 *packed_weights++ = *bias++;
280 } while (--nb != 0);
281 packed_weights += (16 - n);
282 } else {
283 const uint32x4_t vzero = vmovq_n_u32(0);
284 vst1q_u32(packed_weights, vzero); packed_weights += 4;
285 vst1q_u32(packed_weights, vzero); packed_weights += 4;
286 vst1q_u32(packed_weights, vzero); packed_weights += 4;
287 vst1q_u32(packed_weights, vzero); packed_weights += 4;
288 }
289
290 // NR remainder has less than 16 rows so last row is not loaded
291 const uint32_t* w1 = w0 + kc;
292 if XNN_UNPREDICTABLE(n < 2) {
293 w1 = w0;
294 }
295 const uint32_t* w2 = w1 + kc;
296 if XNN_UNPREDICTABLE(n <= 2) {
297 w2 = w1;
298 }
299 const uint32_t* w3 = w2 + kc;
300 if XNN_UNPREDICTABLE(n < 4) {
301 w3 = w2;
302 }
303 const uint32_t* w4 = w3 + kc;
304 if XNN_UNPREDICTABLE(n <= 4) {
305 w4 = w3;
306 }
307 const uint32_t* w5 = w4 + kc;
308 if XNN_UNPREDICTABLE(n < 6) {
309 w5 = w4;
310 }
311 const uint32_t* w6 = w5 + kc;
312 if XNN_UNPREDICTABLE(n <= 6) {
313 w6 = w5;
314 }
315 const uint32_t* w7 = w6 + kc;
316 if XNN_UNPREDICTABLE(n < 8) {
317 w7 = w6;
318 }
319 const uint32_t* w8 = w7 + kc;
320 if XNN_UNPREDICTABLE(n <= 8) {
321 w8 = w7;
322 }
323 const uint32_t* w9 = w8 + kc;
324 if XNN_UNPREDICTABLE(n < 10) {
325 w9 = w8;
326 }
327 const uint32_t* w10 = w9 + kc;
328 if XNN_UNPREDICTABLE(n <= 10) {
329 w10 = w9;
330 }
331 const uint32_t* w11 = w10 + kc;
332 if XNN_UNPREDICTABLE(n < 12) {
333 w11 = w10;
334 }
335 const uint32_t* w12 = w11 + kc;
336 if XNN_UNPREDICTABLE(n <= 12) {
337 w12 = w11;
338 }
339 const uint32_t* w13 = w12 + kc;
340 if XNN_UNPREDICTABLE(n < 14) {
341 w13 = w12;
342 }
343 const uint32_t* w14 = w13 + kc;
344 if XNN_UNPREDICTABLE(n <= 14) {
345 w14 = w13;
346 }
347
348 // KC main loop multiple of 4
349 size_t k = kc;
350 for (; k >= 4; k -= 4) {
351 vtmp0123x0123 = vld4q_lane_u32(w0, vtmp0123x0123, 0); w0 += 4;
352 vtmp0123x0123 = vld4q_lane_u32(w1, vtmp0123x0123, 1); w1 += 4;
353 vtmp0123x0123 = vld4q_lane_u32(w2, vtmp0123x0123, 2); w2 += 4;
354 vtmp0123x0123 = vld4q_lane_u32(w3, vtmp0123x0123, 3); w3 += 4;
355 vtmp0123x4567 = vld4q_lane_u32(w4, vtmp0123x4567, 0); w4 += 4;
356 vtmp0123x4567 = vld4q_lane_u32(w5, vtmp0123x4567, 1); w5 += 4;
357 vtmp0123x4567 = vld4q_lane_u32(w6, vtmp0123x4567, 2); w6 += 4;
358 vtmp0123x4567 = vld4q_lane_u32(w7, vtmp0123x4567, 3); w7 += 4;
359 vtmp0123x89AB = vld4q_lane_u32(w8, vtmp0123x89AB, 0); w8 += 4;
360 vtmp0123x89AB = vld4q_lane_u32(w9, vtmp0123x89AB, 1); w9 += 4;
361 vtmp0123x89AB = vld4q_lane_u32(w10, vtmp0123x89AB, 2); w10 += 4;
362 vtmp0123x89AB = vld4q_lane_u32(w11, vtmp0123x89AB, 3); w11 += 4;
363 vtmp0123xCDEF = vld4q_lane_u32(w12, vtmp0123xCDEF, 0); w12 += 4;
364 vtmp0123xCDEF = vld4q_lane_u32(w13, vtmp0123xCDEF, 1); w13 += 4;
365 vtmp0123xCDEF = vld4q_lane_u32(w14, vtmp0123xCDEF, 2); w14 += 4;
366 vst1q_u32(packed_weights, vtmp0123x0123.val[0]); packed_weights += 4;
367 vst1q_u32(packed_weights, vtmp0123x4567.val[0]); packed_weights += 4;
368 vst1q_u32(packed_weights, vtmp0123x89AB.val[0]); packed_weights += 4;
369 vst1q_u32(packed_weights, vtmp0123xCDEF.val[0]); packed_weights += 4;
370 vst1q_u32(packed_weights, vtmp0123x0123.val[1]); packed_weights += 4;
371 vst1q_u32(packed_weights, vtmp0123x4567.val[1]); packed_weights += 4;
372 vst1q_u32(packed_weights, vtmp0123x89AB.val[1]); packed_weights += 4;
373 vst1q_u32(packed_weights, vtmp0123xCDEF.val[1]); packed_weights += 4;
374 vst1q_u32(packed_weights, vtmp0123x0123.val[2]); packed_weights += 4;
375 vst1q_u32(packed_weights, vtmp0123x4567.val[2]); packed_weights += 4;
376 vst1q_u32(packed_weights, vtmp0123x89AB.val[2]); packed_weights += 4;
377 vst1q_u32(packed_weights, vtmp0123xCDEF.val[2]); packed_weights += 4;
378 vst1q_u32(packed_weights, vtmp0123x0123.val[3]); packed_weights += 4;
379 vst1q_u32(packed_weights, vtmp0123x4567.val[3]); packed_weights += 4;
380 vst1q_u32(packed_weights, vtmp0123x89AB.val[3]); packed_weights += 4;
381 vst1q_u32(packed_weights, vtmp0123xCDEF.val[3]); packed_weights += 4;
382 }
383
384 // KC remainder of 1..3
385 // Same as main loop but ld1, ld2 or ld3
386 if XNN_UNLIKELY(k != 0) {
387 assert(k >= 1);
388 assert(k <= 3);
389 switch (k) {
390 // KC remainder of 1
391 case 1:
392 {
393 uint32x4_t vtmp0x0123 = vdupq_n_u32(0);
394 uint32x4_t vtmp0x4567 = vdupq_n_u32(0);
395 uint32x4_t vtmp0x89AB = vdupq_n_u32(0);
396 uint32x4_t vtmp0xCDEF = vdupq_n_u32(0);
397 vtmp0x0123 = vld1q_lane_u32(w0, vtmp0x0123, 0);
398 vtmp0x0123 = vld1q_lane_u32(w1, vtmp0x0123, 1);
399 vtmp0x0123 = vld1q_lane_u32(w2, vtmp0x0123, 2);
400 vtmp0x0123 = vld1q_lane_u32(w3, vtmp0x0123, 3);
401 vtmp0x4567 = vld1q_lane_u32(w4, vtmp0x4567, 0);
402 vtmp0x4567 = vld1q_lane_u32(w5, vtmp0x4567, 1);
403 vtmp0x4567 = vld1q_lane_u32(w6, vtmp0x4567, 2);
404 vtmp0x4567 = vld1q_lane_u32(w7, vtmp0x4567, 3);
405 vtmp0x89AB = vld1q_lane_u32(w8, vtmp0x89AB, 0);
406 vtmp0x89AB = vld1q_lane_u32(w9, vtmp0x89AB, 1);
407 vtmp0x89AB = vld1q_lane_u32(w10, vtmp0x89AB, 2);
408 vtmp0x89AB = vld1q_lane_u32(w11, vtmp0x89AB, 3);
409 vtmp0xCDEF = vld1q_lane_u32(w12, vtmp0xCDEF, 0);
410 vtmp0xCDEF = vld1q_lane_u32(w13, vtmp0xCDEF, 1);
411 vtmp0xCDEF = vld1q_lane_u32(w14, vtmp0xCDEF, 2);
412 vst1q_u32(packed_weights, vtmp0x0123); packed_weights += 4;
413 vst1q_u32(packed_weights, vtmp0x4567); packed_weights += 4;
414 vst1q_u32(packed_weights, vtmp0x89AB); packed_weights += 4;
415 vst1q_u32(packed_weights, vtmp0xCDEF); packed_weights += 4;
416 break;
417 }
418 // KC remainder of 2
419 case 2:
420 {
421 uint32x4x2_t vtmp01x0123;
422 vtmp01x0123.val[0] = vdupq_n_u32(0);
423 vtmp01x0123.val[1] = vdupq_n_u32(0);
424 uint32x4x2_t vtmp01x4567;
425 vtmp01x4567.val[0] = vdupq_n_u32(0);
426 vtmp01x4567.val[1] = vdupq_n_u32(0);
427 uint32x4x2_t vtmp01x89AB;
428 vtmp01x89AB.val[0] = vdupq_n_u32(0);
429 vtmp01x89AB.val[1] = vdupq_n_u32(0);
430 uint32x4x2_t vtmp01xCDEF;
431 vtmp01xCDEF.val[0] = vdupq_n_u32(0);
432 vtmp01xCDEF.val[1] = vdupq_n_u32(0);
433 vtmp01x0123 = vld2q_lane_u32(w0, vtmp01x0123, 0);
434 vtmp01x0123 = vld2q_lane_u32(w1, vtmp01x0123, 1);
435 vtmp01x0123 = vld2q_lane_u32(w2, vtmp01x0123, 2);
436 vtmp01x0123 = vld2q_lane_u32(w3, vtmp01x0123, 3);
437 vtmp01x4567 = vld2q_lane_u32(w4, vtmp01x4567, 0);
438 vtmp01x4567 = vld2q_lane_u32(w5, vtmp01x4567, 1);
439 vtmp01x4567 = vld2q_lane_u32(w6, vtmp01x4567, 2);
440 vtmp01x4567 = vld2q_lane_u32(w7, vtmp01x4567, 3);
441 vtmp01x89AB = vld2q_lane_u32(w8, vtmp01x89AB, 0);
442 vtmp01x89AB = vld2q_lane_u32(w9, vtmp01x89AB, 1);
443 vtmp01x89AB = vld2q_lane_u32(w10, vtmp01x89AB, 2);
444 vtmp01x89AB = vld2q_lane_u32(w11, vtmp01x89AB, 3);
445 vtmp01xCDEF = vld2q_lane_u32(w12, vtmp01xCDEF, 0);
446 vtmp01xCDEF = vld2q_lane_u32(w13, vtmp01xCDEF, 1);
447 vtmp01xCDEF = vld2q_lane_u32(w14, vtmp01xCDEF, 2);
448 vst1q_u32(packed_weights, vtmp01x0123.val[0]); packed_weights += 4;
449 vst1q_u32(packed_weights, vtmp01x4567.val[0]); packed_weights += 4;
450 vst1q_u32(packed_weights, vtmp01x89AB.val[0]); packed_weights += 4;
451 vst1q_u32(packed_weights, vtmp01xCDEF.val[0]); packed_weights += 4;
452 vst1q_u32(packed_weights, vtmp01x0123.val[1]); packed_weights += 4;
453 vst1q_u32(packed_weights, vtmp01x4567.val[1]); packed_weights += 4;
454 vst1q_u32(packed_weights, vtmp01x89AB.val[1]); packed_weights += 4;
455 vst1q_u32(packed_weights, vtmp01xCDEF.val[1]); packed_weights += 4;
456 break;
457 }
458 // KC remainder of 3
459 case 3:
460 {
461 uint32x4x3_t vtmp012x0123;
462 vtmp012x0123.val[0] = vdupq_n_u32(0);
463 vtmp012x0123.val[1] = vdupq_n_u32(0);
464 vtmp012x0123.val[2] = vdupq_n_u32(0);
465 uint32x4x3_t vtmp012x4567;
466 vtmp012x4567.val[0] = vdupq_n_u32(0);
467 vtmp012x4567.val[1] = vdupq_n_u32(0);
468 vtmp012x4567.val[2] = vdupq_n_u32(0);
469 uint32x4x3_t vtmp012x89AB;
470 vtmp012x89AB.val[0] = vdupq_n_u32(0);
471 vtmp012x89AB.val[1] = vdupq_n_u32(0);
472 vtmp012x89AB.val[2] = vdupq_n_u32(0);
473 uint32x4x3_t vtmp012xCDEF;
474 vtmp012xCDEF.val[0] = vdupq_n_u32(0);
475 vtmp012xCDEF.val[1] = vdupq_n_u32(0);
476 vtmp012xCDEF.val[2] = vdupq_n_u32(0);
477 vtmp012x0123 = vld3q_lane_u32(w0, vtmp012x0123, 0);
478 vtmp012x0123 = vld3q_lane_u32(w1, vtmp012x0123, 1);
479 vtmp012x0123 = vld3q_lane_u32(w2, vtmp012x0123, 2);
480 vtmp012x0123 = vld3q_lane_u32(w3, vtmp012x0123, 3);
481 vtmp012x4567 = vld3q_lane_u32(w4, vtmp012x4567, 0);
482 vtmp012x4567 = vld3q_lane_u32(w5, vtmp012x4567, 1);
483 vtmp012x4567 = vld3q_lane_u32(w6, vtmp012x4567, 2);
484 vtmp012x4567 = vld3q_lane_u32(w7, vtmp012x4567, 3);
485 vtmp012x89AB = vld3q_lane_u32(w8, vtmp012x89AB, 0);
486 vtmp012x89AB = vld3q_lane_u32(w9, vtmp012x89AB, 1);
487 vtmp012x89AB = vld3q_lane_u32(w10, vtmp012x89AB, 2);
488 vtmp012x89AB = vld3q_lane_u32(w11, vtmp012x89AB, 3);
489 vtmp012xCDEF = vld3q_lane_u32(w12, vtmp012xCDEF, 0);
490 vtmp012xCDEF = vld3q_lane_u32(w13, vtmp012xCDEF, 1);
491 vtmp012xCDEF = vld3q_lane_u32(w14, vtmp012xCDEF, 2);
492 vst1q_u32(packed_weights, vtmp012x0123.val[0]); packed_weights += 4;
493 vst1q_u32(packed_weights, vtmp012x4567.val[0]); packed_weights += 4;
494 vst1q_u32(packed_weights, vtmp012x89AB.val[0]); packed_weights += 4;
495 vst1q_u32(packed_weights, vtmp012xCDEF.val[0]); packed_weights += 4;
496 vst1q_u32(packed_weights, vtmp012x0123.val[1]); packed_weights += 4;
497 vst1q_u32(packed_weights, vtmp012x4567.val[1]); packed_weights += 4;
498 vst1q_u32(packed_weights, vtmp012x89AB.val[1]); packed_weights += 4;
499 vst1q_u32(packed_weights, vtmp012xCDEF.val[1]); packed_weights += 4;
500 vst1q_u32(packed_weights, vtmp012x0123.val[2]); packed_weights += 4;
501 vst1q_u32(packed_weights, vtmp012x4567.val[2]); packed_weights += 4;
502 vst1q_u32(packed_weights, vtmp012x89AB.val[2]); packed_weights += 4;
503 vst1q_u32(packed_weights, vtmp012xCDEF.val[2]); packed_weights += 4;
504 break;
505 }
506 default:
507 XNN_UNREACHABLE;
508 }
509 }
510 packed_weights = (uint32_t*) ((uintptr_t) packed_weights + extra_bytes);
511 }
512
513 weights += nc * kc;
514 } while (--g != 0);
515 }