Debian Codegraph @

src/x32-packw/gen/x32-packw-x12-gemm-goi-neon-ld4lane-u8.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/neon.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include <arm_neon.h>
16
17 #include "xnnpack/packw.h"
18
19
20 void xnn_x32_packw_gemm_goi_ukernel_x12__neon_ld4lane_u8(
21 size_t g,
22 size_t nc,
23 size_t kc,
24 size_t nr,
25 size_t kr,
26 size_t sr,
27 const uint32_t* weights,
28 const uint32_t* bias,
29 const void* scale,
30 uint32_t* packed_weights,
31 size_t extra_bytes,
32 const void* params)
33 {
34 assert(g != 0);
35 assert(nc != 0);
36 assert(kc != 0);
37 assert(nr == 12);
38 assert(kr == 1);
39 assert(sr == 1);
40 assert(weights != NULL);
41 assert(packed_weights != NULL);
42 uint32x4x4_t vtmp0123x0123;
43 vtmp0123x0123.val[0] = vdupq_n_u32(0);
44 vtmp0123x0123.val[1] = vdupq_n_u32(0);
45 vtmp0123x0123.val[2] = vdupq_n_u32(0);
46 vtmp0123x0123.val[3] = vdupq_n_u32(0);
47 uint32x4x4_t vtmp4567x0123;
48 vtmp4567x0123.val[0] = vdupq_n_u32(0);
49 vtmp4567x0123.val[1] = vdupq_n_u32(0);
50 vtmp4567x0123.val[2] = vdupq_n_u32(0);
51 vtmp4567x0123.val[3] = vdupq_n_u32(0);
52 uint32x4x4_t vtmp0123x4567;
53 vtmp0123x4567.val[0] = vdupq_n_u32(0);
54 vtmp0123x4567.val[1] = vdupq_n_u32(0);
55 vtmp0123x4567.val[2] = vdupq_n_u32(0);
56 vtmp0123x4567.val[3] = vdupq_n_u32(0);
57 uint32x4x4_t vtmp4567x4567;
58 vtmp4567x4567.val[0] = vdupq_n_u32(0);
59 vtmp4567x4567.val[1] = vdupq_n_u32(0);
60 vtmp4567x4567.val[2] = vdupq_n_u32(0);
61 vtmp4567x4567.val[3] = vdupq_n_u32(0);
62 uint32x4x4_t vtmp0123x89AB;
63 vtmp0123x89AB.val[0] = vdupq_n_u32(0);
64 vtmp0123x89AB.val[1] = vdupq_n_u32(0);
65 vtmp0123x89AB.val[2] = vdupq_n_u32(0);
66 vtmp0123x89AB.val[3] = vdupq_n_u32(0);
67 uint32x4x4_t vtmp4567x89AB;
68 vtmp4567x89AB.val[0] = vdupq_n_u32(0);
69 vtmp4567x89AB.val[1] = vdupq_n_u32(0);
70 vtmp4567x89AB.val[2] = vdupq_n_u32(0);
71 vtmp4567x89AB.val[3] = vdupq_n_u32(0);
72
73 do {
74 // NC main loop multiple of 12
75 const uint32_t* w0 = weights;
76 size_t n = nc;
77
78 for (; n >= 12; n -= 12) {
79 if XNN_LIKELY(bias != NULL) {
80 uint32x4_t vb0 = vld1q_u32(bias); bias += 4;
81 uint32x4_t vb4 = vld1q_u32(bias); bias += 4;
82 uint32x4_t vb8 = vld1q_u32(bias); bias += 4;
83 vst1q_u32(packed_weights, vb0); packed_weights += 4;
84 vst1q_u32(packed_weights, vb4); packed_weights += 4;
85 vst1q_u32(packed_weights, vb8); packed_weights += 4;
86 } else {
87 const uint32x4_t vzero = vmovq_n_u32(0);
88 vst1q_u32(packed_weights, vzero); packed_weights += 4;
89 vst1q_u32(packed_weights, vzero); packed_weights += 4;
90 vst1q_u32(packed_weights, vzero); packed_weights += 4;
91 }
92
93 const uint32_t* w1 = w0 + kc;
94 const uint32_t* w2 = w1 + kc;
95 const uint32_t* w3 = w2 + kc;
96 const uint32_t* w4 = w3 + kc;
97 const uint32_t* w5 = w4 + kc;
98 const uint32_t* w6 = w5 + kc;
99 const uint32_t* w7 = w6 + kc;
100 const uint32_t* w8 = w7 + kc;
101 const uint32_t* w9 = w8 + kc;
102 const uint32_t* w10 = w9 + kc;
103 const uint32_t* w11 = w10 + kc;
104
105 // KC main loop multiple of 8
106 size_t k = kc;
107 for (; k >= 8; k -= 8) {
108 vtmp0123x0123 = vld4q_lane_u32(w0, vtmp0123x0123, 0); w0 += 4;
109 vtmp0123x0123 = vld4q_lane_u32(w1, vtmp0123x0123, 1); w1 += 4;
110 vtmp0123x0123 = vld4q_lane_u32(w2, vtmp0123x0123, 2); w2 += 4;
111 vtmp0123x0123 = vld4q_lane_u32(w3, vtmp0123x0123, 3); w3 += 4;
112 vtmp0123x4567 = vld4q_lane_u32(w4, vtmp0123x4567, 0); w4 += 4;
113 vtmp0123x4567 = vld4q_lane_u32(w5, vtmp0123x4567, 1); w5 += 4;
114 vtmp0123x4567 = vld4q_lane_u32(w6, vtmp0123x4567, 2); w6 += 4;
115 vtmp0123x4567 = vld4q_lane_u32(w7, vtmp0123x4567, 3); w7 += 4;
116 vtmp0123x89AB = vld4q_lane_u32(w8, vtmp0123x89AB, 0); w8 += 4;
117 vtmp0123x89AB = vld4q_lane_u32(w9, vtmp0123x89AB, 1); w9 += 4;
118 vtmp0123x89AB = vld4q_lane_u32(w10, vtmp0123x89AB, 2); w10 += 4;
119 vtmp0123x89AB = vld4q_lane_u32(w11, vtmp0123x89AB, 3); w11 += 4;
120 vtmp4567x0123 = vld4q_lane_u32(w0, vtmp4567x0123, 0); w0 += 4;
121 vtmp4567x0123 = vld4q_lane_u32(w1, vtmp4567x0123, 1); w1 += 4;
122 vtmp4567x0123 = vld4q_lane_u32(w2, vtmp4567x0123, 2); w2 += 4;
123 vtmp4567x0123 = vld4q_lane_u32(w3, vtmp4567x0123, 3); w3 += 4;
124 vtmp4567x4567 = vld4q_lane_u32(w4, vtmp4567x4567, 0); w4 += 4;
125 vtmp4567x4567 = vld4q_lane_u32(w5, vtmp4567x4567, 1); w5 += 4;
126 vtmp4567x4567 = vld4q_lane_u32(w6, vtmp4567x4567, 2); w6 += 4;
127 vtmp4567x4567 = vld4q_lane_u32(w7, vtmp4567x4567, 3); w7 += 4;
128 vtmp4567x89AB = vld4q_lane_u32(w8, vtmp4567x89AB, 0); w8 += 4;
129 vtmp4567x89AB = vld4q_lane_u32(w9, vtmp4567x89AB, 1); w9 += 4;
130 vtmp4567x89AB = vld4q_lane_u32(w10, vtmp4567x89AB, 2); w10 += 4;
131 vtmp4567x89AB = vld4q_lane_u32(w11, vtmp4567x89AB, 3); w11 += 4;
132 vst1q_u32(packed_weights, vtmp0123x0123.val[0]); packed_weights += 4;
133 vst1q_u32(packed_weights, vtmp0123x4567.val[0]); packed_weights += 4;
134 vst1q_u32(packed_weights, vtmp0123x89AB.val[0]); packed_weights += 4;
135 vst1q_u32(packed_weights, vtmp0123x0123.val[1]); packed_weights += 4;
136 vst1q_u32(packed_weights, vtmp0123x4567.val[1]); packed_weights += 4;
137 vst1q_u32(packed_weights, vtmp0123x89AB.val[1]); packed_weights += 4;
138 vst1q_u32(packed_weights, vtmp0123x0123.val[2]); packed_weights += 4;
139 vst1q_u32(packed_weights, vtmp0123x4567.val[2]); packed_weights += 4;
140 vst1q_u32(packed_weights, vtmp0123x89AB.val[2]); packed_weights += 4;
141 vst1q_u32(packed_weights, vtmp0123x0123.val[3]); packed_weights += 4;
142 vst1q_u32(packed_weights, vtmp0123x4567.val[3]); packed_weights += 4;
143 vst1q_u32(packed_weights, vtmp0123x89AB.val[3]); packed_weights += 4;
144 vst1q_u32(packed_weights, vtmp4567x0123.val[0]); packed_weights += 4;
145 vst1q_u32(packed_weights, vtmp4567x4567.val[0]); packed_weights += 4;
146 vst1q_u32(packed_weights, vtmp4567x89AB.val[0]); packed_weights += 4;
147 vst1q_u32(packed_weights, vtmp4567x0123.val[1]); packed_weights += 4;
148 vst1q_u32(packed_weights, vtmp4567x4567.val[1]); packed_weights += 4;
149 vst1q_u32(packed_weights, vtmp4567x89AB.val[1]); packed_weights += 4;
150 vst1q_u32(packed_weights, vtmp4567x0123.val[2]); packed_weights += 4;
151 vst1q_u32(packed_weights, vtmp4567x4567.val[2]); packed_weights += 4;
152 vst1q_u32(packed_weights, vtmp4567x89AB.val[2]); packed_weights += 4;
153 vst1q_u32(packed_weights, vtmp4567x0123.val[3]); packed_weights += 4;
154 vst1q_u32(packed_weights, vtmp4567x4567.val[3]); packed_weights += 4;
155 vst1q_u32(packed_weights, vtmp4567x89AB.val[3]); packed_weights += 4;
156 }
157 // KC remainder multiple of 4
158 if (k >= 4) {
159 vtmp0123x0123 = vld4q_lane_u32(w0, vtmp0123x0123, 0); w0 += 4;
160 vtmp0123x0123 = vld4q_lane_u32(w1, vtmp0123x0123, 1); w1 += 4;
161 vtmp0123x0123 = vld4q_lane_u32(w2, vtmp0123x0123, 2); w2 += 4;
162 vtmp0123x0123 = vld4q_lane_u32(w3, vtmp0123x0123, 3); w3 += 4;
163 vtmp0123x4567 = vld4q_lane_u32(w4, vtmp0123x4567, 0); w4 += 4;
164 vtmp0123x4567 = vld4q_lane_u32(w5, vtmp0123x4567, 1); w5 += 4;
165 vtmp0123x4567 = vld4q_lane_u32(w6, vtmp0123x4567, 2); w6 += 4;
166 vtmp0123x4567 = vld4q_lane_u32(w7, vtmp0123x4567, 3); w7 += 4;
167 vtmp0123x89AB = vld4q_lane_u32(w8, vtmp0123x89AB, 0); w8 += 4;
168 vtmp0123x89AB = vld4q_lane_u32(w9, vtmp0123x89AB, 1); w9 += 4;
169 vtmp0123x89AB = vld4q_lane_u32(w10, vtmp0123x89AB, 2); w10 += 4;
170 vtmp0123x89AB = vld4q_lane_u32(w11, vtmp0123x89AB, 3); w11 += 4;
171 vst1q_u32(packed_weights, vtmp0123x0123.val[0]); packed_weights += 4;
172 vst1q_u32(packed_weights, vtmp0123x4567.val[0]); packed_weights += 4;
173 vst1q_u32(packed_weights, vtmp0123x89AB.val[0]); packed_weights += 4;
174 vst1q_u32(packed_weights, vtmp0123x0123.val[1]); packed_weights += 4;
175 vst1q_u32(packed_weights, vtmp0123x4567.val[1]); packed_weights += 4;
176 vst1q_u32(packed_weights, vtmp0123x89AB.val[1]); packed_weights += 4;
177 vst1q_u32(packed_weights, vtmp0123x0123.val[2]); packed_weights += 4;
178 vst1q_u32(packed_weights, vtmp0123x4567.val[2]); packed_weights += 4;
179 vst1q_u32(packed_weights, vtmp0123x89AB.val[2]); packed_weights += 4;
180 vst1q_u32(packed_weights, vtmp0123x0123.val[3]); packed_weights += 4;
181 vst1q_u32(packed_weights, vtmp0123x4567.val[3]); packed_weights += 4;
182 vst1q_u32(packed_weights, vtmp0123x89AB.val[3]); packed_weights += 4;
183 k -= 4;
184 }
185
186 // KC remainder of 1..3
187 // Same as main loop but ld1, ld2 or ld3
188 if XNN_UNLIKELY(k != 0) {
189 assert(k >= 1);
190 assert(k <= 3);
191 switch (k) {
192 // KC remainder of 1
193 case 1:
194 {
195 uint32x4_t vtmp0x0123 = vdupq_n_u32(0);
196 uint32x4_t vtmp0x4567 = vdupq_n_u32(0);
197 uint32x4_t vtmp0x89AB = vdupq_n_u32(0);
198 vtmp0x0123 = vld1q_lane_u32(w0, vtmp0x0123, 0); w0 += 1;
199 vtmp0x0123 = vld1q_lane_u32(w1, vtmp0x0123, 1); w1 += 1;
200 vtmp0x0123 = vld1q_lane_u32(w2, vtmp0x0123, 2); w2 += 1;
201 vtmp0x0123 = vld1q_lane_u32(w3, vtmp0x0123, 3); w3 += 1;
202 vtmp0x4567 = vld1q_lane_u32(w4, vtmp0x4567, 0); w4 += 1;
203 vtmp0x4567 = vld1q_lane_u32(w5, vtmp0x4567, 1); w5 += 1;
204 vtmp0x4567 = vld1q_lane_u32(w6, vtmp0x4567, 2); w6 += 1;
205 vtmp0x4567 = vld1q_lane_u32(w7, vtmp0x4567, 3); w7 += 1;
206 vtmp0x89AB = vld1q_lane_u32(w8, vtmp0x89AB, 0); w8 += 1;
207 vtmp0x89AB = vld1q_lane_u32(w9, vtmp0x89AB, 1); w9 += 1;
208 vtmp0x89AB = vld1q_lane_u32(w10, vtmp0x89AB, 2); w10 += 1;
209 vtmp0x89AB = vld1q_lane_u32(w11, vtmp0x89AB, 3); w11 += 1;
210 vst1q_u32(packed_weights, vtmp0x0123); packed_weights += 4;
211 vst1q_u32(packed_weights, vtmp0x4567); packed_weights += 4;
212 vst1q_u32(packed_weights, vtmp0x89AB); packed_weights += 4;
213 break;
214 }
215 // KC remainder of 2
216 case 2:
217 {
218 uint32x4x2_t vtmp01x0123;
219 vtmp01x0123.val[0] = vdupq_n_u32(0);
220 vtmp01x0123.val[1] = vdupq_n_u32(0);
221 uint32x4x2_t vtmp01x4567;
222 vtmp01x4567.val[0] = vdupq_n_u32(0);
223 vtmp01x4567.val[1] = vdupq_n_u32(0);
224 uint32x4x2_t vtmp01x89AB;
225 vtmp01x89AB.val[0] = vdupq_n_u32(0);
226 vtmp01x89AB.val[1] = vdupq_n_u32(0);
227 vtmp01x0123 = vld2q_lane_u32(w0, vtmp01x0123, 0); w0 += 2;
228 vtmp01x0123 = vld2q_lane_u32(w1, vtmp01x0123, 1); w1 += 2;
229 vtmp01x0123 = vld2q_lane_u32(w2, vtmp01x0123, 2); w2 += 2;
230 vtmp01x0123 = vld2q_lane_u32(w3, vtmp01x0123, 3); w3 += 2;
231 vtmp01x4567 = vld2q_lane_u32(w4, vtmp01x4567, 0); w4 += 2;
232 vtmp01x4567 = vld2q_lane_u32(w5, vtmp01x4567, 1); w5 += 2;
233 vtmp01x4567 = vld2q_lane_u32(w6, vtmp01x4567, 2); w6 += 2;
234 vtmp01x4567 = vld2q_lane_u32(w7, vtmp01x4567, 3); w7 += 2;
235 vtmp01x89AB = vld2q_lane_u32(w8, vtmp01x89AB, 0); w8 += 2;
236 vtmp01x89AB = vld2q_lane_u32(w9, vtmp01x89AB, 1); w9 += 2;
237 vtmp01x89AB = vld2q_lane_u32(w10, vtmp01x89AB, 2); w10 += 2;
238 vtmp01x89AB = vld2q_lane_u32(w11, vtmp01x89AB, 3); w11 += 2;
239 vst1q_u32(packed_weights, vtmp01x0123.val[0]); packed_weights += 4;
240 vst1q_u32(packed_weights, vtmp01x4567.val[0]); packed_weights += 4;
241 vst1q_u32(packed_weights, vtmp01x89AB.val[0]); packed_weights += 4;
242 vst1q_u32(packed_weights, vtmp01x0123.val[1]); packed_weights += 4;
243 vst1q_u32(packed_weights, vtmp01x4567.val[1]); packed_weights += 4;
244 vst1q_u32(packed_weights, vtmp01x89AB.val[1]); packed_weights += 4;
245 break;
246 }
247 // KC remainder of 3
248 case 3:
249 {
250 uint32x4x3_t vtmp012x0123;
251 vtmp012x0123.val[0] = vdupq_n_u32(0);
252 vtmp012x0123.val[1] = vdupq_n_u32(0);
253 vtmp012x0123.val[2] = vdupq_n_u32(0);
254 uint32x4x3_t vtmp012x4567;
255 vtmp012x4567.val[0] = vdupq_n_u32(0);
256 vtmp012x4567.val[1] = vdupq_n_u32(0);
257 vtmp012x4567.val[2] = vdupq_n_u32(0);
258 uint32x4x3_t vtmp012x89AB;
259 vtmp012x89AB.val[0] = vdupq_n_u32(0);
260 vtmp012x89AB.val[1] = vdupq_n_u32(0);
261 vtmp012x89AB.val[2] = vdupq_n_u32(0);
262 vtmp012x0123 = vld3q_lane_u32(w0, vtmp012x0123, 0); w0 += 3;
263 vtmp012x0123 = vld3q_lane_u32(w1, vtmp012x0123, 1); w1 += 3;
264 vtmp012x0123 = vld3q_lane_u32(w2, vtmp012x0123, 2); w2 += 3;
265 vtmp012x0123 = vld3q_lane_u32(w3, vtmp012x0123, 3); w3 += 3;
266 vtmp012x4567 = vld3q_lane_u32(w4, vtmp012x4567, 0); w4 += 3;
267 vtmp012x4567 = vld3q_lane_u32(w5, vtmp012x4567, 1); w5 += 3;
268 vtmp012x4567 = vld3q_lane_u32(w6, vtmp012x4567, 2); w6 += 3;
269 vtmp012x4567 = vld3q_lane_u32(w7, vtmp012x4567, 3); w7 += 3;
270 vtmp012x89AB = vld3q_lane_u32(w8, vtmp012x89AB, 0); w8 += 3;
271 vtmp012x89AB = vld3q_lane_u32(w9, vtmp012x89AB, 1); w9 += 3;
272 vtmp012x89AB = vld3q_lane_u32(w10, vtmp012x89AB, 2); w10 += 3;
273 vtmp012x89AB = vld3q_lane_u32(w11, vtmp012x89AB, 3); w11 += 3;
274 vst1q_u32(packed_weights, vtmp012x0123.val[0]); packed_weights += 4;
275 vst1q_u32(packed_weights, vtmp012x4567.val[0]); packed_weights += 4;
276 vst1q_u32(packed_weights, vtmp012x89AB.val[0]); packed_weights += 4;
277 vst1q_u32(packed_weights, vtmp012x0123.val[1]); packed_weights += 4;
278 vst1q_u32(packed_weights, vtmp012x4567.val[1]); packed_weights += 4;
279 vst1q_u32(packed_weights, vtmp012x89AB.val[1]); packed_weights += 4;
280 vst1q_u32(packed_weights, vtmp012x0123.val[2]); packed_weights += 4;
281 vst1q_u32(packed_weights, vtmp012x4567.val[2]); packed_weights += 4;
282 vst1q_u32(packed_weights, vtmp012x89AB.val[2]); packed_weights += 4;
283 break;
284 }
285 default:
286 XNN_UNREACHABLE;
287 }
288 }
289 packed_weights = (uint32_t*) ((uintptr_t) packed_weights + extra_bytes);
290 w0 = w11;
291 }
292
293 // NC remainder (1..11)
294 if XNN_UNLIKELY(n != 0) {
295 assert(n >= 1);
296 assert(n <= 11);
297 if XNN_LIKELY(bias != NULL) {
298 size_t nb = n;
299 do {
300 *packed_weights++ = *bias++;
301 } while (--nb != 0);
302 packed_weights += (12 - n);
303 } else {
304 const uint32x4_t vzero = vmovq_n_u32(0);
305 vst1q_u32(packed_weights, vzero); packed_weights += 4;
306 vst1q_u32(packed_weights, vzero); packed_weights += 4;
307 vst1q_u32(packed_weights, vzero); packed_weights += 4;
308 }
309
310 // NR remainder has less than 12 rows so last row is not loaded
311 const uint32_t* w1 = w0 + kc;
312 if XNN_UNPREDICTABLE(n < 2) {
313 w1 = w0;
314 }
315 const uint32_t* w2 = w1 + kc;
316 if XNN_UNPREDICTABLE(n <= 2) {
317 w2 = w1;
318 }
319 const uint32_t* w3 = w2 + kc;
320 if XNN_UNPREDICTABLE(n < 4) {
321 w3 = w2;
322 }
323 const uint32_t* w4 = w3 + kc;
324 if XNN_UNPREDICTABLE(n <= 4) {
325 w4 = w3;
326 }
327 const uint32_t* w5 = w4 + kc;
328 if XNN_UNPREDICTABLE(n < 6) {
329 w5 = w4;
330 }
331 const uint32_t* w6 = w5 + kc;
332 if XNN_UNPREDICTABLE(n <= 6) {
333 w6 = w5;
334 }
335 const uint32_t* w7 = w6 + kc;
336 if XNN_UNPREDICTABLE(n < 8) {
337 w7 = w6;
338 }
339 const uint32_t* w8 = w7 + kc;
340 if XNN_UNPREDICTABLE(n <= 8) {
341 w8 = w7;
342 }
343 const uint32_t* w9 = w8 + kc;
344 if XNN_UNPREDICTABLE(n < 10) {
345 w9 = w8;
346 }
347 const uint32_t* w10 = w9 + kc;
348 if XNN_UNPREDICTABLE(n <= 10) {
349 w10 = w9;
350 }
351
352 // KC main loop multiple of 8
353 size_t k = kc;
354 for (; k >= 8; k -= 8) {
355 vtmp0123x0123 = vld4q_lane_u32(w0, vtmp0123x0123, 0); w0 += 4;
356 vtmp0123x0123 = vld4q_lane_u32(w1, vtmp0123x0123, 1); w1 += 4;
357 vtmp0123x0123 = vld4q_lane_u32(w2, vtmp0123x0123, 2); w2 += 4;
358 vtmp0123x0123 = vld4q_lane_u32(w3, vtmp0123x0123, 3); w3 += 4;
359 vtmp0123x4567 = vld4q_lane_u32(w4, vtmp0123x4567, 0); w4 += 4;
360 vtmp0123x4567 = vld4q_lane_u32(w5, vtmp0123x4567, 1); w5 += 4;
361 vtmp0123x4567 = vld4q_lane_u32(w6, vtmp0123x4567, 2); w6 += 4;
362 vtmp0123x4567 = vld4q_lane_u32(w7, vtmp0123x4567, 3); w7 += 4;
363 vtmp0123x89AB = vld4q_lane_u32(w8, vtmp0123x89AB, 0); w8 += 4;
364 vtmp0123x89AB = vld4q_lane_u32(w9, vtmp0123x89AB, 1); w9 += 4;
365 vtmp0123x89AB = vld4q_lane_u32(w10, vtmp0123x89AB, 2); w10 += 4;
366 vtmp4567x0123 = vld4q_lane_u32(w0, vtmp4567x0123, 0); w0 += 4;
367 vtmp4567x0123 = vld4q_lane_u32(w1, vtmp4567x0123, 1); w1 += 4;
368 vtmp4567x0123 = vld4q_lane_u32(w2, vtmp4567x0123, 2); w2 += 4;
369 vtmp4567x0123 = vld4q_lane_u32(w3, vtmp4567x0123, 3); w3 += 4;
370 vtmp4567x4567 = vld4q_lane_u32(w4, vtmp4567x4567, 0); w4 += 4;
371 vtmp4567x4567 = vld4q_lane_u32(w5, vtmp4567x4567, 1); w5 += 4;
372 vtmp4567x4567 = vld4q_lane_u32(w6, vtmp4567x4567, 2); w6 += 4;
373 vtmp4567x4567 = vld4q_lane_u32(w7, vtmp4567x4567, 3); w7 += 4;
374 vtmp4567x89AB = vld4q_lane_u32(w8, vtmp4567x89AB, 0); w8 += 4;
375 vtmp4567x89AB = vld4q_lane_u32(w9, vtmp4567x89AB, 1); w9 += 4;
376 vtmp4567x89AB = vld4q_lane_u32(w10, vtmp4567x89AB, 2); w10 += 4;
377 vst1q_u32(packed_weights, vtmp0123x0123.val[0]); packed_weights += 4;
378 vst1q_u32(packed_weights, vtmp0123x4567.val[0]); packed_weights += 4;
379 vst1q_u32(packed_weights, vtmp0123x89AB.val[0]); packed_weights += 4;
380 vst1q_u32(packed_weights, vtmp0123x0123.val[1]); packed_weights += 4;
381 vst1q_u32(packed_weights, vtmp0123x4567.val[1]); packed_weights += 4;
382 vst1q_u32(packed_weights, vtmp0123x89AB.val[1]); packed_weights += 4;
383 vst1q_u32(packed_weights, vtmp0123x0123.val[2]); packed_weights += 4;
384 vst1q_u32(packed_weights, vtmp0123x4567.val[2]); packed_weights += 4;
385 vst1q_u32(packed_weights, vtmp0123x89AB.val[2]); packed_weights += 4;
386 vst1q_u32(packed_weights, vtmp0123x0123.val[3]); packed_weights += 4;
387 vst1q_u32(packed_weights, vtmp0123x4567.val[3]); packed_weights += 4;
388 vst1q_u32(packed_weights, vtmp0123x89AB.val[3]); packed_weights += 4;
389 vst1q_u32(packed_weights, vtmp4567x0123.val[0]); packed_weights += 4;
390 vst1q_u32(packed_weights, vtmp4567x4567.val[0]); packed_weights += 4;
391 vst1q_u32(packed_weights, vtmp4567x89AB.val[0]); packed_weights += 4;
392 vst1q_u32(packed_weights, vtmp4567x0123.val[1]); packed_weights += 4;
393 vst1q_u32(packed_weights, vtmp4567x4567.val[1]); packed_weights += 4;
394 vst1q_u32(packed_weights, vtmp4567x89AB.val[1]); packed_weights += 4;
395 vst1q_u32(packed_weights, vtmp4567x0123.val[2]); packed_weights += 4;
396 vst1q_u32(packed_weights, vtmp4567x4567.val[2]); packed_weights += 4;
397 vst1q_u32(packed_weights, vtmp4567x89AB.val[2]); packed_weights += 4;
398 vst1q_u32(packed_weights, vtmp4567x0123.val[3]); packed_weights += 4;
399 vst1q_u32(packed_weights, vtmp4567x4567.val[3]); packed_weights += 4;
400 vst1q_u32(packed_weights, vtmp4567x89AB.val[3]); packed_weights += 4;
401 }
402 // KC remainder multiple of 4
403 if (k >= 4) {
404 vtmp0123x0123 = vld4q_lane_u32(w0, vtmp0123x0123, 0); w0 += 4;
405 vtmp0123x0123 = vld4q_lane_u32(w1, vtmp0123x0123, 1); w1 += 4;
406 vtmp0123x0123 = vld4q_lane_u32(w2, vtmp0123x0123, 2); w2 += 4;
407 vtmp0123x0123 = vld4q_lane_u32(w3, vtmp0123x0123, 3); w3 += 4;
408 vtmp0123x4567 = vld4q_lane_u32(w4, vtmp0123x4567, 0); w4 += 4;
409 vtmp0123x4567 = vld4q_lane_u32(w5, vtmp0123x4567, 1); w5 += 4;
410 vtmp0123x4567 = vld4q_lane_u32(w6, vtmp0123x4567, 2); w6 += 4;
411 vtmp0123x4567 = vld4q_lane_u32(w7, vtmp0123x4567, 3); w7 += 4;
412 vtmp0123x89AB = vld4q_lane_u32(w8, vtmp0123x89AB, 0); w8 += 4;
413 vtmp0123x89AB = vld4q_lane_u32(w9, vtmp0123x89AB, 1); w9 += 4;
414 vtmp0123x89AB = vld4q_lane_u32(w10, vtmp0123x89AB, 2); w10 += 4;
415 vst1q_u32(packed_weights, vtmp0123x0123.val[0]); packed_weights += 4;
416 vst1q_u32(packed_weights, vtmp0123x4567.val[0]); packed_weights += 4;
417 vst1q_u32(packed_weights, vtmp0123x89AB.val[0]); packed_weights += 4;
418 vst1q_u32(packed_weights, vtmp0123x0123.val[1]); packed_weights += 4;
419 vst1q_u32(packed_weights, vtmp0123x4567.val[1]); packed_weights += 4;
420 vst1q_u32(packed_weights, vtmp0123x89AB.val[1]); packed_weights += 4;
421 vst1q_u32(packed_weights, vtmp0123x0123.val[2]); packed_weights += 4;
422 vst1q_u32(packed_weights, vtmp0123x4567.val[2]); packed_weights += 4;
423 vst1q_u32(packed_weights, vtmp0123x89AB.val[2]); packed_weights += 4;
424 vst1q_u32(packed_weights, vtmp0123x0123.val[3]); packed_weights += 4;
425 vst1q_u32(packed_weights, vtmp0123x4567.val[3]); packed_weights += 4;
426 vst1q_u32(packed_weights, vtmp0123x89AB.val[3]); packed_weights += 4;
427 k -= 4;
428 }
429
430 // KC remainder of 1..3
431 // Same as main loop but ld1, ld2 or ld3
432 if XNN_UNLIKELY(k != 0) {
433 assert(k >= 1);
434 assert(k <= 3);
435 switch (k) {
436 // KC remainder of 1
437 case 1:
438 {
439 uint32x4_t vtmp0x0123 = vdupq_n_u32(0);
440 uint32x4_t vtmp0x4567 = vdupq_n_u32(0);
441 uint32x4_t vtmp0x89AB = vdupq_n_u32(0);
442 vtmp0x0123 = vld1q_lane_u32(w0, vtmp0x0123, 0);
443 vtmp0x0123 = vld1q_lane_u32(w1, vtmp0x0123, 1);
444 vtmp0x0123 = vld1q_lane_u32(w2, vtmp0x0123, 2);
445 vtmp0x0123 = vld1q_lane_u32(w3, vtmp0x0123, 3);
446 vtmp0x4567 = vld1q_lane_u32(w4, vtmp0x4567, 0);
447 vtmp0x4567 = vld1q_lane_u32(w5, vtmp0x4567, 1);
448 vtmp0x4567 = vld1q_lane_u32(w6, vtmp0x4567, 2);
449 vtmp0x4567 = vld1q_lane_u32(w7, vtmp0x4567, 3);
450 vtmp0x89AB = vld1q_lane_u32(w8, vtmp0x89AB, 0);
451 vtmp0x89AB = vld1q_lane_u32(w9, vtmp0x89AB, 1);
452 vtmp0x89AB = vld1q_lane_u32(w10, vtmp0x89AB, 2);
453 vst1q_u32(packed_weights, vtmp0x0123); packed_weights += 4;
454 vst1q_u32(packed_weights, vtmp0x4567); packed_weights += 4;
455 vst1q_u32(packed_weights, vtmp0x89AB); packed_weights += 4;
456 break;
457 }
458 // KC remainder of 2
459 case 2:
460 {
461 uint32x4x2_t vtmp01x0123;
462 vtmp01x0123.val[0] = vdupq_n_u32(0);
463 vtmp01x0123.val[1] = vdupq_n_u32(0);
464 uint32x4x2_t vtmp01x4567;
465 vtmp01x4567.val[0] = vdupq_n_u32(0);
466 vtmp01x4567.val[1] = vdupq_n_u32(0);
467 uint32x4x2_t vtmp01x89AB;
468 vtmp01x89AB.val[0] = vdupq_n_u32(0);
469 vtmp01x89AB.val[1] = vdupq_n_u32(0);
470 vtmp01x0123 = vld2q_lane_u32(w0, vtmp01x0123, 0);
471 vtmp01x0123 = vld2q_lane_u32(w1, vtmp01x0123, 1);
472 vtmp01x0123 = vld2q_lane_u32(w2, vtmp01x0123, 2);
473 vtmp01x0123 = vld2q_lane_u32(w3, vtmp01x0123, 3);
474 vtmp01x4567 = vld2q_lane_u32(w4, vtmp01x4567, 0);
475 vtmp01x4567 = vld2q_lane_u32(w5, vtmp01x4567, 1);
476 vtmp01x4567 = vld2q_lane_u32(w6, vtmp01x4567, 2);
477 vtmp01x4567 = vld2q_lane_u32(w7, vtmp01x4567, 3);
478 vtmp01x89AB = vld2q_lane_u32(w8, vtmp01x89AB, 0);
479 vtmp01x89AB = vld2q_lane_u32(w9, vtmp01x89AB, 1);
480 vtmp01x89AB = vld2q_lane_u32(w10, vtmp01x89AB, 2);
481 vst1q_u32(packed_weights, vtmp01x0123.val[0]); packed_weights += 4;
482 vst1q_u32(packed_weights, vtmp01x4567.val[0]); packed_weights += 4;
483 vst1q_u32(packed_weights, vtmp01x89AB.val[0]); packed_weights += 4;
484 vst1q_u32(packed_weights, vtmp01x0123.val[1]); packed_weights += 4;
485 vst1q_u32(packed_weights, vtmp01x4567.val[1]); packed_weights += 4;
486 vst1q_u32(packed_weights, vtmp01x89AB.val[1]); packed_weights += 4;
487 break;
488 }
489 // KC remainder of 3
490 case 3:
491 {
492 uint32x4x3_t vtmp012x0123;
493 vtmp012x0123.val[0] = vdupq_n_u32(0);
494 vtmp012x0123.val[1] = vdupq_n_u32(0);
495 vtmp012x0123.val[2] = vdupq_n_u32(0);
496 uint32x4x3_t vtmp012x4567;
497 vtmp012x4567.val[0] = vdupq_n_u32(0);
498 vtmp012x4567.val[1] = vdupq_n_u32(0);
499 vtmp012x4567.val[2] = vdupq_n_u32(0);
500 uint32x4x3_t vtmp012x89AB;
501 vtmp012x89AB.val[0] = vdupq_n_u32(0);
502 vtmp012x89AB.val[1] = vdupq_n_u32(0);
503 vtmp012x89AB.val[2] = vdupq_n_u32(0);
504 vtmp012x0123 = vld3q_lane_u32(w0, vtmp012x0123, 0);
505 vtmp012x0123 = vld3q_lane_u32(w1, vtmp012x0123, 1);
506 vtmp012x0123 = vld3q_lane_u32(w2, vtmp012x0123, 2);
507 vtmp012x0123 = vld3q_lane_u32(w3, vtmp012x0123, 3);
508 vtmp012x4567 = vld3q_lane_u32(w4, vtmp012x4567, 0);
509 vtmp012x4567 = vld3q_lane_u32(w5, vtmp012x4567, 1);
510 vtmp012x4567 = vld3q_lane_u32(w6, vtmp012x4567, 2);
511 vtmp012x4567 = vld3q_lane_u32(w7, vtmp012x4567, 3);
512 vtmp012x89AB = vld3q_lane_u32(w8, vtmp012x89AB, 0);
513 vtmp012x89AB = vld3q_lane_u32(w9, vtmp012x89AB, 1);
514 vtmp012x89AB = vld3q_lane_u32(w10, vtmp012x89AB, 2);
515 vst1q_u32(packed_weights, vtmp012x0123.val[0]); packed_weights += 4;
516 vst1q_u32(packed_weights, vtmp012x4567.val[0]); packed_weights += 4;
517 vst1q_u32(packed_weights, vtmp012x89AB.val[0]); packed_weights += 4;
518 vst1q_u32(packed_weights, vtmp012x0123.val[1]); packed_weights += 4;
519 vst1q_u32(packed_weights, vtmp012x4567.val[1]); packed_weights += 4;
520 vst1q_u32(packed_weights, vtmp012x89AB.val[1]); packed_weights += 4;
521 vst1q_u32(packed_weights, vtmp012x0123.val[2]); packed_weights += 4;
522 vst1q_u32(packed_weights, vtmp012x4567.val[2]); packed_weights += 4;
523 vst1q_u32(packed_weights, vtmp012x89AB.val[2]); packed_weights += 4;
524 break;
525 }
526 default:
527 XNN_UNREACHABLE;
528 }
529 }
530 packed_weights = (uint32_t*) ((uintptr_t) packed_weights + extra_bytes);
531 }
532
533 weights += nc * kc;
534 } while (--g != 0);
535 }