Debian Codegraph @

src/x32-packw/gen/x32-packw-x8s4-gemm-goi-neon-ld4lane-u8.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/neon.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include <arm_neon.h>
16
17 #include "xnnpack/packw.h"
18
19
20 void xnn_x32_packw_gemm_goi_ukernel_x8s4__neon_ld4lane_u8(
21 size_t g,
22 size_t nc,
23 size_t kc,
24 size_t nr,
25 size_t kr,
26 size_t sr,
27 const uint32_t* weights,
28 const uint32_t* bias,
29 const void* scale,
30 uint32_t* packed_weights,
31 size_t extra_bytes,
32 const void* params)
33 {
34 assert(g != 0);
35 assert(nc != 0);
36 assert(kc != 0);
37 assert(nr == 8);
38 assert(kr == 1);
39 assert(sr == 4);
40 assert(weights != NULL);
41 assert(packed_weights != NULL);
42 uint32x4x4_t vtmp0123x0123;
43 vtmp0123x0123.val[0] = vdupq_n_u32(0);
44 vtmp0123x0123.val[1] = vdupq_n_u32(0);
45 vtmp0123x0123.val[2] = vdupq_n_u32(0);
46 vtmp0123x0123.val[3] = vdupq_n_u32(0);
47 uint32x4x4_t vtmp4567x0123;
48 vtmp4567x0123.val[0] = vdupq_n_u32(0);
49 vtmp4567x0123.val[1] = vdupq_n_u32(0);
50 vtmp4567x0123.val[2] = vdupq_n_u32(0);
51 vtmp4567x0123.val[3] = vdupq_n_u32(0);
52 uint32x4x4_t vtmp0123x4567;
53 vtmp0123x4567.val[0] = vdupq_n_u32(0);
54 vtmp0123x4567.val[1] = vdupq_n_u32(0);
55 vtmp0123x4567.val[2] = vdupq_n_u32(0);
56 vtmp0123x4567.val[3] = vdupq_n_u32(0);
57 uint32x4x4_t vtmp4567x4567;
58 vtmp4567x4567.val[0] = vdupq_n_u32(0);
59 vtmp4567x4567.val[1] = vdupq_n_u32(0);
60 vtmp4567x4567.val[2] = vdupq_n_u32(0);
61 vtmp4567x4567.val[3] = vdupq_n_u32(0);
62
63 do {
64 // NC main loop multiple of 8
65 const uint32_t* w0 = weights;
66 size_t n = nc;
67
68 for (; n >= 8; n -= 8) {
69 if XNN_LIKELY(bias != NULL) {
70 uint32x4_t vb0 = vld1q_u32(bias); bias += 4;
71 uint32x4_t vb4 = vld1q_u32(bias); bias += 4;
72 vst1q_u32(packed_weights, vb0); packed_weights += 4;
73 vst1q_u32(packed_weights, vb4); packed_weights += 4;
74 } else {
75 const uint32x4_t vzero = vmovq_n_u32(0);
76 vst1q_u32(packed_weights, vzero); packed_weights += 4;
77 vst1q_u32(packed_weights, vzero); packed_weights += 4;
78 }
79
80 const uint32_t* w1 = w0 + kc;
81 const uint32_t* w2 = w1 + kc;
82 const uint32_t* w3 = w2 + kc;
83 const uint32_t* w4 = w3 + kc;
84 const uint32_t* w5 = w4 + kc;
85 const uint32_t* w6 = w5 + kc;
86 const uint32_t* w7 = w6 + kc;
87
88 // KC main loop multiple of 8
89 size_t k = kc;
90 for (; k >= 8; k -= 8) {
91 uint32x4_t vsrtmp;
92 vtmp0123x0123 = vld4q_lane_u32(w0, vtmp0123x0123, 0); w0 += 4;
93 vsrtmp = vtmp0123x0123.val[3];
94 vtmp0123x0123.val[3] = vtmp0123x0123.val[2];
95 vtmp0123x0123.val[2] = vtmp0123x0123.val[1];
96 vtmp0123x0123.val[1] = vtmp0123x0123.val[0];
97 vtmp0123x0123.val[0] = vsrtmp;
98 vtmp0123x0123 = vld4q_lane_u32(w1, vtmp0123x0123, 1); w1 += 4;
99 vsrtmp = vtmp0123x0123.val[3];
100 vtmp0123x0123.val[3] = vtmp0123x0123.val[2];
101 vtmp0123x0123.val[2] = vtmp0123x0123.val[1];
102 vtmp0123x0123.val[1] = vtmp0123x0123.val[0];
103 vtmp0123x0123.val[0] = vsrtmp;
104 vtmp0123x0123 = vld4q_lane_u32(w2, vtmp0123x0123, 2); w2 += 4;
105 vsrtmp = vtmp0123x0123.val[3];
106 vtmp0123x0123.val[3] = vtmp0123x0123.val[2];
107 vtmp0123x0123.val[2] = vtmp0123x0123.val[1];
108 vtmp0123x0123.val[1] = vtmp0123x0123.val[0];
109 vtmp0123x0123.val[0] = vsrtmp;
110 vtmp0123x0123 = vld4q_lane_u32(w3, vtmp0123x0123, 3); w3 += 4;
111 vsrtmp = vtmp0123x0123.val[3];
112 vtmp0123x0123.val[3] = vtmp0123x0123.val[2];
113 vtmp0123x0123.val[2] = vtmp0123x0123.val[1];
114 vtmp0123x0123.val[1] = vtmp0123x0123.val[0];
115 vtmp0123x0123.val[0] = vsrtmp;
116 vtmp0123x4567 = vld4q_lane_u32(w4, vtmp0123x4567, 0); w4 += 4;
117 vsrtmp = vtmp0123x4567.val[3];
118 vtmp0123x4567.val[3] = vtmp0123x4567.val[2];
119 vtmp0123x4567.val[2] = vtmp0123x4567.val[1];
120 vtmp0123x4567.val[1] = vtmp0123x4567.val[0];
121 vtmp0123x4567.val[0] = vsrtmp;
122 vtmp0123x4567 = vld4q_lane_u32(w5, vtmp0123x4567, 1); w5 += 4;
123 vsrtmp = vtmp0123x4567.val[3];
124 vtmp0123x4567.val[3] = vtmp0123x4567.val[2];
125 vtmp0123x4567.val[2] = vtmp0123x4567.val[1];
126 vtmp0123x4567.val[1] = vtmp0123x4567.val[0];
127 vtmp0123x4567.val[0] = vsrtmp;
128 vtmp0123x4567 = vld4q_lane_u32(w6, vtmp0123x4567, 2); w6 += 4;
129 vsrtmp = vtmp0123x4567.val[3];
130 vtmp0123x4567.val[3] = vtmp0123x4567.val[2];
131 vtmp0123x4567.val[2] = vtmp0123x4567.val[1];
132 vtmp0123x4567.val[1] = vtmp0123x4567.val[0];
133 vtmp0123x4567.val[0] = vsrtmp;
134 vtmp0123x4567 = vld4q_lane_u32(w7, vtmp0123x4567, 3); w7 += 4;
135 vsrtmp = vtmp0123x4567.val[3];
136 vtmp0123x4567.val[3] = vtmp0123x4567.val[2];
137 vtmp0123x4567.val[2] = vtmp0123x4567.val[1];
138 vtmp0123x4567.val[1] = vtmp0123x4567.val[0];
139 vtmp0123x4567.val[0] = vsrtmp;
140 vtmp4567x0123 = vld4q_lane_u32(w0, vtmp4567x0123, 0); w0 += 4;
141 vsrtmp = vtmp4567x0123.val[3];
142 vtmp4567x0123.val[3] = vtmp4567x0123.val[2];
143 vtmp4567x0123.val[2] = vtmp4567x0123.val[1];
144 vtmp4567x0123.val[1] = vtmp4567x0123.val[0];
145 vtmp4567x0123.val[0] = vsrtmp;
146 vtmp4567x0123 = vld4q_lane_u32(w1, vtmp4567x0123, 1); w1 += 4;
147 vsrtmp = vtmp4567x0123.val[3];
148 vtmp4567x0123.val[3] = vtmp4567x0123.val[2];
149 vtmp4567x0123.val[2] = vtmp4567x0123.val[1];
150 vtmp4567x0123.val[1] = vtmp4567x0123.val[0];
151 vtmp4567x0123.val[0] = vsrtmp;
152 vtmp4567x0123 = vld4q_lane_u32(w2, vtmp4567x0123, 2); w2 += 4;
153 vsrtmp = vtmp4567x0123.val[3];
154 vtmp4567x0123.val[3] = vtmp4567x0123.val[2];
155 vtmp4567x0123.val[2] = vtmp4567x0123.val[1];
156 vtmp4567x0123.val[1] = vtmp4567x0123.val[0];
157 vtmp4567x0123.val[0] = vsrtmp;
158 vtmp4567x0123 = vld4q_lane_u32(w3, vtmp4567x0123, 3); w3 += 4;
159 vsrtmp = vtmp4567x0123.val[3];
160 vtmp4567x0123.val[3] = vtmp4567x0123.val[2];
161 vtmp4567x0123.val[2] = vtmp4567x0123.val[1];
162 vtmp4567x0123.val[1] = vtmp4567x0123.val[0];
163 vtmp4567x0123.val[0] = vsrtmp;
164 vtmp4567x4567 = vld4q_lane_u32(w4, vtmp4567x4567, 0); w4 += 4;
165 vsrtmp = vtmp4567x4567.val[3];
166 vtmp4567x4567.val[3] = vtmp4567x4567.val[2];
167 vtmp4567x4567.val[2] = vtmp4567x4567.val[1];
168 vtmp4567x4567.val[1] = vtmp4567x4567.val[0];
169 vtmp4567x4567.val[0] = vsrtmp;
170 vtmp4567x4567 = vld4q_lane_u32(w5, vtmp4567x4567, 1); w5 += 4;
171 vsrtmp = vtmp4567x4567.val[3];
172 vtmp4567x4567.val[3] = vtmp4567x4567.val[2];
173 vtmp4567x4567.val[2] = vtmp4567x4567.val[1];
174 vtmp4567x4567.val[1] = vtmp4567x4567.val[0];
175 vtmp4567x4567.val[0] = vsrtmp;
176 vtmp4567x4567 = vld4q_lane_u32(w6, vtmp4567x4567, 2); w6 += 4;
177 vsrtmp = vtmp4567x4567.val[3];
178 vtmp4567x4567.val[3] = vtmp4567x4567.val[2];
179 vtmp4567x4567.val[2] = vtmp4567x4567.val[1];
180 vtmp4567x4567.val[1] = vtmp4567x4567.val[0];
181 vtmp4567x4567.val[0] = vsrtmp;
182 vtmp4567x4567 = vld4q_lane_u32(w7, vtmp4567x4567, 3); w7 += 4;
183 vsrtmp = vtmp4567x4567.val[3];
184 vtmp4567x4567.val[3] = vtmp4567x4567.val[2];
185 vtmp4567x4567.val[2] = vtmp4567x4567.val[1];
186 vtmp4567x4567.val[1] = vtmp4567x4567.val[0];
187 vtmp4567x4567.val[0] = vsrtmp;
188 vst1q_u32(packed_weights, vtmp0123x0123.val[0]); packed_weights += 4;
189 vst1q_u32(packed_weights, vtmp0123x4567.val[0]); packed_weights += 4;
190 vst1q_u32(packed_weights, vtmp0123x0123.val[1]); packed_weights += 4;
191 vst1q_u32(packed_weights, vtmp0123x4567.val[1]); packed_weights += 4;
192 vst1q_u32(packed_weights, vtmp0123x0123.val[2]); packed_weights += 4;
193 vst1q_u32(packed_weights, vtmp0123x4567.val[2]); packed_weights += 4;
194 vst1q_u32(packed_weights, vtmp0123x0123.val[3]); packed_weights += 4;
195 vst1q_u32(packed_weights, vtmp0123x4567.val[3]); packed_weights += 4;
196 vst1q_u32(packed_weights, vtmp4567x0123.val[0]); packed_weights += 4;
197 vst1q_u32(packed_weights, vtmp4567x4567.val[0]); packed_weights += 4;
198 vst1q_u32(packed_weights, vtmp4567x0123.val[1]); packed_weights += 4;
199 vst1q_u32(packed_weights, vtmp4567x4567.val[1]); packed_weights += 4;
200 vst1q_u32(packed_weights, vtmp4567x0123.val[2]); packed_weights += 4;
201 vst1q_u32(packed_weights, vtmp4567x4567.val[2]); packed_weights += 4;
202 vst1q_u32(packed_weights, vtmp4567x0123.val[3]); packed_weights += 4;
203 vst1q_u32(packed_weights, vtmp4567x4567.val[3]); packed_weights += 4;
204 }
205 // KC remainder multiple of 4
206 if (k >= 4) {
207 uint32x4_t vsrtmp;
208 vtmp0123x0123 = vld4q_lane_u32(w0, vtmp0123x0123, 0); w0 += 4;
209 vsrtmp = vtmp0123x0123.val[3];
210 vtmp0123x0123.val[3] = vtmp0123x0123.val[2];
211 vtmp0123x0123.val[2] = vtmp0123x0123.val[1];
212 vtmp0123x0123.val[1] = vtmp0123x0123.val[0];
213 vtmp0123x0123.val[0] = vsrtmp;
214 vtmp0123x0123 = vld4q_lane_u32(w1, vtmp0123x0123, 1); w1 += 4;
215 vsrtmp = vtmp0123x0123.val[3];
216 vtmp0123x0123.val[3] = vtmp0123x0123.val[2];
217 vtmp0123x0123.val[2] = vtmp0123x0123.val[1];
218 vtmp0123x0123.val[1] = vtmp0123x0123.val[0];
219 vtmp0123x0123.val[0] = vsrtmp;
220 vtmp0123x0123 = vld4q_lane_u32(w2, vtmp0123x0123, 2); w2 += 4;
221 vsrtmp = vtmp0123x0123.val[3];
222 vtmp0123x0123.val[3] = vtmp0123x0123.val[2];
223 vtmp0123x0123.val[2] = vtmp0123x0123.val[1];
224 vtmp0123x0123.val[1] = vtmp0123x0123.val[0];
225 vtmp0123x0123.val[0] = vsrtmp;
226 vtmp0123x0123 = vld4q_lane_u32(w3, vtmp0123x0123, 3); w3 += 4;
227 vsrtmp = vtmp0123x0123.val[3];
228 vtmp0123x0123.val[3] = vtmp0123x0123.val[2];
229 vtmp0123x0123.val[2] = vtmp0123x0123.val[1];
230 vtmp0123x0123.val[1] = vtmp0123x0123.val[0];
231 vtmp0123x0123.val[0] = vsrtmp;
232 vtmp0123x4567 = vld4q_lane_u32(w4, vtmp0123x4567, 0); w4 += 4;
233 vsrtmp = vtmp0123x4567.val[3];
234 vtmp0123x4567.val[3] = vtmp0123x4567.val[2];
235 vtmp0123x4567.val[2] = vtmp0123x4567.val[1];
236 vtmp0123x4567.val[1] = vtmp0123x4567.val[0];
237 vtmp0123x4567.val[0] = vsrtmp;
238 vtmp0123x4567 = vld4q_lane_u32(w5, vtmp0123x4567, 1); w5 += 4;
239 vsrtmp = vtmp0123x4567.val[3];
240 vtmp0123x4567.val[3] = vtmp0123x4567.val[2];
241 vtmp0123x4567.val[2] = vtmp0123x4567.val[1];
242 vtmp0123x4567.val[1] = vtmp0123x4567.val[0];
243 vtmp0123x4567.val[0] = vsrtmp;
244 vtmp0123x4567 = vld4q_lane_u32(w6, vtmp0123x4567, 2); w6 += 4;
245 vsrtmp = vtmp0123x4567.val[3];
246 vtmp0123x4567.val[3] = vtmp0123x4567.val[2];
247 vtmp0123x4567.val[2] = vtmp0123x4567.val[1];
248 vtmp0123x4567.val[1] = vtmp0123x4567.val[0];
249 vtmp0123x4567.val[0] = vsrtmp;
250 vtmp0123x4567 = vld4q_lane_u32(w7, vtmp0123x4567, 3); w7 += 4;
251 vsrtmp = vtmp0123x4567.val[3];
252 vtmp0123x4567.val[3] = vtmp0123x4567.val[2];
253 vtmp0123x4567.val[2] = vtmp0123x4567.val[1];
254 vtmp0123x4567.val[1] = vtmp0123x4567.val[0];
255 vtmp0123x4567.val[0] = vsrtmp;
256 vst1q_u32(packed_weights, vtmp0123x0123.val[0]); packed_weights += 4;
257 vst1q_u32(packed_weights, vtmp0123x4567.val[0]); packed_weights += 4;
258 vst1q_u32(packed_weights, vtmp0123x0123.val[1]); packed_weights += 4;
259 vst1q_u32(packed_weights, vtmp0123x4567.val[1]); packed_weights += 4;
260 vst1q_u32(packed_weights, vtmp0123x0123.val[2]); packed_weights += 4;
261 vst1q_u32(packed_weights, vtmp0123x4567.val[2]); packed_weights += 4;
262 vst1q_u32(packed_weights, vtmp0123x0123.val[3]); packed_weights += 4;
263 vst1q_u32(packed_weights, vtmp0123x4567.val[3]); packed_weights += 4;
264 k -= 4;
265 }
266
267 // KC remainder of 1..3
268 // Same as main loop but ld1, ld2 or ld3
269 if XNN_UNLIKELY(k != 0) {
270 assert(k >= 1);
271 assert(k <= 3);
272 switch (k) {
273 // KC remainder of 1
274 case 1:
275 {
276 uint32x4_t vtmp0x0123 = vdupq_n_u32(0);
277 uint32x4_t vtmp0x4567 = vdupq_n_u32(0);
278 uint32x4_t vsrtmp;
279 uint32x4_t vsrtmp1x0123 = vdupq_n_u32(0);
280 uint32x4_t vsrtmp2x0123 = vdupq_n_u32(0);
281 uint32x4_t vsrtmp3x0123 = vdupq_n_u32(0);
282 uint32x4_t vsrtmp1x4567 = vdupq_n_u32(0);
283 uint32x4_t vsrtmp2x4567 = vdupq_n_u32(0);
284 uint32x4_t vsrtmp3x4567 = vdupq_n_u32(0);
285 vtmp0x0123 = vld1q_lane_u32(w0, vtmp0x0123, 0); w0 += 1;
286 vsrtmp = vsrtmp3x0123;
287 vsrtmp3x0123 = vsrtmp2x0123;
288 vsrtmp2x0123 = vsrtmp1x0123;
289 vsrtmp1x0123 = vtmp0x0123;
290 vtmp0x0123 = vsrtmp;
291 vtmp0x0123 = vld1q_lane_u32(w1, vtmp0x0123, 1); w1 += 1;
292 vsrtmp = vsrtmp3x0123;
293 vsrtmp3x0123 = vsrtmp2x0123;
294 vsrtmp2x0123 = vsrtmp1x0123;
295 vsrtmp1x0123 = vtmp0x0123;
296 vtmp0x0123 = vsrtmp;
297 vtmp0x0123 = vld1q_lane_u32(w2, vtmp0x0123, 2); w2 += 1;
298 vsrtmp = vsrtmp3x0123;
299 vsrtmp3x0123 = vsrtmp2x0123;
300 vsrtmp2x0123 = vsrtmp1x0123;
301 vsrtmp1x0123 = vtmp0x0123;
302 vtmp0x0123 = vsrtmp;
303 vtmp0x0123 = vld1q_lane_u32(w3, vtmp0x0123, 3); w3 += 1;
304 vsrtmp = vsrtmp3x0123;
305 vsrtmp3x0123 = vsrtmp2x0123;
306 vsrtmp2x0123 = vsrtmp1x0123;
307 vsrtmp1x0123 = vtmp0x0123;
308 vtmp0x0123 = vsrtmp;
309 vtmp0x4567 = vld1q_lane_u32(w4, vtmp0x4567, 0); w4 += 1;
310 vsrtmp = vsrtmp3x4567;
311 vsrtmp3x4567 = vsrtmp2x4567;
312 vsrtmp2x4567 = vsrtmp1x4567;
313 vsrtmp1x4567 = vtmp0x4567;
314 vtmp0x4567 = vsrtmp;
315 vtmp0x4567 = vld1q_lane_u32(w5, vtmp0x4567, 1); w5 += 1;
316 vsrtmp = vsrtmp3x4567;
317 vsrtmp3x4567 = vsrtmp2x4567;
318 vsrtmp2x4567 = vsrtmp1x4567;
319 vsrtmp1x4567 = vtmp0x4567;
320 vtmp0x4567 = vsrtmp;
321 vtmp0x4567 = vld1q_lane_u32(w6, vtmp0x4567, 2); w6 += 1;
322 vsrtmp = vsrtmp3x4567;
323 vsrtmp3x4567 = vsrtmp2x4567;
324 vsrtmp2x4567 = vsrtmp1x4567;
325 vsrtmp1x4567 = vtmp0x4567;
326 vtmp0x4567 = vsrtmp;
327 vtmp0x4567 = vld1q_lane_u32(w7, vtmp0x4567, 3); w7 += 1;
328 vsrtmp = vsrtmp3x4567;
329 vsrtmp3x4567 = vsrtmp2x4567;
330 vsrtmp2x4567 = vsrtmp1x4567;
331 vsrtmp1x4567 = vtmp0x4567;
332 vtmp0x4567 = vsrtmp;
333 vst1q_u32(packed_weights, vtmp0x0123); packed_weights += 4;
334 vst1q_u32(packed_weights, vtmp0x4567); packed_weights += 4;
335 vst1q_u32(packed_weights, vsrtmp1x0123); packed_weights += 4;
336 vst1q_u32(packed_weights, vsrtmp1x4567); packed_weights += 4;
337 vst1q_u32(packed_weights, vsrtmp2x0123); packed_weights += 4;
338 vst1q_u32(packed_weights, vsrtmp2x4567); packed_weights += 4;
339 vst1q_u32(packed_weights, vsrtmp3x0123); packed_weights += 4;
340 vst1q_u32(packed_weights, vsrtmp3x4567); packed_weights += 4;
341 break;
342 }
343 // KC remainder of 2
344 case 2:
345 {
346 uint32x4x2_t vtmp01x0123;
347 vtmp01x0123.val[0] = vdupq_n_u32(0);
348 vtmp01x0123.val[1] = vdupq_n_u32(0);
349 uint32x4x2_t vtmp01x4567;
350 vtmp01x4567.val[0] = vdupq_n_u32(0);
351 vtmp01x4567.val[1] = vdupq_n_u32(0);
352 uint32x4_t vsrtmp;
353 uint32x4_t vsrtmp2x0123 = vdupq_n_u32(0);
354 uint32x4_t vsrtmp3x0123 = vdupq_n_u32(0);
355 uint32x4_t vsrtmp2x4567 = vdupq_n_u32(0);
356 uint32x4_t vsrtmp3x4567 = vdupq_n_u32(0);
357 vtmp01x0123 = vld2q_lane_u32(w0, vtmp01x0123, 0); w0 += 2;
358 vsrtmp = vsrtmp3x0123;
359 vsrtmp3x0123 = vsrtmp2x0123;
360 vsrtmp2x0123 = vtmp01x0123.val[1];
361 vtmp01x0123.val[1] = vtmp01x0123.val[0];
362 vtmp01x0123.val[0] = vsrtmp;
363 vtmp01x0123 = vld2q_lane_u32(w1, vtmp01x0123, 1); w1 += 2;
364 vsrtmp = vsrtmp3x0123;
365 vsrtmp3x0123 = vsrtmp2x0123;
366 vsrtmp2x0123 = vtmp01x0123.val[1];
367 vtmp01x0123.val[1] = vtmp01x0123.val[0];
368 vtmp01x0123.val[0] = vsrtmp;
369 vtmp01x0123 = vld2q_lane_u32(w2, vtmp01x0123, 2); w2 += 2;
370 vsrtmp = vsrtmp3x0123;
371 vsrtmp3x0123 = vsrtmp2x0123;
372 vsrtmp2x0123 = vtmp01x0123.val[1];
373 vtmp01x0123.val[1] = vtmp01x0123.val[0];
374 vtmp01x0123.val[0] = vsrtmp;
375 vtmp01x0123 = vld2q_lane_u32(w3, vtmp01x0123, 3); w3 += 2;
376 vsrtmp = vsrtmp3x0123;
377 vsrtmp3x0123 = vsrtmp2x0123;
378 vsrtmp2x0123 = vtmp01x0123.val[1];
379 vtmp01x0123.val[1] = vtmp01x0123.val[0];
380 vtmp01x0123.val[0] = vsrtmp;
381 vtmp01x4567 = vld2q_lane_u32(w4, vtmp01x4567, 0); w4 += 2;
382 vsrtmp = vsrtmp3x4567;
383 vsrtmp3x4567 = vsrtmp2x4567;
384 vsrtmp2x4567 = vtmp01x4567.val[1];
385 vtmp01x4567.val[1] = vtmp01x4567.val[0];
386 vtmp01x4567.val[0] = vsrtmp;
387 vtmp01x4567 = vld2q_lane_u32(w5, vtmp01x4567, 1); w5 += 2;
388 vsrtmp = vsrtmp3x4567;
389 vsrtmp3x4567 = vsrtmp2x4567;
390 vsrtmp2x4567 = vtmp01x4567.val[1];
391 vtmp01x4567.val[1] = vtmp01x4567.val[0];
392 vtmp01x4567.val[0] = vsrtmp;
393 vtmp01x4567 = vld2q_lane_u32(w6, vtmp01x4567, 2); w6 += 2;
394 vsrtmp = vsrtmp3x4567;
395 vsrtmp3x4567 = vsrtmp2x4567;
396 vsrtmp2x4567 = vtmp01x4567.val[1];
397 vtmp01x4567.val[1] = vtmp01x4567.val[0];
398 vtmp01x4567.val[0] = vsrtmp;
399 vtmp01x4567 = vld2q_lane_u32(w7, vtmp01x4567, 3); w7 += 2;
400 vsrtmp = vsrtmp3x4567;
401 vsrtmp3x4567 = vsrtmp2x4567;
402 vsrtmp2x4567 = vtmp01x4567.val[1];
403 vtmp01x4567.val[1] = vtmp01x4567.val[0];
404 vtmp01x4567.val[0] = vsrtmp;
405 vst1q_u32(packed_weights, vtmp01x0123.val[0]); packed_weights += 4;
406 vst1q_u32(packed_weights, vtmp01x4567.val[0]); packed_weights += 4;
407 vst1q_u32(packed_weights, vtmp01x0123.val[1]); packed_weights += 4;
408 vst1q_u32(packed_weights, vtmp01x4567.val[1]); packed_weights += 4;
409 vst1q_u32(packed_weights, vsrtmp2x0123); packed_weights += 4;
410 vst1q_u32(packed_weights, vsrtmp2x4567); packed_weights += 4;
411 vst1q_u32(packed_weights, vsrtmp3x0123); packed_weights += 4;
412 vst1q_u32(packed_weights, vsrtmp3x4567); packed_weights += 4;
413 break;
414 }
415 // KC remainder of 3
416 case 3:
417 {
418 uint32x4x3_t vtmp012x0123;
419 vtmp012x0123.val[0] = vdupq_n_u32(0);
420 vtmp012x0123.val[1] = vdupq_n_u32(0);
421 vtmp012x0123.val[2] = vdupq_n_u32(0);
422 uint32x4x3_t vtmp012x4567;
423 vtmp012x4567.val[0] = vdupq_n_u32(0);
424 vtmp012x4567.val[1] = vdupq_n_u32(0);
425 vtmp012x4567.val[2] = vdupq_n_u32(0);
426 uint32x4_t vsrtmp;
427 uint32x4_t vsrtmp3x0123 = vdupq_n_u32(0);
428 uint32x4_t vsrtmp3x4567 = vdupq_n_u32(0);
429 vtmp012x0123 = vld3q_lane_u32(w0, vtmp012x0123, 0); w0 += 3;
430 vsrtmp = vsrtmp3x0123;
431 vsrtmp3x0123 = vtmp012x0123.val[2];
432 vtmp012x0123.val[2] = vtmp012x0123.val[1];
433 vtmp012x0123.val[1] = vtmp012x0123.val[0];
434 vtmp012x0123.val[0] = vsrtmp;
435 vtmp012x0123 = vld3q_lane_u32(w1, vtmp012x0123, 1); w1 += 3;
436 vsrtmp = vsrtmp3x0123;
437 vsrtmp3x0123 = vtmp012x0123.val[2];
438 vtmp012x0123.val[2] = vtmp012x0123.val[1];
439 vtmp012x0123.val[1] = vtmp012x0123.val[0];
440 vtmp012x0123.val[0] = vsrtmp;
441 vtmp012x0123 = vld3q_lane_u32(w2, vtmp012x0123, 2); w2 += 3;
442 vsrtmp = vsrtmp3x0123;
443 vsrtmp3x0123 = vtmp012x0123.val[2];
444 vtmp012x0123.val[2] = vtmp012x0123.val[1];
445 vtmp012x0123.val[1] = vtmp012x0123.val[0];
446 vtmp012x0123.val[0] = vsrtmp;
447 vtmp012x0123 = vld3q_lane_u32(w3, vtmp012x0123, 3); w3 += 3;
448 vsrtmp = vsrtmp3x0123;
449 vsrtmp3x0123 = vtmp012x0123.val[2];
450 vtmp012x0123.val[2] = vtmp012x0123.val[1];
451 vtmp012x0123.val[1] = vtmp012x0123.val[0];
452 vtmp012x0123.val[0] = vsrtmp;
453 vtmp012x4567 = vld3q_lane_u32(w4, vtmp012x4567, 0); w4 += 3;
454 vsrtmp = vsrtmp3x4567;
455 vsrtmp3x4567 = vtmp012x4567.val[2];
456 vtmp012x4567.val[2] = vtmp012x4567.val[1];
457 vtmp012x4567.val[1] = vtmp012x4567.val[0];
458 vtmp012x4567.val[0] = vsrtmp;
459 vtmp012x4567 = vld3q_lane_u32(w5, vtmp012x4567, 1); w5 += 3;
460 vsrtmp = vsrtmp3x4567;
461 vsrtmp3x4567 = vtmp012x4567.val[2];
462 vtmp012x4567.val[2] = vtmp012x4567.val[1];
463 vtmp012x4567.val[1] = vtmp012x4567.val[0];
464 vtmp012x4567.val[0] = vsrtmp;
465 vtmp012x4567 = vld3q_lane_u32(w6, vtmp012x4567, 2); w6 += 3;
466 vsrtmp = vsrtmp3x4567;
467 vsrtmp3x4567 = vtmp012x4567.val[2];
468 vtmp012x4567.val[2] = vtmp012x4567.val[1];
469 vtmp012x4567.val[1] = vtmp012x4567.val[0];
470 vtmp012x4567.val[0] = vsrtmp;
471 vtmp012x4567 = vld3q_lane_u32(w7, vtmp012x4567, 3); w7 += 3;
472 vsrtmp = vsrtmp3x4567;
473 vsrtmp3x4567 = vtmp012x4567.val[2];
474 vtmp012x4567.val[2] = vtmp012x4567.val[1];
475 vtmp012x4567.val[1] = vtmp012x4567.val[0];
476 vtmp012x4567.val[0] = vsrtmp;
477 vst1q_u32(packed_weights, vtmp012x0123.val[0]); packed_weights += 4;
478 vst1q_u32(packed_weights, vtmp012x4567.val[0]); packed_weights += 4;
479 vst1q_u32(packed_weights, vtmp012x0123.val[1]); packed_weights += 4;
480 vst1q_u32(packed_weights, vtmp012x4567.val[1]); packed_weights += 4;
481 vst1q_u32(packed_weights, vtmp012x0123.val[2]); packed_weights += 4;
482 vst1q_u32(packed_weights, vtmp012x4567.val[2]); packed_weights += 4;
483 vst1q_u32(packed_weights, vsrtmp3x0123); packed_weights += 4;
484 vst1q_u32(packed_weights, vsrtmp3x4567); packed_weights += 4;
485 break;
486 }
487 default:
488 XNN_UNREACHABLE;
489 }
490 }
491 packed_weights = (uint32_t*) ((uintptr_t) packed_weights + extra_bytes);
492 w0 = w7;
493 }
494
495 // NC remainder (1..7)
496 if XNN_UNLIKELY(n != 0) {
497 assert(n >= 1);
498 assert(n <= 7);
499 if XNN_LIKELY(bias != NULL) {
500 size_t nb = n;
501 do {
502 *packed_weights++ = *bias++;
503 } while (--nb != 0);
504 packed_weights += (8 - n);
505 } else {
506 const uint32x4_t vzero = vmovq_n_u32(0);
507 vst1q_u32(packed_weights, vzero); packed_weights += 4;
508 vst1q_u32(packed_weights, vzero); packed_weights += 4;
509 }
510
511 // NR remainder has less than 8 rows so last row is not loaded
512 const uint32_t* w1 = w0 + kc;
513 if XNN_UNPREDICTABLE(n < 2) {
514 w1 = w0;
515 }
516 const uint32_t* w2 = w1 + kc;
517 if XNN_UNPREDICTABLE(n <= 2) {
518 w2 = w1;
519 }
520 const uint32_t* w3 = w2 + kc;
521 if XNN_UNPREDICTABLE(n < 4) {
522 w3 = w2;
523 }
524 const uint32_t* w4 = w3 + kc;
525 if XNN_UNPREDICTABLE(n <= 4) {
526 w4 = w3;
527 }
528 const uint32_t* w5 = w4 + kc;
529 if XNN_UNPREDICTABLE(n < 6) {
530 w5 = w4;
531 }
532 const uint32_t* w6 = w5 + kc;
533 if XNN_UNPREDICTABLE(n <= 6) {
534 w6 = w5;
535 }
536
537 // KC main loop multiple of 8
538 size_t k = kc;
539 for (; k >= 8; k -= 8) {
540 uint32x4_t vsrtmp;
541 vtmp0123x0123 = vld4q_lane_u32(w0, vtmp0123x0123, 0); w0 += 4;
542 vsrtmp = vtmp0123x0123.val[3];
543 vtmp0123x0123.val[3] = vtmp0123x0123.val[2];
544 vtmp0123x0123.val[2] = vtmp0123x0123.val[1];
545 vtmp0123x0123.val[1] = vtmp0123x0123.val[0];
546 vtmp0123x0123.val[0] = vsrtmp;
547 vtmp0123x0123 = vld4q_lane_u32(w1, vtmp0123x0123, 1); w1 += 4;
548 vsrtmp = vtmp0123x0123.val[3];
549 vtmp0123x0123.val[3] = vtmp0123x0123.val[2];
550 vtmp0123x0123.val[2] = vtmp0123x0123.val[1];
551 vtmp0123x0123.val[1] = vtmp0123x0123.val[0];
552 vtmp0123x0123.val[0] = vsrtmp;
553 vtmp0123x0123 = vld4q_lane_u32(w2, vtmp0123x0123, 2); w2 += 4;
554 vsrtmp = vtmp0123x0123.val[3];
555 vtmp0123x0123.val[3] = vtmp0123x0123.val[2];
556 vtmp0123x0123.val[2] = vtmp0123x0123.val[1];
557 vtmp0123x0123.val[1] = vtmp0123x0123.val[0];
558 vtmp0123x0123.val[0] = vsrtmp;
559 vtmp0123x0123 = vld4q_lane_u32(w3, vtmp0123x0123, 3); w3 += 4;
560 vsrtmp = vtmp0123x0123.val[3];
561 vtmp0123x0123.val[3] = vtmp0123x0123.val[2];
562 vtmp0123x0123.val[2] = vtmp0123x0123.val[1];
563 vtmp0123x0123.val[1] = vtmp0123x0123.val[0];
564 vtmp0123x0123.val[0] = vsrtmp;
565 vtmp0123x4567 = vld4q_lane_u32(w4, vtmp0123x4567, 0); w4 += 4;
566 vsrtmp = vtmp0123x4567.val[3];
567 vtmp0123x4567.val[3] = vtmp0123x4567.val[2];
568 vtmp0123x4567.val[2] = vtmp0123x4567.val[1];
569 vtmp0123x4567.val[1] = vtmp0123x4567.val[0];
570 vtmp0123x4567.val[0] = vsrtmp;
571 vtmp0123x4567 = vld4q_lane_u32(w5, vtmp0123x4567, 1); w5 += 4;
572 vsrtmp = vtmp0123x4567.val[3];
573 vtmp0123x4567.val[3] = vtmp0123x4567.val[2];
574 vtmp0123x4567.val[2] = vtmp0123x4567.val[1];
575 vtmp0123x4567.val[1] = vtmp0123x4567.val[0];
576 vtmp0123x4567.val[0] = vsrtmp;
577 vtmp0123x4567 = vld4q_lane_u32(w6, vtmp0123x4567, 2); w6 += 4;
578 vsrtmp = vtmp0123x4567.val[3];
579 vtmp0123x4567.val[3] = vtmp0123x4567.val[2];
580 vtmp0123x4567.val[2] = vtmp0123x4567.val[1];
581 vtmp0123x4567.val[1] = vtmp0123x4567.val[0];
582 vtmp0123x4567.val[0] = vsrtmp;
583 vsrtmp = vtmp0123x4567.val[3];
584 vtmp0123x4567.val[3] = vtmp0123x4567.val[2];
585 vtmp0123x4567.val[2] = vtmp0123x4567.val[1];
586 vtmp0123x4567.val[1] = vtmp0123x4567.val[0];
587 vtmp0123x4567.val[0] = vsrtmp;
588 vtmp4567x0123 = vld4q_lane_u32(w0, vtmp4567x0123, 0); w0 += 4;
589 vsrtmp = vtmp4567x0123.val[3];
590 vtmp4567x0123.val[3] = vtmp4567x0123.val[2];
591 vtmp4567x0123.val[2] = vtmp4567x0123.val[1];
592 vtmp4567x0123.val[1] = vtmp4567x0123.val[0];
593 vtmp4567x0123.val[0] = vsrtmp;
594 vtmp4567x0123 = vld4q_lane_u32(w1, vtmp4567x0123, 1); w1 += 4;
595 vsrtmp = vtmp4567x0123.val[3];
596 vtmp4567x0123.val[3] = vtmp4567x0123.val[2];
597 vtmp4567x0123.val[2] = vtmp4567x0123.val[1];
598 vtmp4567x0123.val[1] = vtmp4567x0123.val[0];
599 vtmp4567x0123.val[0] = vsrtmp;
600 vtmp4567x0123 = vld4q_lane_u32(w2, vtmp4567x0123, 2); w2 += 4;
601 vsrtmp = vtmp4567x0123.val[3];
602 vtmp4567x0123.val[3] = vtmp4567x0123.val[2];
603 vtmp4567x0123.val[2] = vtmp4567x0123.val[1];
604 vtmp4567x0123.val[1] = vtmp4567x0123.val[0];
605 vtmp4567x0123.val[0] = vsrtmp;
606 vtmp4567x0123 = vld4q_lane_u32(w3, vtmp4567x0123, 3); w3 += 4;
607 vsrtmp = vtmp4567x0123.val[3];
608 vtmp4567x0123.val[3] = vtmp4567x0123.val[2];
609 vtmp4567x0123.val[2] = vtmp4567x0123.val[1];
610 vtmp4567x0123.val[1] = vtmp4567x0123.val[0];
611 vtmp4567x0123.val[0] = vsrtmp;
612 vtmp4567x4567 = vld4q_lane_u32(w4, vtmp4567x4567, 0); w4 += 4;
613 vsrtmp = vtmp4567x4567.val[3];
614 vtmp4567x4567.val[3] = vtmp4567x4567.val[2];
615 vtmp4567x4567.val[2] = vtmp4567x4567.val[1];
616 vtmp4567x4567.val[1] = vtmp4567x4567.val[0];
617 vtmp4567x4567.val[0] = vsrtmp;
618 vtmp4567x4567 = vld4q_lane_u32(w5, vtmp4567x4567, 1); w5 += 4;
619 vsrtmp = vtmp4567x4567.val[3];
620 vtmp4567x4567.val[3] = vtmp4567x4567.val[2];
621 vtmp4567x4567.val[2] = vtmp4567x4567.val[1];
622 vtmp4567x4567.val[1] = vtmp4567x4567.val[0];
623 vtmp4567x4567.val[0] = vsrtmp;
624 vtmp4567x4567 = vld4q_lane_u32(w6, vtmp4567x4567, 2); w6 += 4;
625 vsrtmp = vtmp4567x4567.val[3];
626 vtmp4567x4567.val[3] = vtmp4567x4567.val[2];
627 vtmp4567x4567.val[2] = vtmp4567x4567.val[1];
628 vtmp4567x4567.val[1] = vtmp4567x4567.val[0];
629 vtmp4567x4567.val[0] = vsrtmp;
630 vsrtmp = vtmp4567x4567.val[3];
631 vtmp4567x4567.val[3] = vtmp4567x4567.val[2];
632 vtmp4567x4567.val[2] = vtmp4567x4567.val[1];
633 vtmp4567x4567.val[1] = vtmp4567x4567.val[0];
634 vtmp4567x4567.val[0] = vsrtmp;
635 vst1q_u32(packed_weights, vtmp0123x0123.val[0]); packed_weights += 4;
636 vst1q_u32(packed_weights, vtmp0123x4567.val[0]); packed_weights += 4;
637 vst1q_u32(packed_weights, vtmp0123x0123.val[1]); packed_weights += 4;
638 vst1q_u32(packed_weights, vtmp0123x4567.val[1]); packed_weights += 4;
639 vst1q_u32(packed_weights, vtmp0123x0123.val[2]); packed_weights += 4;
640 vst1q_u32(packed_weights, vtmp0123x4567.val[2]); packed_weights += 4;
641 vst1q_u32(packed_weights, vtmp0123x0123.val[3]); packed_weights += 4;
642 vst1q_u32(packed_weights, vtmp0123x4567.val[3]); packed_weights += 4;
643 vst1q_u32(packed_weights, vtmp4567x0123.val[0]); packed_weights += 4;
644 vst1q_u32(packed_weights, vtmp4567x4567.val[0]); packed_weights += 4;
645 vst1q_u32(packed_weights, vtmp4567x0123.val[1]); packed_weights += 4;
646 vst1q_u32(packed_weights, vtmp4567x4567.val[1]); packed_weights += 4;
647 vst1q_u32(packed_weights, vtmp4567x0123.val[2]); packed_weights += 4;
648 vst1q_u32(packed_weights, vtmp4567x4567.val[2]); packed_weights += 4;
649 vst1q_u32(packed_weights, vtmp4567x0123.val[3]); packed_weights += 4;
650 vst1q_u32(packed_weights, vtmp4567x4567.val[3]); packed_weights += 4;
651 }
652 // KC remainder multiple of 4
653 if (k >= 4) {
654 uint32x4_t vsrtmp;
655 vtmp0123x0123 = vld4q_lane_u32(w0, vtmp0123x0123, 0); w0 += 4;
656 vsrtmp = vtmp0123x0123.val[3];
657 vtmp0123x0123.val[3] = vtmp0123x0123.val[2];
658 vtmp0123x0123.val[2] = vtmp0123x0123.val[1];
659 vtmp0123x0123.val[1] = vtmp0123x0123.val[0];
660 vtmp0123x0123.val[0] = vsrtmp;
661 vtmp0123x0123 = vld4q_lane_u32(w1, vtmp0123x0123, 1); w1 += 4;
662 vsrtmp = vtmp0123x0123.val[3];
663 vtmp0123x0123.val[3] = vtmp0123x0123.val[2];
664 vtmp0123x0123.val[2] = vtmp0123x0123.val[1];
665 vtmp0123x0123.val[1] = vtmp0123x0123.val[0];
666 vtmp0123x0123.val[0] = vsrtmp;
667 vtmp0123x0123 = vld4q_lane_u32(w2, vtmp0123x0123, 2); w2 += 4;
668 vsrtmp = vtmp0123x0123.val[3];
669 vtmp0123x0123.val[3] = vtmp0123x0123.val[2];
670 vtmp0123x0123.val[2] = vtmp0123x0123.val[1];
671 vtmp0123x0123.val[1] = vtmp0123x0123.val[0];
672 vtmp0123x0123.val[0] = vsrtmp;
673 vtmp0123x0123 = vld4q_lane_u32(w3, vtmp0123x0123, 3); w3 += 4;
674 vsrtmp = vtmp0123x0123.val[3];
675 vtmp0123x0123.val[3] = vtmp0123x0123.val[2];
676 vtmp0123x0123.val[2] = vtmp0123x0123.val[1];
677 vtmp0123x0123.val[1] = vtmp0123x0123.val[0];
678 vtmp0123x0123.val[0] = vsrtmp;
679 vtmp0123x4567 = vld4q_lane_u32(w4, vtmp0123x4567, 0); w4 += 4;
680 vsrtmp = vtmp0123x4567.val[3];
681 vtmp0123x4567.val[3] = vtmp0123x4567.val[2];
682 vtmp0123x4567.val[2] = vtmp0123x4567.val[1];
683 vtmp0123x4567.val[1] = vtmp0123x4567.val[0];
684 vtmp0123x4567.val[0] = vsrtmp;
685 vtmp0123x4567 = vld4q_lane_u32(w5, vtmp0123x4567, 1); w5 += 4;
686 vsrtmp = vtmp0123x4567.val[3];
687 vtmp0123x4567.val[3] = vtmp0123x4567.val[2];
688 vtmp0123x4567.val[2] = vtmp0123x4567.val[1];
689 vtmp0123x4567.val[1] = vtmp0123x4567.val[0];
690 vtmp0123x4567.val[0] = vsrtmp;
691 vtmp0123x4567 = vld4q_lane_u32(w6, vtmp0123x4567, 2); w6 += 4;
692 vsrtmp = vtmp0123x4567.val[3];
693 vtmp0123x4567.val[3] = vtmp0123x4567.val[2];
694 vtmp0123x4567.val[2] = vtmp0123x4567.val[1];
695 vtmp0123x4567.val[1] = vtmp0123x4567.val[0];
696 vtmp0123x4567.val[0] = vsrtmp;
697 vsrtmp = vtmp0123x4567.val[3];
698 vtmp0123x4567.val[3] = vtmp0123x4567.val[2];
699 vtmp0123x4567.val[2] = vtmp0123x4567.val[1];
700 vtmp0123x4567.val[1] = vtmp0123x4567.val[0];
701 vtmp0123x4567.val[0] = vsrtmp;
702 vst1q_u32(packed_weights, vtmp0123x0123.val[0]); packed_weights += 4;
703 vst1q_u32(packed_weights, vtmp0123x4567.val[0]); packed_weights += 4;
704 vst1q_u32(packed_weights, vtmp0123x0123.val[1]); packed_weights += 4;
705 vst1q_u32(packed_weights, vtmp0123x4567.val[1]); packed_weights += 4;
706 vst1q_u32(packed_weights, vtmp0123x0123.val[2]); packed_weights += 4;
707 vst1q_u32(packed_weights, vtmp0123x4567.val[2]); packed_weights += 4;
708 vst1q_u32(packed_weights, vtmp0123x0123.val[3]); packed_weights += 4;
709 vst1q_u32(packed_weights, vtmp0123x4567.val[3]); packed_weights += 4;
710 k -= 4;
711 }
712
713 // KC remainder of 1..3
714 // Same as main loop but ld1, ld2 or ld3
715 if XNN_UNLIKELY(k != 0) {
716 assert(k >= 1);
717 assert(k <= 3);
718 switch (k) {
719 // KC remainder of 1
720 case 1:
721 {
722 uint32x4_t vtmp0x0123 = vdupq_n_u32(0);
723 uint32x4_t vtmp0x4567 = vdupq_n_u32(0);
724 uint32x4_t vsrtmp;
725 uint32x4_t vsrtmp1x0123 = vdupq_n_u32(0);
726 uint32x4_t vsrtmp2x0123 = vdupq_n_u32(0);
727 uint32x4_t vsrtmp3x0123 = vdupq_n_u32(0);
728 uint32x4_t vsrtmp1x4567 = vdupq_n_u32(0);
729 uint32x4_t vsrtmp2x4567 = vdupq_n_u32(0);
730 uint32x4_t vsrtmp3x4567 = vdupq_n_u32(0);
731 vtmp0x0123 = vld1q_lane_u32(w0, vtmp0x0123, 0);
732 vsrtmp = vsrtmp3x0123;
733 vsrtmp3x0123 = vsrtmp2x0123;
734 vsrtmp2x0123 = vsrtmp1x0123;
735 vsrtmp1x0123 = vtmp0x0123;
736 vtmp0x0123 = vsrtmp;
737 vtmp0x0123 = vld1q_lane_u32(w1, vtmp0x0123, 1);
738 vsrtmp = vsrtmp3x0123;
739 vsrtmp3x0123 = vsrtmp2x0123;
740 vsrtmp2x0123 = vsrtmp1x0123;
741 vsrtmp1x0123 = vtmp0x0123;
742 vtmp0x0123 = vsrtmp;
743 vtmp0x0123 = vld1q_lane_u32(w2, vtmp0x0123, 2);
744 vsrtmp = vsrtmp3x0123;
745 vsrtmp3x0123 = vsrtmp2x0123;
746 vsrtmp2x0123 = vsrtmp1x0123;
747 vsrtmp1x0123 = vtmp0x0123;
748 vtmp0x0123 = vsrtmp;
749 vtmp0x0123 = vld1q_lane_u32(w3, vtmp0x0123, 3);
750 vsrtmp = vsrtmp3x0123;
751 vsrtmp3x0123 = vsrtmp2x0123;
752 vsrtmp2x0123 = vsrtmp1x0123;
753 vsrtmp1x0123 = vtmp0x0123;
754 vtmp0x0123 = vsrtmp;
755 vtmp0x4567 = vld1q_lane_u32(w4, vtmp0x4567, 0);
756 vsrtmp = vsrtmp3x4567;
757 vsrtmp3x4567 = vsrtmp2x4567;
758 vsrtmp2x4567 = vsrtmp1x4567;
759 vsrtmp1x4567 = vtmp0x4567;
760 vtmp0x4567 = vsrtmp;
761 vtmp0x4567 = vld1q_lane_u32(w5, vtmp0x4567, 1);
762 vsrtmp = vsrtmp3x4567;
763 vsrtmp3x4567 = vsrtmp2x4567;
764 vsrtmp2x4567 = vsrtmp1x4567;
765 vsrtmp1x4567 = vtmp0x4567;
766 vtmp0x4567 = vsrtmp;
767 vtmp0x4567 = vld1q_lane_u32(w6, vtmp0x4567, 2);
768 vsrtmp = vsrtmp3x4567;
769 vsrtmp3x4567 = vsrtmp2x4567;
770 vsrtmp2x4567 = vsrtmp1x4567;
771 vsrtmp1x4567 = vtmp0x4567;
772 vtmp0x4567 = vsrtmp;
773 vsrtmp = vsrtmp3x4567;
774 vsrtmp3x4567 = vsrtmp2x4567;
775 vsrtmp2x4567 = vsrtmp1x4567;
776 vsrtmp1x4567 = vtmp0x4567;
777 vtmp0x4567 = vsrtmp;
778 vst1q_u32(packed_weights, vtmp0x0123); packed_weights += 4;
779 vst1q_u32(packed_weights, vtmp0x4567); packed_weights += 4;
780 vst1q_u32(packed_weights, vsrtmp1x0123); packed_weights += 4;
781 vst1q_u32(packed_weights, vsrtmp1x4567); packed_weights += 4;
782 vst1q_u32(packed_weights, vsrtmp2x0123); packed_weights += 4;
783 vst1q_u32(packed_weights, vsrtmp2x4567); packed_weights += 4;
784 vst1q_u32(packed_weights, vsrtmp3x0123); packed_weights += 4;
785 vst1q_u32(packed_weights, vsrtmp3x4567); packed_weights += 4;
786 break;
787 }
788 // KC remainder of 2
789 case 2:
790 {
791 uint32x4x2_t vtmp01x0123;
792 vtmp01x0123.val[0] = vdupq_n_u32(0);
793 vtmp01x0123.val[1] = vdupq_n_u32(0);
794 uint32x4x2_t vtmp01x4567;
795 vtmp01x4567.val[0] = vdupq_n_u32(0);
796 vtmp01x4567.val[1] = vdupq_n_u32(0);
797 uint32x4_t vsrtmp;
798 uint32x4_t vsrtmp2x0123 = vdupq_n_u32(0);
799 uint32x4_t vsrtmp3x0123 = vdupq_n_u32(0);
800 uint32x4_t vsrtmp2x4567 = vdupq_n_u32(0);
801 uint32x4_t vsrtmp3x4567 = vdupq_n_u32(0);
802 vtmp01x0123 = vld2q_lane_u32(w0, vtmp01x0123, 0);
803 vsrtmp = vsrtmp3x0123;
804 vsrtmp3x0123 = vsrtmp2x0123;
805 vsrtmp2x0123 = vtmp01x0123.val[1];
806 vtmp01x0123.val[1] = vtmp01x0123.val[0];
807 vtmp01x0123.val[0] = vsrtmp;
808 vtmp01x0123 = vld2q_lane_u32(w1, vtmp01x0123, 1);
809 vsrtmp = vsrtmp3x0123;
810 vsrtmp3x0123 = vsrtmp2x0123;
811 vsrtmp2x0123 = vtmp01x0123.val[1];
812 vtmp01x0123.val[1] = vtmp01x0123.val[0];
813 vtmp01x0123.val[0] = vsrtmp;
814 vtmp01x0123 = vld2q_lane_u32(w2, vtmp01x0123, 2);
815 vsrtmp = vsrtmp3x0123;
816 vsrtmp3x0123 = vsrtmp2x0123;
817 vsrtmp2x0123 = vtmp01x0123.val[1];
818 vtmp01x0123.val[1] = vtmp01x0123.val[0];
819 vtmp01x0123.val[0] = vsrtmp;
820 vtmp01x0123 = vld2q_lane_u32(w3, vtmp01x0123, 3);
821 vsrtmp = vsrtmp3x0123;
822 vsrtmp3x0123 = vsrtmp2x0123;
823 vsrtmp2x0123 = vtmp01x0123.val[1];
824 vtmp01x0123.val[1] = vtmp01x0123.val[0];
825 vtmp01x0123.val[0] = vsrtmp;
826 vtmp01x4567 = vld2q_lane_u32(w4, vtmp01x4567, 0);
827 vsrtmp = vsrtmp3x4567;
828 vsrtmp3x4567 = vsrtmp2x4567;
829 vsrtmp2x4567 = vtmp01x4567.val[1];
830 vtmp01x4567.val[1] = vtmp01x4567.val[0];
831 vtmp01x4567.val[0] = vsrtmp;
832 vtmp01x4567 = vld2q_lane_u32(w5, vtmp01x4567, 1);
833 vsrtmp = vsrtmp3x4567;
834 vsrtmp3x4567 = vsrtmp2x4567;
835 vsrtmp2x4567 = vtmp01x4567.val[1];
836 vtmp01x4567.val[1] = vtmp01x4567.val[0];
837 vtmp01x4567.val[0] = vsrtmp;
838 vtmp01x4567 = vld2q_lane_u32(w6, vtmp01x4567, 2);
839 vsrtmp = vsrtmp3x4567;
840 vsrtmp3x4567 = vsrtmp2x4567;
841 vsrtmp2x4567 = vtmp01x4567.val[1];
842 vtmp01x4567.val[1] = vtmp01x4567.val[0];
843 vtmp01x4567.val[0] = vsrtmp;
844 vsrtmp = vsrtmp3x4567;
845 vsrtmp3x4567 = vsrtmp2x4567;
846 vsrtmp2x4567 = vtmp01x4567.val[1];
847 vtmp01x4567.val[1] = vtmp01x4567.val[0];
848 vtmp01x4567.val[0] = vsrtmp;
849 vst1q_u32(packed_weights, vtmp01x0123.val[0]); packed_weights += 4;
850 vst1q_u32(packed_weights, vtmp01x4567.val[0]); packed_weights += 4;
851 vst1q_u32(packed_weights, vtmp01x0123.val[1]); packed_weights += 4;
852 vst1q_u32(packed_weights, vtmp01x4567.val[1]); packed_weights += 4;
853 vst1q_u32(packed_weights, vsrtmp2x0123); packed_weights += 4;
854 vst1q_u32(packed_weights, vsrtmp2x4567); packed_weights += 4;
855 vst1q_u32(packed_weights, vsrtmp3x0123); packed_weights += 4;
856 vst1q_u32(packed_weights, vsrtmp3x4567); packed_weights += 4;
857 break;
858 }
859 // KC remainder of 3
860 case 3:
861 {
862 uint32x4x3_t vtmp012x0123;
863 vtmp012x0123.val[0] = vdupq_n_u32(0);
864 vtmp012x0123.val[1] = vdupq_n_u32(0);
865 vtmp012x0123.val[2] = vdupq_n_u32(0);
866 uint32x4x3_t vtmp012x4567;
867 vtmp012x4567.val[0] = vdupq_n_u32(0);
868 vtmp012x4567.val[1] = vdupq_n_u32(0);
869 vtmp012x4567.val[2] = vdupq_n_u32(0);
870 uint32x4_t vsrtmp;
871 uint32x4_t vsrtmp3x0123 = vdupq_n_u32(0);
872 uint32x4_t vsrtmp3x4567 = vdupq_n_u32(0);
873 vtmp012x0123 = vld3q_lane_u32(w0, vtmp012x0123, 0);
874 vsrtmp = vsrtmp3x0123;
875 vsrtmp3x0123 = vtmp012x0123.val[2];
876 vtmp012x0123.val[2] = vtmp012x0123.val[1];
877 vtmp012x0123.val[1] = vtmp012x0123.val[0];
878 vtmp012x0123.val[0] = vsrtmp;
879 vtmp012x0123 = vld3q_lane_u32(w1, vtmp012x0123, 1);
880 vsrtmp = vsrtmp3x0123;
881 vsrtmp3x0123 = vtmp012x0123.val[2];
882 vtmp012x0123.val[2] = vtmp012x0123.val[1];
883 vtmp012x0123.val[1] = vtmp012x0123.val[0];
884 vtmp012x0123.val[0] = vsrtmp;
885 vtmp012x0123 = vld3q_lane_u32(w2, vtmp012x0123, 2);
886 vsrtmp = vsrtmp3x0123;
887 vsrtmp3x0123 = vtmp012x0123.val[2];
888 vtmp012x0123.val[2] = vtmp012x0123.val[1];
889 vtmp012x0123.val[1] = vtmp012x0123.val[0];
890 vtmp012x0123.val[0] = vsrtmp;
891 vtmp012x0123 = vld3q_lane_u32(w3, vtmp012x0123, 3);
892 vsrtmp = vsrtmp3x0123;
893 vsrtmp3x0123 = vtmp012x0123.val[2];
894 vtmp012x0123.val[2] = vtmp012x0123.val[1];
895 vtmp012x0123.val[1] = vtmp012x0123.val[0];
896 vtmp012x0123.val[0] = vsrtmp;
897 vtmp012x4567 = vld3q_lane_u32(w4, vtmp012x4567, 0);
898 vsrtmp = vsrtmp3x4567;
899 vsrtmp3x4567 = vtmp012x4567.val[2];
900 vtmp012x4567.val[2] = vtmp012x4567.val[1];
901 vtmp012x4567.val[1] = vtmp012x4567.val[0];
902 vtmp012x4567.val[0] = vsrtmp;
903 vtmp012x4567 = vld3q_lane_u32(w5, vtmp012x4567, 1);
904 vsrtmp = vsrtmp3x4567;
905 vsrtmp3x4567 = vtmp012x4567.val[2];
906 vtmp012x4567.val[2] = vtmp012x4567.val[1];
907 vtmp012x4567.val[1] = vtmp012x4567.val[0];
908 vtmp012x4567.val[0] = vsrtmp;
909 vtmp012x4567 = vld3q_lane_u32(w6, vtmp012x4567, 2);
910 vsrtmp = vsrtmp3x4567;
911 vsrtmp3x4567 = vtmp012x4567.val[2];
912 vtmp012x4567.val[2] = vtmp012x4567.val[1];
913 vtmp012x4567.val[1] = vtmp012x4567.val[0];
914 vtmp012x4567.val[0] = vsrtmp;
915 vsrtmp = vsrtmp3x4567;
916 vsrtmp3x4567 = vtmp012x4567.val[2];
917 vtmp012x4567.val[2] = vtmp012x4567.val[1];
918 vtmp012x4567.val[1] = vtmp012x4567.val[0];
919 vtmp012x4567.val[0] = vsrtmp;
920 vst1q_u32(packed_weights, vtmp012x0123.val[0]); packed_weights += 4;
921 vst1q_u32(packed_weights, vtmp012x4567.val[0]); packed_weights += 4;
922 vst1q_u32(packed_weights, vtmp012x0123.val[1]); packed_weights += 4;
923 vst1q_u32(packed_weights, vtmp012x4567.val[1]); packed_weights += 4;
924 vst1q_u32(packed_weights, vtmp012x0123.val[2]); packed_weights += 4;
925 vst1q_u32(packed_weights, vtmp012x4567.val[2]); packed_weights += 4;
926 vst1q_u32(packed_weights, vsrtmp3x0123); packed_weights += 4;
927 vst1q_u32(packed_weights, vsrtmp3x4567); packed_weights += 4;
928 break;
929 }
930 default:
931 XNN_UNREACHABLE;
932 }
933 }
934 packed_weights = (uint32_t*) ((uintptr_t) packed_weights + extra_bytes);
935 }
936
937 weights += nc * kc;
938 } while (--g != 0);
939 }