Debian Codegraph @

src/x32-packw/gen/x32-packw-x8s4-gemm-goi-neon-ld4lane-u4.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/neon.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include <arm_neon.h>
16
17 #include "xnnpack/packw.h"
18
19
20 void xnn_x32_packw_gemm_goi_ukernel_x8s4__neon_ld4lane_u4(
21 size_t g,
22 size_t nc,
23 size_t kc,
24 size_t nr,
25 size_t kr,
26 size_t sr,
27 const uint32_t* weights,
28 const uint32_t* bias,
29 const void* scale,
30 uint32_t* packed_weights,
31 size_t extra_bytes,
32 const void* params)
33 {
34 assert(g != 0);
35 assert(nc != 0);
36 assert(kc != 0);
37 assert(nr == 8);
38 assert(kr == 1);
39 assert(sr == 4);
40 assert(weights != NULL);
41 assert(packed_weights != NULL);
42 uint32x4x4_t vtmp0123x0123;
43 vtmp0123x0123.val[0] = vdupq_n_u32(0);
44 vtmp0123x0123.val[1] = vdupq_n_u32(0);
45 vtmp0123x0123.val[2] = vdupq_n_u32(0);
46 vtmp0123x0123.val[3] = vdupq_n_u32(0);
47 uint32x4x4_t vtmp0123x4567;
48 vtmp0123x4567.val[0] = vdupq_n_u32(0);
49 vtmp0123x4567.val[1] = vdupq_n_u32(0);
50 vtmp0123x4567.val[2] = vdupq_n_u32(0);
51 vtmp0123x4567.val[3] = vdupq_n_u32(0);
52
53 do {
54 // NC main loop multiple of 8
55 const uint32_t* w0 = weights;
56 size_t n = nc;
57
58 for (; n >= 8; n -= 8) {
59 if XNN_LIKELY(bias != NULL) {
60 uint32x4_t vb0 = vld1q_u32(bias); bias += 4;
61 uint32x4_t vb4 = vld1q_u32(bias); bias += 4;
62 vst1q_u32(packed_weights, vb0); packed_weights += 4;
63 vst1q_u32(packed_weights, vb4); packed_weights += 4;
64 } else {
65 const uint32x4_t vzero = vmovq_n_u32(0);
66 vst1q_u32(packed_weights, vzero); packed_weights += 4;
67 vst1q_u32(packed_weights, vzero); packed_weights += 4;
68 }
69
70 const uint32_t* w1 = w0 + kc;
71 const uint32_t* w2 = w1 + kc;
72 const uint32_t* w3 = w2 + kc;
73 const uint32_t* w4 = w3 + kc;
74 const uint32_t* w5 = w4 + kc;
75 const uint32_t* w6 = w5 + kc;
76 const uint32_t* w7 = w6 + kc;
77
78 // KC main loop multiple of 4
79 size_t k = kc;
80 for (; k >= 4; k -= 4) {
81 uint32x4_t vsrtmp;
82 vtmp0123x0123 = vld4q_lane_u32(w0, vtmp0123x0123, 0); w0 += 4;
83 vsrtmp = vtmp0123x0123.val[3];
84 vtmp0123x0123.val[3] = vtmp0123x0123.val[2];
85 vtmp0123x0123.val[2] = vtmp0123x0123.val[1];
86 vtmp0123x0123.val[1] = vtmp0123x0123.val[0];
87 vtmp0123x0123.val[0] = vsrtmp;
88 vtmp0123x0123 = vld4q_lane_u32(w1, vtmp0123x0123, 1); w1 += 4;
89 vsrtmp = vtmp0123x0123.val[3];
90 vtmp0123x0123.val[3] = vtmp0123x0123.val[2];
91 vtmp0123x0123.val[2] = vtmp0123x0123.val[1];
92 vtmp0123x0123.val[1] = vtmp0123x0123.val[0];
93 vtmp0123x0123.val[0] = vsrtmp;
94 vtmp0123x0123 = vld4q_lane_u32(w2, vtmp0123x0123, 2); w2 += 4;
95 vsrtmp = vtmp0123x0123.val[3];
96 vtmp0123x0123.val[3] = vtmp0123x0123.val[2];
97 vtmp0123x0123.val[2] = vtmp0123x0123.val[1];
98 vtmp0123x0123.val[1] = vtmp0123x0123.val[0];
99 vtmp0123x0123.val[0] = vsrtmp;
100 vtmp0123x0123 = vld4q_lane_u32(w3, vtmp0123x0123, 3); w3 += 4;
101 vsrtmp = vtmp0123x0123.val[3];
102 vtmp0123x0123.val[3] = vtmp0123x0123.val[2];
103 vtmp0123x0123.val[2] = vtmp0123x0123.val[1];
104 vtmp0123x0123.val[1] = vtmp0123x0123.val[0];
105 vtmp0123x0123.val[0] = vsrtmp;
106 vtmp0123x4567 = vld4q_lane_u32(w4, vtmp0123x4567, 0); w4 += 4;
107 vsrtmp = vtmp0123x4567.val[3];
108 vtmp0123x4567.val[3] = vtmp0123x4567.val[2];
109 vtmp0123x4567.val[2] = vtmp0123x4567.val[1];
110 vtmp0123x4567.val[1] = vtmp0123x4567.val[0];
111 vtmp0123x4567.val[0] = vsrtmp;
112 vtmp0123x4567 = vld4q_lane_u32(w5, vtmp0123x4567, 1); w5 += 4;
113 vsrtmp = vtmp0123x4567.val[3];
114 vtmp0123x4567.val[3] = vtmp0123x4567.val[2];
115 vtmp0123x4567.val[2] = vtmp0123x4567.val[1];
116 vtmp0123x4567.val[1] = vtmp0123x4567.val[0];
117 vtmp0123x4567.val[0] = vsrtmp;
118 vtmp0123x4567 = vld4q_lane_u32(w6, vtmp0123x4567, 2); w6 += 4;
119 vsrtmp = vtmp0123x4567.val[3];
120 vtmp0123x4567.val[3] = vtmp0123x4567.val[2];
121 vtmp0123x4567.val[2] = vtmp0123x4567.val[1];
122 vtmp0123x4567.val[1] = vtmp0123x4567.val[0];
123 vtmp0123x4567.val[0] = vsrtmp;
124 vtmp0123x4567 = vld4q_lane_u32(w7, vtmp0123x4567, 3); w7 += 4;
125 vsrtmp = vtmp0123x4567.val[3];
126 vtmp0123x4567.val[3] = vtmp0123x4567.val[2];
127 vtmp0123x4567.val[2] = vtmp0123x4567.val[1];
128 vtmp0123x4567.val[1] = vtmp0123x4567.val[0];
129 vtmp0123x4567.val[0] = vsrtmp;
130 vst1q_u32(packed_weights, vtmp0123x0123.val[0]); packed_weights += 4;
131 vst1q_u32(packed_weights, vtmp0123x4567.val[0]); packed_weights += 4;
132 vst1q_u32(packed_weights, vtmp0123x0123.val[1]); packed_weights += 4;
133 vst1q_u32(packed_weights, vtmp0123x4567.val[1]); packed_weights += 4;
134 vst1q_u32(packed_weights, vtmp0123x0123.val[2]); packed_weights += 4;
135 vst1q_u32(packed_weights, vtmp0123x4567.val[2]); packed_weights += 4;
136 vst1q_u32(packed_weights, vtmp0123x0123.val[3]); packed_weights += 4;
137 vst1q_u32(packed_weights, vtmp0123x4567.val[3]); packed_weights += 4;
138 }
139
140 // KC remainder of 1..3
141 // Same as main loop but ld1, ld2 or ld3
142 if XNN_UNLIKELY(k != 0) {
143 assert(k >= 1);
144 assert(k <= 3);
145 switch (k) {
146 // KC remainder of 1
147 case 1:
148 {
149 uint32x4_t vtmp0x0123 = vdupq_n_u32(0);
150 uint32x4_t vtmp0x4567 = vdupq_n_u32(0);
151 uint32x4_t vsrtmp;
152 uint32x4_t vsrtmp1x0123 = vdupq_n_u32(0);
153 uint32x4_t vsrtmp2x0123 = vdupq_n_u32(0);
154 uint32x4_t vsrtmp3x0123 = vdupq_n_u32(0);
155 uint32x4_t vsrtmp1x4567 = vdupq_n_u32(0);
156 uint32x4_t vsrtmp2x4567 = vdupq_n_u32(0);
157 uint32x4_t vsrtmp3x4567 = vdupq_n_u32(0);
158 vtmp0x0123 = vld1q_lane_u32(w0, vtmp0x0123, 0); w0 += 1;
159 vsrtmp = vsrtmp3x0123;
160 vsrtmp3x0123 = vsrtmp2x0123;
161 vsrtmp2x0123 = vsrtmp1x0123;
162 vsrtmp1x0123 = vtmp0x0123;
163 vtmp0x0123 = vsrtmp;
164 vtmp0x0123 = vld1q_lane_u32(w1, vtmp0x0123, 1); w1 += 1;
165 vsrtmp = vsrtmp3x0123;
166 vsrtmp3x0123 = vsrtmp2x0123;
167 vsrtmp2x0123 = vsrtmp1x0123;
168 vsrtmp1x0123 = vtmp0x0123;
169 vtmp0x0123 = vsrtmp;
170 vtmp0x0123 = vld1q_lane_u32(w2, vtmp0x0123, 2); w2 += 1;
171 vsrtmp = vsrtmp3x0123;
172 vsrtmp3x0123 = vsrtmp2x0123;
173 vsrtmp2x0123 = vsrtmp1x0123;
174 vsrtmp1x0123 = vtmp0x0123;
175 vtmp0x0123 = vsrtmp;
176 vtmp0x0123 = vld1q_lane_u32(w3, vtmp0x0123, 3); w3 += 1;
177 vsrtmp = vsrtmp3x0123;
178 vsrtmp3x0123 = vsrtmp2x0123;
179 vsrtmp2x0123 = vsrtmp1x0123;
180 vsrtmp1x0123 = vtmp0x0123;
181 vtmp0x0123 = vsrtmp;
182 vtmp0x4567 = vld1q_lane_u32(w4, vtmp0x4567, 0); w4 += 1;
183 vsrtmp = vsrtmp3x4567;
184 vsrtmp3x4567 = vsrtmp2x4567;
185 vsrtmp2x4567 = vsrtmp1x4567;
186 vsrtmp1x4567 = vtmp0x4567;
187 vtmp0x4567 = vsrtmp;
188 vtmp0x4567 = vld1q_lane_u32(w5, vtmp0x4567, 1); w5 += 1;
189 vsrtmp = vsrtmp3x4567;
190 vsrtmp3x4567 = vsrtmp2x4567;
191 vsrtmp2x4567 = vsrtmp1x4567;
192 vsrtmp1x4567 = vtmp0x4567;
193 vtmp0x4567 = vsrtmp;
194 vtmp0x4567 = vld1q_lane_u32(w6, vtmp0x4567, 2); w6 += 1;
195 vsrtmp = vsrtmp3x4567;
196 vsrtmp3x4567 = vsrtmp2x4567;
197 vsrtmp2x4567 = vsrtmp1x4567;
198 vsrtmp1x4567 = vtmp0x4567;
199 vtmp0x4567 = vsrtmp;
200 vtmp0x4567 = vld1q_lane_u32(w7, vtmp0x4567, 3); w7 += 1;
201 vsrtmp = vsrtmp3x4567;
202 vsrtmp3x4567 = vsrtmp2x4567;
203 vsrtmp2x4567 = vsrtmp1x4567;
204 vsrtmp1x4567 = vtmp0x4567;
205 vtmp0x4567 = vsrtmp;
206 vst1q_u32(packed_weights, vtmp0x0123); packed_weights += 4;
207 vst1q_u32(packed_weights, vtmp0x4567); packed_weights += 4;
208 vst1q_u32(packed_weights, vsrtmp1x0123); packed_weights += 4;
209 vst1q_u32(packed_weights, vsrtmp1x4567); packed_weights += 4;
210 vst1q_u32(packed_weights, vsrtmp2x0123); packed_weights += 4;
211 vst1q_u32(packed_weights, vsrtmp2x4567); packed_weights += 4;
212 vst1q_u32(packed_weights, vsrtmp3x0123); packed_weights += 4;
213 vst1q_u32(packed_weights, vsrtmp3x4567); packed_weights += 4;
214 break;
215 }
216 // KC remainder of 2
217 case 2:
218 {
219 uint32x4x2_t vtmp01x0123;
220 vtmp01x0123.val[0] = vdupq_n_u32(0);
221 vtmp01x0123.val[1] = vdupq_n_u32(0);
222 uint32x4x2_t vtmp01x4567;
223 vtmp01x4567.val[0] = vdupq_n_u32(0);
224 vtmp01x4567.val[1] = vdupq_n_u32(0);
225 uint32x4_t vsrtmp;
226 uint32x4_t vsrtmp2x0123 = vdupq_n_u32(0);
227 uint32x4_t vsrtmp3x0123 = vdupq_n_u32(0);
228 uint32x4_t vsrtmp2x4567 = vdupq_n_u32(0);
229 uint32x4_t vsrtmp3x4567 = vdupq_n_u32(0);
230 vtmp01x0123 = vld2q_lane_u32(w0, vtmp01x0123, 0); w0 += 2;
231 vsrtmp = vsrtmp3x0123;
232 vsrtmp3x0123 = vsrtmp2x0123;
233 vsrtmp2x0123 = vtmp01x0123.val[1];
234 vtmp01x0123.val[1] = vtmp01x0123.val[0];
235 vtmp01x0123.val[0] = vsrtmp;
236 vtmp01x0123 = vld2q_lane_u32(w1, vtmp01x0123, 1); w1 += 2;
237 vsrtmp = vsrtmp3x0123;
238 vsrtmp3x0123 = vsrtmp2x0123;
239 vsrtmp2x0123 = vtmp01x0123.val[1];
240 vtmp01x0123.val[1] = vtmp01x0123.val[0];
241 vtmp01x0123.val[0] = vsrtmp;
242 vtmp01x0123 = vld2q_lane_u32(w2, vtmp01x0123, 2); w2 += 2;
243 vsrtmp = vsrtmp3x0123;
244 vsrtmp3x0123 = vsrtmp2x0123;
245 vsrtmp2x0123 = vtmp01x0123.val[1];
246 vtmp01x0123.val[1] = vtmp01x0123.val[0];
247 vtmp01x0123.val[0] = vsrtmp;
248 vtmp01x0123 = vld2q_lane_u32(w3, vtmp01x0123, 3); w3 += 2;
249 vsrtmp = vsrtmp3x0123;
250 vsrtmp3x0123 = vsrtmp2x0123;
251 vsrtmp2x0123 = vtmp01x0123.val[1];
252 vtmp01x0123.val[1] = vtmp01x0123.val[0];
253 vtmp01x0123.val[0] = vsrtmp;
254 vtmp01x4567 = vld2q_lane_u32(w4, vtmp01x4567, 0); w4 += 2;
255 vsrtmp = vsrtmp3x4567;
256 vsrtmp3x4567 = vsrtmp2x4567;
257 vsrtmp2x4567 = vtmp01x4567.val[1];
258 vtmp01x4567.val[1] = vtmp01x4567.val[0];
259 vtmp01x4567.val[0] = vsrtmp;
260 vtmp01x4567 = vld2q_lane_u32(w5, vtmp01x4567, 1); w5 += 2;
261 vsrtmp = vsrtmp3x4567;
262 vsrtmp3x4567 = vsrtmp2x4567;
263 vsrtmp2x4567 = vtmp01x4567.val[1];
264 vtmp01x4567.val[1] = vtmp01x4567.val[0];
265 vtmp01x4567.val[0] = vsrtmp;
266 vtmp01x4567 = vld2q_lane_u32(w6, vtmp01x4567, 2); w6 += 2;
267 vsrtmp = vsrtmp3x4567;
268 vsrtmp3x4567 = vsrtmp2x4567;
269 vsrtmp2x4567 = vtmp01x4567.val[1];
270 vtmp01x4567.val[1] = vtmp01x4567.val[0];
271 vtmp01x4567.val[0] = vsrtmp;
272 vtmp01x4567 = vld2q_lane_u32(w7, vtmp01x4567, 3); w7 += 2;
273 vsrtmp = vsrtmp3x4567;
274 vsrtmp3x4567 = vsrtmp2x4567;
275 vsrtmp2x4567 = vtmp01x4567.val[1];
276 vtmp01x4567.val[1] = vtmp01x4567.val[0];
277 vtmp01x4567.val[0] = vsrtmp;
278 vst1q_u32(packed_weights, vtmp01x0123.val[0]); packed_weights += 4;
279 vst1q_u32(packed_weights, vtmp01x4567.val[0]); packed_weights += 4;
280 vst1q_u32(packed_weights, vtmp01x0123.val[1]); packed_weights += 4;
281 vst1q_u32(packed_weights, vtmp01x4567.val[1]); packed_weights += 4;
282 vst1q_u32(packed_weights, vsrtmp2x0123); packed_weights += 4;
283 vst1q_u32(packed_weights, vsrtmp2x4567); packed_weights += 4;
284 vst1q_u32(packed_weights, vsrtmp3x0123); packed_weights += 4;
285 vst1q_u32(packed_weights, vsrtmp3x4567); packed_weights += 4;
286 break;
287 }
288 // KC remainder of 3
289 case 3:
290 {
291 uint32x4x3_t vtmp012x0123;
292 vtmp012x0123.val[0] = vdupq_n_u32(0);
293 vtmp012x0123.val[1] = vdupq_n_u32(0);
294 vtmp012x0123.val[2] = vdupq_n_u32(0);
295 uint32x4x3_t vtmp012x4567;
296 vtmp012x4567.val[0] = vdupq_n_u32(0);
297 vtmp012x4567.val[1] = vdupq_n_u32(0);
298 vtmp012x4567.val[2] = vdupq_n_u32(0);
299 uint32x4_t vsrtmp;
300 uint32x4_t vsrtmp3x0123 = vdupq_n_u32(0);
301 uint32x4_t vsrtmp3x4567 = vdupq_n_u32(0);
302 vtmp012x0123 = vld3q_lane_u32(w0, vtmp012x0123, 0); w0 += 3;
303 vsrtmp = vsrtmp3x0123;
304 vsrtmp3x0123 = vtmp012x0123.val[2];
305 vtmp012x0123.val[2] = vtmp012x0123.val[1];
306 vtmp012x0123.val[1] = vtmp012x0123.val[0];
307 vtmp012x0123.val[0] = vsrtmp;
308 vtmp012x0123 = vld3q_lane_u32(w1, vtmp012x0123, 1); w1 += 3;
309 vsrtmp = vsrtmp3x0123;
310 vsrtmp3x0123 = vtmp012x0123.val[2];
311 vtmp012x0123.val[2] = vtmp012x0123.val[1];
312 vtmp012x0123.val[1] = vtmp012x0123.val[0];
313 vtmp012x0123.val[0] = vsrtmp;
314 vtmp012x0123 = vld3q_lane_u32(w2, vtmp012x0123, 2); w2 += 3;
315 vsrtmp = vsrtmp3x0123;
316 vsrtmp3x0123 = vtmp012x0123.val[2];
317 vtmp012x0123.val[2] = vtmp012x0123.val[1];
318 vtmp012x0123.val[1] = vtmp012x0123.val[0];
319 vtmp012x0123.val[0] = vsrtmp;
320 vtmp012x0123 = vld3q_lane_u32(w3, vtmp012x0123, 3); w3 += 3;
321 vsrtmp = vsrtmp3x0123;
322 vsrtmp3x0123 = vtmp012x0123.val[2];
323 vtmp012x0123.val[2] = vtmp012x0123.val[1];
324 vtmp012x0123.val[1] = vtmp012x0123.val[0];
325 vtmp012x0123.val[0] = vsrtmp;
326 vtmp012x4567 = vld3q_lane_u32(w4, vtmp012x4567, 0); w4 += 3;
327 vsrtmp = vsrtmp3x4567;
328 vsrtmp3x4567 = vtmp012x4567.val[2];
329 vtmp012x4567.val[2] = vtmp012x4567.val[1];
330 vtmp012x4567.val[1] = vtmp012x4567.val[0];
331 vtmp012x4567.val[0] = vsrtmp;
332 vtmp012x4567 = vld3q_lane_u32(w5, vtmp012x4567, 1); w5 += 3;
333 vsrtmp = vsrtmp3x4567;
334 vsrtmp3x4567 = vtmp012x4567.val[2];
335 vtmp012x4567.val[2] = vtmp012x4567.val[1];
336 vtmp012x4567.val[1] = vtmp012x4567.val[0];
337 vtmp012x4567.val[0] = vsrtmp;
338 vtmp012x4567 = vld3q_lane_u32(w6, vtmp012x4567, 2); w6 += 3;
339 vsrtmp = vsrtmp3x4567;
340 vsrtmp3x4567 = vtmp012x4567.val[2];
341 vtmp012x4567.val[2] = vtmp012x4567.val[1];
342 vtmp012x4567.val[1] = vtmp012x4567.val[0];
343 vtmp012x4567.val[0] = vsrtmp;
344 vtmp012x4567 = vld3q_lane_u32(w7, vtmp012x4567, 3); w7 += 3;
345 vsrtmp = vsrtmp3x4567;
346 vsrtmp3x4567 = vtmp012x4567.val[2];
347 vtmp012x4567.val[2] = vtmp012x4567.val[1];
348 vtmp012x4567.val[1] = vtmp012x4567.val[0];
349 vtmp012x4567.val[0] = vsrtmp;
350 vst1q_u32(packed_weights, vtmp012x0123.val[0]); packed_weights += 4;
351 vst1q_u32(packed_weights, vtmp012x4567.val[0]); packed_weights += 4;
352 vst1q_u32(packed_weights, vtmp012x0123.val[1]); packed_weights += 4;
353 vst1q_u32(packed_weights, vtmp012x4567.val[1]); packed_weights += 4;
354 vst1q_u32(packed_weights, vtmp012x0123.val[2]); packed_weights += 4;
355 vst1q_u32(packed_weights, vtmp012x4567.val[2]); packed_weights += 4;
356 vst1q_u32(packed_weights, vsrtmp3x0123); packed_weights += 4;
357 vst1q_u32(packed_weights, vsrtmp3x4567); packed_weights += 4;
358 break;
359 }
360 default:
361 XNN_UNREACHABLE;
362 }
363 }
364 packed_weights = (uint32_t*) ((uintptr_t) packed_weights + extra_bytes);
365 w0 = w7;
366 }
367
368 // NC remainder (1..7)
369 if XNN_UNLIKELY(n != 0) {
370 assert(n >= 1);
371 assert(n <= 7);
372 if XNN_LIKELY(bias != NULL) {
373 size_t nb = n;
374 do {
375 *packed_weights++ = *bias++;
376 } while (--nb != 0);
377 packed_weights += (8 - n);
378 } else {
379 const uint32x4_t vzero = vmovq_n_u32(0);
380 vst1q_u32(packed_weights, vzero); packed_weights += 4;
381 vst1q_u32(packed_weights, vzero); packed_weights += 4;
382 }
383
384 // NR remainder has less than 8 rows so last row is not loaded
385 const uint32_t* w1 = w0 + kc;
386 if XNN_UNPREDICTABLE(n < 2) {
387 w1 = w0;
388 }
389 const uint32_t* w2 = w1 + kc;
390 if XNN_UNPREDICTABLE(n <= 2) {
391 w2 = w1;
392 }
393 const uint32_t* w3 = w2 + kc;
394 if XNN_UNPREDICTABLE(n < 4) {
395 w3 = w2;
396 }
397 const uint32_t* w4 = w3 + kc;
398 if XNN_UNPREDICTABLE(n <= 4) {
399 w4 = w3;
400 }
401 const uint32_t* w5 = w4 + kc;
402 if XNN_UNPREDICTABLE(n < 6) {
403 w5 = w4;
404 }
405 const uint32_t* w6 = w5 + kc;
406 if XNN_UNPREDICTABLE(n <= 6) {
407 w6 = w5;
408 }
409
410 // KC main loop multiple of 4
411 size_t k = kc;
412 for (; k >= 4; k -= 4) {
413 uint32x4_t vsrtmp;
414 vtmp0123x0123 = vld4q_lane_u32(w0, vtmp0123x0123, 0); w0 += 4;
415 vsrtmp = vtmp0123x0123.val[3];
416 vtmp0123x0123.val[3] = vtmp0123x0123.val[2];
417 vtmp0123x0123.val[2] = vtmp0123x0123.val[1];
418 vtmp0123x0123.val[1] = vtmp0123x0123.val[0];
419 vtmp0123x0123.val[0] = vsrtmp;
420 vtmp0123x0123 = vld4q_lane_u32(w1, vtmp0123x0123, 1); w1 += 4;
421 vsrtmp = vtmp0123x0123.val[3];
422 vtmp0123x0123.val[3] = vtmp0123x0123.val[2];
423 vtmp0123x0123.val[2] = vtmp0123x0123.val[1];
424 vtmp0123x0123.val[1] = vtmp0123x0123.val[0];
425 vtmp0123x0123.val[0] = vsrtmp;
426 vtmp0123x0123 = vld4q_lane_u32(w2, vtmp0123x0123, 2); w2 += 4;
427 vsrtmp = vtmp0123x0123.val[3];
428 vtmp0123x0123.val[3] = vtmp0123x0123.val[2];
429 vtmp0123x0123.val[2] = vtmp0123x0123.val[1];
430 vtmp0123x0123.val[1] = vtmp0123x0123.val[0];
431 vtmp0123x0123.val[0] = vsrtmp;
432 vtmp0123x0123 = vld4q_lane_u32(w3, vtmp0123x0123, 3); w3 += 4;
433 vsrtmp = vtmp0123x0123.val[3];
434 vtmp0123x0123.val[3] = vtmp0123x0123.val[2];
435 vtmp0123x0123.val[2] = vtmp0123x0123.val[1];
436 vtmp0123x0123.val[1] = vtmp0123x0123.val[0];
437 vtmp0123x0123.val[0] = vsrtmp;
438 vtmp0123x4567 = vld4q_lane_u32(w4, vtmp0123x4567, 0); w4 += 4;
439 vsrtmp = vtmp0123x4567.val[3];
440 vtmp0123x4567.val[3] = vtmp0123x4567.val[2];
441 vtmp0123x4567.val[2] = vtmp0123x4567.val[1];
442 vtmp0123x4567.val[1] = vtmp0123x4567.val[0];
443 vtmp0123x4567.val[0] = vsrtmp;
444 vtmp0123x4567 = vld4q_lane_u32(w5, vtmp0123x4567, 1); w5 += 4;
445 vsrtmp = vtmp0123x4567.val[3];
446 vtmp0123x4567.val[3] = vtmp0123x4567.val[2];
447 vtmp0123x4567.val[2] = vtmp0123x4567.val[1];
448 vtmp0123x4567.val[1] = vtmp0123x4567.val[0];
449 vtmp0123x4567.val[0] = vsrtmp;
450 vtmp0123x4567 = vld4q_lane_u32(w6, vtmp0123x4567, 2); w6 += 4;
451 vsrtmp = vtmp0123x4567.val[3];
452 vtmp0123x4567.val[3] = vtmp0123x4567.val[2];
453 vtmp0123x4567.val[2] = vtmp0123x4567.val[1];
454 vtmp0123x4567.val[1] = vtmp0123x4567.val[0];
455 vtmp0123x4567.val[0] = vsrtmp;
456 vsrtmp = vtmp0123x4567.val[3];
457 vtmp0123x4567.val[3] = vtmp0123x4567.val[2];
458 vtmp0123x4567.val[2] = vtmp0123x4567.val[1];
459 vtmp0123x4567.val[1] = vtmp0123x4567.val[0];
460 vtmp0123x4567.val[0] = vsrtmp;
461 vst1q_u32(packed_weights, vtmp0123x0123.val[0]); packed_weights += 4;
462 vst1q_u32(packed_weights, vtmp0123x4567.val[0]); packed_weights += 4;
463 vst1q_u32(packed_weights, vtmp0123x0123.val[1]); packed_weights += 4;
464 vst1q_u32(packed_weights, vtmp0123x4567.val[1]); packed_weights += 4;
465 vst1q_u32(packed_weights, vtmp0123x0123.val[2]); packed_weights += 4;
466 vst1q_u32(packed_weights, vtmp0123x4567.val[2]); packed_weights += 4;
467 vst1q_u32(packed_weights, vtmp0123x0123.val[3]); packed_weights += 4;
468 vst1q_u32(packed_weights, vtmp0123x4567.val[3]); packed_weights += 4;
469 }
470
471 // KC remainder of 1..3
472 // Same as main loop but ld1, ld2 or ld3
473 if XNN_UNLIKELY(k != 0) {
474 assert(k >= 1);
475 assert(k <= 3);
476 switch (k) {
477 // KC remainder of 1
478 case 1:
479 {
480 uint32x4_t vtmp0x0123 = vdupq_n_u32(0);
481 uint32x4_t vtmp0x4567 = vdupq_n_u32(0);
482 uint32x4_t vsrtmp;
483 uint32x4_t vsrtmp1x0123 = vdupq_n_u32(0);
484 uint32x4_t vsrtmp2x0123 = vdupq_n_u32(0);
485 uint32x4_t vsrtmp3x0123 = vdupq_n_u32(0);
486 uint32x4_t vsrtmp1x4567 = vdupq_n_u32(0);
487 uint32x4_t vsrtmp2x4567 = vdupq_n_u32(0);
488 uint32x4_t vsrtmp3x4567 = vdupq_n_u32(0);
489 vtmp0x0123 = vld1q_lane_u32(w0, vtmp0x0123, 0);
490 vsrtmp = vsrtmp3x0123;
491 vsrtmp3x0123 = vsrtmp2x0123;
492 vsrtmp2x0123 = vsrtmp1x0123;
493 vsrtmp1x0123 = vtmp0x0123;
494 vtmp0x0123 = vsrtmp;
495 vtmp0x0123 = vld1q_lane_u32(w1, vtmp0x0123, 1);
496 vsrtmp = vsrtmp3x0123;
497 vsrtmp3x0123 = vsrtmp2x0123;
498 vsrtmp2x0123 = vsrtmp1x0123;
499 vsrtmp1x0123 = vtmp0x0123;
500 vtmp0x0123 = vsrtmp;
501 vtmp0x0123 = vld1q_lane_u32(w2, vtmp0x0123, 2);
502 vsrtmp = vsrtmp3x0123;
503 vsrtmp3x0123 = vsrtmp2x0123;
504 vsrtmp2x0123 = vsrtmp1x0123;
505 vsrtmp1x0123 = vtmp0x0123;
506 vtmp0x0123 = vsrtmp;
507 vtmp0x0123 = vld1q_lane_u32(w3, vtmp0x0123, 3);
508 vsrtmp = vsrtmp3x0123;
509 vsrtmp3x0123 = vsrtmp2x0123;
510 vsrtmp2x0123 = vsrtmp1x0123;
511 vsrtmp1x0123 = vtmp0x0123;
512 vtmp0x0123 = vsrtmp;
513 vtmp0x4567 = vld1q_lane_u32(w4, vtmp0x4567, 0);
514 vsrtmp = vsrtmp3x4567;
515 vsrtmp3x4567 = vsrtmp2x4567;
516 vsrtmp2x4567 = vsrtmp1x4567;
517 vsrtmp1x4567 = vtmp0x4567;
518 vtmp0x4567 = vsrtmp;
519 vtmp0x4567 = vld1q_lane_u32(w5, vtmp0x4567, 1);
520 vsrtmp = vsrtmp3x4567;
521 vsrtmp3x4567 = vsrtmp2x4567;
522 vsrtmp2x4567 = vsrtmp1x4567;
523 vsrtmp1x4567 = vtmp0x4567;
524 vtmp0x4567 = vsrtmp;
525 vtmp0x4567 = vld1q_lane_u32(w6, vtmp0x4567, 2);
526 vsrtmp = vsrtmp3x4567;
527 vsrtmp3x4567 = vsrtmp2x4567;
528 vsrtmp2x4567 = vsrtmp1x4567;
529 vsrtmp1x4567 = vtmp0x4567;
530 vtmp0x4567 = vsrtmp;
531 vsrtmp = vsrtmp3x4567;
532 vsrtmp3x4567 = vsrtmp2x4567;
533 vsrtmp2x4567 = vsrtmp1x4567;
534 vsrtmp1x4567 = vtmp0x4567;
535 vtmp0x4567 = vsrtmp;
536 vst1q_u32(packed_weights, vtmp0x0123); packed_weights += 4;
537 vst1q_u32(packed_weights, vtmp0x4567); packed_weights += 4;
538 vst1q_u32(packed_weights, vsrtmp1x0123); packed_weights += 4;
539 vst1q_u32(packed_weights, vsrtmp1x4567); packed_weights += 4;
540 vst1q_u32(packed_weights, vsrtmp2x0123); packed_weights += 4;
541 vst1q_u32(packed_weights, vsrtmp2x4567); packed_weights += 4;
542 vst1q_u32(packed_weights, vsrtmp3x0123); packed_weights += 4;
543 vst1q_u32(packed_weights, vsrtmp3x4567); packed_weights += 4;
544 break;
545 }
546 // KC remainder of 2
547 case 2:
548 {
549 uint32x4x2_t vtmp01x0123;
550 vtmp01x0123.val[0] = vdupq_n_u32(0);
551 vtmp01x0123.val[1] = vdupq_n_u32(0);
552 uint32x4x2_t vtmp01x4567;
553 vtmp01x4567.val[0] = vdupq_n_u32(0);
554 vtmp01x4567.val[1] = vdupq_n_u32(0);
555 uint32x4_t vsrtmp;
556 uint32x4_t vsrtmp2x0123 = vdupq_n_u32(0);
557 uint32x4_t vsrtmp3x0123 = vdupq_n_u32(0);
558 uint32x4_t vsrtmp2x4567 = vdupq_n_u32(0);
559 uint32x4_t vsrtmp3x4567 = vdupq_n_u32(0);
560 vtmp01x0123 = vld2q_lane_u32(w0, vtmp01x0123, 0);
561 vsrtmp = vsrtmp3x0123;
562 vsrtmp3x0123 = vsrtmp2x0123;
563 vsrtmp2x0123 = vtmp01x0123.val[1];
564 vtmp01x0123.val[1] = vtmp01x0123.val[0];
565 vtmp01x0123.val[0] = vsrtmp;
566 vtmp01x0123 = vld2q_lane_u32(w1, vtmp01x0123, 1);
567 vsrtmp = vsrtmp3x0123;
568 vsrtmp3x0123 = vsrtmp2x0123;
569 vsrtmp2x0123 = vtmp01x0123.val[1];
570 vtmp01x0123.val[1] = vtmp01x0123.val[0];
571 vtmp01x0123.val[0] = vsrtmp;
572 vtmp01x0123 = vld2q_lane_u32(w2, vtmp01x0123, 2);
573 vsrtmp = vsrtmp3x0123;
574 vsrtmp3x0123 = vsrtmp2x0123;
575 vsrtmp2x0123 = vtmp01x0123.val[1];
576 vtmp01x0123.val[1] = vtmp01x0123.val[0];
577 vtmp01x0123.val[0] = vsrtmp;
578 vtmp01x0123 = vld2q_lane_u32(w3, vtmp01x0123, 3);
579 vsrtmp = vsrtmp3x0123;
580 vsrtmp3x0123 = vsrtmp2x0123;
581 vsrtmp2x0123 = vtmp01x0123.val[1];
582 vtmp01x0123.val[1] = vtmp01x0123.val[0];
583 vtmp01x0123.val[0] = vsrtmp;
584 vtmp01x4567 = vld2q_lane_u32(w4, vtmp01x4567, 0);
585 vsrtmp = vsrtmp3x4567;
586 vsrtmp3x4567 = vsrtmp2x4567;
587 vsrtmp2x4567 = vtmp01x4567.val[1];
588 vtmp01x4567.val[1] = vtmp01x4567.val[0];
589 vtmp01x4567.val[0] = vsrtmp;
590 vtmp01x4567 = vld2q_lane_u32(w5, vtmp01x4567, 1);
591 vsrtmp = vsrtmp3x4567;
592 vsrtmp3x4567 = vsrtmp2x4567;
593 vsrtmp2x4567 = vtmp01x4567.val[1];
594 vtmp01x4567.val[1] = vtmp01x4567.val[0];
595 vtmp01x4567.val[0] = vsrtmp;
596 vtmp01x4567 = vld2q_lane_u32(w6, vtmp01x4567, 2);
597 vsrtmp = vsrtmp3x4567;
598 vsrtmp3x4567 = vsrtmp2x4567;
599 vsrtmp2x4567 = vtmp01x4567.val[1];
600 vtmp01x4567.val[1] = vtmp01x4567.val[0];
601 vtmp01x4567.val[0] = vsrtmp;
602 vsrtmp = vsrtmp3x4567;
603 vsrtmp3x4567 = vsrtmp2x4567;
604 vsrtmp2x4567 = vtmp01x4567.val[1];
605 vtmp01x4567.val[1] = vtmp01x4567.val[0];
606 vtmp01x4567.val[0] = vsrtmp;
607 vst1q_u32(packed_weights, vtmp01x0123.val[0]); packed_weights += 4;
608 vst1q_u32(packed_weights, vtmp01x4567.val[0]); packed_weights += 4;
609 vst1q_u32(packed_weights, vtmp01x0123.val[1]); packed_weights += 4;
610 vst1q_u32(packed_weights, vtmp01x4567.val[1]); packed_weights += 4;
611 vst1q_u32(packed_weights, vsrtmp2x0123); packed_weights += 4;
612 vst1q_u32(packed_weights, vsrtmp2x4567); packed_weights += 4;
613 vst1q_u32(packed_weights, vsrtmp3x0123); packed_weights += 4;
614 vst1q_u32(packed_weights, vsrtmp3x4567); packed_weights += 4;
615 break;
616 }
617 // KC remainder of 3
618 case 3:
619 {
620 uint32x4x3_t vtmp012x0123;
621 vtmp012x0123.val[0] = vdupq_n_u32(0);
622 vtmp012x0123.val[1] = vdupq_n_u32(0);
623 vtmp012x0123.val[2] = vdupq_n_u32(0);
624 uint32x4x3_t vtmp012x4567;
625 vtmp012x4567.val[0] = vdupq_n_u32(0);
626 vtmp012x4567.val[1] = vdupq_n_u32(0);
627 vtmp012x4567.val[2] = vdupq_n_u32(0);
628 uint32x4_t vsrtmp;
629 uint32x4_t vsrtmp3x0123 = vdupq_n_u32(0);
630 uint32x4_t vsrtmp3x4567 = vdupq_n_u32(0);
631 vtmp012x0123 = vld3q_lane_u32(w0, vtmp012x0123, 0);
632 vsrtmp = vsrtmp3x0123;
633 vsrtmp3x0123 = vtmp012x0123.val[2];
634 vtmp012x0123.val[2] = vtmp012x0123.val[1];
635 vtmp012x0123.val[1] = vtmp012x0123.val[0];
636 vtmp012x0123.val[0] = vsrtmp;
637 vtmp012x0123 = vld3q_lane_u32(w1, vtmp012x0123, 1);
638 vsrtmp = vsrtmp3x0123;
639 vsrtmp3x0123 = vtmp012x0123.val[2];
640 vtmp012x0123.val[2] = vtmp012x0123.val[1];
641 vtmp012x0123.val[1] = vtmp012x0123.val[0];
642 vtmp012x0123.val[0] = vsrtmp;
643 vtmp012x0123 = vld3q_lane_u32(w2, vtmp012x0123, 2);
644 vsrtmp = vsrtmp3x0123;
645 vsrtmp3x0123 = vtmp012x0123.val[2];
646 vtmp012x0123.val[2] = vtmp012x0123.val[1];
647 vtmp012x0123.val[1] = vtmp012x0123.val[0];
648 vtmp012x0123.val[0] = vsrtmp;
649 vtmp012x0123 = vld3q_lane_u32(w3, vtmp012x0123, 3);
650 vsrtmp = vsrtmp3x0123;
651 vsrtmp3x0123 = vtmp012x0123.val[2];
652 vtmp012x0123.val[2] = vtmp012x0123.val[1];
653 vtmp012x0123.val[1] = vtmp012x0123.val[0];
654 vtmp012x0123.val[0] = vsrtmp;
655 vtmp012x4567 = vld3q_lane_u32(w4, vtmp012x4567, 0);
656 vsrtmp = vsrtmp3x4567;
657 vsrtmp3x4567 = vtmp012x4567.val[2];
658 vtmp012x4567.val[2] = vtmp012x4567.val[1];
659 vtmp012x4567.val[1] = vtmp012x4567.val[0];
660 vtmp012x4567.val[0] = vsrtmp;
661 vtmp012x4567 = vld3q_lane_u32(w5, vtmp012x4567, 1);
662 vsrtmp = vsrtmp3x4567;
663 vsrtmp3x4567 = vtmp012x4567.val[2];
664 vtmp012x4567.val[2] = vtmp012x4567.val[1];
665 vtmp012x4567.val[1] = vtmp012x4567.val[0];
666 vtmp012x4567.val[0] = vsrtmp;
667 vtmp012x4567 = vld3q_lane_u32(w6, vtmp012x4567, 2);
668 vsrtmp = vsrtmp3x4567;
669 vsrtmp3x4567 = vtmp012x4567.val[2];
670 vtmp012x4567.val[2] = vtmp012x4567.val[1];
671 vtmp012x4567.val[1] = vtmp012x4567.val[0];
672 vtmp012x4567.val[0] = vsrtmp;
673 vsrtmp = vsrtmp3x4567;
674 vsrtmp3x4567 = vtmp012x4567.val[2];
675 vtmp012x4567.val[2] = vtmp012x4567.val[1];
676 vtmp012x4567.val[1] = vtmp012x4567.val[0];
677 vtmp012x4567.val[0] = vsrtmp;
678 vst1q_u32(packed_weights, vtmp012x0123.val[0]); packed_weights += 4;
679 vst1q_u32(packed_weights, vtmp012x4567.val[0]); packed_weights += 4;
680 vst1q_u32(packed_weights, vtmp012x0123.val[1]); packed_weights += 4;
681 vst1q_u32(packed_weights, vtmp012x4567.val[1]); packed_weights += 4;
682 vst1q_u32(packed_weights, vtmp012x0123.val[2]); packed_weights += 4;
683 vst1q_u32(packed_weights, vtmp012x4567.val[2]); packed_weights += 4;
684 vst1q_u32(packed_weights, vsrtmp3x0123); packed_weights += 4;
685 vst1q_u32(packed_weights, vsrtmp3x4567); packed_weights += 4;
686 break;
687 }
688 default:
689 XNN_UNREACHABLE;
690 }
691 }
692 packed_weights = (uint32_t*) ((uintptr_t) packed_weights + extra_bytes);
693 }
694
695 weights += nc * kc;
696 } while (--g != 0);
697 }