Debian Codegraph @

src/x32-packw/gen/x32-packw-x16-gemm-goi-neon-ld4lane-u8-prfm.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/neon.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include <arm_neon.h>
16
17 #include "xnnpack/packw.h"
18 #include "xnnpack/prefetch.h"
19
20
21 void xnn_x32_packw_gemm_goi_ukernel_x16__neon_ld4lane_u8_prfm(
22 size_t g,
23 size_t nc,
24 size_t kc,
25 size_t nr,
26 size_t kr,
27 size_t sr,
28 const uint32_t* weights,
29 const uint32_t* bias,
30 const void* scale,
31 uint32_t* packed_weights,
32 size_t extra_bytes,
33 const void* params)
34 {
35 assert(g != 0);
36 assert(nc != 0);
37 assert(kc != 0);
38 assert(nr == 16);
39 assert(kr == 1);
40 assert(sr == 1);
41 assert(weights != NULL);
42 assert(packed_weights != NULL);
43 uint32x4x4_t vtmp0123x0123;
44 vtmp0123x0123.val[0] = vdupq_n_u32(0);
45 vtmp0123x0123.val[1] = vdupq_n_u32(0);
46 vtmp0123x0123.val[2] = vdupq_n_u32(0);
47 vtmp0123x0123.val[3] = vdupq_n_u32(0);
48 uint32x4x4_t vtmp4567x0123;
49 vtmp4567x0123.val[0] = vdupq_n_u32(0);
50 vtmp4567x0123.val[1] = vdupq_n_u32(0);
51 vtmp4567x0123.val[2] = vdupq_n_u32(0);
52 vtmp4567x0123.val[3] = vdupq_n_u32(0);
53 uint32x4x4_t vtmp0123x4567;
54 vtmp0123x4567.val[0] = vdupq_n_u32(0);
55 vtmp0123x4567.val[1] = vdupq_n_u32(0);
56 vtmp0123x4567.val[2] = vdupq_n_u32(0);
57 vtmp0123x4567.val[3] = vdupq_n_u32(0);
58 uint32x4x4_t vtmp4567x4567;
59 vtmp4567x4567.val[0] = vdupq_n_u32(0);
60 vtmp4567x4567.val[1] = vdupq_n_u32(0);
61 vtmp4567x4567.val[2] = vdupq_n_u32(0);
62 vtmp4567x4567.val[3] = vdupq_n_u32(0);
63 uint32x4x4_t vtmp0123x89AB;
64 vtmp0123x89AB.val[0] = vdupq_n_u32(0);
65 vtmp0123x89AB.val[1] = vdupq_n_u32(0);
66 vtmp0123x89AB.val[2] = vdupq_n_u32(0);
67 vtmp0123x89AB.val[3] = vdupq_n_u32(0);
68 uint32x4x4_t vtmp4567x89AB;
69 vtmp4567x89AB.val[0] = vdupq_n_u32(0);
70 vtmp4567x89AB.val[1] = vdupq_n_u32(0);
71 vtmp4567x89AB.val[2] = vdupq_n_u32(0);
72 vtmp4567x89AB.val[3] = vdupq_n_u32(0);
73 uint32x4x4_t vtmp0123xCDEF;
74 vtmp0123xCDEF.val[0] = vdupq_n_u32(0);
75 vtmp0123xCDEF.val[1] = vdupq_n_u32(0);
76 vtmp0123xCDEF.val[2] = vdupq_n_u32(0);
77 vtmp0123xCDEF.val[3] = vdupq_n_u32(0);
78 uint32x4x4_t vtmp4567xCDEF;
79 vtmp4567xCDEF.val[0] = vdupq_n_u32(0);
80 vtmp4567xCDEF.val[1] = vdupq_n_u32(0);
81 vtmp4567xCDEF.val[2] = vdupq_n_u32(0);
82 vtmp4567xCDEF.val[3] = vdupq_n_u32(0);
83
84 do {
85 // NC main loop multiple of 16
86 const uint32_t* w0 = weights;
87 size_t n = nc;
88
89 for (; n >= 16; n -= 16) {
90 if XNN_LIKELY(bias != NULL) {
91 uint32x4_t vb0 = vld1q_u32(bias); bias += 4;
92 uint32x4_t vb4 = vld1q_u32(bias); bias += 4;
93 uint32x4_t vb8 = vld1q_u32(bias); bias += 4;
94 uint32x4_t vb12 = vld1q_u32(bias); bias += 4;
95 vst1q_u32(packed_weights, vb0); packed_weights += 4;
96 vst1q_u32(packed_weights, vb4); packed_weights += 4;
97 vst1q_u32(packed_weights, vb8); packed_weights += 4;
98 vst1q_u32(packed_weights, vb12); packed_weights += 4;
99 } else {
100 const uint32x4_t vzero = vmovq_n_u32(0);
101 vst1q_u32(packed_weights, vzero); packed_weights += 4;
102 vst1q_u32(packed_weights, vzero); packed_weights += 4;
103 vst1q_u32(packed_weights, vzero); packed_weights += 4;
104 vst1q_u32(packed_weights, vzero); packed_weights += 4;
105 }
106
107 const uint32_t* w1 = w0 + kc;
108 const uint32_t* w2 = w1 + kc;
109 const uint32_t* w3 = w2 + kc;
110 const uint32_t* w4 = w3 + kc;
111 const uint32_t* w5 = w4 + kc;
112 const uint32_t* w6 = w5 + kc;
113 const uint32_t* w7 = w6 + kc;
114 const uint32_t* w8 = w7 + kc;
115 const uint32_t* w9 = w8 + kc;
116 const uint32_t* w10 = w9 + kc;
117 const uint32_t* w11 = w10 + kc;
118 const uint32_t* w12 = w11 + kc;
119 const uint32_t* w13 = w12 + kc;
120 const uint32_t* w14 = w13 + kc;
121 const uint32_t* w15 = w14 + kc;
122 xnn_prefetch_to_l1((const int8_t*) w0);
123 xnn_prefetch_to_l1((const int8_t*) w0 + 64);
124 xnn_prefetch_to_l1((const int8_t*) w1);
125 xnn_prefetch_to_l1((const int8_t*) w1 + 64);
126 xnn_prefetch_to_l1((const int8_t*) w2);
127 xnn_prefetch_to_l1((const int8_t*) w2 + 64);
128 xnn_prefetch_to_l1((const int8_t*) w3);
129 xnn_prefetch_to_l1((const int8_t*) w3 + 64);
130 xnn_prefetch_to_l1((const int8_t*) w4);
131 xnn_prefetch_to_l1((const int8_t*) w4 + 64);
132 xnn_prefetch_to_l1((const int8_t*) w5);
133 xnn_prefetch_to_l1((const int8_t*) w5 + 64);
134 xnn_prefetch_to_l1((const int8_t*) w6);
135 xnn_prefetch_to_l1((const int8_t*) w6 + 64);
136 xnn_prefetch_to_l1((const int8_t*) w7);
137 xnn_prefetch_to_l1((const int8_t*) w7 + 64);
138 xnn_prefetch_to_l1((const int8_t*) w8);
139 xnn_prefetch_to_l1((const int8_t*) w8 + 64);
140 xnn_prefetch_to_l1((const int8_t*) w9);
141 xnn_prefetch_to_l1((const int8_t*) w9 + 64);
142 xnn_prefetch_to_l1((const int8_t*) w10);
143 xnn_prefetch_to_l1((const int8_t*) w10 + 64);
144 xnn_prefetch_to_l1((const int8_t*) w11);
145 xnn_prefetch_to_l1((const int8_t*) w11 + 64);
146 xnn_prefetch_to_l1((const int8_t*) w12);
147 xnn_prefetch_to_l1((const int8_t*) w12 + 64);
148 xnn_prefetch_to_l1((const int8_t*) w13);
149 xnn_prefetch_to_l1((const int8_t*) w13 + 64);
150 xnn_prefetch_to_l1((const int8_t*) w14);
151 xnn_prefetch_to_l1((const int8_t*) w14 + 64);
152 xnn_prefetch_to_l1((const int8_t*) w15);
153 xnn_prefetch_to_l1((const int8_t*) w15 + 64);
154
155 // KC main loop multiple of 8
156 size_t k = kc;
157 for (; k >= 8; k -= 8) {
158 vtmp0123x0123 = vld4q_lane_u32(w0, vtmp0123x0123, 0); w0 += 4;
159 vtmp0123x0123 = vld4q_lane_u32(w1, vtmp0123x0123, 1); w1 += 4;
160 vtmp0123x0123 = vld4q_lane_u32(w2, vtmp0123x0123, 2); w2 += 4;
161 vtmp0123x0123 = vld4q_lane_u32(w3, vtmp0123x0123, 3); w3 += 4;
162 vtmp0123x4567 = vld4q_lane_u32(w4, vtmp0123x4567, 0); w4 += 4;
163 vtmp0123x4567 = vld4q_lane_u32(w5, vtmp0123x4567, 1); w5 += 4;
164 vtmp0123x4567 = vld4q_lane_u32(w6, vtmp0123x4567, 2); w6 += 4;
165 vtmp0123x4567 = vld4q_lane_u32(w7, vtmp0123x4567, 3); w7 += 4;
166 vtmp0123x89AB = vld4q_lane_u32(w8, vtmp0123x89AB, 0); w8 += 4;
167 vtmp0123x89AB = vld4q_lane_u32(w9, vtmp0123x89AB, 1); w9 += 4;
168 vtmp0123x89AB = vld4q_lane_u32(w10, vtmp0123x89AB, 2); w10 += 4;
169 vtmp0123x89AB = vld4q_lane_u32(w11, vtmp0123x89AB, 3); w11 += 4;
170 vtmp0123xCDEF = vld4q_lane_u32(w12, vtmp0123xCDEF, 0); w12 += 4;
171 vtmp0123xCDEF = vld4q_lane_u32(w13, vtmp0123xCDEF, 1); w13 += 4;
172 vtmp0123xCDEF = vld4q_lane_u32(w14, vtmp0123xCDEF, 2); w14 += 4;
173 vtmp0123xCDEF = vld4q_lane_u32(w15, vtmp0123xCDEF, 3); w15 += 4;
174 vtmp4567x0123 = vld4q_lane_u32(w0, vtmp4567x0123, 0); w0 += 4;
175 vtmp4567x0123 = vld4q_lane_u32(w1, vtmp4567x0123, 1); w1 += 4;
176 vtmp4567x0123 = vld4q_lane_u32(w2, vtmp4567x0123, 2); w2 += 4;
177 vtmp4567x0123 = vld4q_lane_u32(w3, vtmp4567x0123, 3); w3 += 4;
178 vtmp4567x4567 = vld4q_lane_u32(w4, vtmp4567x4567, 0); w4 += 4;
179 vtmp4567x4567 = vld4q_lane_u32(w5, vtmp4567x4567, 1); w5 += 4;
180 vtmp4567x4567 = vld4q_lane_u32(w6, vtmp4567x4567, 2); w6 += 4;
181 vtmp4567x4567 = vld4q_lane_u32(w7, vtmp4567x4567, 3); w7 += 4;
182 vtmp4567x89AB = vld4q_lane_u32(w8, vtmp4567x89AB, 0); w8 += 4;
183 vtmp4567x89AB = vld4q_lane_u32(w9, vtmp4567x89AB, 1); w9 += 4;
184 vtmp4567x89AB = vld4q_lane_u32(w10, vtmp4567x89AB, 2); w10 += 4;
185 vtmp4567x89AB = vld4q_lane_u32(w11, vtmp4567x89AB, 3); w11 += 4;
186 vtmp4567xCDEF = vld4q_lane_u32(w12, vtmp4567xCDEF, 0); w12 += 4;
187 vtmp4567xCDEF = vld4q_lane_u32(w13, vtmp4567xCDEF, 1); w13 += 4;
188 vtmp4567xCDEF = vld4q_lane_u32(w14, vtmp4567xCDEF, 2); w14 += 4;
189 vtmp4567xCDEF = vld4q_lane_u32(w15, vtmp4567xCDEF, 3); w15 += 4;
190 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
191 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
192 xnn_prefetch_to_l1((const int8_t*) w2 + 128);
193 xnn_prefetch_to_l1((const int8_t*) w3 + 128);
194 xnn_prefetch_to_l1((const int8_t*) w4 + 128);
195 xnn_prefetch_to_l1((const int8_t*) w5 + 128);
196 xnn_prefetch_to_l1((const int8_t*) w6 + 128);
197 xnn_prefetch_to_l1((const int8_t*) w7 + 128);
198 xnn_prefetch_to_l1((const int8_t*) w8 + 128);
199 xnn_prefetch_to_l1((const int8_t*) w9 + 128);
200 xnn_prefetch_to_l1((const int8_t*) w10 + 128);
201 xnn_prefetch_to_l1((const int8_t*) w11 + 128);
202 xnn_prefetch_to_l1((const int8_t*) w12 + 128);
203 xnn_prefetch_to_l1((const int8_t*) w13 + 128);
204 xnn_prefetch_to_l1((const int8_t*) w14 + 128);
205 xnn_prefetch_to_l1((const int8_t*) w15 + 128);
206 vst1q_u32(packed_weights, vtmp0123x0123.val[0]); packed_weights += 4;
207 vst1q_u32(packed_weights, vtmp0123x4567.val[0]); packed_weights += 4;
208 vst1q_u32(packed_weights, vtmp0123x89AB.val[0]); packed_weights += 4;
209 vst1q_u32(packed_weights, vtmp0123xCDEF.val[0]); packed_weights += 4;
210 vst1q_u32(packed_weights, vtmp0123x0123.val[1]); packed_weights += 4;
211 vst1q_u32(packed_weights, vtmp0123x4567.val[1]); packed_weights += 4;
212 vst1q_u32(packed_weights, vtmp0123x89AB.val[1]); packed_weights += 4;
213 vst1q_u32(packed_weights, vtmp0123xCDEF.val[1]); packed_weights += 4;
214 vst1q_u32(packed_weights, vtmp0123x0123.val[2]); packed_weights += 4;
215 vst1q_u32(packed_weights, vtmp0123x4567.val[2]); packed_weights += 4;
216 vst1q_u32(packed_weights, vtmp0123x89AB.val[2]); packed_weights += 4;
217 vst1q_u32(packed_weights, vtmp0123xCDEF.val[2]); packed_weights += 4;
218 vst1q_u32(packed_weights, vtmp0123x0123.val[3]); packed_weights += 4;
219 vst1q_u32(packed_weights, vtmp0123x4567.val[3]); packed_weights += 4;
220 vst1q_u32(packed_weights, vtmp0123x89AB.val[3]); packed_weights += 4;
221 vst1q_u32(packed_weights, vtmp0123xCDEF.val[3]); packed_weights += 4;
222 vst1q_u32(packed_weights, vtmp4567x0123.val[0]); packed_weights += 4;
223 vst1q_u32(packed_weights, vtmp4567x4567.val[0]); packed_weights += 4;
224 vst1q_u32(packed_weights, vtmp4567x89AB.val[0]); packed_weights += 4;
225 vst1q_u32(packed_weights, vtmp4567xCDEF.val[0]); packed_weights += 4;
226 vst1q_u32(packed_weights, vtmp4567x0123.val[1]); packed_weights += 4;
227 vst1q_u32(packed_weights, vtmp4567x4567.val[1]); packed_weights += 4;
228 vst1q_u32(packed_weights, vtmp4567x89AB.val[1]); packed_weights += 4;
229 vst1q_u32(packed_weights, vtmp4567xCDEF.val[1]); packed_weights += 4;
230 vst1q_u32(packed_weights, vtmp4567x0123.val[2]); packed_weights += 4;
231 vst1q_u32(packed_weights, vtmp4567x4567.val[2]); packed_weights += 4;
232 vst1q_u32(packed_weights, vtmp4567x89AB.val[2]); packed_weights += 4;
233 vst1q_u32(packed_weights, vtmp4567xCDEF.val[2]); packed_weights += 4;
234 vst1q_u32(packed_weights, vtmp4567x0123.val[3]); packed_weights += 4;
235 vst1q_u32(packed_weights, vtmp4567x4567.val[3]); packed_weights += 4;
236 vst1q_u32(packed_weights, vtmp4567x89AB.val[3]); packed_weights += 4;
237 vst1q_u32(packed_weights, vtmp4567xCDEF.val[3]); packed_weights += 4;
238 }
239 // KC remainder multiple of 4
240 if (k >= 4) {
241 vtmp0123x0123 = vld4q_lane_u32(w0, vtmp0123x0123, 0); w0 += 4;
242 vtmp0123x0123 = vld4q_lane_u32(w1, vtmp0123x0123, 1); w1 += 4;
243 vtmp0123x0123 = vld4q_lane_u32(w2, vtmp0123x0123, 2); w2 += 4;
244 vtmp0123x0123 = vld4q_lane_u32(w3, vtmp0123x0123, 3); w3 += 4;
245 vtmp0123x4567 = vld4q_lane_u32(w4, vtmp0123x4567, 0); w4 += 4;
246 vtmp0123x4567 = vld4q_lane_u32(w5, vtmp0123x4567, 1); w5 += 4;
247 vtmp0123x4567 = vld4q_lane_u32(w6, vtmp0123x4567, 2); w6 += 4;
248 vtmp0123x4567 = vld4q_lane_u32(w7, vtmp0123x4567, 3); w7 += 4;
249 vtmp0123x89AB = vld4q_lane_u32(w8, vtmp0123x89AB, 0); w8 += 4;
250 vtmp0123x89AB = vld4q_lane_u32(w9, vtmp0123x89AB, 1); w9 += 4;
251 vtmp0123x89AB = vld4q_lane_u32(w10, vtmp0123x89AB, 2); w10 += 4;
252 vtmp0123x89AB = vld4q_lane_u32(w11, vtmp0123x89AB, 3); w11 += 4;
253 vtmp0123xCDEF = vld4q_lane_u32(w12, vtmp0123xCDEF, 0); w12 += 4;
254 vtmp0123xCDEF = vld4q_lane_u32(w13, vtmp0123xCDEF, 1); w13 += 4;
255 vtmp0123xCDEF = vld4q_lane_u32(w14, vtmp0123xCDEF, 2); w14 += 4;
256 vtmp0123xCDEF = vld4q_lane_u32(w15, vtmp0123xCDEF, 3); w15 += 4;
257 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
258 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
259 xnn_prefetch_to_l1((const int8_t*) w2 + 128);
260 xnn_prefetch_to_l1((const int8_t*) w3 + 128);
261 xnn_prefetch_to_l1((const int8_t*) w4 + 128);
262 xnn_prefetch_to_l1((const int8_t*) w5 + 128);
263 xnn_prefetch_to_l1((const int8_t*) w6 + 128);
264 xnn_prefetch_to_l1((const int8_t*) w7 + 128);
265 xnn_prefetch_to_l1((const int8_t*) w8 + 128);
266 xnn_prefetch_to_l1((const int8_t*) w9 + 128);
267 xnn_prefetch_to_l1((const int8_t*) w10 + 128);
268 xnn_prefetch_to_l1((const int8_t*) w11 + 128);
269 xnn_prefetch_to_l1((const int8_t*) w12 + 128);
270 xnn_prefetch_to_l1((const int8_t*) w13 + 128);
271 xnn_prefetch_to_l1((const int8_t*) w14 + 128);
272 xnn_prefetch_to_l1((const int8_t*) w15 + 128);
273 vst1q_u32(packed_weights, vtmp0123x0123.val[0]); packed_weights += 4;
274 vst1q_u32(packed_weights, vtmp0123x4567.val[0]); packed_weights += 4;
275 vst1q_u32(packed_weights, vtmp0123x89AB.val[0]); packed_weights += 4;
276 vst1q_u32(packed_weights, vtmp0123xCDEF.val[0]); packed_weights += 4;
277 vst1q_u32(packed_weights, vtmp0123x0123.val[1]); packed_weights += 4;
278 vst1q_u32(packed_weights, vtmp0123x4567.val[1]); packed_weights += 4;
279 vst1q_u32(packed_weights, vtmp0123x89AB.val[1]); packed_weights += 4;
280 vst1q_u32(packed_weights, vtmp0123xCDEF.val[1]); packed_weights += 4;
281 vst1q_u32(packed_weights, vtmp0123x0123.val[2]); packed_weights += 4;
282 vst1q_u32(packed_weights, vtmp0123x4567.val[2]); packed_weights += 4;
283 vst1q_u32(packed_weights, vtmp0123x89AB.val[2]); packed_weights += 4;
284 vst1q_u32(packed_weights, vtmp0123xCDEF.val[2]); packed_weights += 4;
285 vst1q_u32(packed_weights, vtmp0123x0123.val[3]); packed_weights += 4;
286 vst1q_u32(packed_weights, vtmp0123x4567.val[3]); packed_weights += 4;
287 vst1q_u32(packed_weights, vtmp0123x89AB.val[3]); packed_weights += 4;
288 vst1q_u32(packed_weights, vtmp0123xCDEF.val[3]); packed_weights += 4;
289 k -= 4;
290 }
291
292 // KC remainder of 1..3
293 // Same as main loop but ld1, ld2 or ld3
294 if XNN_UNLIKELY(k != 0) {
295 assert(k >= 1);
296 assert(k <= 3);
297 switch (k) {
298 // KC remainder of 1
299 case 1:
300 {
301 uint32x4_t vtmp0x0123 = vdupq_n_u32(0);
302 uint32x4_t vtmp0x4567 = vdupq_n_u32(0);
303 uint32x4_t vtmp0x89AB = vdupq_n_u32(0);
304 uint32x4_t vtmp0xCDEF = vdupq_n_u32(0);
305 vtmp0x0123 = vld1q_lane_u32(w0, vtmp0x0123, 0); w0 += 1;
306 vtmp0x0123 = vld1q_lane_u32(w1, vtmp0x0123, 1); w1 += 1;
307 vtmp0x0123 = vld1q_lane_u32(w2, vtmp0x0123, 2); w2 += 1;
308 vtmp0x0123 = vld1q_lane_u32(w3, vtmp0x0123, 3); w3 += 1;
309 vtmp0x4567 = vld1q_lane_u32(w4, vtmp0x4567, 0); w4 += 1;
310 vtmp0x4567 = vld1q_lane_u32(w5, vtmp0x4567, 1); w5 += 1;
311 vtmp0x4567 = vld1q_lane_u32(w6, vtmp0x4567, 2); w6 += 1;
312 vtmp0x4567 = vld1q_lane_u32(w7, vtmp0x4567, 3); w7 += 1;
313 vtmp0x89AB = vld1q_lane_u32(w8, vtmp0x89AB, 0); w8 += 1;
314 vtmp0x89AB = vld1q_lane_u32(w9, vtmp0x89AB, 1); w9 += 1;
315 vtmp0x89AB = vld1q_lane_u32(w10, vtmp0x89AB, 2); w10 += 1;
316 vtmp0x89AB = vld1q_lane_u32(w11, vtmp0x89AB, 3); w11 += 1;
317 vtmp0xCDEF = vld1q_lane_u32(w12, vtmp0xCDEF, 0); w12 += 1;
318 vtmp0xCDEF = vld1q_lane_u32(w13, vtmp0xCDEF, 1); w13 += 1;
319 vtmp0xCDEF = vld1q_lane_u32(w14, vtmp0xCDEF, 2); w14 += 1;
320 vtmp0xCDEF = vld1q_lane_u32(w15, vtmp0xCDEF, 3); w15 += 1;
321 vst1q_u32(packed_weights, vtmp0x0123); packed_weights += 4;
322 vst1q_u32(packed_weights, vtmp0x4567); packed_weights += 4;
323 vst1q_u32(packed_weights, vtmp0x89AB); packed_weights += 4;
324 vst1q_u32(packed_weights, vtmp0xCDEF); packed_weights += 4;
325 break;
326 }
327 // KC remainder of 2
328 case 2:
329 {
330 uint32x4x2_t vtmp01x0123;
331 vtmp01x0123.val[0] = vdupq_n_u32(0);
332 vtmp01x0123.val[1] = vdupq_n_u32(0);
333 uint32x4x2_t vtmp01x4567;
334 vtmp01x4567.val[0] = vdupq_n_u32(0);
335 vtmp01x4567.val[1] = vdupq_n_u32(0);
336 uint32x4x2_t vtmp01x89AB;
337 vtmp01x89AB.val[0] = vdupq_n_u32(0);
338 vtmp01x89AB.val[1] = vdupq_n_u32(0);
339 uint32x4x2_t vtmp01xCDEF;
340 vtmp01xCDEF.val[0] = vdupq_n_u32(0);
341 vtmp01xCDEF.val[1] = vdupq_n_u32(0);
342 vtmp01x0123 = vld2q_lane_u32(w0, vtmp01x0123, 0); w0 += 2;
343 vtmp01x0123 = vld2q_lane_u32(w1, vtmp01x0123, 1); w1 += 2;
344 vtmp01x0123 = vld2q_lane_u32(w2, vtmp01x0123, 2); w2 += 2;
345 vtmp01x0123 = vld2q_lane_u32(w3, vtmp01x0123, 3); w3 += 2;
346 vtmp01x4567 = vld2q_lane_u32(w4, vtmp01x4567, 0); w4 += 2;
347 vtmp01x4567 = vld2q_lane_u32(w5, vtmp01x4567, 1); w5 += 2;
348 vtmp01x4567 = vld2q_lane_u32(w6, vtmp01x4567, 2); w6 += 2;
349 vtmp01x4567 = vld2q_lane_u32(w7, vtmp01x4567, 3); w7 += 2;
350 vtmp01x89AB = vld2q_lane_u32(w8, vtmp01x89AB, 0); w8 += 2;
351 vtmp01x89AB = vld2q_lane_u32(w9, vtmp01x89AB, 1); w9 += 2;
352 vtmp01x89AB = vld2q_lane_u32(w10, vtmp01x89AB, 2); w10 += 2;
353 vtmp01x89AB = vld2q_lane_u32(w11, vtmp01x89AB, 3); w11 += 2;
354 vtmp01xCDEF = vld2q_lane_u32(w12, vtmp01xCDEF, 0); w12 += 2;
355 vtmp01xCDEF = vld2q_lane_u32(w13, vtmp01xCDEF, 1); w13 += 2;
356 vtmp01xCDEF = vld2q_lane_u32(w14, vtmp01xCDEF, 2); w14 += 2;
357 vtmp01xCDEF = vld2q_lane_u32(w15, vtmp01xCDEF, 3); w15 += 2;
358 vst1q_u32(packed_weights, vtmp01x0123.val[0]); packed_weights += 4;
359 vst1q_u32(packed_weights, vtmp01x4567.val[0]); packed_weights += 4;
360 vst1q_u32(packed_weights, vtmp01x89AB.val[0]); packed_weights += 4;
361 vst1q_u32(packed_weights, vtmp01xCDEF.val[0]); packed_weights += 4;
362 vst1q_u32(packed_weights, vtmp01x0123.val[1]); packed_weights += 4;
363 vst1q_u32(packed_weights, vtmp01x4567.val[1]); packed_weights += 4;
364 vst1q_u32(packed_weights, vtmp01x89AB.val[1]); packed_weights += 4;
365 vst1q_u32(packed_weights, vtmp01xCDEF.val[1]); packed_weights += 4;
366 break;
367 }
368 // KC remainder of 3
369 case 3:
370 {
371 uint32x4x3_t vtmp012x0123;
372 vtmp012x0123.val[0] = vdupq_n_u32(0);
373 vtmp012x0123.val[1] = vdupq_n_u32(0);
374 vtmp012x0123.val[2] = vdupq_n_u32(0);
375 uint32x4x3_t vtmp012x4567;
376 vtmp012x4567.val[0] = vdupq_n_u32(0);
377 vtmp012x4567.val[1] = vdupq_n_u32(0);
378 vtmp012x4567.val[2] = vdupq_n_u32(0);
379 uint32x4x3_t vtmp012x89AB;
380 vtmp012x89AB.val[0] = vdupq_n_u32(0);
381 vtmp012x89AB.val[1] = vdupq_n_u32(0);
382 vtmp012x89AB.val[2] = vdupq_n_u32(0);
383 uint32x4x3_t vtmp012xCDEF;
384 vtmp012xCDEF.val[0] = vdupq_n_u32(0);
385 vtmp012xCDEF.val[1] = vdupq_n_u32(0);
386 vtmp012xCDEF.val[2] = vdupq_n_u32(0);
387 vtmp012x0123 = vld3q_lane_u32(w0, vtmp012x0123, 0); w0 += 3;
388 vtmp012x0123 = vld3q_lane_u32(w1, vtmp012x0123, 1); w1 += 3;
389 vtmp012x0123 = vld3q_lane_u32(w2, vtmp012x0123, 2); w2 += 3;
390 vtmp012x0123 = vld3q_lane_u32(w3, vtmp012x0123, 3); w3 += 3;
391 vtmp012x4567 = vld3q_lane_u32(w4, vtmp012x4567, 0); w4 += 3;
392 vtmp012x4567 = vld3q_lane_u32(w5, vtmp012x4567, 1); w5 += 3;
393 vtmp012x4567 = vld3q_lane_u32(w6, vtmp012x4567, 2); w6 += 3;
394 vtmp012x4567 = vld3q_lane_u32(w7, vtmp012x4567, 3); w7 += 3;
395 vtmp012x89AB = vld3q_lane_u32(w8, vtmp012x89AB, 0); w8 += 3;
396 vtmp012x89AB = vld3q_lane_u32(w9, vtmp012x89AB, 1); w9 += 3;
397 vtmp012x89AB = vld3q_lane_u32(w10, vtmp012x89AB, 2); w10 += 3;
398 vtmp012x89AB = vld3q_lane_u32(w11, vtmp012x89AB, 3); w11 += 3;
399 vtmp012xCDEF = vld3q_lane_u32(w12, vtmp012xCDEF, 0); w12 += 3;
400 vtmp012xCDEF = vld3q_lane_u32(w13, vtmp012xCDEF, 1); w13 += 3;
401 vtmp012xCDEF = vld3q_lane_u32(w14, vtmp012xCDEF, 2); w14 += 3;
402 vtmp012xCDEF = vld3q_lane_u32(w15, vtmp012xCDEF, 3); w15 += 3;
403 vst1q_u32(packed_weights, vtmp012x0123.val[0]); packed_weights += 4;
404 vst1q_u32(packed_weights, vtmp012x4567.val[0]); packed_weights += 4;
405 vst1q_u32(packed_weights, vtmp012x89AB.val[0]); packed_weights += 4;
406 vst1q_u32(packed_weights, vtmp012xCDEF.val[0]); packed_weights += 4;
407 vst1q_u32(packed_weights, vtmp012x0123.val[1]); packed_weights += 4;
408 vst1q_u32(packed_weights, vtmp012x4567.val[1]); packed_weights += 4;
409 vst1q_u32(packed_weights, vtmp012x89AB.val[1]); packed_weights += 4;
410 vst1q_u32(packed_weights, vtmp012xCDEF.val[1]); packed_weights += 4;
411 vst1q_u32(packed_weights, vtmp012x0123.val[2]); packed_weights += 4;
412 vst1q_u32(packed_weights, vtmp012x4567.val[2]); packed_weights += 4;
413 vst1q_u32(packed_weights, vtmp012x89AB.val[2]); packed_weights += 4;
414 vst1q_u32(packed_weights, vtmp012xCDEF.val[2]); packed_weights += 4;
415 break;
416 }
417 default:
418 XNN_UNREACHABLE;
419 }
420 }
421 packed_weights = (uint32_t*) ((uintptr_t) packed_weights + extra_bytes);
422 w0 = w15;
423 }
424
425 // NC remainder (1..15)
426 if XNN_UNLIKELY(n != 0) {
427 assert(n >= 1);
428 assert(n <= 15);
429 if XNN_LIKELY(bias != NULL) {
430 size_t nb = n;
431 do {
432 *packed_weights++ = *bias++;
433 } while (--nb != 0);
434 packed_weights += (16 - n);
435 } else {
436 const uint32x4_t vzero = vmovq_n_u32(0);
437 vst1q_u32(packed_weights, vzero); packed_weights += 4;
438 vst1q_u32(packed_weights, vzero); packed_weights += 4;
439 vst1q_u32(packed_weights, vzero); packed_weights += 4;
440 vst1q_u32(packed_weights, vzero); packed_weights += 4;
441 }
442
443 // NR remainder has less than 16 rows so last row is not loaded
444 const uint32_t* w1 = w0 + kc;
445 if XNN_UNPREDICTABLE(n < 2) {
446 w1 = w0;
447 }
448 const uint32_t* w2 = w1 + kc;
449 if XNN_UNPREDICTABLE(n <= 2) {
450 w2 = w1;
451 }
452 const uint32_t* w3 = w2 + kc;
453 if XNN_UNPREDICTABLE(n < 4) {
454 w3 = w2;
455 }
456 const uint32_t* w4 = w3 + kc;
457 if XNN_UNPREDICTABLE(n <= 4) {
458 w4 = w3;
459 }
460 const uint32_t* w5 = w4 + kc;
461 if XNN_UNPREDICTABLE(n < 6) {
462 w5 = w4;
463 }
464 const uint32_t* w6 = w5 + kc;
465 if XNN_UNPREDICTABLE(n <= 6) {
466 w6 = w5;
467 }
468 const uint32_t* w7 = w6 + kc;
469 if XNN_UNPREDICTABLE(n < 8) {
470 w7 = w6;
471 }
472 const uint32_t* w8 = w7 + kc;
473 if XNN_UNPREDICTABLE(n <= 8) {
474 w8 = w7;
475 }
476 const uint32_t* w9 = w8 + kc;
477 if XNN_UNPREDICTABLE(n < 10) {
478 w9 = w8;
479 }
480 const uint32_t* w10 = w9 + kc;
481 if XNN_UNPREDICTABLE(n <= 10) {
482 w10 = w9;
483 }
484 const uint32_t* w11 = w10 + kc;
485 if XNN_UNPREDICTABLE(n < 12) {
486 w11 = w10;
487 }
488 const uint32_t* w12 = w11 + kc;
489 if XNN_UNPREDICTABLE(n <= 12) {
490 w12 = w11;
491 }
492 const uint32_t* w13 = w12 + kc;
493 if XNN_UNPREDICTABLE(n < 14) {
494 w13 = w12;
495 }
496 const uint32_t* w14 = w13 + kc;
497 if XNN_UNPREDICTABLE(n <= 14) {
498 w14 = w13;
499 }
500
501 // KC main loop multiple of 8
502 size_t k = kc;
503 for (; k >= 8; k -= 8) {
504 vtmp0123x0123 = vld4q_lane_u32(w0, vtmp0123x0123, 0); w0 += 4;
505 vtmp0123x0123 = vld4q_lane_u32(w1, vtmp0123x0123, 1); w1 += 4;
506 vtmp0123x0123 = vld4q_lane_u32(w2, vtmp0123x0123, 2); w2 += 4;
507 vtmp0123x0123 = vld4q_lane_u32(w3, vtmp0123x0123, 3); w3 += 4;
508 vtmp0123x4567 = vld4q_lane_u32(w4, vtmp0123x4567, 0); w4 += 4;
509 vtmp0123x4567 = vld4q_lane_u32(w5, vtmp0123x4567, 1); w5 += 4;
510 vtmp0123x4567 = vld4q_lane_u32(w6, vtmp0123x4567, 2); w6 += 4;
511 vtmp0123x4567 = vld4q_lane_u32(w7, vtmp0123x4567, 3); w7 += 4;
512 vtmp0123x89AB = vld4q_lane_u32(w8, vtmp0123x89AB, 0); w8 += 4;
513 vtmp0123x89AB = vld4q_lane_u32(w9, vtmp0123x89AB, 1); w9 += 4;
514 vtmp0123x89AB = vld4q_lane_u32(w10, vtmp0123x89AB, 2); w10 += 4;
515 vtmp0123x89AB = vld4q_lane_u32(w11, vtmp0123x89AB, 3); w11 += 4;
516 vtmp0123xCDEF = vld4q_lane_u32(w12, vtmp0123xCDEF, 0); w12 += 4;
517 vtmp0123xCDEF = vld4q_lane_u32(w13, vtmp0123xCDEF, 1); w13 += 4;
518 vtmp0123xCDEF = vld4q_lane_u32(w14, vtmp0123xCDEF, 2); w14 += 4;
519 vtmp4567x0123 = vld4q_lane_u32(w0, vtmp4567x0123, 0); w0 += 4;
520 vtmp4567x0123 = vld4q_lane_u32(w1, vtmp4567x0123, 1); w1 += 4;
521 vtmp4567x0123 = vld4q_lane_u32(w2, vtmp4567x0123, 2); w2 += 4;
522 vtmp4567x0123 = vld4q_lane_u32(w3, vtmp4567x0123, 3); w3 += 4;
523 vtmp4567x4567 = vld4q_lane_u32(w4, vtmp4567x4567, 0); w4 += 4;
524 vtmp4567x4567 = vld4q_lane_u32(w5, vtmp4567x4567, 1); w5 += 4;
525 vtmp4567x4567 = vld4q_lane_u32(w6, vtmp4567x4567, 2); w6 += 4;
526 vtmp4567x4567 = vld4q_lane_u32(w7, vtmp4567x4567, 3); w7 += 4;
527 vtmp4567x89AB = vld4q_lane_u32(w8, vtmp4567x89AB, 0); w8 += 4;
528 vtmp4567x89AB = vld4q_lane_u32(w9, vtmp4567x89AB, 1); w9 += 4;
529 vtmp4567x89AB = vld4q_lane_u32(w10, vtmp4567x89AB, 2); w10 += 4;
530 vtmp4567x89AB = vld4q_lane_u32(w11, vtmp4567x89AB, 3); w11 += 4;
531 vtmp4567xCDEF = vld4q_lane_u32(w12, vtmp4567xCDEF, 0); w12 += 4;
532 vtmp4567xCDEF = vld4q_lane_u32(w13, vtmp4567xCDEF, 1); w13 += 4;
533 vtmp4567xCDEF = vld4q_lane_u32(w14, vtmp4567xCDEF, 2); w14 += 4;
534 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
535 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
536 xnn_prefetch_to_l1((const int8_t*) w2 + 128);
537 xnn_prefetch_to_l1((const int8_t*) w3 + 128);
538 xnn_prefetch_to_l1((const int8_t*) w4 + 128);
539 xnn_prefetch_to_l1((const int8_t*) w5 + 128);
540 xnn_prefetch_to_l1((const int8_t*) w6 + 128);
541 xnn_prefetch_to_l1((const int8_t*) w7 + 128);
542 xnn_prefetch_to_l1((const int8_t*) w8 + 128);
543 xnn_prefetch_to_l1((const int8_t*) w9 + 128);
544 xnn_prefetch_to_l1((const int8_t*) w10 + 128);
545 xnn_prefetch_to_l1((const int8_t*) w11 + 128);
546 xnn_prefetch_to_l1((const int8_t*) w12 + 128);
547 xnn_prefetch_to_l1((const int8_t*) w13 + 128);
548 xnn_prefetch_to_l1((const int8_t*) w14 + 128);
549 vst1q_u32(packed_weights, vtmp0123x0123.val[0]); packed_weights += 4;
550 vst1q_u32(packed_weights, vtmp0123x4567.val[0]); packed_weights += 4;
551 vst1q_u32(packed_weights, vtmp0123x89AB.val[0]); packed_weights += 4;
552 vst1q_u32(packed_weights, vtmp0123xCDEF.val[0]); packed_weights += 4;
553 vst1q_u32(packed_weights, vtmp0123x0123.val[1]); packed_weights += 4;
554 vst1q_u32(packed_weights, vtmp0123x4567.val[1]); packed_weights += 4;
555 vst1q_u32(packed_weights, vtmp0123x89AB.val[1]); packed_weights += 4;
556 vst1q_u32(packed_weights, vtmp0123xCDEF.val[1]); packed_weights += 4;
557 vst1q_u32(packed_weights, vtmp0123x0123.val[2]); packed_weights += 4;
558 vst1q_u32(packed_weights, vtmp0123x4567.val[2]); packed_weights += 4;
559 vst1q_u32(packed_weights, vtmp0123x89AB.val[2]); packed_weights += 4;
560 vst1q_u32(packed_weights, vtmp0123xCDEF.val[2]); packed_weights += 4;
561 vst1q_u32(packed_weights, vtmp0123x0123.val[3]); packed_weights += 4;
562 vst1q_u32(packed_weights, vtmp0123x4567.val[3]); packed_weights += 4;
563 vst1q_u32(packed_weights, vtmp0123x89AB.val[3]); packed_weights += 4;
564 vst1q_u32(packed_weights, vtmp0123xCDEF.val[3]); packed_weights += 4;
565 vst1q_u32(packed_weights, vtmp4567x0123.val[0]); packed_weights += 4;
566 vst1q_u32(packed_weights, vtmp4567x4567.val[0]); packed_weights += 4;
567 vst1q_u32(packed_weights, vtmp4567x89AB.val[0]); packed_weights += 4;
568 vst1q_u32(packed_weights, vtmp4567xCDEF.val[0]); packed_weights += 4;
569 vst1q_u32(packed_weights, vtmp4567x0123.val[1]); packed_weights += 4;
570 vst1q_u32(packed_weights, vtmp4567x4567.val[1]); packed_weights += 4;
571 vst1q_u32(packed_weights, vtmp4567x89AB.val[1]); packed_weights += 4;
572 vst1q_u32(packed_weights, vtmp4567xCDEF.val[1]); packed_weights += 4;
573 vst1q_u32(packed_weights, vtmp4567x0123.val[2]); packed_weights += 4;
574 vst1q_u32(packed_weights, vtmp4567x4567.val[2]); packed_weights += 4;
575 vst1q_u32(packed_weights, vtmp4567x89AB.val[2]); packed_weights += 4;
576 vst1q_u32(packed_weights, vtmp4567xCDEF.val[2]); packed_weights += 4;
577 vst1q_u32(packed_weights, vtmp4567x0123.val[3]); packed_weights += 4;
578 vst1q_u32(packed_weights, vtmp4567x4567.val[3]); packed_weights += 4;
579 vst1q_u32(packed_weights, vtmp4567x89AB.val[3]); packed_weights += 4;
580 vst1q_u32(packed_weights, vtmp4567xCDEF.val[3]); packed_weights += 4;
581 }
582 // KC remainder multiple of 4
583 if (k >= 4) {
584 vtmp0123x0123 = vld4q_lane_u32(w0, vtmp0123x0123, 0); w0 += 4;
585 vtmp0123x0123 = vld4q_lane_u32(w1, vtmp0123x0123, 1); w1 += 4;
586 vtmp0123x0123 = vld4q_lane_u32(w2, vtmp0123x0123, 2); w2 += 4;
587 vtmp0123x0123 = vld4q_lane_u32(w3, vtmp0123x0123, 3); w3 += 4;
588 vtmp0123x4567 = vld4q_lane_u32(w4, vtmp0123x4567, 0); w4 += 4;
589 vtmp0123x4567 = vld4q_lane_u32(w5, vtmp0123x4567, 1); w5 += 4;
590 vtmp0123x4567 = vld4q_lane_u32(w6, vtmp0123x4567, 2); w6 += 4;
591 vtmp0123x4567 = vld4q_lane_u32(w7, vtmp0123x4567, 3); w7 += 4;
592 vtmp0123x89AB = vld4q_lane_u32(w8, vtmp0123x89AB, 0); w8 += 4;
593 vtmp0123x89AB = vld4q_lane_u32(w9, vtmp0123x89AB, 1); w9 += 4;
594 vtmp0123x89AB = vld4q_lane_u32(w10, vtmp0123x89AB, 2); w10 += 4;
595 vtmp0123x89AB = vld4q_lane_u32(w11, vtmp0123x89AB, 3); w11 += 4;
596 vtmp0123xCDEF = vld4q_lane_u32(w12, vtmp0123xCDEF, 0); w12 += 4;
597 vtmp0123xCDEF = vld4q_lane_u32(w13, vtmp0123xCDEF, 1); w13 += 4;
598 vtmp0123xCDEF = vld4q_lane_u32(w14, vtmp0123xCDEF, 2); w14 += 4;
599 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
600 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
601 xnn_prefetch_to_l1((const int8_t*) w2 + 128);
602 xnn_prefetch_to_l1((const int8_t*) w3 + 128);
603 xnn_prefetch_to_l1((const int8_t*) w4 + 128);
604 xnn_prefetch_to_l1((const int8_t*) w5 + 128);
605 xnn_prefetch_to_l1((const int8_t*) w6 + 128);
606 xnn_prefetch_to_l1((const int8_t*) w7 + 128);
607 xnn_prefetch_to_l1((const int8_t*) w8 + 128);
608 xnn_prefetch_to_l1((const int8_t*) w9 + 128);
609 xnn_prefetch_to_l1((const int8_t*) w10 + 128);
610 xnn_prefetch_to_l1((const int8_t*) w11 + 128);
611 xnn_prefetch_to_l1((const int8_t*) w12 + 128);
612 xnn_prefetch_to_l1((const int8_t*) w13 + 128);
613 xnn_prefetch_to_l1((const int8_t*) w14 + 128);
614 vst1q_u32(packed_weights, vtmp0123x0123.val[0]); packed_weights += 4;
615 vst1q_u32(packed_weights, vtmp0123x4567.val[0]); packed_weights += 4;
616 vst1q_u32(packed_weights, vtmp0123x89AB.val[0]); packed_weights += 4;
617 vst1q_u32(packed_weights, vtmp0123xCDEF.val[0]); packed_weights += 4;
618 vst1q_u32(packed_weights, vtmp0123x0123.val[1]); packed_weights += 4;
619 vst1q_u32(packed_weights, vtmp0123x4567.val[1]); packed_weights += 4;
620 vst1q_u32(packed_weights, vtmp0123x89AB.val[1]); packed_weights += 4;
621 vst1q_u32(packed_weights, vtmp0123xCDEF.val[1]); packed_weights += 4;
622 vst1q_u32(packed_weights, vtmp0123x0123.val[2]); packed_weights += 4;
623 vst1q_u32(packed_weights, vtmp0123x4567.val[2]); packed_weights += 4;
624 vst1q_u32(packed_weights, vtmp0123x89AB.val[2]); packed_weights += 4;
625 vst1q_u32(packed_weights, vtmp0123xCDEF.val[2]); packed_weights += 4;
626 vst1q_u32(packed_weights, vtmp0123x0123.val[3]); packed_weights += 4;
627 vst1q_u32(packed_weights, vtmp0123x4567.val[3]); packed_weights += 4;
628 vst1q_u32(packed_weights, vtmp0123x89AB.val[3]); packed_weights += 4;
629 vst1q_u32(packed_weights, vtmp0123xCDEF.val[3]); packed_weights += 4;
630 k -= 4;
631 }
632
633 // KC remainder of 1..3
634 // Same as main loop but ld1, ld2 or ld3
635 if XNN_UNLIKELY(k != 0) {
636 assert(k >= 1);
637 assert(k <= 3);
638 switch (k) {
639 // KC remainder of 1
640 case 1:
641 {
642 uint32x4_t vtmp0x0123 = vdupq_n_u32(0);
643 uint32x4_t vtmp0x4567 = vdupq_n_u32(0);
644 uint32x4_t vtmp0x89AB = vdupq_n_u32(0);
645 uint32x4_t vtmp0xCDEF = vdupq_n_u32(0);
646 vtmp0x0123 = vld1q_lane_u32(w0, vtmp0x0123, 0);
647 vtmp0x0123 = vld1q_lane_u32(w1, vtmp0x0123, 1);
648 vtmp0x0123 = vld1q_lane_u32(w2, vtmp0x0123, 2);
649 vtmp0x0123 = vld1q_lane_u32(w3, vtmp0x0123, 3);
650 vtmp0x4567 = vld1q_lane_u32(w4, vtmp0x4567, 0);
651 vtmp0x4567 = vld1q_lane_u32(w5, vtmp0x4567, 1);
652 vtmp0x4567 = vld1q_lane_u32(w6, vtmp0x4567, 2);
653 vtmp0x4567 = vld1q_lane_u32(w7, vtmp0x4567, 3);
654 vtmp0x89AB = vld1q_lane_u32(w8, vtmp0x89AB, 0);
655 vtmp0x89AB = vld1q_lane_u32(w9, vtmp0x89AB, 1);
656 vtmp0x89AB = vld1q_lane_u32(w10, vtmp0x89AB, 2);
657 vtmp0x89AB = vld1q_lane_u32(w11, vtmp0x89AB, 3);
658 vtmp0xCDEF = vld1q_lane_u32(w12, vtmp0xCDEF, 0);
659 vtmp0xCDEF = vld1q_lane_u32(w13, vtmp0xCDEF, 1);
660 vtmp0xCDEF = vld1q_lane_u32(w14, vtmp0xCDEF, 2);
661 vst1q_u32(packed_weights, vtmp0x0123); packed_weights += 4;
662 vst1q_u32(packed_weights, vtmp0x4567); packed_weights += 4;
663 vst1q_u32(packed_weights, vtmp0x89AB); packed_weights += 4;
664 vst1q_u32(packed_weights, vtmp0xCDEF); packed_weights += 4;
665 break;
666 }
667 // KC remainder of 2
668 case 2:
669 {
670 uint32x4x2_t vtmp01x0123;
671 vtmp01x0123.val[0] = vdupq_n_u32(0);
672 vtmp01x0123.val[1] = vdupq_n_u32(0);
673 uint32x4x2_t vtmp01x4567;
674 vtmp01x4567.val[0] = vdupq_n_u32(0);
675 vtmp01x4567.val[1] = vdupq_n_u32(0);
676 uint32x4x2_t vtmp01x89AB;
677 vtmp01x89AB.val[0] = vdupq_n_u32(0);
678 vtmp01x89AB.val[1] = vdupq_n_u32(0);
679 uint32x4x2_t vtmp01xCDEF;
680 vtmp01xCDEF.val[0] = vdupq_n_u32(0);
681 vtmp01xCDEF.val[1] = vdupq_n_u32(0);
682 vtmp01x0123 = vld2q_lane_u32(w0, vtmp01x0123, 0);
683 vtmp01x0123 = vld2q_lane_u32(w1, vtmp01x0123, 1);
684 vtmp01x0123 = vld2q_lane_u32(w2, vtmp01x0123, 2);
685 vtmp01x0123 = vld2q_lane_u32(w3, vtmp01x0123, 3);
686 vtmp01x4567 = vld2q_lane_u32(w4, vtmp01x4567, 0);
687 vtmp01x4567 = vld2q_lane_u32(w5, vtmp01x4567, 1);
688 vtmp01x4567 = vld2q_lane_u32(w6, vtmp01x4567, 2);
689 vtmp01x4567 = vld2q_lane_u32(w7, vtmp01x4567, 3);
690 vtmp01x89AB = vld2q_lane_u32(w8, vtmp01x89AB, 0);
691 vtmp01x89AB = vld2q_lane_u32(w9, vtmp01x89AB, 1);
692 vtmp01x89AB = vld2q_lane_u32(w10, vtmp01x89AB, 2);
693 vtmp01x89AB = vld2q_lane_u32(w11, vtmp01x89AB, 3);
694 vtmp01xCDEF = vld2q_lane_u32(w12, vtmp01xCDEF, 0);
695 vtmp01xCDEF = vld2q_lane_u32(w13, vtmp01xCDEF, 1);
696 vtmp01xCDEF = vld2q_lane_u32(w14, vtmp01xCDEF, 2);
697 vst1q_u32(packed_weights, vtmp01x0123.val[0]); packed_weights += 4;
698 vst1q_u32(packed_weights, vtmp01x4567.val[0]); packed_weights += 4;
699 vst1q_u32(packed_weights, vtmp01x89AB.val[0]); packed_weights += 4;
700 vst1q_u32(packed_weights, vtmp01xCDEF.val[0]); packed_weights += 4;
701 vst1q_u32(packed_weights, vtmp01x0123.val[1]); packed_weights += 4;
702 vst1q_u32(packed_weights, vtmp01x4567.val[1]); packed_weights += 4;
703 vst1q_u32(packed_weights, vtmp01x89AB.val[1]); packed_weights += 4;
704 vst1q_u32(packed_weights, vtmp01xCDEF.val[1]); packed_weights += 4;
705 break;
706 }
707 // KC remainder of 3
708 case 3:
709 {
710 uint32x4x3_t vtmp012x0123;
711 vtmp012x0123.val[0] = vdupq_n_u32(0);
712 vtmp012x0123.val[1] = vdupq_n_u32(0);
713 vtmp012x0123.val[2] = vdupq_n_u32(0);
714 uint32x4x3_t vtmp012x4567;
715 vtmp012x4567.val[0] = vdupq_n_u32(0);
716 vtmp012x4567.val[1] = vdupq_n_u32(0);
717 vtmp012x4567.val[2] = vdupq_n_u32(0);
718 uint32x4x3_t vtmp012x89AB;
719 vtmp012x89AB.val[0] = vdupq_n_u32(0);
720 vtmp012x89AB.val[1] = vdupq_n_u32(0);
721 vtmp012x89AB.val[2] = vdupq_n_u32(0);
722 uint32x4x3_t vtmp012xCDEF;
723 vtmp012xCDEF.val[0] = vdupq_n_u32(0);
724 vtmp012xCDEF.val[1] = vdupq_n_u32(0);
725 vtmp012xCDEF.val[2] = vdupq_n_u32(0);
726 vtmp012x0123 = vld3q_lane_u32(w0, vtmp012x0123, 0);
727 vtmp012x0123 = vld3q_lane_u32(w1, vtmp012x0123, 1);
728 vtmp012x0123 = vld3q_lane_u32(w2, vtmp012x0123, 2);
729 vtmp012x0123 = vld3q_lane_u32(w3, vtmp012x0123, 3);
730 vtmp012x4567 = vld3q_lane_u32(w4, vtmp012x4567, 0);
731 vtmp012x4567 = vld3q_lane_u32(w5, vtmp012x4567, 1);
732 vtmp012x4567 = vld3q_lane_u32(w6, vtmp012x4567, 2);
733 vtmp012x4567 = vld3q_lane_u32(w7, vtmp012x4567, 3);
734 vtmp012x89AB = vld3q_lane_u32(w8, vtmp012x89AB, 0);
735 vtmp012x89AB = vld3q_lane_u32(w9, vtmp012x89AB, 1);
736 vtmp012x89AB = vld3q_lane_u32(w10, vtmp012x89AB, 2);
737 vtmp012x89AB = vld3q_lane_u32(w11, vtmp012x89AB, 3);
738 vtmp012xCDEF = vld3q_lane_u32(w12, vtmp012xCDEF, 0);
739 vtmp012xCDEF = vld3q_lane_u32(w13, vtmp012xCDEF, 1);
740 vtmp012xCDEF = vld3q_lane_u32(w14, vtmp012xCDEF, 2);
741 vst1q_u32(packed_weights, vtmp012x0123.val[0]); packed_weights += 4;
742 vst1q_u32(packed_weights, vtmp012x4567.val[0]); packed_weights += 4;
743 vst1q_u32(packed_weights, vtmp012x89AB.val[0]); packed_weights += 4;
744 vst1q_u32(packed_weights, vtmp012xCDEF.val[0]); packed_weights += 4;
745 vst1q_u32(packed_weights, vtmp012x0123.val[1]); packed_weights += 4;
746 vst1q_u32(packed_weights, vtmp012x4567.val[1]); packed_weights += 4;
747 vst1q_u32(packed_weights, vtmp012x89AB.val[1]); packed_weights += 4;
748 vst1q_u32(packed_weights, vtmp012xCDEF.val[1]); packed_weights += 4;
749 vst1q_u32(packed_weights, vtmp012x0123.val[2]); packed_weights += 4;
750 vst1q_u32(packed_weights, vtmp012x4567.val[2]); packed_weights += 4;
751 vst1q_u32(packed_weights, vtmp012x89AB.val[2]); packed_weights += 4;
752 vst1q_u32(packed_weights, vtmp012xCDEF.val[2]); packed_weights += 4;
753 break;
754 }
755 default:
756 XNN_UNREACHABLE;
757 }
758 }
759 packed_weights = (uint32_t*) ((uintptr_t) packed_weights + extra_bytes);
760 }
761
762 weights += nc * kc;
763 } while (--g != 0);
764 }