Debian Codegraph @

src/x32-packw/gen/x32-packw-x8s4-gemm-goi-neon-ld4lane-u8-prfm.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/neon.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include <arm_neon.h>
16
17 #include "xnnpack/packw.h"
18 #include "xnnpack/prefetch.h"
19
20
21 void xnn_x32_packw_gemm_goi_ukernel_x8s4__neon_ld4lane_u8_prfm(
22 size_t g,
23 size_t nc,
24 size_t kc,
25 size_t nr,
26 size_t kr,
27 size_t sr,
28 const uint32_t* weights,
29 const uint32_t* bias,
30 const void* scale,
31 uint32_t* packed_weights,
32 size_t extra_bytes,
33 const void* params)
34 {
35 assert(g != 0);
36 assert(nc != 0);
37 assert(kc != 0);
38 assert(nr == 8);
39 assert(kr == 1);
40 assert(sr == 4);
41 assert(weights != NULL);
42 assert(packed_weights != NULL);
43 uint32x4x4_t vtmp0123x0123;
44 vtmp0123x0123.val[0] = vdupq_n_u32(0);
45 vtmp0123x0123.val[1] = vdupq_n_u32(0);
46 vtmp0123x0123.val[2] = vdupq_n_u32(0);
47 vtmp0123x0123.val[3] = vdupq_n_u32(0);
48 uint32x4x4_t vtmp4567x0123;
49 vtmp4567x0123.val[0] = vdupq_n_u32(0);
50 vtmp4567x0123.val[1] = vdupq_n_u32(0);
51 vtmp4567x0123.val[2] = vdupq_n_u32(0);
52 vtmp4567x0123.val[3] = vdupq_n_u32(0);
53 uint32x4x4_t vtmp0123x4567;
54 vtmp0123x4567.val[0] = vdupq_n_u32(0);
55 vtmp0123x4567.val[1] = vdupq_n_u32(0);
56 vtmp0123x4567.val[2] = vdupq_n_u32(0);
57 vtmp0123x4567.val[3] = vdupq_n_u32(0);
58 uint32x4x4_t vtmp4567x4567;
59 vtmp4567x4567.val[0] = vdupq_n_u32(0);
60 vtmp4567x4567.val[1] = vdupq_n_u32(0);
61 vtmp4567x4567.val[2] = vdupq_n_u32(0);
62 vtmp4567x4567.val[3] = vdupq_n_u32(0);
63
64 do {
65 // NC main loop multiple of 8
66 const uint32_t* w0 = weights;
67 size_t n = nc;
68
69 for (; n >= 8; n -= 8) {
70 if XNN_LIKELY(bias != NULL) {
71 uint32x4_t vb0 = vld1q_u32(bias); bias += 4;
72 uint32x4_t vb4 = vld1q_u32(bias); bias += 4;
73 vst1q_u32(packed_weights, vb0); packed_weights += 4;
74 vst1q_u32(packed_weights, vb4); packed_weights += 4;
75 } else {
76 const uint32x4_t vzero = vmovq_n_u32(0);
77 vst1q_u32(packed_weights, vzero); packed_weights += 4;
78 vst1q_u32(packed_weights, vzero); packed_weights += 4;
79 }
80
81 const uint32_t* w1 = w0 + kc;
82 const uint32_t* w2 = w1 + kc;
83 const uint32_t* w3 = w2 + kc;
84 const uint32_t* w4 = w3 + kc;
85 const uint32_t* w5 = w4 + kc;
86 const uint32_t* w6 = w5 + kc;
87 const uint32_t* w7 = w6 + kc;
88 xnn_prefetch_to_l1((const int8_t*) w0);
89 xnn_prefetch_to_l1((const int8_t*) w0 + 64);
90 xnn_prefetch_to_l1((const int8_t*) w1);
91 xnn_prefetch_to_l1((const int8_t*) w1 + 64);
92 xnn_prefetch_to_l1((const int8_t*) w2);
93 xnn_prefetch_to_l1((const int8_t*) w2 + 64);
94 xnn_prefetch_to_l1((const int8_t*) w3);
95 xnn_prefetch_to_l1((const int8_t*) w3 + 64);
96 xnn_prefetch_to_l1((const int8_t*) w4);
97 xnn_prefetch_to_l1((const int8_t*) w4 + 64);
98 xnn_prefetch_to_l1((const int8_t*) w5);
99 xnn_prefetch_to_l1((const int8_t*) w5 + 64);
100 xnn_prefetch_to_l1((const int8_t*) w6);
101 xnn_prefetch_to_l1((const int8_t*) w6 + 64);
102 xnn_prefetch_to_l1((const int8_t*) w7);
103 xnn_prefetch_to_l1((const int8_t*) w7 + 64);
104
105 // KC main loop multiple of 8
106 size_t k = kc;
107 for (; k >= 8; k -= 8) {
108 uint32x4_t vsrtmp;
109 vtmp0123x0123 = vld4q_lane_u32(w0, vtmp0123x0123, 0); w0 += 4;
110 vsrtmp = vtmp0123x0123.val[3];
111 vtmp0123x0123.val[3] = vtmp0123x0123.val[2];
112 vtmp0123x0123.val[2] = vtmp0123x0123.val[1];
113 vtmp0123x0123.val[1] = vtmp0123x0123.val[0];
114 vtmp0123x0123.val[0] = vsrtmp;
115 vtmp0123x0123 = vld4q_lane_u32(w1, vtmp0123x0123, 1); w1 += 4;
116 vsrtmp = vtmp0123x0123.val[3];
117 vtmp0123x0123.val[3] = vtmp0123x0123.val[2];
118 vtmp0123x0123.val[2] = vtmp0123x0123.val[1];
119 vtmp0123x0123.val[1] = vtmp0123x0123.val[0];
120 vtmp0123x0123.val[0] = vsrtmp;
121 vtmp0123x0123 = vld4q_lane_u32(w2, vtmp0123x0123, 2); w2 += 4;
122 vsrtmp = vtmp0123x0123.val[3];
123 vtmp0123x0123.val[3] = vtmp0123x0123.val[2];
124 vtmp0123x0123.val[2] = vtmp0123x0123.val[1];
125 vtmp0123x0123.val[1] = vtmp0123x0123.val[0];
126 vtmp0123x0123.val[0] = vsrtmp;
127 vtmp0123x0123 = vld4q_lane_u32(w3, vtmp0123x0123, 3); w3 += 4;
128 vsrtmp = vtmp0123x0123.val[3];
129 vtmp0123x0123.val[3] = vtmp0123x0123.val[2];
130 vtmp0123x0123.val[2] = vtmp0123x0123.val[1];
131 vtmp0123x0123.val[1] = vtmp0123x0123.val[0];
132 vtmp0123x0123.val[0] = vsrtmp;
133 vtmp0123x4567 = vld4q_lane_u32(w4, vtmp0123x4567, 0); w4 += 4;
134 vsrtmp = vtmp0123x4567.val[3];
135 vtmp0123x4567.val[3] = vtmp0123x4567.val[2];
136 vtmp0123x4567.val[2] = vtmp0123x4567.val[1];
137 vtmp0123x4567.val[1] = vtmp0123x4567.val[0];
138 vtmp0123x4567.val[0] = vsrtmp;
139 vtmp0123x4567 = vld4q_lane_u32(w5, vtmp0123x4567, 1); w5 += 4;
140 vsrtmp = vtmp0123x4567.val[3];
141 vtmp0123x4567.val[3] = vtmp0123x4567.val[2];
142 vtmp0123x4567.val[2] = vtmp0123x4567.val[1];
143 vtmp0123x4567.val[1] = vtmp0123x4567.val[0];
144 vtmp0123x4567.val[0] = vsrtmp;
145 vtmp0123x4567 = vld4q_lane_u32(w6, vtmp0123x4567, 2); w6 += 4;
146 vsrtmp = vtmp0123x4567.val[3];
147 vtmp0123x4567.val[3] = vtmp0123x4567.val[2];
148 vtmp0123x4567.val[2] = vtmp0123x4567.val[1];
149 vtmp0123x4567.val[1] = vtmp0123x4567.val[0];
150 vtmp0123x4567.val[0] = vsrtmp;
151 vtmp0123x4567 = vld4q_lane_u32(w7, vtmp0123x4567, 3); w7 += 4;
152 vsrtmp = vtmp0123x4567.val[3];
153 vtmp0123x4567.val[3] = vtmp0123x4567.val[2];
154 vtmp0123x4567.val[2] = vtmp0123x4567.val[1];
155 vtmp0123x4567.val[1] = vtmp0123x4567.val[0];
156 vtmp0123x4567.val[0] = vsrtmp;
157 vtmp4567x0123 = vld4q_lane_u32(w0, vtmp4567x0123, 0); w0 += 4;
158 vsrtmp = vtmp4567x0123.val[3];
159 vtmp4567x0123.val[3] = vtmp4567x0123.val[2];
160 vtmp4567x0123.val[2] = vtmp4567x0123.val[1];
161 vtmp4567x0123.val[1] = vtmp4567x0123.val[0];
162 vtmp4567x0123.val[0] = vsrtmp;
163 vtmp4567x0123 = vld4q_lane_u32(w1, vtmp4567x0123, 1); w1 += 4;
164 vsrtmp = vtmp4567x0123.val[3];
165 vtmp4567x0123.val[3] = vtmp4567x0123.val[2];
166 vtmp4567x0123.val[2] = vtmp4567x0123.val[1];
167 vtmp4567x0123.val[1] = vtmp4567x0123.val[0];
168 vtmp4567x0123.val[0] = vsrtmp;
169 vtmp4567x0123 = vld4q_lane_u32(w2, vtmp4567x0123, 2); w2 += 4;
170 vsrtmp = vtmp4567x0123.val[3];
171 vtmp4567x0123.val[3] = vtmp4567x0123.val[2];
172 vtmp4567x0123.val[2] = vtmp4567x0123.val[1];
173 vtmp4567x0123.val[1] = vtmp4567x0123.val[0];
174 vtmp4567x0123.val[0] = vsrtmp;
175 vtmp4567x0123 = vld4q_lane_u32(w3, vtmp4567x0123, 3); w3 += 4;
176 vsrtmp = vtmp4567x0123.val[3];
177 vtmp4567x0123.val[3] = vtmp4567x0123.val[2];
178 vtmp4567x0123.val[2] = vtmp4567x0123.val[1];
179 vtmp4567x0123.val[1] = vtmp4567x0123.val[0];
180 vtmp4567x0123.val[0] = vsrtmp;
181 vtmp4567x4567 = vld4q_lane_u32(w4, vtmp4567x4567, 0); w4 += 4;
182 vsrtmp = vtmp4567x4567.val[3];
183 vtmp4567x4567.val[3] = vtmp4567x4567.val[2];
184 vtmp4567x4567.val[2] = vtmp4567x4567.val[1];
185 vtmp4567x4567.val[1] = vtmp4567x4567.val[0];
186 vtmp4567x4567.val[0] = vsrtmp;
187 vtmp4567x4567 = vld4q_lane_u32(w5, vtmp4567x4567, 1); w5 += 4;
188 vsrtmp = vtmp4567x4567.val[3];
189 vtmp4567x4567.val[3] = vtmp4567x4567.val[2];
190 vtmp4567x4567.val[2] = vtmp4567x4567.val[1];
191 vtmp4567x4567.val[1] = vtmp4567x4567.val[0];
192 vtmp4567x4567.val[0] = vsrtmp;
193 vtmp4567x4567 = vld4q_lane_u32(w6, vtmp4567x4567, 2); w6 += 4;
194 vsrtmp = vtmp4567x4567.val[3];
195 vtmp4567x4567.val[3] = vtmp4567x4567.val[2];
196 vtmp4567x4567.val[2] = vtmp4567x4567.val[1];
197 vtmp4567x4567.val[1] = vtmp4567x4567.val[0];
198 vtmp4567x4567.val[0] = vsrtmp;
199 vtmp4567x4567 = vld4q_lane_u32(w7, vtmp4567x4567, 3); w7 += 4;
200 vsrtmp = vtmp4567x4567.val[3];
201 vtmp4567x4567.val[3] = vtmp4567x4567.val[2];
202 vtmp4567x4567.val[2] = vtmp4567x4567.val[1];
203 vtmp4567x4567.val[1] = vtmp4567x4567.val[0];
204 vtmp4567x4567.val[0] = vsrtmp;
205 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
206 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
207 xnn_prefetch_to_l1((const int8_t*) w2 + 128);
208 xnn_prefetch_to_l1((const int8_t*) w3 + 128);
209 xnn_prefetch_to_l1((const int8_t*) w4 + 128);
210 xnn_prefetch_to_l1((const int8_t*) w5 + 128);
211 xnn_prefetch_to_l1((const int8_t*) w6 + 128);
212 xnn_prefetch_to_l1((const int8_t*) w7 + 128);
213 vst1q_u32(packed_weights, vtmp0123x0123.val[0]); packed_weights += 4;
214 vst1q_u32(packed_weights, vtmp0123x4567.val[0]); packed_weights += 4;
215 vst1q_u32(packed_weights, vtmp0123x0123.val[1]); packed_weights += 4;
216 vst1q_u32(packed_weights, vtmp0123x4567.val[1]); packed_weights += 4;
217 vst1q_u32(packed_weights, vtmp0123x0123.val[2]); packed_weights += 4;
218 vst1q_u32(packed_weights, vtmp0123x4567.val[2]); packed_weights += 4;
219 vst1q_u32(packed_weights, vtmp0123x0123.val[3]); packed_weights += 4;
220 vst1q_u32(packed_weights, vtmp0123x4567.val[3]); packed_weights += 4;
221 vst1q_u32(packed_weights, vtmp4567x0123.val[0]); packed_weights += 4;
222 vst1q_u32(packed_weights, vtmp4567x4567.val[0]); packed_weights += 4;
223 vst1q_u32(packed_weights, vtmp4567x0123.val[1]); packed_weights += 4;
224 vst1q_u32(packed_weights, vtmp4567x4567.val[1]); packed_weights += 4;
225 vst1q_u32(packed_weights, vtmp4567x0123.val[2]); packed_weights += 4;
226 vst1q_u32(packed_weights, vtmp4567x4567.val[2]); packed_weights += 4;
227 vst1q_u32(packed_weights, vtmp4567x0123.val[3]); packed_weights += 4;
228 vst1q_u32(packed_weights, vtmp4567x4567.val[3]); packed_weights += 4;
229 }
230 // KC remainder multiple of 4
231 if (k >= 4) {
232 uint32x4_t vsrtmp;
233 vtmp0123x0123 = vld4q_lane_u32(w0, vtmp0123x0123, 0); w0 += 4;
234 vsrtmp = vtmp0123x0123.val[3];
235 vtmp0123x0123.val[3] = vtmp0123x0123.val[2];
236 vtmp0123x0123.val[2] = vtmp0123x0123.val[1];
237 vtmp0123x0123.val[1] = vtmp0123x0123.val[0];
238 vtmp0123x0123.val[0] = vsrtmp;
239 vtmp0123x0123 = vld4q_lane_u32(w1, vtmp0123x0123, 1); w1 += 4;
240 vsrtmp = vtmp0123x0123.val[3];
241 vtmp0123x0123.val[3] = vtmp0123x0123.val[2];
242 vtmp0123x0123.val[2] = vtmp0123x0123.val[1];
243 vtmp0123x0123.val[1] = vtmp0123x0123.val[0];
244 vtmp0123x0123.val[0] = vsrtmp;
245 vtmp0123x0123 = vld4q_lane_u32(w2, vtmp0123x0123, 2); w2 += 4;
246 vsrtmp = vtmp0123x0123.val[3];
247 vtmp0123x0123.val[3] = vtmp0123x0123.val[2];
248 vtmp0123x0123.val[2] = vtmp0123x0123.val[1];
249 vtmp0123x0123.val[1] = vtmp0123x0123.val[0];
250 vtmp0123x0123.val[0] = vsrtmp;
251 vtmp0123x0123 = vld4q_lane_u32(w3, vtmp0123x0123, 3); w3 += 4;
252 vsrtmp = vtmp0123x0123.val[3];
253 vtmp0123x0123.val[3] = vtmp0123x0123.val[2];
254 vtmp0123x0123.val[2] = vtmp0123x0123.val[1];
255 vtmp0123x0123.val[1] = vtmp0123x0123.val[0];
256 vtmp0123x0123.val[0] = vsrtmp;
257 vtmp0123x4567 = vld4q_lane_u32(w4, vtmp0123x4567, 0); w4 += 4;
258 vsrtmp = vtmp0123x4567.val[3];
259 vtmp0123x4567.val[3] = vtmp0123x4567.val[2];
260 vtmp0123x4567.val[2] = vtmp0123x4567.val[1];
261 vtmp0123x4567.val[1] = vtmp0123x4567.val[0];
262 vtmp0123x4567.val[0] = vsrtmp;
263 vtmp0123x4567 = vld4q_lane_u32(w5, vtmp0123x4567, 1); w5 += 4;
264 vsrtmp = vtmp0123x4567.val[3];
265 vtmp0123x4567.val[3] = vtmp0123x4567.val[2];
266 vtmp0123x4567.val[2] = vtmp0123x4567.val[1];
267 vtmp0123x4567.val[1] = vtmp0123x4567.val[0];
268 vtmp0123x4567.val[0] = vsrtmp;
269 vtmp0123x4567 = vld4q_lane_u32(w6, vtmp0123x4567, 2); w6 += 4;
270 vsrtmp = vtmp0123x4567.val[3];
271 vtmp0123x4567.val[3] = vtmp0123x4567.val[2];
272 vtmp0123x4567.val[2] = vtmp0123x4567.val[1];
273 vtmp0123x4567.val[1] = vtmp0123x4567.val[0];
274 vtmp0123x4567.val[0] = vsrtmp;
275 vtmp0123x4567 = vld4q_lane_u32(w7, vtmp0123x4567, 3); w7 += 4;
276 vsrtmp = vtmp0123x4567.val[3];
277 vtmp0123x4567.val[3] = vtmp0123x4567.val[2];
278 vtmp0123x4567.val[2] = vtmp0123x4567.val[1];
279 vtmp0123x4567.val[1] = vtmp0123x4567.val[0];
280 vtmp0123x4567.val[0] = vsrtmp;
281 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
282 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
283 xnn_prefetch_to_l1((const int8_t*) w2 + 128);
284 xnn_prefetch_to_l1((const int8_t*) w3 + 128);
285 xnn_prefetch_to_l1((const int8_t*) w4 + 128);
286 xnn_prefetch_to_l1((const int8_t*) w5 + 128);
287 xnn_prefetch_to_l1((const int8_t*) w6 + 128);
288 xnn_prefetch_to_l1((const int8_t*) w7 + 128);
289 vst1q_u32(packed_weights, vtmp0123x0123.val[0]); packed_weights += 4;
290 vst1q_u32(packed_weights, vtmp0123x4567.val[0]); packed_weights += 4;
291 vst1q_u32(packed_weights, vtmp0123x0123.val[1]); packed_weights += 4;
292 vst1q_u32(packed_weights, vtmp0123x4567.val[1]); packed_weights += 4;
293 vst1q_u32(packed_weights, vtmp0123x0123.val[2]); packed_weights += 4;
294 vst1q_u32(packed_weights, vtmp0123x4567.val[2]); packed_weights += 4;
295 vst1q_u32(packed_weights, vtmp0123x0123.val[3]); packed_weights += 4;
296 vst1q_u32(packed_weights, vtmp0123x4567.val[3]); packed_weights += 4;
297 k -= 4;
298 }
299
300 // KC remainder of 1..3
301 // Same as main loop but ld1, ld2 or ld3
302 if XNN_UNLIKELY(k != 0) {
303 assert(k >= 1);
304 assert(k <= 3);
305 switch (k) {
306 // KC remainder of 1
307 case 1:
308 {
309 uint32x4_t vtmp0x0123 = vdupq_n_u32(0);
310 uint32x4_t vtmp0x4567 = vdupq_n_u32(0);
311 uint32x4_t vsrtmp;
312 uint32x4_t vsrtmp1x0123 = vdupq_n_u32(0);
313 uint32x4_t vsrtmp2x0123 = vdupq_n_u32(0);
314 uint32x4_t vsrtmp3x0123 = vdupq_n_u32(0);
315 uint32x4_t vsrtmp1x4567 = vdupq_n_u32(0);
316 uint32x4_t vsrtmp2x4567 = vdupq_n_u32(0);
317 uint32x4_t vsrtmp3x4567 = vdupq_n_u32(0);
318 vtmp0x0123 = vld1q_lane_u32(w0, vtmp0x0123, 0); w0 += 1;
319 vsrtmp = vsrtmp3x0123;
320 vsrtmp3x0123 = vsrtmp2x0123;
321 vsrtmp2x0123 = vsrtmp1x0123;
322 vsrtmp1x0123 = vtmp0x0123;
323 vtmp0x0123 = vsrtmp;
324 vtmp0x0123 = vld1q_lane_u32(w1, vtmp0x0123, 1); w1 += 1;
325 vsrtmp = vsrtmp3x0123;
326 vsrtmp3x0123 = vsrtmp2x0123;
327 vsrtmp2x0123 = vsrtmp1x0123;
328 vsrtmp1x0123 = vtmp0x0123;
329 vtmp0x0123 = vsrtmp;
330 vtmp0x0123 = vld1q_lane_u32(w2, vtmp0x0123, 2); w2 += 1;
331 vsrtmp = vsrtmp3x0123;
332 vsrtmp3x0123 = vsrtmp2x0123;
333 vsrtmp2x0123 = vsrtmp1x0123;
334 vsrtmp1x0123 = vtmp0x0123;
335 vtmp0x0123 = vsrtmp;
336 vtmp0x0123 = vld1q_lane_u32(w3, vtmp0x0123, 3); w3 += 1;
337 vsrtmp = vsrtmp3x0123;
338 vsrtmp3x0123 = vsrtmp2x0123;
339 vsrtmp2x0123 = vsrtmp1x0123;
340 vsrtmp1x0123 = vtmp0x0123;
341 vtmp0x0123 = vsrtmp;
342 vtmp0x4567 = vld1q_lane_u32(w4, vtmp0x4567, 0); w4 += 1;
343 vsrtmp = vsrtmp3x4567;
344 vsrtmp3x4567 = vsrtmp2x4567;
345 vsrtmp2x4567 = vsrtmp1x4567;
346 vsrtmp1x4567 = vtmp0x4567;
347 vtmp0x4567 = vsrtmp;
348 vtmp0x4567 = vld1q_lane_u32(w5, vtmp0x4567, 1); w5 += 1;
349 vsrtmp = vsrtmp3x4567;
350 vsrtmp3x4567 = vsrtmp2x4567;
351 vsrtmp2x4567 = vsrtmp1x4567;
352 vsrtmp1x4567 = vtmp0x4567;
353 vtmp0x4567 = vsrtmp;
354 vtmp0x4567 = vld1q_lane_u32(w6, vtmp0x4567, 2); w6 += 1;
355 vsrtmp = vsrtmp3x4567;
356 vsrtmp3x4567 = vsrtmp2x4567;
357 vsrtmp2x4567 = vsrtmp1x4567;
358 vsrtmp1x4567 = vtmp0x4567;
359 vtmp0x4567 = vsrtmp;
360 vtmp0x4567 = vld1q_lane_u32(w7, vtmp0x4567, 3); w7 += 1;
361 vsrtmp = vsrtmp3x4567;
362 vsrtmp3x4567 = vsrtmp2x4567;
363 vsrtmp2x4567 = vsrtmp1x4567;
364 vsrtmp1x4567 = vtmp0x4567;
365 vtmp0x4567 = vsrtmp;
366 vst1q_u32(packed_weights, vtmp0x0123); packed_weights += 4;
367 vst1q_u32(packed_weights, vtmp0x4567); packed_weights += 4;
368 vst1q_u32(packed_weights, vsrtmp1x0123); packed_weights += 4;
369 vst1q_u32(packed_weights, vsrtmp1x4567); packed_weights += 4;
370 vst1q_u32(packed_weights, vsrtmp2x0123); packed_weights += 4;
371 vst1q_u32(packed_weights, vsrtmp2x4567); packed_weights += 4;
372 vst1q_u32(packed_weights, vsrtmp3x0123); packed_weights += 4;
373 vst1q_u32(packed_weights, vsrtmp3x4567); packed_weights += 4;
374 break;
375 }
376 // KC remainder of 2
377 case 2:
378 {
379 uint32x4x2_t vtmp01x0123;
380 vtmp01x0123.val[0] = vdupq_n_u32(0);
381 vtmp01x0123.val[1] = vdupq_n_u32(0);
382 uint32x4x2_t vtmp01x4567;
383 vtmp01x4567.val[0] = vdupq_n_u32(0);
384 vtmp01x4567.val[1] = vdupq_n_u32(0);
385 uint32x4_t vsrtmp;
386 uint32x4_t vsrtmp2x0123 = vdupq_n_u32(0);
387 uint32x4_t vsrtmp3x0123 = vdupq_n_u32(0);
388 uint32x4_t vsrtmp2x4567 = vdupq_n_u32(0);
389 uint32x4_t vsrtmp3x4567 = vdupq_n_u32(0);
390 vtmp01x0123 = vld2q_lane_u32(w0, vtmp01x0123, 0); w0 += 2;
391 vsrtmp = vsrtmp3x0123;
392 vsrtmp3x0123 = vsrtmp2x0123;
393 vsrtmp2x0123 = vtmp01x0123.val[1];
394 vtmp01x0123.val[1] = vtmp01x0123.val[0];
395 vtmp01x0123.val[0] = vsrtmp;
396 vtmp01x0123 = vld2q_lane_u32(w1, vtmp01x0123, 1); w1 += 2;
397 vsrtmp = vsrtmp3x0123;
398 vsrtmp3x0123 = vsrtmp2x0123;
399 vsrtmp2x0123 = vtmp01x0123.val[1];
400 vtmp01x0123.val[1] = vtmp01x0123.val[0];
401 vtmp01x0123.val[0] = vsrtmp;
402 vtmp01x0123 = vld2q_lane_u32(w2, vtmp01x0123, 2); w2 += 2;
403 vsrtmp = vsrtmp3x0123;
404 vsrtmp3x0123 = vsrtmp2x0123;
405 vsrtmp2x0123 = vtmp01x0123.val[1];
406 vtmp01x0123.val[1] = vtmp01x0123.val[0];
407 vtmp01x0123.val[0] = vsrtmp;
408 vtmp01x0123 = vld2q_lane_u32(w3, vtmp01x0123, 3); w3 += 2;
409 vsrtmp = vsrtmp3x0123;
410 vsrtmp3x0123 = vsrtmp2x0123;
411 vsrtmp2x0123 = vtmp01x0123.val[1];
412 vtmp01x0123.val[1] = vtmp01x0123.val[0];
413 vtmp01x0123.val[0] = vsrtmp;
414 vtmp01x4567 = vld2q_lane_u32(w4, vtmp01x4567, 0); w4 += 2;
415 vsrtmp = vsrtmp3x4567;
416 vsrtmp3x4567 = vsrtmp2x4567;
417 vsrtmp2x4567 = vtmp01x4567.val[1];
418 vtmp01x4567.val[1] = vtmp01x4567.val[0];
419 vtmp01x4567.val[0] = vsrtmp;
420 vtmp01x4567 = vld2q_lane_u32(w5, vtmp01x4567, 1); w5 += 2;
421 vsrtmp = vsrtmp3x4567;
422 vsrtmp3x4567 = vsrtmp2x4567;
423 vsrtmp2x4567 = vtmp01x4567.val[1];
424 vtmp01x4567.val[1] = vtmp01x4567.val[0];
425 vtmp01x4567.val[0] = vsrtmp;
426 vtmp01x4567 = vld2q_lane_u32(w6, vtmp01x4567, 2); w6 += 2;
427 vsrtmp = vsrtmp3x4567;
428 vsrtmp3x4567 = vsrtmp2x4567;
429 vsrtmp2x4567 = vtmp01x4567.val[1];
430 vtmp01x4567.val[1] = vtmp01x4567.val[0];
431 vtmp01x4567.val[0] = vsrtmp;
432 vtmp01x4567 = vld2q_lane_u32(w7, vtmp01x4567, 3); w7 += 2;
433 vsrtmp = vsrtmp3x4567;
434 vsrtmp3x4567 = vsrtmp2x4567;
435 vsrtmp2x4567 = vtmp01x4567.val[1];
436 vtmp01x4567.val[1] = vtmp01x4567.val[0];
437 vtmp01x4567.val[0] = vsrtmp;
438 vst1q_u32(packed_weights, vtmp01x0123.val[0]); packed_weights += 4;
439 vst1q_u32(packed_weights, vtmp01x4567.val[0]); packed_weights += 4;
440 vst1q_u32(packed_weights, vtmp01x0123.val[1]); packed_weights += 4;
441 vst1q_u32(packed_weights, vtmp01x4567.val[1]); packed_weights += 4;
442 vst1q_u32(packed_weights, vsrtmp2x0123); packed_weights += 4;
443 vst1q_u32(packed_weights, vsrtmp2x4567); packed_weights += 4;
444 vst1q_u32(packed_weights, vsrtmp3x0123); packed_weights += 4;
445 vst1q_u32(packed_weights, vsrtmp3x4567); packed_weights += 4;
446 break;
447 }
448 // KC remainder of 3
449 case 3:
450 {
451 uint32x4x3_t vtmp012x0123;
452 vtmp012x0123.val[0] = vdupq_n_u32(0);
453 vtmp012x0123.val[1] = vdupq_n_u32(0);
454 vtmp012x0123.val[2] = vdupq_n_u32(0);
455 uint32x4x3_t vtmp012x4567;
456 vtmp012x4567.val[0] = vdupq_n_u32(0);
457 vtmp012x4567.val[1] = vdupq_n_u32(0);
458 vtmp012x4567.val[2] = vdupq_n_u32(0);
459 uint32x4_t vsrtmp;
460 uint32x4_t vsrtmp3x0123 = vdupq_n_u32(0);
461 uint32x4_t vsrtmp3x4567 = vdupq_n_u32(0);
462 vtmp012x0123 = vld3q_lane_u32(w0, vtmp012x0123, 0); w0 += 3;
463 vsrtmp = vsrtmp3x0123;
464 vsrtmp3x0123 = vtmp012x0123.val[2];
465 vtmp012x0123.val[2] = vtmp012x0123.val[1];
466 vtmp012x0123.val[1] = vtmp012x0123.val[0];
467 vtmp012x0123.val[0] = vsrtmp;
468 vtmp012x0123 = vld3q_lane_u32(w1, vtmp012x0123, 1); w1 += 3;
469 vsrtmp = vsrtmp3x0123;
470 vsrtmp3x0123 = vtmp012x0123.val[2];
471 vtmp012x0123.val[2] = vtmp012x0123.val[1];
472 vtmp012x0123.val[1] = vtmp012x0123.val[0];
473 vtmp012x0123.val[0] = vsrtmp;
474 vtmp012x0123 = vld3q_lane_u32(w2, vtmp012x0123, 2); w2 += 3;
475 vsrtmp = vsrtmp3x0123;
476 vsrtmp3x0123 = vtmp012x0123.val[2];
477 vtmp012x0123.val[2] = vtmp012x0123.val[1];
478 vtmp012x0123.val[1] = vtmp012x0123.val[0];
479 vtmp012x0123.val[0] = vsrtmp;
480 vtmp012x0123 = vld3q_lane_u32(w3, vtmp012x0123, 3); w3 += 3;
481 vsrtmp = vsrtmp3x0123;
482 vsrtmp3x0123 = vtmp012x0123.val[2];
483 vtmp012x0123.val[2] = vtmp012x0123.val[1];
484 vtmp012x0123.val[1] = vtmp012x0123.val[0];
485 vtmp012x0123.val[0] = vsrtmp;
486 vtmp012x4567 = vld3q_lane_u32(w4, vtmp012x4567, 0); w4 += 3;
487 vsrtmp = vsrtmp3x4567;
488 vsrtmp3x4567 = vtmp012x4567.val[2];
489 vtmp012x4567.val[2] = vtmp012x4567.val[1];
490 vtmp012x4567.val[1] = vtmp012x4567.val[0];
491 vtmp012x4567.val[0] = vsrtmp;
492 vtmp012x4567 = vld3q_lane_u32(w5, vtmp012x4567, 1); w5 += 3;
493 vsrtmp = vsrtmp3x4567;
494 vsrtmp3x4567 = vtmp012x4567.val[2];
495 vtmp012x4567.val[2] = vtmp012x4567.val[1];
496 vtmp012x4567.val[1] = vtmp012x4567.val[0];
497 vtmp012x4567.val[0] = vsrtmp;
498 vtmp012x4567 = vld3q_lane_u32(w6, vtmp012x4567, 2); w6 += 3;
499 vsrtmp = vsrtmp3x4567;
500 vsrtmp3x4567 = vtmp012x4567.val[2];
501 vtmp012x4567.val[2] = vtmp012x4567.val[1];
502 vtmp012x4567.val[1] = vtmp012x4567.val[0];
503 vtmp012x4567.val[0] = vsrtmp;
504 vtmp012x4567 = vld3q_lane_u32(w7, vtmp012x4567, 3); w7 += 3;
505 vsrtmp = vsrtmp3x4567;
506 vsrtmp3x4567 = vtmp012x4567.val[2];
507 vtmp012x4567.val[2] = vtmp012x4567.val[1];
508 vtmp012x4567.val[1] = vtmp012x4567.val[0];
509 vtmp012x4567.val[0] = vsrtmp;
510 vst1q_u32(packed_weights, vtmp012x0123.val[0]); packed_weights += 4;
511 vst1q_u32(packed_weights, vtmp012x4567.val[0]); packed_weights += 4;
512 vst1q_u32(packed_weights, vtmp012x0123.val[1]); packed_weights += 4;
513 vst1q_u32(packed_weights, vtmp012x4567.val[1]); packed_weights += 4;
514 vst1q_u32(packed_weights, vtmp012x0123.val[2]); packed_weights += 4;
515 vst1q_u32(packed_weights, vtmp012x4567.val[2]); packed_weights += 4;
516 vst1q_u32(packed_weights, vsrtmp3x0123); packed_weights += 4;
517 vst1q_u32(packed_weights, vsrtmp3x4567); packed_weights += 4;
518 break;
519 }
520 default:
521 XNN_UNREACHABLE;
522 }
523 }
524 packed_weights = (uint32_t*) ((uintptr_t) packed_weights + extra_bytes);
525 w0 = w7;
526 }
527
528 // NC remainder (1..7)
529 if XNN_UNLIKELY(n != 0) {
530 assert(n >= 1);
531 assert(n <= 7);
532 if XNN_LIKELY(bias != NULL) {
533 size_t nb = n;
534 do {
535 *packed_weights++ = *bias++;
536 } while (--nb != 0);
537 packed_weights += (8 - n);
538 } else {
539 const uint32x4_t vzero = vmovq_n_u32(0);
540 vst1q_u32(packed_weights, vzero); packed_weights += 4;
541 vst1q_u32(packed_weights, vzero); packed_weights += 4;
542 }
543
544 // NR remainder has less than 8 rows so last row is not loaded
545 const uint32_t* w1 = w0 + kc;
546 if XNN_UNPREDICTABLE(n < 2) {
547 w1 = w0;
548 }
549 const uint32_t* w2 = w1 + kc;
550 if XNN_UNPREDICTABLE(n <= 2) {
551 w2 = w1;
552 }
553 const uint32_t* w3 = w2 + kc;
554 if XNN_UNPREDICTABLE(n < 4) {
555 w3 = w2;
556 }
557 const uint32_t* w4 = w3 + kc;
558 if XNN_UNPREDICTABLE(n <= 4) {
559 w4 = w3;
560 }
561 const uint32_t* w5 = w4 + kc;
562 if XNN_UNPREDICTABLE(n < 6) {
563 w5 = w4;
564 }
565 const uint32_t* w6 = w5 + kc;
566 if XNN_UNPREDICTABLE(n <= 6) {
567 w6 = w5;
568 }
569
570 // KC main loop multiple of 8
571 size_t k = kc;
572 for (; k >= 8; k -= 8) {
573 uint32x4_t vsrtmp;
574 vtmp0123x0123 = vld4q_lane_u32(w0, vtmp0123x0123, 0); w0 += 4;
575 vsrtmp = vtmp0123x0123.val[3];
576 vtmp0123x0123.val[3] = vtmp0123x0123.val[2];
577 vtmp0123x0123.val[2] = vtmp0123x0123.val[1];
578 vtmp0123x0123.val[1] = vtmp0123x0123.val[0];
579 vtmp0123x0123.val[0] = vsrtmp;
580 vtmp0123x0123 = vld4q_lane_u32(w1, vtmp0123x0123, 1); w1 += 4;
581 vsrtmp = vtmp0123x0123.val[3];
582 vtmp0123x0123.val[3] = vtmp0123x0123.val[2];
583 vtmp0123x0123.val[2] = vtmp0123x0123.val[1];
584 vtmp0123x0123.val[1] = vtmp0123x0123.val[0];
585 vtmp0123x0123.val[0] = vsrtmp;
586 vtmp0123x0123 = vld4q_lane_u32(w2, vtmp0123x0123, 2); w2 += 4;
587 vsrtmp = vtmp0123x0123.val[3];
588 vtmp0123x0123.val[3] = vtmp0123x0123.val[2];
589 vtmp0123x0123.val[2] = vtmp0123x0123.val[1];
590 vtmp0123x0123.val[1] = vtmp0123x0123.val[0];
591 vtmp0123x0123.val[0] = vsrtmp;
592 vtmp0123x0123 = vld4q_lane_u32(w3, vtmp0123x0123, 3); w3 += 4;
593 vsrtmp = vtmp0123x0123.val[3];
594 vtmp0123x0123.val[3] = vtmp0123x0123.val[2];
595 vtmp0123x0123.val[2] = vtmp0123x0123.val[1];
596 vtmp0123x0123.val[1] = vtmp0123x0123.val[0];
597 vtmp0123x0123.val[0] = vsrtmp;
598 vtmp0123x4567 = vld4q_lane_u32(w4, vtmp0123x4567, 0); w4 += 4;
599 vsrtmp = vtmp0123x4567.val[3];
600 vtmp0123x4567.val[3] = vtmp0123x4567.val[2];
601 vtmp0123x4567.val[2] = vtmp0123x4567.val[1];
602 vtmp0123x4567.val[1] = vtmp0123x4567.val[0];
603 vtmp0123x4567.val[0] = vsrtmp;
604 vtmp0123x4567 = vld4q_lane_u32(w5, vtmp0123x4567, 1); w5 += 4;
605 vsrtmp = vtmp0123x4567.val[3];
606 vtmp0123x4567.val[3] = vtmp0123x4567.val[2];
607 vtmp0123x4567.val[2] = vtmp0123x4567.val[1];
608 vtmp0123x4567.val[1] = vtmp0123x4567.val[0];
609 vtmp0123x4567.val[0] = vsrtmp;
610 vtmp0123x4567 = vld4q_lane_u32(w6, vtmp0123x4567, 2); w6 += 4;
611 vsrtmp = vtmp0123x4567.val[3];
612 vtmp0123x4567.val[3] = vtmp0123x4567.val[2];
613 vtmp0123x4567.val[2] = vtmp0123x4567.val[1];
614 vtmp0123x4567.val[1] = vtmp0123x4567.val[0];
615 vtmp0123x4567.val[0] = vsrtmp;
616 vsrtmp = vtmp0123x4567.val[3];
617 vtmp0123x4567.val[3] = vtmp0123x4567.val[2];
618 vtmp0123x4567.val[2] = vtmp0123x4567.val[1];
619 vtmp0123x4567.val[1] = vtmp0123x4567.val[0];
620 vtmp0123x4567.val[0] = vsrtmp;
621 vtmp4567x0123 = vld4q_lane_u32(w0, vtmp4567x0123, 0); w0 += 4;
622 vsrtmp = vtmp4567x0123.val[3];
623 vtmp4567x0123.val[3] = vtmp4567x0123.val[2];
624 vtmp4567x0123.val[2] = vtmp4567x0123.val[1];
625 vtmp4567x0123.val[1] = vtmp4567x0123.val[0];
626 vtmp4567x0123.val[0] = vsrtmp;
627 vtmp4567x0123 = vld4q_lane_u32(w1, vtmp4567x0123, 1); w1 += 4;
628 vsrtmp = vtmp4567x0123.val[3];
629 vtmp4567x0123.val[3] = vtmp4567x0123.val[2];
630 vtmp4567x0123.val[2] = vtmp4567x0123.val[1];
631 vtmp4567x0123.val[1] = vtmp4567x0123.val[0];
632 vtmp4567x0123.val[0] = vsrtmp;
633 vtmp4567x0123 = vld4q_lane_u32(w2, vtmp4567x0123, 2); w2 += 4;
634 vsrtmp = vtmp4567x0123.val[3];
635 vtmp4567x0123.val[3] = vtmp4567x0123.val[2];
636 vtmp4567x0123.val[2] = vtmp4567x0123.val[1];
637 vtmp4567x0123.val[1] = vtmp4567x0123.val[0];
638 vtmp4567x0123.val[0] = vsrtmp;
639 vtmp4567x0123 = vld4q_lane_u32(w3, vtmp4567x0123, 3); w3 += 4;
640 vsrtmp = vtmp4567x0123.val[3];
641 vtmp4567x0123.val[3] = vtmp4567x0123.val[2];
642 vtmp4567x0123.val[2] = vtmp4567x0123.val[1];
643 vtmp4567x0123.val[1] = vtmp4567x0123.val[0];
644 vtmp4567x0123.val[0] = vsrtmp;
645 vtmp4567x4567 = vld4q_lane_u32(w4, vtmp4567x4567, 0); w4 += 4;
646 vsrtmp = vtmp4567x4567.val[3];
647 vtmp4567x4567.val[3] = vtmp4567x4567.val[2];
648 vtmp4567x4567.val[2] = vtmp4567x4567.val[1];
649 vtmp4567x4567.val[1] = vtmp4567x4567.val[0];
650 vtmp4567x4567.val[0] = vsrtmp;
651 vtmp4567x4567 = vld4q_lane_u32(w5, vtmp4567x4567, 1); w5 += 4;
652 vsrtmp = vtmp4567x4567.val[3];
653 vtmp4567x4567.val[3] = vtmp4567x4567.val[2];
654 vtmp4567x4567.val[2] = vtmp4567x4567.val[1];
655 vtmp4567x4567.val[1] = vtmp4567x4567.val[0];
656 vtmp4567x4567.val[0] = vsrtmp;
657 vtmp4567x4567 = vld4q_lane_u32(w6, vtmp4567x4567, 2); w6 += 4;
658 vsrtmp = vtmp4567x4567.val[3];
659 vtmp4567x4567.val[3] = vtmp4567x4567.val[2];
660 vtmp4567x4567.val[2] = vtmp4567x4567.val[1];
661 vtmp4567x4567.val[1] = vtmp4567x4567.val[0];
662 vtmp4567x4567.val[0] = vsrtmp;
663 vsrtmp = vtmp4567x4567.val[3];
664 vtmp4567x4567.val[3] = vtmp4567x4567.val[2];
665 vtmp4567x4567.val[2] = vtmp4567x4567.val[1];
666 vtmp4567x4567.val[1] = vtmp4567x4567.val[0];
667 vtmp4567x4567.val[0] = vsrtmp;
668 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
669 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
670 xnn_prefetch_to_l1((const int8_t*) w2 + 128);
671 xnn_prefetch_to_l1((const int8_t*) w3 + 128);
672 xnn_prefetch_to_l1((const int8_t*) w4 + 128);
673 xnn_prefetch_to_l1((const int8_t*) w5 + 128);
674 xnn_prefetch_to_l1((const int8_t*) w6 + 128);
675 vst1q_u32(packed_weights, vtmp0123x0123.val[0]); packed_weights += 4;
676 vst1q_u32(packed_weights, vtmp0123x4567.val[0]); packed_weights += 4;
677 vst1q_u32(packed_weights, vtmp0123x0123.val[1]); packed_weights += 4;
678 vst1q_u32(packed_weights, vtmp0123x4567.val[1]); packed_weights += 4;
679 vst1q_u32(packed_weights, vtmp0123x0123.val[2]); packed_weights += 4;
680 vst1q_u32(packed_weights, vtmp0123x4567.val[2]); packed_weights += 4;
681 vst1q_u32(packed_weights, vtmp0123x0123.val[3]); packed_weights += 4;
682 vst1q_u32(packed_weights, vtmp0123x4567.val[3]); packed_weights += 4;
683 vst1q_u32(packed_weights, vtmp4567x0123.val[0]); packed_weights += 4;
684 vst1q_u32(packed_weights, vtmp4567x4567.val[0]); packed_weights += 4;
685 vst1q_u32(packed_weights, vtmp4567x0123.val[1]); packed_weights += 4;
686 vst1q_u32(packed_weights, vtmp4567x4567.val[1]); packed_weights += 4;
687 vst1q_u32(packed_weights, vtmp4567x0123.val[2]); packed_weights += 4;
688 vst1q_u32(packed_weights, vtmp4567x4567.val[2]); packed_weights += 4;
689 vst1q_u32(packed_weights, vtmp4567x0123.val[3]); packed_weights += 4;
690 vst1q_u32(packed_weights, vtmp4567x4567.val[3]); packed_weights += 4;
691 }
692 // KC remainder multiple of 4
693 if (k >= 4) {
694 uint32x4_t vsrtmp;
695 vtmp0123x0123 = vld4q_lane_u32(w0, vtmp0123x0123, 0); w0 += 4;
696 vsrtmp = vtmp0123x0123.val[3];
697 vtmp0123x0123.val[3] = vtmp0123x0123.val[2];
698 vtmp0123x0123.val[2] = vtmp0123x0123.val[1];
699 vtmp0123x0123.val[1] = vtmp0123x0123.val[0];
700 vtmp0123x0123.val[0] = vsrtmp;
701 vtmp0123x0123 = vld4q_lane_u32(w1, vtmp0123x0123, 1); w1 += 4;
702 vsrtmp = vtmp0123x0123.val[3];
703 vtmp0123x0123.val[3] = vtmp0123x0123.val[2];
704 vtmp0123x0123.val[2] = vtmp0123x0123.val[1];
705 vtmp0123x0123.val[1] = vtmp0123x0123.val[0];
706 vtmp0123x0123.val[0] = vsrtmp;
707 vtmp0123x0123 = vld4q_lane_u32(w2, vtmp0123x0123, 2); w2 += 4;
708 vsrtmp = vtmp0123x0123.val[3];
709 vtmp0123x0123.val[3] = vtmp0123x0123.val[2];
710 vtmp0123x0123.val[2] = vtmp0123x0123.val[1];
711 vtmp0123x0123.val[1] = vtmp0123x0123.val[0];
712 vtmp0123x0123.val[0] = vsrtmp;
713 vtmp0123x0123 = vld4q_lane_u32(w3, vtmp0123x0123, 3); w3 += 4;
714 vsrtmp = vtmp0123x0123.val[3];
715 vtmp0123x0123.val[3] = vtmp0123x0123.val[2];
716 vtmp0123x0123.val[2] = vtmp0123x0123.val[1];
717 vtmp0123x0123.val[1] = vtmp0123x0123.val[0];
718 vtmp0123x0123.val[0] = vsrtmp;
719 vtmp0123x4567 = vld4q_lane_u32(w4, vtmp0123x4567, 0); w4 += 4;
720 vsrtmp = vtmp0123x4567.val[3];
721 vtmp0123x4567.val[3] = vtmp0123x4567.val[2];
722 vtmp0123x4567.val[2] = vtmp0123x4567.val[1];
723 vtmp0123x4567.val[1] = vtmp0123x4567.val[0];
724 vtmp0123x4567.val[0] = vsrtmp;
725 vtmp0123x4567 = vld4q_lane_u32(w5, vtmp0123x4567, 1); w5 += 4;
726 vsrtmp = vtmp0123x4567.val[3];
727 vtmp0123x4567.val[3] = vtmp0123x4567.val[2];
728 vtmp0123x4567.val[2] = vtmp0123x4567.val[1];
729 vtmp0123x4567.val[1] = vtmp0123x4567.val[0];
730 vtmp0123x4567.val[0] = vsrtmp;
731 vtmp0123x4567 = vld4q_lane_u32(w6, vtmp0123x4567, 2); w6 += 4;
732 vsrtmp = vtmp0123x4567.val[3];
733 vtmp0123x4567.val[3] = vtmp0123x4567.val[2];
734 vtmp0123x4567.val[2] = vtmp0123x4567.val[1];
735 vtmp0123x4567.val[1] = vtmp0123x4567.val[0];
736 vtmp0123x4567.val[0] = vsrtmp;
737 vsrtmp = vtmp0123x4567.val[3];
738 vtmp0123x4567.val[3] = vtmp0123x4567.val[2];
739 vtmp0123x4567.val[2] = vtmp0123x4567.val[1];
740 vtmp0123x4567.val[1] = vtmp0123x4567.val[0];
741 vtmp0123x4567.val[0] = vsrtmp;
742 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
743 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
744 xnn_prefetch_to_l1((const int8_t*) w2 + 128);
745 xnn_prefetch_to_l1((const int8_t*) w3 + 128);
746 xnn_prefetch_to_l1((const int8_t*) w4 + 128);
747 xnn_prefetch_to_l1((const int8_t*) w5 + 128);
748 xnn_prefetch_to_l1((const int8_t*) w6 + 128);
749 vst1q_u32(packed_weights, vtmp0123x0123.val[0]); packed_weights += 4;
750 vst1q_u32(packed_weights, vtmp0123x4567.val[0]); packed_weights += 4;
751 vst1q_u32(packed_weights, vtmp0123x0123.val[1]); packed_weights += 4;
752 vst1q_u32(packed_weights, vtmp0123x4567.val[1]); packed_weights += 4;
753 vst1q_u32(packed_weights, vtmp0123x0123.val[2]); packed_weights += 4;
754 vst1q_u32(packed_weights, vtmp0123x4567.val[2]); packed_weights += 4;
755 vst1q_u32(packed_weights, vtmp0123x0123.val[3]); packed_weights += 4;
756 vst1q_u32(packed_weights, vtmp0123x4567.val[3]); packed_weights += 4;
757 k -= 4;
758 }
759
760 // KC remainder of 1..3
761 // Same as main loop but ld1, ld2 or ld3
762 if XNN_UNLIKELY(k != 0) {
763 assert(k >= 1);
764 assert(k <= 3);
765 switch (k) {
766 // KC remainder of 1
767 case 1:
768 {
769 uint32x4_t vtmp0x0123 = vdupq_n_u32(0);
770 uint32x4_t vtmp0x4567 = vdupq_n_u32(0);
771 uint32x4_t vsrtmp;
772 uint32x4_t vsrtmp1x0123 = vdupq_n_u32(0);
773 uint32x4_t vsrtmp2x0123 = vdupq_n_u32(0);
774 uint32x4_t vsrtmp3x0123 = vdupq_n_u32(0);
775 uint32x4_t vsrtmp1x4567 = vdupq_n_u32(0);
776 uint32x4_t vsrtmp2x4567 = vdupq_n_u32(0);
777 uint32x4_t vsrtmp3x4567 = vdupq_n_u32(0);
778 vtmp0x0123 = vld1q_lane_u32(w0, vtmp0x0123, 0);
779 vsrtmp = vsrtmp3x0123;
780 vsrtmp3x0123 = vsrtmp2x0123;
781 vsrtmp2x0123 = vsrtmp1x0123;
782 vsrtmp1x0123 = vtmp0x0123;
783 vtmp0x0123 = vsrtmp;
784 vtmp0x0123 = vld1q_lane_u32(w1, vtmp0x0123, 1);
785 vsrtmp = vsrtmp3x0123;
786 vsrtmp3x0123 = vsrtmp2x0123;
787 vsrtmp2x0123 = vsrtmp1x0123;
788 vsrtmp1x0123 = vtmp0x0123;
789 vtmp0x0123 = vsrtmp;
790 vtmp0x0123 = vld1q_lane_u32(w2, vtmp0x0123, 2);
791 vsrtmp = vsrtmp3x0123;
792 vsrtmp3x0123 = vsrtmp2x0123;
793 vsrtmp2x0123 = vsrtmp1x0123;
794 vsrtmp1x0123 = vtmp0x0123;
795 vtmp0x0123 = vsrtmp;
796 vtmp0x0123 = vld1q_lane_u32(w3, vtmp0x0123, 3);
797 vsrtmp = vsrtmp3x0123;
798 vsrtmp3x0123 = vsrtmp2x0123;
799 vsrtmp2x0123 = vsrtmp1x0123;
800 vsrtmp1x0123 = vtmp0x0123;
801 vtmp0x0123 = vsrtmp;
802 vtmp0x4567 = vld1q_lane_u32(w4, vtmp0x4567, 0);
803 vsrtmp = vsrtmp3x4567;
804 vsrtmp3x4567 = vsrtmp2x4567;
805 vsrtmp2x4567 = vsrtmp1x4567;
806 vsrtmp1x4567 = vtmp0x4567;
807 vtmp0x4567 = vsrtmp;
808 vtmp0x4567 = vld1q_lane_u32(w5, vtmp0x4567, 1);
809 vsrtmp = vsrtmp3x4567;
810 vsrtmp3x4567 = vsrtmp2x4567;
811 vsrtmp2x4567 = vsrtmp1x4567;
812 vsrtmp1x4567 = vtmp0x4567;
813 vtmp0x4567 = vsrtmp;
814 vtmp0x4567 = vld1q_lane_u32(w6, vtmp0x4567, 2);
815 vsrtmp = vsrtmp3x4567;
816 vsrtmp3x4567 = vsrtmp2x4567;
817 vsrtmp2x4567 = vsrtmp1x4567;
818 vsrtmp1x4567 = vtmp0x4567;
819 vtmp0x4567 = vsrtmp;
820 vsrtmp = vsrtmp3x4567;
821 vsrtmp3x4567 = vsrtmp2x4567;
822 vsrtmp2x4567 = vsrtmp1x4567;
823 vsrtmp1x4567 = vtmp0x4567;
824 vtmp0x4567 = vsrtmp;
825 vst1q_u32(packed_weights, vtmp0x0123); packed_weights += 4;
826 vst1q_u32(packed_weights, vtmp0x4567); packed_weights += 4;
827 vst1q_u32(packed_weights, vsrtmp1x0123); packed_weights += 4;
828 vst1q_u32(packed_weights, vsrtmp1x4567); packed_weights += 4;
829 vst1q_u32(packed_weights, vsrtmp2x0123); packed_weights += 4;
830 vst1q_u32(packed_weights, vsrtmp2x4567); packed_weights += 4;
831 vst1q_u32(packed_weights, vsrtmp3x0123); packed_weights += 4;
832 vst1q_u32(packed_weights, vsrtmp3x4567); packed_weights += 4;
833 break;
834 }
835 // KC remainder of 2
836 case 2:
837 {
838 uint32x4x2_t vtmp01x0123;
839 vtmp01x0123.val[0] = vdupq_n_u32(0);
840 vtmp01x0123.val[1] = vdupq_n_u32(0);
841 uint32x4x2_t vtmp01x4567;
842 vtmp01x4567.val[0] = vdupq_n_u32(0);
843 vtmp01x4567.val[1] = vdupq_n_u32(0);
844 uint32x4_t vsrtmp;
845 uint32x4_t vsrtmp2x0123 = vdupq_n_u32(0);
846 uint32x4_t vsrtmp3x0123 = vdupq_n_u32(0);
847 uint32x4_t vsrtmp2x4567 = vdupq_n_u32(0);
848 uint32x4_t vsrtmp3x4567 = vdupq_n_u32(0);
849 vtmp01x0123 = vld2q_lane_u32(w0, vtmp01x0123, 0);
850 vsrtmp = vsrtmp3x0123;
851 vsrtmp3x0123 = vsrtmp2x0123;
852 vsrtmp2x0123 = vtmp01x0123.val[1];
853 vtmp01x0123.val[1] = vtmp01x0123.val[0];
854 vtmp01x0123.val[0] = vsrtmp;
855 vtmp01x0123 = vld2q_lane_u32(w1, vtmp01x0123, 1);
856 vsrtmp = vsrtmp3x0123;
857 vsrtmp3x0123 = vsrtmp2x0123;
858 vsrtmp2x0123 = vtmp01x0123.val[1];
859 vtmp01x0123.val[1] = vtmp01x0123.val[0];
860 vtmp01x0123.val[0] = vsrtmp;
861 vtmp01x0123 = vld2q_lane_u32(w2, vtmp01x0123, 2);
862 vsrtmp = vsrtmp3x0123;
863 vsrtmp3x0123 = vsrtmp2x0123;
864 vsrtmp2x0123 = vtmp01x0123.val[1];
865 vtmp01x0123.val[1] = vtmp01x0123.val[0];
866 vtmp01x0123.val[0] = vsrtmp;
867 vtmp01x0123 = vld2q_lane_u32(w3, vtmp01x0123, 3);
868 vsrtmp = vsrtmp3x0123;
869 vsrtmp3x0123 = vsrtmp2x0123;
870 vsrtmp2x0123 = vtmp01x0123.val[1];
871 vtmp01x0123.val[1] = vtmp01x0123.val[0];
872 vtmp01x0123.val[0] = vsrtmp;
873 vtmp01x4567 = vld2q_lane_u32(w4, vtmp01x4567, 0);
874 vsrtmp = vsrtmp3x4567;
875 vsrtmp3x4567 = vsrtmp2x4567;
876 vsrtmp2x4567 = vtmp01x4567.val[1];
877 vtmp01x4567.val[1] = vtmp01x4567.val[0];
878 vtmp01x4567.val[0] = vsrtmp;
879 vtmp01x4567 = vld2q_lane_u32(w5, vtmp01x4567, 1);
880 vsrtmp = vsrtmp3x4567;
881 vsrtmp3x4567 = vsrtmp2x4567;
882 vsrtmp2x4567 = vtmp01x4567.val[1];
883 vtmp01x4567.val[1] = vtmp01x4567.val[0];
884 vtmp01x4567.val[0] = vsrtmp;
885 vtmp01x4567 = vld2q_lane_u32(w6, vtmp01x4567, 2);
886 vsrtmp = vsrtmp3x4567;
887 vsrtmp3x4567 = vsrtmp2x4567;
888 vsrtmp2x4567 = vtmp01x4567.val[1];
889 vtmp01x4567.val[1] = vtmp01x4567.val[0];
890 vtmp01x4567.val[0] = vsrtmp;
891 vsrtmp = vsrtmp3x4567;
892 vsrtmp3x4567 = vsrtmp2x4567;
893 vsrtmp2x4567 = vtmp01x4567.val[1];
894 vtmp01x4567.val[1] = vtmp01x4567.val[0];
895 vtmp01x4567.val[0] = vsrtmp;
896 vst1q_u32(packed_weights, vtmp01x0123.val[0]); packed_weights += 4;
897 vst1q_u32(packed_weights, vtmp01x4567.val[0]); packed_weights += 4;
898 vst1q_u32(packed_weights, vtmp01x0123.val[1]); packed_weights += 4;
899 vst1q_u32(packed_weights, vtmp01x4567.val[1]); packed_weights += 4;
900 vst1q_u32(packed_weights, vsrtmp2x0123); packed_weights += 4;
901 vst1q_u32(packed_weights, vsrtmp2x4567); packed_weights += 4;
902 vst1q_u32(packed_weights, vsrtmp3x0123); packed_weights += 4;
903 vst1q_u32(packed_weights, vsrtmp3x4567); packed_weights += 4;
904 break;
905 }
906 // KC remainder of 3
907 case 3:
908 {
909 uint32x4x3_t vtmp012x0123;
910 vtmp012x0123.val[0] = vdupq_n_u32(0);
911 vtmp012x0123.val[1] = vdupq_n_u32(0);
912 vtmp012x0123.val[2] = vdupq_n_u32(0);
913 uint32x4x3_t vtmp012x4567;
914 vtmp012x4567.val[0] = vdupq_n_u32(0);
915 vtmp012x4567.val[1] = vdupq_n_u32(0);
916 vtmp012x4567.val[2] = vdupq_n_u32(0);
917 uint32x4_t vsrtmp;
918 uint32x4_t vsrtmp3x0123 = vdupq_n_u32(0);
919 uint32x4_t vsrtmp3x4567 = vdupq_n_u32(0);
920 vtmp012x0123 = vld3q_lane_u32(w0, vtmp012x0123, 0);
921 vsrtmp = vsrtmp3x0123;
922 vsrtmp3x0123 = vtmp012x0123.val[2];
923 vtmp012x0123.val[2] = vtmp012x0123.val[1];
924 vtmp012x0123.val[1] = vtmp012x0123.val[0];
925 vtmp012x0123.val[0] = vsrtmp;
926 vtmp012x0123 = vld3q_lane_u32(w1, vtmp012x0123, 1);
927 vsrtmp = vsrtmp3x0123;
928 vsrtmp3x0123 = vtmp012x0123.val[2];
929 vtmp012x0123.val[2] = vtmp012x0123.val[1];
930 vtmp012x0123.val[1] = vtmp012x0123.val[0];
931 vtmp012x0123.val[0] = vsrtmp;
932 vtmp012x0123 = vld3q_lane_u32(w2, vtmp012x0123, 2);
933 vsrtmp = vsrtmp3x0123;
934 vsrtmp3x0123 = vtmp012x0123.val[2];
935 vtmp012x0123.val[2] = vtmp012x0123.val[1];
936 vtmp012x0123.val[1] = vtmp012x0123.val[0];
937 vtmp012x0123.val[0] = vsrtmp;
938 vtmp012x0123 = vld3q_lane_u32(w3, vtmp012x0123, 3);
939 vsrtmp = vsrtmp3x0123;
940 vsrtmp3x0123 = vtmp012x0123.val[2];
941 vtmp012x0123.val[2] = vtmp012x0123.val[1];
942 vtmp012x0123.val[1] = vtmp012x0123.val[0];
943 vtmp012x0123.val[0] = vsrtmp;
944 vtmp012x4567 = vld3q_lane_u32(w4, vtmp012x4567, 0);
945 vsrtmp = vsrtmp3x4567;
946 vsrtmp3x4567 = vtmp012x4567.val[2];
947 vtmp012x4567.val[2] = vtmp012x4567.val[1];
948 vtmp012x4567.val[1] = vtmp012x4567.val[0];
949 vtmp012x4567.val[0] = vsrtmp;
950 vtmp012x4567 = vld3q_lane_u32(w5, vtmp012x4567, 1);
951 vsrtmp = vsrtmp3x4567;
952 vsrtmp3x4567 = vtmp012x4567.val[2];
953 vtmp012x4567.val[2] = vtmp012x4567.val[1];
954 vtmp012x4567.val[1] = vtmp012x4567.val[0];
955 vtmp012x4567.val[0] = vsrtmp;
956 vtmp012x4567 = vld3q_lane_u32(w6, vtmp012x4567, 2);
957 vsrtmp = vsrtmp3x4567;
958 vsrtmp3x4567 = vtmp012x4567.val[2];
959 vtmp012x4567.val[2] = vtmp012x4567.val[1];
960 vtmp012x4567.val[1] = vtmp012x4567.val[0];
961 vtmp012x4567.val[0] = vsrtmp;
962 vsrtmp = vsrtmp3x4567;
963 vsrtmp3x4567 = vtmp012x4567.val[2];
964 vtmp012x4567.val[2] = vtmp012x4567.val[1];
965 vtmp012x4567.val[1] = vtmp012x4567.val[0];
966 vtmp012x4567.val[0] = vsrtmp;
967 vst1q_u32(packed_weights, vtmp012x0123.val[0]); packed_weights += 4;
968 vst1q_u32(packed_weights, vtmp012x4567.val[0]); packed_weights += 4;
969 vst1q_u32(packed_weights, vtmp012x0123.val[1]); packed_weights += 4;
970 vst1q_u32(packed_weights, vtmp012x4567.val[1]); packed_weights += 4;
971 vst1q_u32(packed_weights, vtmp012x0123.val[2]); packed_weights += 4;
972 vst1q_u32(packed_weights, vtmp012x4567.val[2]); packed_weights += 4;
973 vst1q_u32(packed_weights, vsrtmp3x0123); packed_weights += 4;
974 vst1q_u32(packed_weights, vsrtmp3x4567); packed_weights += 4;
975 break;
976 }
977 default:
978 XNN_UNREACHABLE;
979 }
980 }
981 packed_weights = (uint32_t*) ((uintptr_t) packed_weights + extra_bytes);
982 }
983
984 weights += nc * kc;
985 } while (--g != 0);
986 }