Debian Codegraph @

src/x32-packw/gen/x32-packw-x16-gemm-goi-neon-ld4lane-u4-prfm.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/neon.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include <arm_neon.h>
16
17 #include "xnnpack/packw.h"
18 #include "xnnpack/prefetch.h"
19
20
21 void xnn_x32_packw_gemm_goi_ukernel_x16__neon_ld4lane_u4_prfm(
22 size_t g,
23 size_t nc,
24 size_t kc,
25 size_t nr,
26 size_t kr,
27 size_t sr,
28 const uint32_t* weights,
29 const uint32_t* bias,
30 const void* scale,
31 uint32_t* packed_weights,
32 size_t extra_bytes,
33 const void* params)
34 {
35 assert(g != 0);
36 assert(nc != 0);
37 assert(kc != 0);
38 assert(nr == 16);
39 assert(kr == 1);
40 assert(sr == 1);
41 assert(weights != NULL);
42 assert(packed_weights != NULL);
43 uint32x4x4_t vtmp0123x0123;
44 vtmp0123x0123.val[0] = vdupq_n_u32(0);
45 vtmp0123x0123.val[1] = vdupq_n_u32(0);
46 vtmp0123x0123.val[2] = vdupq_n_u32(0);
47 vtmp0123x0123.val[3] = vdupq_n_u32(0);
48 uint32x4x4_t vtmp0123x4567;
49 vtmp0123x4567.val[0] = vdupq_n_u32(0);
50 vtmp0123x4567.val[1] = vdupq_n_u32(0);
51 vtmp0123x4567.val[2] = vdupq_n_u32(0);
52 vtmp0123x4567.val[3] = vdupq_n_u32(0);
53 uint32x4x4_t vtmp0123x89AB;
54 vtmp0123x89AB.val[0] = vdupq_n_u32(0);
55 vtmp0123x89AB.val[1] = vdupq_n_u32(0);
56 vtmp0123x89AB.val[2] = vdupq_n_u32(0);
57 vtmp0123x89AB.val[3] = vdupq_n_u32(0);
58 uint32x4x4_t vtmp0123xCDEF;
59 vtmp0123xCDEF.val[0] = vdupq_n_u32(0);
60 vtmp0123xCDEF.val[1] = vdupq_n_u32(0);
61 vtmp0123xCDEF.val[2] = vdupq_n_u32(0);
62 vtmp0123xCDEF.val[3] = vdupq_n_u32(0);
63
64 do {
65 // NC main loop multiple of 16
66 const uint32_t* w0 = weights;
67 size_t n = nc;
68
69 for (; n >= 16; n -= 16) {
70 if XNN_LIKELY(bias != NULL) {
71 uint32x4_t vb0 = vld1q_u32(bias); bias += 4;
72 uint32x4_t vb4 = vld1q_u32(bias); bias += 4;
73 uint32x4_t vb8 = vld1q_u32(bias); bias += 4;
74 uint32x4_t vb12 = vld1q_u32(bias); bias += 4;
75 vst1q_u32(packed_weights, vb0); packed_weights += 4;
76 vst1q_u32(packed_weights, vb4); packed_weights += 4;
77 vst1q_u32(packed_weights, vb8); packed_weights += 4;
78 vst1q_u32(packed_weights, vb12); packed_weights += 4;
79 } else {
80 const uint32x4_t vzero = vmovq_n_u32(0);
81 vst1q_u32(packed_weights, vzero); packed_weights += 4;
82 vst1q_u32(packed_weights, vzero); packed_weights += 4;
83 vst1q_u32(packed_weights, vzero); packed_weights += 4;
84 vst1q_u32(packed_weights, vzero); packed_weights += 4;
85 }
86
87 const uint32_t* w1 = w0 + kc;
88 const uint32_t* w2 = w1 + kc;
89 const uint32_t* w3 = w2 + kc;
90 const uint32_t* w4 = w3 + kc;
91 const uint32_t* w5 = w4 + kc;
92 const uint32_t* w6 = w5 + kc;
93 const uint32_t* w7 = w6 + kc;
94 const uint32_t* w8 = w7 + kc;
95 const uint32_t* w9 = w8 + kc;
96 const uint32_t* w10 = w9 + kc;
97 const uint32_t* w11 = w10 + kc;
98 const uint32_t* w12 = w11 + kc;
99 const uint32_t* w13 = w12 + kc;
100 const uint32_t* w14 = w13 + kc;
101 const uint32_t* w15 = w14 + kc;
102 xnn_prefetch_to_l1((const int8_t*) w0);
103 xnn_prefetch_to_l1((const int8_t*) w0 + 64);
104 xnn_prefetch_to_l1((const int8_t*) w1);
105 xnn_prefetch_to_l1((const int8_t*) w1 + 64);
106 xnn_prefetch_to_l1((const int8_t*) w2);
107 xnn_prefetch_to_l1((const int8_t*) w2 + 64);
108 xnn_prefetch_to_l1((const int8_t*) w3);
109 xnn_prefetch_to_l1((const int8_t*) w3 + 64);
110 xnn_prefetch_to_l1((const int8_t*) w4);
111 xnn_prefetch_to_l1((const int8_t*) w4 + 64);
112 xnn_prefetch_to_l1((const int8_t*) w5);
113 xnn_prefetch_to_l1((const int8_t*) w5 + 64);
114 xnn_prefetch_to_l1((const int8_t*) w6);
115 xnn_prefetch_to_l1((const int8_t*) w6 + 64);
116 xnn_prefetch_to_l1((const int8_t*) w7);
117 xnn_prefetch_to_l1((const int8_t*) w7 + 64);
118 xnn_prefetch_to_l1((const int8_t*) w8);
119 xnn_prefetch_to_l1((const int8_t*) w8 + 64);
120 xnn_prefetch_to_l1((const int8_t*) w9);
121 xnn_prefetch_to_l1((const int8_t*) w9 + 64);
122 xnn_prefetch_to_l1((const int8_t*) w10);
123 xnn_prefetch_to_l1((const int8_t*) w10 + 64);
124 xnn_prefetch_to_l1((const int8_t*) w11);
125 xnn_prefetch_to_l1((const int8_t*) w11 + 64);
126 xnn_prefetch_to_l1((const int8_t*) w12);
127 xnn_prefetch_to_l1((const int8_t*) w12 + 64);
128 xnn_prefetch_to_l1((const int8_t*) w13);
129 xnn_prefetch_to_l1((const int8_t*) w13 + 64);
130 xnn_prefetch_to_l1((const int8_t*) w14);
131 xnn_prefetch_to_l1((const int8_t*) w14 + 64);
132 xnn_prefetch_to_l1((const int8_t*) w15);
133 xnn_prefetch_to_l1((const int8_t*) w15 + 64);
134
135 // KC main loop multiple of 4
136 size_t k = kc;
137 for (; k >= 4; k -= 4) {
138 vtmp0123x0123 = vld4q_lane_u32(w0, vtmp0123x0123, 0); w0 += 4;
139 vtmp0123x0123 = vld4q_lane_u32(w1, vtmp0123x0123, 1); w1 += 4;
140 vtmp0123x0123 = vld4q_lane_u32(w2, vtmp0123x0123, 2); w2 += 4;
141 vtmp0123x0123 = vld4q_lane_u32(w3, vtmp0123x0123, 3); w3 += 4;
142 vtmp0123x4567 = vld4q_lane_u32(w4, vtmp0123x4567, 0); w4 += 4;
143 vtmp0123x4567 = vld4q_lane_u32(w5, vtmp0123x4567, 1); w5 += 4;
144 vtmp0123x4567 = vld4q_lane_u32(w6, vtmp0123x4567, 2); w6 += 4;
145 vtmp0123x4567 = vld4q_lane_u32(w7, vtmp0123x4567, 3); w7 += 4;
146 vtmp0123x89AB = vld4q_lane_u32(w8, vtmp0123x89AB, 0); w8 += 4;
147 vtmp0123x89AB = vld4q_lane_u32(w9, vtmp0123x89AB, 1); w9 += 4;
148 vtmp0123x89AB = vld4q_lane_u32(w10, vtmp0123x89AB, 2); w10 += 4;
149 vtmp0123x89AB = vld4q_lane_u32(w11, vtmp0123x89AB, 3); w11 += 4;
150 vtmp0123xCDEF = vld4q_lane_u32(w12, vtmp0123xCDEF, 0); w12 += 4;
151 vtmp0123xCDEF = vld4q_lane_u32(w13, vtmp0123xCDEF, 1); w13 += 4;
152 vtmp0123xCDEF = vld4q_lane_u32(w14, vtmp0123xCDEF, 2); w14 += 4;
153 vtmp0123xCDEF = vld4q_lane_u32(w15, vtmp0123xCDEF, 3); w15 += 4;
154 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
155 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
156 xnn_prefetch_to_l1((const int8_t*) w2 + 128);
157 xnn_prefetch_to_l1((const int8_t*) w3 + 128);
158 xnn_prefetch_to_l1((const int8_t*) w4 + 128);
159 xnn_prefetch_to_l1((const int8_t*) w5 + 128);
160 xnn_prefetch_to_l1((const int8_t*) w6 + 128);
161 xnn_prefetch_to_l1((const int8_t*) w7 + 128);
162 xnn_prefetch_to_l1((const int8_t*) w8 + 128);
163 xnn_prefetch_to_l1((const int8_t*) w9 + 128);
164 xnn_prefetch_to_l1((const int8_t*) w10 + 128);
165 xnn_prefetch_to_l1((const int8_t*) w11 + 128);
166 xnn_prefetch_to_l1((const int8_t*) w12 + 128);
167 xnn_prefetch_to_l1((const int8_t*) w13 + 128);
168 xnn_prefetch_to_l1((const int8_t*) w14 + 128);
169 xnn_prefetch_to_l1((const int8_t*) w15 + 128);
170 vst1q_u32(packed_weights, vtmp0123x0123.val[0]); packed_weights += 4;
171 vst1q_u32(packed_weights, vtmp0123x4567.val[0]); packed_weights += 4;
172 vst1q_u32(packed_weights, vtmp0123x89AB.val[0]); packed_weights += 4;
173 vst1q_u32(packed_weights, vtmp0123xCDEF.val[0]); packed_weights += 4;
174 vst1q_u32(packed_weights, vtmp0123x0123.val[1]); packed_weights += 4;
175 vst1q_u32(packed_weights, vtmp0123x4567.val[1]); packed_weights += 4;
176 vst1q_u32(packed_weights, vtmp0123x89AB.val[1]); packed_weights += 4;
177 vst1q_u32(packed_weights, vtmp0123xCDEF.val[1]); packed_weights += 4;
178 vst1q_u32(packed_weights, vtmp0123x0123.val[2]); packed_weights += 4;
179 vst1q_u32(packed_weights, vtmp0123x4567.val[2]); packed_weights += 4;
180 vst1q_u32(packed_weights, vtmp0123x89AB.val[2]); packed_weights += 4;
181 vst1q_u32(packed_weights, vtmp0123xCDEF.val[2]); packed_weights += 4;
182 vst1q_u32(packed_weights, vtmp0123x0123.val[3]); packed_weights += 4;
183 vst1q_u32(packed_weights, vtmp0123x4567.val[3]); packed_weights += 4;
184 vst1q_u32(packed_weights, vtmp0123x89AB.val[3]); packed_weights += 4;
185 vst1q_u32(packed_weights, vtmp0123xCDEF.val[3]); packed_weights += 4;
186 }
187
188 // KC remainder of 1..3
189 // Same as main loop but ld1, ld2 or ld3
190 if XNN_UNLIKELY(k != 0) {
191 assert(k >= 1);
192 assert(k <= 3);
193 switch (k) {
194 // KC remainder of 1
195 case 1:
196 {
197 uint32x4_t vtmp0x0123 = vdupq_n_u32(0);
198 uint32x4_t vtmp0x4567 = vdupq_n_u32(0);
199 uint32x4_t vtmp0x89AB = vdupq_n_u32(0);
200 uint32x4_t vtmp0xCDEF = vdupq_n_u32(0);
201 vtmp0x0123 = vld1q_lane_u32(w0, vtmp0x0123, 0); w0 += 1;
202 vtmp0x0123 = vld1q_lane_u32(w1, vtmp0x0123, 1); w1 += 1;
203 vtmp0x0123 = vld1q_lane_u32(w2, vtmp0x0123, 2); w2 += 1;
204 vtmp0x0123 = vld1q_lane_u32(w3, vtmp0x0123, 3); w3 += 1;
205 vtmp0x4567 = vld1q_lane_u32(w4, vtmp0x4567, 0); w4 += 1;
206 vtmp0x4567 = vld1q_lane_u32(w5, vtmp0x4567, 1); w5 += 1;
207 vtmp0x4567 = vld1q_lane_u32(w6, vtmp0x4567, 2); w6 += 1;
208 vtmp0x4567 = vld1q_lane_u32(w7, vtmp0x4567, 3); w7 += 1;
209 vtmp0x89AB = vld1q_lane_u32(w8, vtmp0x89AB, 0); w8 += 1;
210 vtmp0x89AB = vld1q_lane_u32(w9, vtmp0x89AB, 1); w9 += 1;
211 vtmp0x89AB = vld1q_lane_u32(w10, vtmp0x89AB, 2); w10 += 1;
212 vtmp0x89AB = vld1q_lane_u32(w11, vtmp0x89AB, 3); w11 += 1;
213 vtmp0xCDEF = vld1q_lane_u32(w12, vtmp0xCDEF, 0); w12 += 1;
214 vtmp0xCDEF = vld1q_lane_u32(w13, vtmp0xCDEF, 1); w13 += 1;
215 vtmp0xCDEF = vld1q_lane_u32(w14, vtmp0xCDEF, 2); w14 += 1;
216 vtmp0xCDEF = vld1q_lane_u32(w15, vtmp0xCDEF, 3); w15 += 1;
217 vst1q_u32(packed_weights, vtmp0x0123); packed_weights += 4;
218 vst1q_u32(packed_weights, vtmp0x4567); packed_weights += 4;
219 vst1q_u32(packed_weights, vtmp0x89AB); packed_weights += 4;
220 vst1q_u32(packed_weights, vtmp0xCDEF); packed_weights += 4;
221 break;
222 }
223 // KC remainder of 2
224 case 2:
225 {
226 uint32x4x2_t vtmp01x0123;
227 vtmp01x0123.val[0] = vdupq_n_u32(0);
228 vtmp01x0123.val[1] = vdupq_n_u32(0);
229 uint32x4x2_t vtmp01x4567;
230 vtmp01x4567.val[0] = vdupq_n_u32(0);
231 vtmp01x4567.val[1] = vdupq_n_u32(0);
232 uint32x4x2_t vtmp01x89AB;
233 vtmp01x89AB.val[0] = vdupq_n_u32(0);
234 vtmp01x89AB.val[1] = vdupq_n_u32(0);
235 uint32x4x2_t vtmp01xCDEF;
236 vtmp01xCDEF.val[0] = vdupq_n_u32(0);
237 vtmp01xCDEF.val[1] = vdupq_n_u32(0);
238 vtmp01x0123 = vld2q_lane_u32(w0, vtmp01x0123, 0); w0 += 2;
239 vtmp01x0123 = vld2q_lane_u32(w1, vtmp01x0123, 1); w1 += 2;
240 vtmp01x0123 = vld2q_lane_u32(w2, vtmp01x0123, 2); w2 += 2;
241 vtmp01x0123 = vld2q_lane_u32(w3, vtmp01x0123, 3); w3 += 2;
242 vtmp01x4567 = vld2q_lane_u32(w4, vtmp01x4567, 0); w4 += 2;
243 vtmp01x4567 = vld2q_lane_u32(w5, vtmp01x4567, 1); w5 += 2;
244 vtmp01x4567 = vld2q_lane_u32(w6, vtmp01x4567, 2); w6 += 2;
245 vtmp01x4567 = vld2q_lane_u32(w7, vtmp01x4567, 3); w7 += 2;
246 vtmp01x89AB = vld2q_lane_u32(w8, vtmp01x89AB, 0); w8 += 2;
247 vtmp01x89AB = vld2q_lane_u32(w9, vtmp01x89AB, 1); w9 += 2;
248 vtmp01x89AB = vld2q_lane_u32(w10, vtmp01x89AB, 2); w10 += 2;
249 vtmp01x89AB = vld2q_lane_u32(w11, vtmp01x89AB, 3); w11 += 2;
250 vtmp01xCDEF = vld2q_lane_u32(w12, vtmp01xCDEF, 0); w12 += 2;
251 vtmp01xCDEF = vld2q_lane_u32(w13, vtmp01xCDEF, 1); w13 += 2;
252 vtmp01xCDEF = vld2q_lane_u32(w14, vtmp01xCDEF, 2); w14 += 2;
253 vtmp01xCDEF = vld2q_lane_u32(w15, vtmp01xCDEF, 3); w15 += 2;
254 vst1q_u32(packed_weights, vtmp01x0123.val[0]); packed_weights += 4;
255 vst1q_u32(packed_weights, vtmp01x4567.val[0]); packed_weights += 4;
256 vst1q_u32(packed_weights, vtmp01x89AB.val[0]); packed_weights += 4;
257 vst1q_u32(packed_weights, vtmp01xCDEF.val[0]); packed_weights += 4;
258 vst1q_u32(packed_weights, vtmp01x0123.val[1]); packed_weights += 4;
259 vst1q_u32(packed_weights, vtmp01x4567.val[1]); packed_weights += 4;
260 vst1q_u32(packed_weights, vtmp01x89AB.val[1]); packed_weights += 4;
261 vst1q_u32(packed_weights, vtmp01xCDEF.val[1]); packed_weights += 4;
262 break;
263 }
264 // KC remainder of 3
265 case 3:
266 {
267 uint32x4x3_t vtmp012x0123;
268 vtmp012x0123.val[0] = vdupq_n_u32(0);
269 vtmp012x0123.val[1] = vdupq_n_u32(0);
270 vtmp012x0123.val[2] = vdupq_n_u32(0);
271 uint32x4x3_t vtmp012x4567;
272 vtmp012x4567.val[0] = vdupq_n_u32(0);
273 vtmp012x4567.val[1] = vdupq_n_u32(0);
274 vtmp012x4567.val[2] = vdupq_n_u32(0);
275 uint32x4x3_t vtmp012x89AB;
276 vtmp012x89AB.val[0] = vdupq_n_u32(0);
277 vtmp012x89AB.val[1] = vdupq_n_u32(0);
278 vtmp012x89AB.val[2] = vdupq_n_u32(0);
279 uint32x4x3_t vtmp012xCDEF;
280 vtmp012xCDEF.val[0] = vdupq_n_u32(0);
281 vtmp012xCDEF.val[1] = vdupq_n_u32(0);
282 vtmp012xCDEF.val[2] = vdupq_n_u32(0);
283 vtmp012x0123 = vld3q_lane_u32(w0, vtmp012x0123, 0); w0 += 3;
284 vtmp012x0123 = vld3q_lane_u32(w1, vtmp012x0123, 1); w1 += 3;
285 vtmp012x0123 = vld3q_lane_u32(w2, vtmp012x0123, 2); w2 += 3;
286 vtmp012x0123 = vld3q_lane_u32(w3, vtmp012x0123, 3); w3 += 3;
287 vtmp012x4567 = vld3q_lane_u32(w4, vtmp012x4567, 0); w4 += 3;
288 vtmp012x4567 = vld3q_lane_u32(w5, vtmp012x4567, 1); w5 += 3;
289 vtmp012x4567 = vld3q_lane_u32(w6, vtmp012x4567, 2); w6 += 3;
290 vtmp012x4567 = vld3q_lane_u32(w7, vtmp012x4567, 3); w7 += 3;
291 vtmp012x89AB = vld3q_lane_u32(w8, vtmp012x89AB, 0); w8 += 3;
292 vtmp012x89AB = vld3q_lane_u32(w9, vtmp012x89AB, 1); w9 += 3;
293 vtmp012x89AB = vld3q_lane_u32(w10, vtmp012x89AB, 2); w10 += 3;
294 vtmp012x89AB = vld3q_lane_u32(w11, vtmp012x89AB, 3); w11 += 3;
295 vtmp012xCDEF = vld3q_lane_u32(w12, vtmp012xCDEF, 0); w12 += 3;
296 vtmp012xCDEF = vld3q_lane_u32(w13, vtmp012xCDEF, 1); w13 += 3;
297 vtmp012xCDEF = vld3q_lane_u32(w14, vtmp012xCDEF, 2); w14 += 3;
298 vtmp012xCDEF = vld3q_lane_u32(w15, vtmp012xCDEF, 3); w15 += 3;
299 vst1q_u32(packed_weights, vtmp012x0123.val[0]); packed_weights += 4;
300 vst1q_u32(packed_weights, vtmp012x4567.val[0]); packed_weights += 4;
301 vst1q_u32(packed_weights, vtmp012x89AB.val[0]); packed_weights += 4;
302 vst1q_u32(packed_weights, vtmp012xCDEF.val[0]); packed_weights += 4;
303 vst1q_u32(packed_weights, vtmp012x0123.val[1]); packed_weights += 4;
304 vst1q_u32(packed_weights, vtmp012x4567.val[1]); packed_weights += 4;
305 vst1q_u32(packed_weights, vtmp012x89AB.val[1]); packed_weights += 4;
306 vst1q_u32(packed_weights, vtmp012xCDEF.val[1]); packed_weights += 4;
307 vst1q_u32(packed_weights, vtmp012x0123.val[2]); packed_weights += 4;
308 vst1q_u32(packed_weights, vtmp012x4567.val[2]); packed_weights += 4;
309 vst1q_u32(packed_weights, vtmp012x89AB.val[2]); packed_weights += 4;
310 vst1q_u32(packed_weights, vtmp012xCDEF.val[2]); packed_weights += 4;
311 break;
312 }
313 default:
314 XNN_UNREACHABLE;
315 }
316 }
317 packed_weights = (uint32_t*) ((uintptr_t) packed_weights + extra_bytes);
318 w0 = w15;
319 }
320
321 // NC remainder (1..15)
322 if XNN_UNLIKELY(n != 0) {
323 assert(n >= 1);
324 assert(n <= 15);
325 if XNN_LIKELY(bias != NULL) {
326 size_t nb = n;
327 do {
328 *packed_weights++ = *bias++;
329 } while (--nb != 0);
330 packed_weights += (16 - n);
331 } else {
332 const uint32x4_t vzero = vmovq_n_u32(0);
333 vst1q_u32(packed_weights, vzero); packed_weights += 4;
334 vst1q_u32(packed_weights, vzero); packed_weights += 4;
335 vst1q_u32(packed_weights, vzero); packed_weights += 4;
336 vst1q_u32(packed_weights, vzero); packed_weights += 4;
337 }
338
339 // NR remainder has less than 16 rows so last row is not loaded
340 const uint32_t* w1 = w0 + kc;
341 if XNN_UNPREDICTABLE(n < 2) {
342 w1 = w0;
343 }
344 const uint32_t* w2 = w1 + kc;
345 if XNN_UNPREDICTABLE(n <= 2) {
346 w2 = w1;
347 }
348 const uint32_t* w3 = w2 + kc;
349 if XNN_UNPREDICTABLE(n < 4) {
350 w3 = w2;
351 }
352 const uint32_t* w4 = w3 + kc;
353 if XNN_UNPREDICTABLE(n <= 4) {
354 w4 = w3;
355 }
356 const uint32_t* w5 = w4 + kc;
357 if XNN_UNPREDICTABLE(n < 6) {
358 w5 = w4;
359 }
360 const uint32_t* w6 = w5 + kc;
361 if XNN_UNPREDICTABLE(n <= 6) {
362 w6 = w5;
363 }
364 const uint32_t* w7 = w6 + kc;
365 if XNN_UNPREDICTABLE(n < 8) {
366 w7 = w6;
367 }
368 const uint32_t* w8 = w7 + kc;
369 if XNN_UNPREDICTABLE(n <= 8) {
370 w8 = w7;
371 }
372 const uint32_t* w9 = w8 + kc;
373 if XNN_UNPREDICTABLE(n < 10) {
374 w9 = w8;
375 }
376 const uint32_t* w10 = w9 + kc;
377 if XNN_UNPREDICTABLE(n <= 10) {
378 w10 = w9;
379 }
380 const uint32_t* w11 = w10 + kc;
381 if XNN_UNPREDICTABLE(n < 12) {
382 w11 = w10;
383 }
384 const uint32_t* w12 = w11 + kc;
385 if XNN_UNPREDICTABLE(n <= 12) {
386 w12 = w11;
387 }
388 const uint32_t* w13 = w12 + kc;
389 if XNN_UNPREDICTABLE(n < 14) {
390 w13 = w12;
391 }
392 const uint32_t* w14 = w13 + kc;
393 if XNN_UNPREDICTABLE(n <= 14) {
394 w14 = w13;
395 }
396
397 // KC main loop multiple of 4
398 size_t k = kc;
399 for (; k >= 4; k -= 4) {
400 vtmp0123x0123 = vld4q_lane_u32(w0, vtmp0123x0123, 0); w0 += 4;
401 vtmp0123x0123 = vld4q_lane_u32(w1, vtmp0123x0123, 1); w1 += 4;
402 vtmp0123x0123 = vld4q_lane_u32(w2, vtmp0123x0123, 2); w2 += 4;
403 vtmp0123x0123 = vld4q_lane_u32(w3, vtmp0123x0123, 3); w3 += 4;
404 vtmp0123x4567 = vld4q_lane_u32(w4, vtmp0123x4567, 0); w4 += 4;
405 vtmp0123x4567 = vld4q_lane_u32(w5, vtmp0123x4567, 1); w5 += 4;
406 vtmp0123x4567 = vld4q_lane_u32(w6, vtmp0123x4567, 2); w6 += 4;
407 vtmp0123x4567 = vld4q_lane_u32(w7, vtmp0123x4567, 3); w7 += 4;
408 vtmp0123x89AB = vld4q_lane_u32(w8, vtmp0123x89AB, 0); w8 += 4;
409 vtmp0123x89AB = vld4q_lane_u32(w9, vtmp0123x89AB, 1); w9 += 4;
410 vtmp0123x89AB = vld4q_lane_u32(w10, vtmp0123x89AB, 2); w10 += 4;
411 vtmp0123x89AB = vld4q_lane_u32(w11, vtmp0123x89AB, 3); w11 += 4;
412 vtmp0123xCDEF = vld4q_lane_u32(w12, vtmp0123xCDEF, 0); w12 += 4;
413 vtmp0123xCDEF = vld4q_lane_u32(w13, vtmp0123xCDEF, 1); w13 += 4;
414 vtmp0123xCDEF = vld4q_lane_u32(w14, vtmp0123xCDEF, 2); w14 += 4;
415 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
416 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
417 xnn_prefetch_to_l1((const int8_t*) w2 + 128);
418 xnn_prefetch_to_l1((const int8_t*) w3 + 128);
419 xnn_prefetch_to_l1((const int8_t*) w4 + 128);
420 xnn_prefetch_to_l1((const int8_t*) w5 + 128);
421 xnn_prefetch_to_l1((const int8_t*) w6 + 128);
422 xnn_prefetch_to_l1((const int8_t*) w7 + 128);
423 xnn_prefetch_to_l1((const int8_t*) w8 + 128);
424 xnn_prefetch_to_l1((const int8_t*) w9 + 128);
425 xnn_prefetch_to_l1((const int8_t*) w10 + 128);
426 xnn_prefetch_to_l1((const int8_t*) w11 + 128);
427 xnn_prefetch_to_l1((const int8_t*) w12 + 128);
428 xnn_prefetch_to_l1((const int8_t*) w13 + 128);
429 xnn_prefetch_to_l1((const int8_t*) w14 + 128);
430 vst1q_u32(packed_weights, vtmp0123x0123.val[0]); packed_weights += 4;
431 vst1q_u32(packed_weights, vtmp0123x4567.val[0]); packed_weights += 4;
432 vst1q_u32(packed_weights, vtmp0123x89AB.val[0]); packed_weights += 4;
433 vst1q_u32(packed_weights, vtmp0123xCDEF.val[0]); packed_weights += 4;
434 vst1q_u32(packed_weights, vtmp0123x0123.val[1]); packed_weights += 4;
435 vst1q_u32(packed_weights, vtmp0123x4567.val[1]); packed_weights += 4;
436 vst1q_u32(packed_weights, vtmp0123x89AB.val[1]); packed_weights += 4;
437 vst1q_u32(packed_weights, vtmp0123xCDEF.val[1]); packed_weights += 4;
438 vst1q_u32(packed_weights, vtmp0123x0123.val[2]); packed_weights += 4;
439 vst1q_u32(packed_weights, vtmp0123x4567.val[2]); packed_weights += 4;
440 vst1q_u32(packed_weights, vtmp0123x89AB.val[2]); packed_weights += 4;
441 vst1q_u32(packed_weights, vtmp0123xCDEF.val[2]); packed_weights += 4;
442 vst1q_u32(packed_weights, vtmp0123x0123.val[3]); packed_weights += 4;
443 vst1q_u32(packed_weights, vtmp0123x4567.val[3]); packed_weights += 4;
444 vst1q_u32(packed_weights, vtmp0123x89AB.val[3]); packed_weights += 4;
445 vst1q_u32(packed_weights, vtmp0123xCDEF.val[3]); packed_weights += 4;
446 }
447
448 // KC remainder of 1..3
449 // Same as main loop but ld1, ld2 or ld3
450 if XNN_UNLIKELY(k != 0) {
451 assert(k >= 1);
452 assert(k <= 3);
453 switch (k) {
454 // KC remainder of 1
455 case 1:
456 {
457 uint32x4_t vtmp0x0123 = vdupq_n_u32(0);
458 uint32x4_t vtmp0x4567 = vdupq_n_u32(0);
459 uint32x4_t vtmp0x89AB = vdupq_n_u32(0);
460 uint32x4_t vtmp0xCDEF = vdupq_n_u32(0);
461 vtmp0x0123 = vld1q_lane_u32(w0, vtmp0x0123, 0);
462 vtmp0x0123 = vld1q_lane_u32(w1, vtmp0x0123, 1);
463 vtmp0x0123 = vld1q_lane_u32(w2, vtmp0x0123, 2);
464 vtmp0x0123 = vld1q_lane_u32(w3, vtmp0x0123, 3);
465 vtmp0x4567 = vld1q_lane_u32(w4, vtmp0x4567, 0);
466 vtmp0x4567 = vld1q_lane_u32(w5, vtmp0x4567, 1);
467 vtmp0x4567 = vld1q_lane_u32(w6, vtmp0x4567, 2);
468 vtmp0x4567 = vld1q_lane_u32(w7, vtmp0x4567, 3);
469 vtmp0x89AB = vld1q_lane_u32(w8, vtmp0x89AB, 0);
470 vtmp0x89AB = vld1q_lane_u32(w9, vtmp0x89AB, 1);
471 vtmp0x89AB = vld1q_lane_u32(w10, vtmp0x89AB, 2);
472 vtmp0x89AB = vld1q_lane_u32(w11, vtmp0x89AB, 3);
473 vtmp0xCDEF = vld1q_lane_u32(w12, vtmp0xCDEF, 0);
474 vtmp0xCDEF = vld1q_lane_u32(w13, vtmp0xCDEF, 1);
475 vtmp0xCDEF = vld1q_lane_u32(w14, vtmp0xCDEF, 2);
476 vst1q_u32(packed_weights, vtmp0x0123); packed_weights += 4;
477 vst1q_u32(packed_weights, vtmp0x4567); packed_weights += 4;
478 vst1q_u32(packed_weights, vtmp0x89AB); packed_weights += 4;
479 vst1q_u32(packed_weights, vtmp0xCDEF); packed_weights += 4;
480 break;
481 }
482 // KC remainder of 2
483 case 2:
484 {
485 uint32x4x2_t vtmp01x0123;
486 vtmp01x0123.val[0] = vdupq_n_u32(0);
487 vtmp01x0123.val[1] = vdupq_n_u32(0);
488 uint32x4x2_t vtmp01x4567;
489 vtmp01x4567.val[0] = vdupq_n_u32(0);
490 vtmp01x4567.val[1] = vdupq_n_u32(0);
491 uint32x4x2_t vtmp01x89AB;
492 vtmp01x89AB.val[0] = vdupq_n_u32(0);
493 vtmp01x89AB.val[1] = vdupq_n_u32(0);
494 uint32x4x2_t vtmp01xCDEF;
495 vtmp01xCDEF.val[0] = vdupq_n_u32(0);
496 vtmp01xCDEF.val[1] = vdupq_n_u32(0);
497 vtmp01x0123 = vld2q_lane_u32(w0, vtmp01x0123, 0);
498 vtmp01x0123 = vld2q_lane_u32(w1, vtmp01x0123, 1);
499 vtmp01x0123 = vld2q_lane_u32(w2, vtmp01x0123, 2);
500 vtmp01x0123 = vld2q_lane_u32(w3, vtmp01x0123, 3);
501 vtmp01x4567 = vld2q_lane_u32(w4, vtmp01x4567, 0);
502 vtmp01x4567 = vld2q_lane_u32(w5, vtmp01x4567, 1);
503 vtmp01x4567 = vld2q_lane_u32(w6, vtmp01x4567, 2);
504 vtmp01x4567 = vld2q_lane_u32(w7, vtmp01x4567, 3);
505 vtmp01x89AB = vld2q_lane_u32(w8, vtmp01x89AB, 0);
506 vtmp01x89AB = vld2q_lane_u32(w9, vtmp01x89AB, 1);
507 vtmp01x89AB = vld2q_lane_u32(w10, vtmp01x89AB, 2);
508 vtmp01x89AB = vld2q_lane_u32(w11, vtmp01x89AB, 3);
509 vtmp01xCDEF = vld2q_lane_u32(w12, vtmp01xCDEF, 0);
510 vtmp01xCDEF = vld2q_lane_u32(w13, vtmp01xCDEF, 1);
511 vtmp01xCDEF = vld2q_lane_u32(w14, vtmp01xCDEF, 2);
512 vst1q_u32(packed_weights, vtmp01x0123.val[0]); packed_weights += 4;
513 vst1q_u32(packed_weights, vtmp01x4567.val[0]); packed_weights += 4;
514 vst1q_u32(packed_weights, vtmp01x89AB.val[0]); packed_weights += 4;
515 vst1q_u32(packed_weights, vtmp01xCDEF.val[0]); packed_weights += 4;
516 vst1q_u32(packed_weights, vtmp01x0123.val[1]); packed_weights += 4;
517 vst1q_u32(packed_weights, vtmp01x4567.val[1]); packed_weights += 4;
518 vst1q_u32(packed_weights, vtmp01x89AB.val[1]); packed_weights += 4;
519 vst1q_u32(packed_weights, vtmp01xCDEF.val[1]); packed_weights += 4;
520 break;
521 }
522 // KC remainder of 3
523 case 3:
524 {
525 uint32x4x3_t vtmp012x0123;
526 vtmp012x0123.val[0] = vdupq_n_u32(0);
527 vtmp012x0123.val[1] = vdupq_n_u32(0);
528 vtmp012x0123.val[2] = vdupq_n_u32(0);
529 uint32x4x3_t vtmp012x4567;
530 vtmp012x4567.val[0] = vdupq_n_u32(0);
531 vtmp012x4567.val[1] = vdupq_n_u32(0);
532 vtmp012x4567.val[2] = vdupq_n_u32(0);
533 uint32x4x3_t vtmp012x89AB;
534 vtmp012x89AB.val[0] = vdupq_n_u32(0);
535 vtmp012x89AB.val[1] = vdupq_n_u32(0);
536 vtmp012x89AB.val[2] = vdupq_n_u32(0);
537 uint32x4x3_t vtmp012xCDEF;
538 vtmp012xCDEF.val[0] = vdupq_n_u32(0);
539 vtmp012xCDEF.val[1] = vdupq_n_u32(0);
540 vtmp012xCDEF.val[2] = vdupq_n_u32(0);
541 vtmp012x0123 = vld3q_lane_u32(w0, vtmp012x0123, 0);
542 vtmp012x0123 = vld3q_lane_u32(w1, vtmp012x0123, 1);
543 vtmp012x0123 = vld3q_lane_u32(w2, vtmp012x0123, 2);
544 vtmp012x0123 = vld3q_lane_u32(w3, vtmp012x0123, 3);
545 vtmp012x4567 = vld3q_lane_u32(w4, vtmp012x4567, 0);
546 vtmp012x4567 = vld3q_lane_u32(w5, vtmp012x4567, 1);
547 vtmp012x4567 = vld3q_lane_u32(w6, vtmp012x4567, 2);
548 vtmp012x4567 = vld3q_lane_u32(w7, vtmp012x4567, 3);
549 vtmp012x89AB = vld3q_lane_u32(w8, vtmp012x89AB, 0);
550 vtmp012x89AB = vld3q_lane_u32(w9, vtmp012x89AB, 1);
551 vtmp012x89AB = vld3q_lane_u32(w10, vtmp012x89AB, 2);
552 vtmp012x89AB = vld3q_lane_u32(w11, vtmp012x89AB, 3);
553 vtmp012xCDEF = vld3q_lane_u32(w12, vtmp012xCDEF, 0);
554 vtmp012xCDEF = vld3q_lane_u32(w13, vtmp012xCDEF, 1);
555 vtmp012xCDEF = vld3q_lane_u32(w14, vtmp012xCDEF, 2);
556 vst1q_u32(packed_weights, vtmp012x0123.val[0]); packed_weights += 4;
557 vst1q_u32(packed_weights, vtmp012x4567.val[0]); packed_weights += 4;
558 vst1q_u32(packed_weights, vtmp012x89AB.val[0]); packed_weights += 4;
559 vst1q_u32(packed_weights, vtmp012xCDEF.val[0]); packed_weights += 4;
560 vst1q_u32(packed_weights, vtmp012x0123.val[1]); packed_weights += 4;
561 vst1q_u32(packed_weights, vtmp012x4567.val[1]); packed_weights += 4;
562 vst1q_u32(packed_weights, vtmp012x89AB.val[1]); packed_weights += 4;
563 vst1q_u32(packed_weights, vtmp012xCDEF.val[1]); packed_weights += 4;
564 vst1q_u32(packed_weights, vtmp012x0123.val[2]); packed_weights += 4;
565 vst1q_u32(packed_weights, vtmp012x4567.val[2]); packed_weights += 4;
566 vst1q_u32(packed_weights, vtmp012x89AB.val[2]); packed_weights += 4;
567 vst1q_u32(packed_weights, vtmp012xCDEF.val[2]); packed_weights += 4;
568 break;
569 }
570 default:
571 XNN_UNREACHABLE;
572 }
573 }
574 packed_weights = (uint32_t*) ((uintptr_t) packed_weights + extra_bytes);
575 }
576
577 weights += nc * kc;
578 } while (--g != 0);
579 }