Debian Codegraph @

src/x32-packw/gen/x32-packw-x12-gemm-goi-neon-ld4lane-u8-prfm.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/neon.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include <arm_neon.h>
16
17 #include "xnnpack/packw.h"
18 #include "xnnpack/prefetch.h"
19
20
21 void xnn_x32_packw_gemm_goi_ukernel_x12__neon_ld4lane_u8_prfm(
22 size_t g,
23 size_t nc,
24 size_t kc,
25 size_t nr,
26 size_t kr,
27 size_t sr,
28 const uint32_t* weights,
29 const uint32_t* bias,
30 const void* scale,
31 uint32_t* packed_weights,
32 size_t extra_bytes,
33 const void* params)
34 {
35 assert(g != 0);
36 assert(nc != 0);
37 assert(kc != 0);
38 assert(nr == 12);
39 assert(kr == 1);
40 assert(sr == 1);
41 assert(weights != NULL);
42 assert(packed_weights != NULL);
43 uint32x4x4_t vtmp0123x0123;
44 vtmp0123x0123.val[0] = vdupq_n_u32(0);
45 vtmp0123x0123.val[1] = vdupq_n_u32(0);
46 vtmp0123x0123.val[2] = vdupq_n_u32(0);
47 vtmp0123x0123.val[3] = vdupq_n_u32(0);
48 uint32x4x4_t vtmp4567x0123;
49 vtmp4567x0123.val[0] = vdupq_n_u32(0);
50 vtmp4567x0123.val[1] = vdupq_n_u32(0);
51 vtmp4567x0123.val[2] = vdupq_n_u32(0);
52 vtmp4567x0123.val[3] = vdupq_n_u32(0);
53 uint32x4x4_t vtmp0123x4567;
54 vtmp0123x4567.val[0] = vdupq_n_u32(0);
55 vtmp0123x4567.val[1] = vdupq_n_u32(0);
56 vtmp0123x4567.val[2] = vdupq_n_u32(0);
57 vtmp0123x4567.val[3] = vdupq_n_u32(0);
58 uint32x4x4_t vtmp4567x4567;
59 vtmp4567x4567.val[0] = vdupq_n_u32(0);
60 vtmp4567x4567.val[1] = vdupq_n_u32(0);
61 vtmp4567x4567.val[2] = vdupq_n_u32(0);
62 vtmp4567x4567.val[3] = vdupq_n_u32(0);
63 uint32x4x4_t vtmp0123x89AB;
64 vtmp0123x89AB.val[0] = vdupq_n_u32(0);
65 vtmp0123x89AB.val[1] = vdupq_n_u32(0);
66 vtmp0123x89AB.val[2] = vdupq_n_u32(0);
67 vtmp0123x89AB.val[3] = vdupq_n_u32(0);
68 uint32x4x4_t vtmp4567x89AB;
69 vtmp4567x89AB.val[0] = vdupq_n_u32(0);
70 vtmp4567x89AB.val[1] = vdupq_n_u32(0);
71 vtmp4567x89AB.val[2] = vdupq_n_u32(0);
72 vtmp4567x89AB.val[3] = vdupq_n_u32(0);
73
74 do {
75 // NC main loop multiple of 12
76 const uint32_t* w0 = weights;
77 size_t n = nc;
78
79 for (; n >= 12; n -= 12) {
80 if XNN_LIKELY(bias != NULL) {
81 uint32x4_t vb0 = vld1q_u32(bias); bias += 4;
82 uint32x4_t vb4 = vld1q_u32(bias); bias += 4;
83 uint32x4_t vb8 = vld1q_u32(bias); bias += 4;
84 vst1q_u32(packed_weights, vb0); packed_weights += 4;
85 vst1q_u32(packed_weights, vb4); packed_weights += 4;
86 vst1q_u32(packed_weights, vb8); packed_weights += 4;
87 } else {
88 const uint32x4_t vzero = vmovq_n_u32(0);
89 vst1q_u32(packed_weights, vzero); packed_weights += 4;
90 vst1q_u32(packed_weights, vzero); packed_weights += 4;
91 vst1q_u32(packed_weights, vzero); packed_weights += 4;
92 }
93
94 const uint32_t* w1 = w0 + kc;
95 const uint32_t* w2 = w1 + kc;
96 const uint32_t* w3 = w2 + kc;
97 const uint32_t* w4 = w3 + kc;
98 const uint32_t* w5 = w4 + kc;
99 const uint32_t* w6 = w5 + kc;
100 const uint32_t* w7 = w6 + kc;
101 const uint32_t* w8 = w7 + kc;
102 const uint32_t* w9 = w8 + kc;
103 const uint32_t* w10 = w9 + kc;
104 const uint32_t* w11 = w10 + kc;
105 xnn_prefetch_to_l1((const int8_t*) w0);
106 xnn_prefetch_to_l1((const int8_t*) w0 + 64);
107 xnn_prefetch_to_l1((const int8_t*) w1);
108 xnn_prefetch_to_l1((const int8_t*) w1 + 64);
109 xnn_prefetch_to_l1((const int8_t*) w2);
110 xnn_prefetch_to_l1((const int8_t*) w2 + 64);
111 xnn_prefetch_to_l1((const int8_t*) w3);
112 xnn_prefetch_to_l1((const int8_t*) w3 + 64);
113 xnn_prefetch_to_l1((const int8_t*) w4);
114 xnn_prefetch_to_l1((const int8_t*) w4 + 64);
115 xnn_prefetch_to_l1((const int8_t*) w5);
116 xnn_prefetch_to_l1((const int8_t*) w5 + 64);
117 xnn_prefetch_to_l1((const int8_t*) w6);
118 xnn_prefetch_to_l1((const int8_t*) w6 + 64);
119 xnn_prefetch_to_l1((const int8_t*) w7);
120 xnn_prefetch_to_l1((const int8_t*) w7 + 64);
121 xnn_prefetch_to_l1((const int8_t*) w8);
122 xnn_prefetch_to_l1((const int8_t*) w8 + 64);
123 xnn_prefetch_to_l1((const int8_t*) w9);
124 xnn_prefetch_to_l1((const int8_t*) w9 + 64);
125 xnn_prefetch_to_l1((const int8_t*) w10);
126 xnn_prefetch_to_l1((const int8_t*) w10 + 64);
127 xnn_prefetch_to_l1((const int8_t*) w11);
128 xnn_prefetch_to_l1((const int8_t*) w11 + 64);
129
130 // KC main loop multiple of 8
131 size_t k = kc;
132 for (; k >= 8; k -= 8) {
133 vtmp0123x0123 = vld4q_lane_u32(w0, vtmp0123x0123, 0); w0 += 4;
134 vtmp0123x0123 = vld4q_lane_u32(w1, vtmp0123x0123, 1); w1 += 4;
135 vtmp0123x0123 = vld4q_lane_u32(w2, vtmp0123x0123, 2); w2 += 4;
136 vtmp0123x0123 = vld4q_lane_u32(w3, vtmp0123x0123, 3); w3 += 4;
137 vtmp0123x4567 = vld4q_lane_u32(w4, vtmp0123x4567, 0); w4 += 4;
138 vtmp0123x4567 = vld4q_lane_u32(w5, vtmp0123x4567, 1); w5 += 4;
139 vtmp0123x4567 = vld4q_lane_u32(w6, vtmp0123x4567, 2); w6 += 4;
140 vtmp0123x4567 = vld4q_lane_u32(w7, vtmp0123x4567, 3); w7 += 4;
141 vtmp0123x89AB = vld4q_lane_u32(w8, vtmp0123x89AB, 0); w8 += 4;
142 vtmp0123x89AB = vld4q_lane_u32(w9, vtmp0123x89AB, 1); w9 += 4;
143 vtmp0123x89AB = vld4q_lane_u32(w10, vtmp0123x89AB, 2); w10 += 4;
144 vtmp0123x89AB = vld4q_lane_u32(w11, vtmp0123x89AB, 3); w11 += 4;
145 vtmp4567x0123 = vld4q_lane_u32(w0, vtmp4567x0123, 0); w0 += 4;
146 vtmp4567x0123 = vld4q_lane_u32(w1, vtmp4567x0123, 1); w1 += 4;
147 vtmp4567x0123 = vld4q_lane_u32(w2, vtmp4567x0123, 2); w2 += 4;
148 vtmp4567x0123 = vld4q_lane_u32(w3, vtmp4567x0123, 3); w3 += 4;
149 vtmp4567x4567 = vld4q_lane_u32(w4, vtmp4567x4567, 0); w4 += 4;
150 vtmp4567x4567 = vld4q_lane_u32(w5, vtmp4567x4567, 1); w5 += 4;
151 vtmp4567x4567 = vld4q_lane_u32(w6, vtmp4567x4567, 2); w6 += 4;
152 vtmp4567x4567 = vld4q_lane_u32(w7, vtmp4567x4567, 3); w7 += 4;
153 vtmp4567x89AB = vld4q_lane_u32(w8, vtmp4567x89AB, 0); w8 += 4;
154 vtmp4567x89AB = vld4q_lane_u32(w9, vtmp4567x89AB, 1); w9 += 4;
155 vtmp4567x89AB = vld4q_lane_u32(w10, vtmp4567x89AB, 2); w10 += 4;
156 vtmp4567x89AB = vld4q_lane_u32(w11, vtmp4567x89AB, 3); w11 += 4;
157 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
158 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
159 xnn_prefetch_to_l1((const int8_t*) w2 + 128);
160 xnn_prefetch_to_l1((const int8_t*) w3 + 128);
161 xnn_prefetch_to_l1((const int8_t*) w4 + 128);
162 xnn_prefetch_to_l1((const int8_t*) w5 + 128);
163 xnn_prefetch_to_l1((const int8_t*) w6 + 128);
164 xnn_prefetch_to_l1((const int8_t*) w7 + 128);
165 xnn_prefetch_to_l1((const int8_t*) w8 + 128);
166 xnn_prefetch_to_l1((const int8_t*) w9 + 128);
167 xnn_prefetch_to_l1((const int8_t*) w10 + 128);
168 xnn_prefetch_to_l1((const int8_t*) w11 + 128);
169 vst1q_u32(packed_weights, vtmp0123x0123.val[0]); packed_weights += 4;
170 vst1q_u32(packed_weights, vtmp0123x4567.val[0]); packed_weights += 4;
171 vst1q_u32(packed_weights, vtmp0123x89AB.val[0]); packed_weights += 4;
172 vst1q_u32(packed_weights, vtmp0123x0123.val[1]); packed_weights += 4;
173 vst1q_u32(packed_weights, vtmp0123x4567.val[1]); packed_weights += 4;
174 vst1q_u32(packed_weights, vtmp0123x89AB.val[1]); packed_weights += 4;
175 vst1q_u32(packed_weights, vtmp0123x0123.val[2]); packed_weights += 4;
176 vst1q_u32(packed_weights, vtmp0123x4567.val[2]); packed_weights += 4;
177 vst1q_u32(packed_weights, vtmp0123x89AB.val[2]); packed_weights += 4;
178 vst1q_u32(packed_weights, vtmp0123x0123.val[3]); packed_weights += 4;
179 vst1q_u32(packed_weights, vtmp0123x4567.val[3]); packed_weights += 4;
180 vst1q_u32(packed_weights, vtmp0123x89AB.val[3]); packed_weights += 4;
181 vst1q_u32(packed_weights, vtmp4567x0123.val[0]); packed_weights += 4;
182 vst1q_u32(packed_weights, vtmp4567x4567.val[0]); packed_weights += 4;
183 vst1q_u32(packed_weights, vtmp4567x89AB.val[0]); packed_weights += 4;
184 vst1q_u32(packed_weights, vtmp4567x0123.val[1]); packed_weights += 4;
185 vst1q_u32(packed_weights, vtmp4567x4567.val[1]); packed_weights += 4;
186 vst1q_u32(packed_weights, vtmp4567x89AB.val[1]); packed_weights += 4;
187 vst1q_u32(packed_weights, vtmp4567x0123.val[2]); packed_weights += 4;
188 vst1q_u32(packed_weights, vtmp4567x4567.val[2]); packed_weights += 4;
189 vst1q_u32(packed_weights, vtmp4567x89AB.val[2]); packed_weights += 4;
190 vst1q_u32(packed_weights, vtmp4567x0123.val[3]); packed_weights += 4;
191 vst1q_u32(packed_weights, vtmp4567x4567.val[3]); packed_weights += 4;
192 vst1q_u32(packed_weights, vtmp4567x89AB.val[3]); packed_weights += 4;
193 }
194 // KC remainder multiple of 4
195 if (k >= 4) {
196 vtmp0123x0123 = vld4q_lane_u32(w0, vtmp0123x0123, 0); w0 += 4;
197 vtmp0123x0123 = vld4q_lane_u32(w1, vtmp0123x0123, 1); w1 += 4;
198 vtmp0123x0123 = vld4q_lane_u32(w2, vtmp0123x0123, 2); w2 += 4;
199 vtmp0123x0123 = vld4q_lane_u32(w3, vtmp0123x0123, 3); w3 += 4;
200 vtmp0123x4567 = vld4q_lane_u32(w4, vtmp0123x4567, 0); w4 += 4;
201 vtmp0123x4567 = vld4q_lane_u32(w5, vtmp0123x4567, 1); w5 += 4;
202 vtmp0123x4567 = vld4q_lane_u32(w6, vtmp0123x4567, 2); w6 += 4;
203 vtmp0123x4567 = vld4q_lane_u32(w7, vtmp0123x4567, 3); w7 += 4;
204 vtmp0123x89AB = vld4q_lane_u32(w8, vtmp0123x89AB, 0); w8 += 4;
205 vtmp0123x89AB = vld4q_lane_u32(w9, vtmp0123x89AB, 1); w9 += 4;
206 vtmp0123x89AB = vld4q_lane_u32(w10, vtmp0123x89AB, 2); w10 += 4;
207 vtmp0123x89AB = vld4q_lane_u32(w11, vtmp0123x89AB, 3); w11 += 4;
208 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
209 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
210 xnn_prefetch_to_l1((const int8_t*) w2 + 128);
211 xnn_prefetch_to_l1((const int8_t*) w3 + 128);
212 xnn_prefetch_to_l1((const int8_t*) w4 + 128);
213 xnn_prefetch_to_l1((const int8_t*) w5 + 128);
214 xnn_prefetch_to_l1((const int8_t*) w6 + 128);
215 xnn_prefetch_to_l1((const int8_t*) w7 + 128);
216 xnn_prefetch_to_l1((const int8_t*) w8 + 128);
217 xnn_prefetch_to_l1((const int8_t*) w9 + 128);
218 xnn_prefetch_to_l1((const int8_t*) w10 + 128);
219 xnn_prefetch_to_l1((const int8_t*) w11 + 128);
220 vst1q_u32(packed_weights, vtmp0123x0123.val[0]); packed_weights += 4;
221 vst1q_u32(packed_weights, vtmp0123x4567.val[0]); packed_weights += 4;
222 vst1q_u32(packed_weights, vtmp0123x89AB.val[0]); packed_weights += 4;
223 vst1q_u32(packed_weights, vtmp0123x0123.val[1]); packed_weights += 4;
224 vst1q_u32(packed_weights, vtmp0123x4567.val[1]); packed_weights += 4;
225 vst1q_u32(packed_weights, vtmp0123x89AB.val[1]); packed_weights += 4;
226 vst1q_u32(packed_weights, vtmp0123x0123.val[2]); packed_weights += 4;
227 vst1q_u32(packed_weights, vtmp0123x4567.val[2]); packed_weights += 4;
228 vst1q_u32(packed_weights, vtmp0123x89AB.val[2]); packed_weights += 4;
229 vst1q_u32(packed_weights, vtmp0123x0123.val[3]); packed_weights += 4;
230 vst1q_u32(packed_weights, vtmp0123x4567.val[3]); packed_weights += 4;
231 vst1q_u32(packed_weights, vtmp0123x89AB.val[3]); packed_weights += 4;
232 k -= 4;
233 }
234
235 // KC remainder of 1..3
236 // Same as main loop but ld1, ld2 or ld3
237 if XNN_UNLIKELY(k != 0) {
238 assert(k >= 1);
239 assert(k <= 3);
240 switch (k) {
241 // KC remainder of 1
242 case 1:
243 {
244 uint32x4_t vtmp0x0123 = vdupq_n_u32(0);
245 uint32x4_t vtmp0x4567 = vdupq_n_u32(0);
246 uint32x4_t vtmp0x89AB = vdupq_n_u32(0);
247 vtmp0x0123 = vld1q_lane_u32(w0, vtmp0x0123, 0); w0 += 1;
248 vtmp0x0123 = vld1q_lane_u32(w1, vtmp0x0123, 1); w1 += 1;
249 vtmp0x0123 = vld1q_lane_u32(w2, vtmp0x0123, 2); w2 += 1;
250 vtmp0x0123 = vld1q_lane_u32(w3, vtmp0x0123, 3); w3 += 1;
251 vtmp0x4567 = vld1q_lane_u32(w4, vtmp0x4567, 0); w4 += 1;
252 vtmp0x4567 = vld1q_lane_u32(w5, vtmp0x4567, 1); w5 += 1;
253 vtmp0x4567 = vld1q_lane_u32(w6, vtmp0x4567, 2); w6 += 1;
254 vtmp0x4567 = vld1q_lane_u32(w7, vtmp0x4567, 3); w7 += 1;
255 vtmp0x89AB = vld1q_lane_u32(w8, vtmp0x89AB, 0); w8 += 1;
256 vtmp0x89AB = vld1q_lane_u32(w9, vtmp0x89AB, 1); w9 += 1;
257 vtmp0x89AB = vld1q_lane_u32(w10, vtmp0x89AB, 2); w10 += 1;
258 vtmp0x89AB = vld1q_lane_u32(w11, vtmp0x89AB, 3); w11 += 1;
259 vst1q_u32(packed_weights, vtmp0x0123); packed_weights += 4;
260 vst1q_u32(packed_weights, vtmp0x4567); packed_weights += 4;
261 vst1q_u32(packed_weights, vtmp0x89AB); packed_weights += 4;
262 break;
263 }
264 // KC remainder of 2
265 case 2:
266 {
267 uint32x4x2_t vtmp01x0123;
268 vtmp01x0123.val[0] = vdupq_n_u32(0);
269 vtmp01x0123.val[1] = vdupq_n_u32(0);
270 uint32x4x2_t vtmp01x4567;
271 vtmp01x4567.val[0] = vdupq_n_u32(0);
272 vtmp01x4567.val[1] = vdupq_n_u32(0);
273 uint32x4x2_t vtmp01x89AB;
274 vtmp01x89AB.val[0] = vdupq_n_u32(0);
275 vtmp01x89AB.val[1] = vdupq_n_u32(0);
276 vtmp01x0123 = vld2q_lane_u32(w0, vtmp01x0123, 0); w0 += 2;
277 vtmp01x0123 = vld2q_lane_u32(w1, vtmp01x0123, 1); w1 += 2;
278 vtmp01x0123 = vld2q_lane_u32(w2, vtmp01x0123, 2); w2 += 2;
279 vtmp01x0123 = vld2q_lane_u32(w3, vtmp01x0123, 3); w3 += 2;
280 vtmp01x4567 = vld2q_lane_u32(w4, vtmp01x4567, 0); w4 += 2;
281 vtmp01x4567 = vld2q_lane_u32(w5, vtmp01x4567, 1); w5 += 2;
282 vtmp01x4567 = vld2q_lane_u32(w6, vtmp01x4567, 2); w6 += 2;
283 vtmp01x4567 = vld2q_lane_u32(w7, vtmp01x4567, 3); w7 += 2;
284 vtmp01x89AB = vld2q_lane_u32(w8, vtmp01x89AB, 0); w8 += 2;
285 vtmp01x89AB = vld2q_lane_u32(w9, vtmp01x89AB, 1); w9 += 2;
286 vtmp01x89AB = vld2q_lane_u32(w10, vtmp01x89AB, 2); w10 += 2;
287 vtmp01x89AB = vld2q_lane_u32(w11, vtmp01x89AB, 3); w11 += 2;
288 vst1q_u32(packed_weights, vtmp01x0123.val[0]); packed_weights += 4;
289 vst1q_u32(packed_weights, vtmp01x4567.val[0]); packed_weights += 4;
290 vst1q_u32(packed_weights, vtmp01x89AB.val[0]); packed_weights += 4;
291 vst1q_u32(packed_weights, vtmp01x0123.val[1]); packed_weights += 4;
292 vst1q_u32(packed_weights, vtmp01x4567.val[1]); packed_weights += 4;
293 vst1q_u32(packed_weights, vtmp01x89AB.val[1]); packed_weights += 4;
294 break;
295 }
296 // KC remainder of 3
297 case 3:
298 {
299 uint32x4x3_t vtmp012x0123;
300 vtmp012x0123.val[0] = vdupq_n_u32(0);
301 vtmp012x0123.val[1] = vdupq_n_u32(0);
302 vtmp012x0123.val[2] = vdupq_n_u32(0);
303 uint32x4x3_t vtmp012x4567;
304 vtmp012x4567.val[0] = vdupq_n_u32(0);
305 vtmp012x4567.val[1] = vdupq_n_u32(0);
306 vtmp012x4567.val[2] = vdupq_n_u32(0);
307 uint32x4x3_t vtmp012x89AB;
308 vtmp012x89AB.val[0] = vdupq_n_u32(0);
309 vtmp012x89AB.val[1] = vdupq_n_u32(0);
310 vtmp012x89AB.val[2] = vdupq_n_u32(0);
311 vtmp012x0123 = vld3q_lane_u32(w0, vtmp012x0123, 0); w0 += 3;
312 vtmp012x0123 = vld3q_lane_u32(w1, vtmp012x0123, 1); w1 += 3;
313 vtmp012x0123 = vld3q_lane_u32(w2, vtmp012x0123, 2); w2 += 3;
314 vtmp012x0123 = vld3q_lane_u32(w3, vtmp012x0123, 3); w3 += 3;
315 vtmp012x4567 = vld3q_lane_u32(w4, vtmp012x4567, 0); w4 += 3;
316 vtmp012x4567 = vld3q_lane_u32(w5, vtmp012x4567, 1); w5 += 3;
317 vtmp012x4567 = vld3q_lane_u32(w6, vtmp012x4567, 2); w6 += 3;
318 vtmp012x4567 = vld3q_lane_u32(w7, vtmp012x4567, 3); w7 += 3;
319 vtmp012x89AB = vld3q_lane_u32(w8, vtmp012x89AB, 0); w8 += 3;
320 vtmp012x89AB = vld3q_lane_u32(w9, vtmp012x89AB, 1); w9 += 3;
321 vtmp012x89AB = vld3q_lane_u32(w10, vtmp012x89AB, 2); w10 += 3;
322 vtmp012x89AB = vld3q_lane_u32(w11, vtmp012x89AB, 3); w11 += 3;
323 vst1q_u32(packed_weights, vtmp012x0123.val[0]); packed_weights += 4;
324 vst1q_u32(packed_weights, vtmp012x4567.val[0]); packed_weights += 4;
325 vst1q_u32(packed_weights, vtmp012x89AB.val[0]); packed_weights += 4;
326 vst1q_u32(packed_weights, vtmp012x0123.val[1]); packed_weights += 4;
327 vst1q_u32(packed_weights, vtmp012x4567.val[1]); packed_weights += 4;
328 vst1q_u32(packed_weights, vtmp012x89AB.val[1]); packed_weights += 4;
329 vst1q_u32(packed_weights, vtmp012x0123.val[2]); packed_weights += 4;
330 vst1q_u32(packed_weights, vtmp012x4567.val[2]); packed_weights += 4;
331 vst1q_u32(packed_weights, vtmp012x89AB.val[2]); packed_weights += 4;
332 break;
333 }
334 default:
335 XNN_UNREACHABLE;
336 }
337 }
338 packed_weights = (uint32_t*) ((uintptr_t) packed_weights + extra_bytes);
339 w0 = w11;
340 }
341
342 // NC remainder (1..11)
343 if XNN_UNLIKELY(n != 0) {
344 assert(n >= 1);
345 assert(n <= 11);
346 if XNN_LIKELY(bias != NULL) {
347 size_t nb = n;
348 do {
349 *packed_weights++ = *bias++;
350 } while (--nb != 0);
351 packed_weights += (12 - n);
352 } else {
353 const uint32x4_t vzero = vmovq_n_u32(0);
354 vst1q_u32(packed_weights, vzero); packed_weights += 4;
355 vst1q_u32(packed_weights, vzero); packed_weights += 4;
356 vst1q_u32(packed_weights, vzero); packed_weights += 4;
357 }
358
359 // NR remainder has less than 12 rows so last row is not loaded
360 const uint32_t* w1 = w0 + kc;
361 if XNN_UNPREDICTABLE(n < 2) {
362 w1 = w0;
363 }
364 const uint32_t* w2 = w1 + kc;
365 if XNN_UNPREDICTABLE(n <= 2) {
366 w2 = w1;
367 }
368 const uint32_t* w3 = w2 + kc;
369 if XNN_UNPREDICTABLE(n < 4) {
370 w3 = w2;
371 }
372 const uint32_t* w4 = w3 + kc;
373 if XNN_UNPREDICTABLE(n <= 4) {
374 w4 = w3;
375 }
376 const uint32_t* w5 = w4 + kc;
377 if XNN_UNPREDICTABLE(n < 6) {
378 w5 = w4;
379 }
380 const uint32_t* w6 = w5 + kc;
381 if XNN_UNPREDICTABLE(n <= 6) {
382 w6 = w5;
383 }
384 const uint32_t* w7 = w6 + kc;
385 if XNN_UNPREDICTABLE(n < 8) {
386 w7 = w6;
387 }
388 const uint32_t* w8 = w7 + kc;
389 if XNN_UNPREDICTABLE(n <= 8) {
390 w8 = w7;
391 }
392 const uint32_t* w9 = w8 + kc;
393 if XNN_UNPREDICTABLE(n < 10) {
394 w9 = w8;
395 }
396 const uint32_t* w10 = w9 + kc;
397 if XNN_UNPREDICTABLE(n <= 10) {
398 w10 = w9;
399 }
400
401 // KC main loop multiple of 8
402 size_t k = kc;
403 for (; k >= 8; k -= 8) {
404 vtmp0123x0123 = vld4q_lane_u32(w0, vtmp0123x0123, 0); w0 += 4;
405 vtmp0123x0123 = vld4q_lane_u32(w1, vtmp0123x0123, 1); w1 += 4;
406 vtmp0123x0123 = vld4q_lane_u32(w2, vtmp0123x0123, 2); w2 += 4;
407 vtmp0123x0123 = vld4q_lane_u32(w3, vtmp0123x0123, 3); w3 += 4;
408 vtmp0123x4567 = vld4q_lane_u32(w4, vtmp0123x4567, 0); w4 += 4;
409 vtmp0123x4567 = vld4q_lane_u32(w5, vtmp0123x4567, 1); w5 += 4;
410 vtmp0123x4567 = vld4q_lane_u32(w6, vtmp0123x4567, 2); w6 += 4;
411 vtmp0123x4567 = vld4q_lane_u32(w7, vtmp0123x4567, 3); w7 += 4;
412 vtmp0123x89AB = vld4q_lane_u32(w8, vtmp0123x89AB, 0); w8 += 4;
413 vtmp0123x89AB = vld4q_lane_u32(w9, vtmp0123x89AB, 1); w9 += 4;
414 vtmp0123x89AB = vld4q_lane_u32(w10, vtmp0123x89AB, 2); w10 += 4;
415 vtmp4567x0123 = vld4q_lane_u32(w0, vtmp4567x0123, 0); w0 += 4;
416 vtmp4567x0123 = vld4q_lane_u32(w1, vtmp4567x0123, 1); w1 += 4;
417 vtmp4567x0123 = vld4q_lane_u32(w2, vtmp4567x0123, 2); w2 += 4;
418 vtmp4567x0123 = vld4q_lane_u32(w3, vtmp4567x0123, 3); w3 += 4;
419 vtmp4567x4567 = vld4q_lane_u32(w4, vtmp4567x4567, 0); w4 += 4;
420 vtmp4567x4567 = vld4q_lane_u32(w5, vtmp4567x4567, 1); w5 += 4;
421 vtmp4567x4567 = vld4q_lane_u32(w6, vtmp4567x4567, 2); w6 += 4;
422 vtmp4567x4567 = vld4q_lane_u32(w7, vtmp4567x4567, 3); w7 += 4;
423 vtmp4567x89AB = vld4q_lane_u32(w8, vtmp4567x89AB, 0); w8 += 4;
424 vtmp4567x89AB = vld4q_lane_u32(w9, vtmp4567x89AB, 1); w9 += 4;
425 vtmp4567x89AB = vld4q_lane_u32(w10, vtmp4567x89AB, 2); w10 += 4;
426 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
427 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
428 xnn_prefetch_to_l1((const int8_t*) w2 + 128);
429 xnn_prefetch_to_l1((const int8_t*) w3 + 128);
430 xnn_prefetch_to_l1((const int8_t*) w4 + 128);
431 xnn_prefetch_to_l1((const int8_t*) w5 + 128);
432 xnn_prefetch_to_l1((const int8_t*) w6 + 128);
433 xnn_prefetch_to_l1((const int8_t*) w7 + 128);
434 xnn_prefetch_to_l1((const int8_t*) w8 + 128);
435 xnn_prefetch_to_l1((const int8_t*) w9 + 128);
436 xnn_prefetch_to_l1((const int8_t*) w10 + 128);
437 vst1q_u32(packed_weights, vtmp0123x0123.val[0]); packed_weights += 4;
438 vst1q_u32(packed_weights, vtmp0123x4567.val[0]); packed_weights += 4;
439 vst1q_u32(packed_weights, vtmp0123x89AB.val[0]); packed_weights += 4;
440 vst1q_u32(packed_weights, vtmp0123x0123.val[1]); packed_weights += 4;
441 vst1q_u32(packed_weights, vtmp0123x4567.val[1]); packed_weights += 4;
442 vst1q_u32(packed_weights, vtmp0123x89AB.val[1]); packed_weights += 4;
443 vst1q_u32(packed_weights, vtmp0123x0123.val[2]); packed_weights += 4;
444 vst1q_u32(packed_weights, vtmp0123x4567.val[2]); packed_weights += 4;
445 vst1q_u32(packed_weights, vtmp0123x89AB.val[2]); packed_weights += 4;
446 vst1q_u32(packed_weights, vtmp0123x0123.val[3]); packed_weights += 4;
447 vst1q_u32(packed_weights, vtmp0123x4567.val[3]); packed_weights += 4;
448 vst1q_u32(packed_weights, vtmp0123x89AB.val[3]); packed_weights += 4;
449 vst1q_u32(packed_weights, vtmp4567x0123.val[0]); packed_weights += 4;
450 vst1q_u32(packed_weights, vtmp4567x4567.val[0]); packed_weights += 4;
451 vst1q_u32(packed_weights, vtmp4567x89AB.val[0]); packed_weights += 4;
452 vst1q_u32(packed_weights, vtmp4567x0123.val[1]); packed_weights += 4;
453 vst1q_u32(packed_weights, vtmp4567x4567.val[1]); packed_weights += 4;
454 vst1q_u32(packed_weights, vtmp4567x89AB.val[1]); packed_weights += 4;
455 vst1q_u32(packed_weights, vtmp4567x0123.val[2]); packed_weights += 4;
456 vst1q_u32(packed_weights, vtmp4567x4567.val[2]); packed_weights += 4;
457 vst1q_u32(packed_weights, vtmp4567x89AB.val[2]); packed_weights += 4;
458 vst1q_u32(packed_weights, vtmp4567x0123.val[3]); packed_weights += 4;
459 vst1q_u32(packed_weights, vtmp4567x4567.val[3]); packed_weights += 4;
460 vst1q_u32(packed_weights, vtmp4567x89AB.val[3]); packed_weights += 4;
461 }
462 // KC remainder multiple of 4
463 if (k >= 4) {
464 vtmp0123x0123 = vld4q_lane_u32(w0, vtmp0123x0123, 0); w0 += 4;
465 vtmp0123x0123 = vld4q_lane_u32(w1, vtmp0123x0123, 1); w1 += 4;
466 vtmp0123x0123 = vld4q_lane_u32(w2, vtmp0123x0123, 2); w2 += 4;
467 vtmp0123x0123 = vld4q_lane_u32(w3, vtmp0123x0123, 3); w3 += 4;
468 vtmp0123x4567 = vld4q_lane_u32(w4, vtmp0123x4567, 0); w4 += 4;
469 vtmp0123x4567 = vld4q_lane_u32(w5, vtmp0123x4567, 1); w5 += 4;
470 vtmp0123x4567 = vld4q_lane_u32(w6, vtmp0123x4567, 2); w6 += 4;
471 vtmp0123x4567 = vld4q_lane_u32(w7, vtmp0123x4567, 3); w7 += 4;
472 vtmp0123x89AB = vld4q_lane_u32(w8, vtmp0123x89AB, 0); w8 += 4;
473 vtmp0123x89AB = vld4q_lane_u32(w9, vtmp0123x89AB, 1); w9 += 4;
474 vtmp0123x89AB = vld4q_lane_u32(w10, vtmp0123x89AB, 2); w10 += 4;
475 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
476 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
477 xnn_prefetch_to_l1((const int8_t*) w2 + 128);
478 xnn_prefetch_to_l1((const int8_t*) w3 + 128);
479 xnn_prefetch_to_l1((const int8_t*) w4 + 128);
480 xnn_prefetch_to_l1((const int8_t*) w5 + 128);
481 xnn_prefetch_to_l1((const int8_t*) w6 + 128);
482 xnn_prefetch_to_l1((const int8_t*) w7 + 128);
483 xnn_prefetch_to_l1((const int8_t*) w8 + 128);
484 xnn_prefetch_to_l1((const int8_t*) w9 + 128);
485 xnn_prefetch_to_l1((const int8_t*) w10 + 128);
486 vst1q_u32(packed_weights, vtmp0123x0123.val[0]); packed_weights += 4;
487 vst1q_u32(packed_weights, vtmp0123x4567.val[0]); packed_weights += 4;
488 vst1q_u32(packed_weights, vtmp0123x89AB.val[0]); packed_weights += 4;
489 vst1q_u32(packed_weights, vtmp0123x0123.val[1]); packed_weights += 4;
490 vst1q_u32(packed_weights, vtmp0123x4567.val[1]); packed_weights += 4;
491 vst1q_u32(packed_weights, vtmp0123x89AB.val[1]); packed_weights += 4;
492 vst1q_u32(packed_weights, vtmp0123x0123.val[2]); packed_weights += 4;
493 vst1q_u32(packed_weights, vtmp0123x4567.val[2]); packed_weights += 4;
494 vst1q_u32(packed_weights, vtmp0123x89AB.val[2]); packed_weights += 4;
495 vst1q_u32(packed_weights, vtmp0123x0123.val[3]); packed_weights += 4;
496 vst1q_u32(packed_weights, vtmp0123x4567.val[3]); packed_weights += 4;
497 vst1q_u32(packed_weights, vtmp0123x89AB.val[3]); packed_weights += 4;
498 k -= 4;
499 }
500
501 // KC remainder of 1..3
502 // Same as main loop but ld1, ld2 or ld3
503 if XNN_UNLIKELY(k != 0) {
504 assert(k >= 1);
505 assert(k <= 3);
506 switch (k) {
507 // KC remainder of 1
508 case 1:
509 {
510 uint32x4_t vtmp0x0123 = vdupq_n_u32(0);
511 uint32x4_t vtmp0x4567 = vdupq_n_u32(0);
512 uint32x4_t vtmp0x89AB = vdupq_n_u32(0);
513 vtmp0x0123 = vld1q_lane_u32(w0, vtmp0x0123, 0);
514 vtmp0x0123 = vld1q_lane_u32(w1, vtmp0x0123, 1);
515 vtmp0x0123 = vld1q_lane_u32(w2, vtmp0x0123, 2);
516 vtmp0x0123 = vld1q_lane_u32(w3, vtmp0x0123, 3);
517 vtmp0x4567 = vld1q_lane_u32(w4, vtmp0x4567, 0);
518 vtmp0x4567 = vld1q_lane_u32(w5, vtmp0x4567, 1);
519 vtmp0x4567 = vld1q_lane_u32(w6, vtmp0x4567, 2);
520 vtmp0x4567 = vld1q_lane_u32(w7, vtmp0x4567, 3);
521 vtmp0x89AB = vld1q_lane_u32(w8, vtmp0x89AB, 0);
522 vtmp0x89AB = vld1q_lane_u32(w9, vtmp0x89AB, 1);
523 vtmp0x89AB = vld1q_lane_u32(w10, vtmp0x89AB, 2);
524 vst1q_u32(packed_weights, vtmp0x0123); packed_weights += 4;
525 vst1q_u32(packed_weights, vtmp0x4567); packed_weights += 4;
526 vst1q_u32(packed_weights, vtmp0x89AB); packed_weights += 4;
527 break;
528 }
529 // KC remainder of 2
530 case 2:
531 {
532 uint32x4x2_t vtmp01x0123;
533 vtmp01x0123.val[0] = vdupq_n_u32(0);
534 vtmp01x0123.val[1] = vdupq_n_u32(0);
535 uint32x4x2_t vtmp01x4567;
536 vtmp01x4567.val[0] = vdupq_n_u32(0);
537 vtmp01x4567.val[1] = vdupq_n_u32(0);
538 uint32x4x2_t vtmp01x89AB;
539 vtmp01x89AB.val[0] = vdupq_n_u32(0);
540 vtmp01x89AB.val[1] = vdupq_n_u32(0);
541 vtmp01x0123 = vld2q_lane_u32(w0, vtmp01x0123, 0);
542 vtmp01x0123 = vld2q_lane_u32(w1, vtmp01x0123, 1);
543 vtmp01x0123 = vld2q_lane_u32(w2, vtmp01x0123, 2);
544 vtmp01x0123 = vld2q_lane_u32(w3, vtmp01x0123, 3);
545 vtmp01x4567 = vld2q_lane_u32(w4, vtmp01x4567, 0);
546 vtmp01x4567 = vld2q_lane_u32(w5, vtmp01x4567, 1);
547 vtmp01x4567 = vld2q_lane_u32(w6, vtmp01x4567, 2);
548 vtmp01x4567 = vld2q_lane_u32(w7, vtmp01x4567, 3);
549 vtmp01x89AB = vld2q_lane_u32(w8, vtmp01x89AB, 0);
550 vtmp01x89AB = vld2q_lane_u32(w9, vtmp01x89AB, 1);
551 vtmp01x89AB = vld2q_lane_u32(w10, vtmp01x89AB, 2);
552 vst1q_u32(packed_weights, vtmp01x0123.val[0]); packed_weights += 4;
553 vst1q_u32(packed_weights, vtmp01x4567.val[0]); packed_weights += 4;
554 vst1q_u32(packed_weights, vtmp01x89AB.val[0]); packed_weights += 4;
555 vst1q_u32(packed_weights, vtmp01x0123.val[1]); packed_weights += 4;
556 vst1q_u32(packed_weights, vtmp01x4567.val[1]); packed_weights += 4;
557 vst1q_u32(packed_weights, vtmp01x89AB.val[1]); packed_weights += 4;
558 break;
559 }
560 // KC remainder of 3
561 case 3:
562 {
563 uint32x4x3_t vtmp012x0123;
564 vtmp012x0123.val[0] = vdupq_n_u32(0);
565 vtmp012x0123.val[1] = vdupq_n_u32(0);
566 vtmp012x0123.val[2] = vdupq_n_u32(0);
567 uint32x4x3_t vtmp012x4567;
568 vtmp012x4567.val[0] = vdupq_n_u32(0);
569 vtmp012x4567.val[1] = vdupq_n_u32(0);
570 vtmp012x4567.val[2] = vdupq_n_u32(0);
571 uint32x4x3_t vtmp012x89AB;
572 vtmp012x89AB.val[0] = vdupq_n_u32(0);
573 vtmp012x89AB.val[1] = vdupq_n_u32(0);
574 vtmp012x89AB.val[2] = vdupq_n_u32(0);
575 vtmp012x0123 = vld3q_lane_u32(w0, vtmp012x0123, 0);
576 vtmp012x0123 = vld3q_lane_u32(w1, vtmp012x0123, 1);
577 vtmp012x0123 = vld3q_lane_u32(w2, vtmp012x0123, 2);
578 vtmp012x0123 = vld3q_lane_u32(w3, vtmp012x0123, 3);
579 vtmp012x4567 = vld3q_lane_u32(w4, vtmp012x4567, 0);
580 vtmp012x4567 = vld3q_lane_u32(w5, vtmp012x4567, 1);
581 vtmp012x4567 = vld3q_lane_u32(w6, vtmp012x4567, 2);
582 vtmp012x4567 = vld3q_lane_u32(w7, vtmp012x4567, 3);
583 vtmp012x89AB = vld3q_lane_u32(w8, vtmp012x89AB, 0);
584 vtmp012x89AB = vld3q_lane_u32(w9, vtmp012x89AB, 1);
585 vtmp012x89AB = vld3q_lane_u32(w10, vtmp012x89AB, 2);
586 vst1q_u32(packed_weights, vtmp012x0123.val[0]); packed_weights += 4;
587 vst1q_u32(packed_weights, vtmp012x4567.val[0]); packed_weights += 4;
588 vst1q_u32(packed_weights, vtmp012x89AB.val[0]); packed_weights += 4;
589 vst1q_u32(packed_weights, vtmp012x0123.val[1]); packed_weights += 4;
590 vst1q_u32(packed_weights, vtmp012x4567.val[1]); packed_weights += 4;
591 vst1q_u32(packed_weights, vtmp012x89AB.val[1]); packed_weights += 4;
592 vst1q_u32(packed_weights, vtmp012x0123.val[2]); packed_weights += 4;
593 vst1q_u32(packed_weights, vtmp012x4567.val[2]); packed_weights += 4;
594 vst1q_u32(packed_weights, vtmp012x89AB.val[2]); packed_weights += 4;
595 break;
596 }
597 default:
598 XNN_UNREACHABLE;
599 }
600 }
601 packed_weights = (uint32_t*) ((uintptr_t) packed_weights + extra_bytes);
602 }
603
604 weights += nc * kc;
605 } while (--g != 0);
606 }