Debian Codegraph @

src/x32-packw/gen/x32-packw-x8-gemm-goi-neon-ld4lane-u4-prfm.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/neon.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include <arm_neon.h>
16
17 #include "xnnpack/packw.h"
18 #include "xnnpack/prefetch.h"
19
20
21 void xnn_x32_packw_gemm_goi_ukernel_x8__neon_ld4lane_u4_prfm(
22 size_t g,
23 size_t nc,
24 size_t kc,
25 size_t nr,
26 size_t kr,
27 size_t sr,
28 const uint32_t* weights,
29 const uint32_t* bias,
30 const void* scale,
31 uint32_t* packed_weights,
32 size_t extra_bytes,
33 const void* params)
34 {
35 assert(g != 0);
36 assert(nc != 0);
37 assert(kc != 0);
38 assert(nr == 8);
39 assert(kr == 1);
40 assert(sr == 1);
41 assert(weights != NULL);
42 assert(packed_weights != NULL);
43 uint32x4x4_t vtmp0123x0123;
44 vtmp0123x0123.val[0] = vdupq_n_u32(0);
45 vtmp0123x0123.val[1] = vdupq_n_u32(0);
46 vtmp0123x0123.val[2] = vdupq_n_u32(0);
47 vtmp0123x0123.val[3] = vdupq_n_u32(0);
48 uint32x4x4_t vtmp0123x4567;
49 vtmp0123x4567.val[0] = vdupq_n_u32(0);
50 vtmp0123x4567.val[1] = vdupq_n_u32(0);
51 vtmp0123x4567.val[2] = vdupq_n_u32(0);
52 vtmp0123x4567.val[3] = vdupq_n_u32(0);
53
54 do {
55 // NC main loop multiple of 8
56 const uint32_t* w0 = weights;
57 size_t n = nc;
58
59 for (; n >= 8; n -= 8) {
60 if XNN_LIKELY(bias != NULL) {
61 uint32x4_t vb0 = vld1q_u32(bias); bias += 4;
62 uint32x4_t vb4 = vld1q_u32(bias); bias += 4;
63 vst1q_u32(packed_weights, vb0); packed_weights += 4;
64 vst1q_u32(packed_weights, vb4); packed_weights += 4;
65 } else {
66 const uint32x4_t vzero = vmovq_n_u32(0);
67 vst1q_u32(packed_weights, vzero); packed_weights += 4;
68 vst1q_u32(packed_weights, vzero); packed_weights += 4;
69 }
70
71 const uint32_t* w1 = w0 + kc;
72 const uint32_t* w2 = w1 + kc;
73 const uint32_t* w3 = w2 + kc;
74 const uint32_t* w4 = w3 + kc;
75 const uint32_t* w5 = w4 + kc;
76 const uint32_t* w6 = w5 + kc;
77 const uint32_t* w7 = w6 + kc;
78 xnn_prefetch_to_l1((const int8_t*) w0);
79 xnn_prefetch_to_l1((const int8_t*) w0 + 64);
80 xnn_prefetch_to_l1((const int8_t*) w1);
81 xnn_prefetch_to_l1((const int8_t*) w1 + 64);
82 xnn_prefetch_to_l1((const int8_t*) w2);
83 xnn_prefetch_to_l1((const int8_t*) w2 + 64);
84 xnn_prefetch_to_l1((const int8_t*) w3);
85 xnn_prefetch_to_l1((const int8_t*) w3 + 64);
86 xnn_prefetch_to_l1((const int8_t*) w4);
87 xnn_prefetch_to_l1((const int8_t*) w4 + 64);
88 xnn_prefetch_to_l1((const int8_t*) w5);
89 xnn_prefetch_to_l1((const int8_t*) w5 + 64);
90 xnn_prefetch_to_l1((const int8_t*) w6);
91 xnn_prefetch_to_l1((const int8_t*) w6 + 64);
92 xnn_prefetch_to_l1((const int8_t*) w7);
93 xnn_prefetch_to_l1((const int8_t*) w7 + 64);
94
95 // KC main loop multiple of 4
96 size_t k = kc;
97 for (; k >= 4; k -= 4) {
98 vtmp0123x0123 = vld4q_lane_u32(w0, vtmp0123x0123, 0); w0 += 4;
99 vtmp0123x0123 = vld4q_lane_u32(w1, vtmp0123x0123, 1); w1 += 4;
100 vtmp0123x0123 = vld4q_lane_u32(w2, vtmp0123x0123, 2); w2 += 4;
101 vtmp0123x0123 = vld4q_lane_u32(w3, vtmp0123x0123, 3); w3 += 4;
102 vtmp0123x4567 = vld4q_lane_u32(w4, vtmp0123x4567, 0); w4 += 4;
103 vtmp0123x4567 = vld4q_lane_u32(w5, vtmp0123x4567, 1); w5 += 4;
104 vtmp0123x4567 = vld4q_lane_u32(w6, vtmp0123x4567, 2); w6 += 4;
105 vtmp0123x4567 = vld4q_lane_u32(w7, vtmp0123x4567, 3); w7 += 4;
106 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
107 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
108 xnn_prefetch_to_l1((const int8_t*) w2 + 128);
109 xnn_prefetch_to_l1((const int8_t*) w3 + 128);
110 xnn_prefetch_to_l1((const int8_t*) w4 + 128);
111 xnn_prefetch_to_l1((const int8_t*) w5 + 128);
112 xnn_prefetch_to_l1((const int8_t*) w6 + 128);
113 xnn_prefetch_to_l1((const int8_t*) w7 + 128);
114 vst1q_u32(packed_weights, vtmp0123x0123.val[0]); packed_weights += 4;
115 vst1q_u32(packed_weights, vtmp0123x4567.val[0]); packed_weights += 4;
116 vst1q_u32(packed_weights, vtmp0123x0123.val[1]); packed_weights += 4;
117 vst1q_u32(packed_weights, vtmp0123x4567.val[1]); packed_weights += 4;
118 vst1q_u32(packed_weights, vtmp0123x0123.val[2]); packed_weights += 4;
119 vst1q_u32(packed_weights, vtmp0123x4567.val[2]); packed_weights += 4;
120 vst1q_u32(packed_weights, vtmp0123x0123.val[3]); packed_weights += 4;
121 vst1q_u32(packed_weights, vtmp0123x4567.val[3]); packed_weights += 4;
122 }
123
124 // KC remainder of 1..3
125 // Same as main loop but ld1, ld2 or ld3
126 if XNN_UNLIKELY(k != 0) {
127 assert(k >= 1);
128 assert(k <= 3);
129 switch (k) {
130 // KC remainder of 1
131 case 1:
132 {
133 uint32x4_t vtmp0x0123 = vdupq_n_u32(0);
134 uint32x4_t vtmp0x4567 = vdupq_n_u32(0);
135 vtmp0x0123 = vld1q_lane_u32(w0, vtmp0x0123, 0); w0 += 1;
136 vtmp0x0123 = vld1q_lane_u32(w1, vtmp0x0123, 1); w1 += 1;
137 vtmp0x0123 = vld1q_lane_u32(w2, vtmp0x0123, 2); w2 += 1;
138 vtmp0x0123 = vld1q_lane_u32(w3, vtmp0x0123, 3); w3 += 1;
139 vtmp0x4567 = vld1q_lane_u32(w4, vtmp0x4567, 0); w4 += 1;
140 vtmp0x4567 = vld1q_lane_u32(w5, vtmp0x4567, 1); w5 += 1;
141 vtmp0x4567 = vld1q_lane_u32(w6, vtmp0x4567, 2); w6 += 1;
142 vtmp0x4567 = vld1q_lane_u32(w7, vtmp0x4567, 3); w7 += 1;
143 vst1q_u32(packed_weights, vtmp0x0123); packed_weights += 4;
144 vst1q_u32(packed_weights, vtmp0x4567); packed_weights += 4;
145 break;
146 }
147 // KC remainder of 2
148 case 2:
149 {
150 uint32x4x2_t vtmp01x0123;
151 vtmp01x0123.val[0] = vdupq_n_u32(0);
152 vtmp01x0123.val[1] = vdupq_n_u32(0);
153 uint32x4x2_t vtmp01x4567;
154 vtmp01x4567.val[0] = vdupq_n_u32(0);
155 vtmp01x4567.val[1] = vdupq_n_u32(0);
156 vtmp01x0123 = vld2q_lane_u32(w0, vtmp01x0123, 0); w0 += 2;
157 vtmp01x0123 = vld2q_lane_u32(w1, vtmp01x0123, 1); w1 += 2;
158 vtmp01x0123 = vld2q_lane_u32(w2, vtmp01x0123, 2); w2 += 2;
159 vtmp01x0123 = vld2q_lane_u32(w3, vtmp01x0123, 3); w3 += 2;
160 vtmp01x4567 = vld2q_lane_u32(w4, vtmp01x4567, 0); w4 += 2;
161 vtmp01x4567 = vld2q_lane_u32(w5, vtmp01x4567, 1); w5 += 2;
162 vtmp01x4567 = vld2q_lane_u32(w6, vtmp01x4567, 2); w6 += 2;
163 vtmp01x4567 = vld2q_lane_u32(w7, vtmp01x4567, 3); w7 += 2;
164 vst1q_u32(packed_weights, vtmp01x0123.val[0]); packed_weights += 4;
165 vst1q_u32(packed_weights, vtmp01x4567.val[0]); packed_weights += 4;
166 vst1q_u32(packed_weights, vtmp01x0123.val[1]); packed_weights += 4;
167 vst1q_u32(packed_weights, vtmp01x4567.val[1]); packed_weights += 4;
168 break;
169 }
170 // KC remainder of 3
171 case 3:
172 {
173 uint32x4x3_t vtmp012x0123;
174 vtmp012x0123.val[0] = vdupq_n_u32(0);
175 vtmp012x0123.val[1] = vdupq_n_u32(0);
176 vtmp012x0123.val[2] = vdupq_n_u32(0);
177 uint32x4x3_t vtmp012x4567;
178 vtmp012x4567.val[0] = vdupq_n_u32(0);
179 vtmp012x4567.val[1] = vdupq_n_u32(0);
180 vtmp012x4567.val[2] = vdupq_n_u32(0);
181 vtmp012x0123 = vld3q_lane_u32(w0, vtmp012x0123, 0); w0 += 3;
182 vtmp012x0123 = vld3q_lane_u32(w1, vtmp012x0123, 1); w1 += 3;
183 vtmp012x0123 = vld3q_lane_u32(w2, vtmp012x0123, 2); w2 += 3;
184 vtmp012x0123 = vld3q_lane_u32(w3, vtmp012x0123, 3); w3 += 3;
185 vtmp012x4567 = vld3q_lane_u32(w4, vtmp012x4567, 0); w4 += 3;
186 vtmp012x4567 = vld3q_lane_u32(w5, vtmp012x4567, 1); w5 += 3;
187 vtmp012x4567 = vld3q_lane_u32(w6, vtmp012x4567, 2); w6 += 3;
188 vtmp012x4567 = vld3q_lane_u32(w7, vtmp012x4567, 3); w7 += 3;
189 vst1q_u32(packed_weights, vtmp012x0123.val[0]); packed_weights += 4;
190 vst1q_u32(packed_weights, vtmp012x4567.val[0]); packed_weights += 4;
191 vst1q_u32(packed_weights, vtmp012x0123.val[1]); packed_weights += 4;
192 vst1q_u32(packed_weights, vtmp012x4567.val[1]); packed_weights += 4;
193 vst1q_u32(packed_weights, vtmp012x0123.val[2]); packed_weights += 4;
194 vst1q_u32(packed_weights, vtmp012x4567.val[2]); packed_weights += 4;
195 break;
196 }
197 default:
198 XNN_UNREACHABLE;
199 }
200 }
201 packed_weights = (uint32_t*) ((uintptr_t) packed_weights + extra_bytes);
202 w0 = w7;
203 }
204
205 // NC remainder (1..7)
206 if XNN_UNLIKELY(n != 0) {
207 assert(n >= 1);
208 assert(n <= 7);
209 if XNN_LIKELY(bias != NULL) {
210 size_t nb = n;
211 do {
212 *packed_weights++ = *bias++;
213 } while (--nb != 0);
214 packed_weights += (8 - n);
215 } else {
216 const uint32x4_t vzero = vmovq_n_u32(0);
217 vst1q_u32(packed_weights, vzero); packed_weights += 4;
218 vst1q_u32(packed_weights, vzero); packed_weights += 4;
219 }
220
221 // NR remainder has less than 8 rows so last row is not loaded
222 const uint32_t* w1 = w0 + kc;
223 if XNN_UNPREDICTABLE(n < 2) {
224 w1 = w0;
225 }
226 const uint32_t* w2 = w1 + kc;
227 if XNN_UNPREDICTABLE(n <= 2) {
228 w2 = w1;
229 }
230 const uint32_t* w3 = w2 + kc;
231 if XNN_UNPREDICTABLE(n < 4) {
232 w3 = w2;
233 }
234 const uint32_t* w4 = w3 + kc;
235 if XNN_UNPREDICTABLE(n <= 4) {
236 w4 = w3;
237 }
238 const uint32_t* w5 = w4 + kc;
239 if XNN_UNPREDICTABLE(n < 6) {
240 w5 = w4;
241 }
242 const uint32_t* w6 = w5 + kc;
243 if XNN_UNPREDICTABLE(n <= 6) {
244 w6 = w5;
245 }
246
247 // KC main loop multiple of 4
248 size_t k = kc;
249 for (; k >= 4; k -= 4) {
250 vtmp0123x0123 = vld4q_lane_u32(w0, vtmp0123x0123, 0); w0 += 4;
251 vtmp0123x0123 = vld4q_lane_u32(w1, vtmp0123x0123, 1); w1 += 4;
252 vtmp0123x0123 = vld4q_lane_u32(w2, vtmp0123x0123, 2); w2 += 4;
253 vtmp0123x0123 = vld4q_lane_u32(w3, vtmp0123x0123, 3); w3 += 4;
254 vtmp0123x4567 = vld4q_lane_u32(w4, vtmp0123x4567, 0); w4 += 4;
255 vtmp0123x4567 = vld4q_lane_u32(w5, vtmp0123x4567, 1); w5 += 4;
256 vtmp0123x4567 = vld4q_lane_u32(w6, vtmp0123x4567, 2); w6 += 4;
257 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
258 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
259 xnn_prefetch_to_l1((const int8_t*) w2 + 128);
260 xnn_prefetch_to_l1((const int8_t*) w3 + 128);
261 xnn_prefetch_to_l1((const int8_t*) w4 + 128);
262 xnn_prefetch_to_l1((const int8_t*) w5 + 128);
263 xnn_prefetch_to_l1((const int8_t*) w6 + 128);
264 vst1q_u32(packed_weights, vtmp0123x0123.val[0]); packed_weights += 4;
265 vst1q_u32(packed_weights, vtmp0123x4567.val[0]); packed_weights += 4;
266 vst1q_u32(packed_weights, vtmp0123x0123.val[1]); packed_weights += 4;
267 vst1q_u32(packed_weights, vtmp0123x4567.val[1]); packed_weights += 4;
268 vst1q_u32(packed_weights, vtmp0123x0123.val[2]); packed_weights += 4;
269 vst1q_u32(packed_weights, vtmp0123x4567.val[2]); packed_weights += 4;
270 vst1q_u32(packed_weights, vtmp0123x0123.val[3]); packed_weights += 4;
271 vst1q_u32(packed_weights, vtmp0123x4567.val[3]); packed_weights += 4;
272 }
273
274 // KC remainder of 1..3
275 // Same as main loop but ld1, ld2 or ld3
276 if XNN_UNLIKELY(k != 0) {
277 assert(k >= 1);
278 assert(k <= 3);
279 switch (k) {
280 // KC remainder of 1
281 case 1:
282 {
283 uint32x4_t vtmp0x0123 = vdupq_n_u32(0);
284 uint32x4_t vtmp0x4567 = vdupq_n_u32(0);
285 vtmp0x0123 = vld1q_lane_u32(w0, vtmp0x0123, 0);
286 vtmp0x0123 = vld1q_lane_u32(w1, vtmp0x0123, 1);
287 vtmp0x0123 = vld1q_lane_u32(w2, vtmp0x0123, 2);
288 vtmp0x0123 = vld1q_lane_u32(w3, vtmp0x0123, 3);
289 vtmp0x4567 = vld1q_lane_u32(w4, vtmp0x4567, 0);
290 vtmp0x4567 = vld1q_lane_u32(w5, vtmp0x4567, 1);
291 vtmp0x4567 = vld1q_lane_u32(w6, vtmp0x4567, 2);
292 vst1q_u32(packed_weights, vtmp0x0123); packed_weights += 4;
293 vst1q_u32(packed_weights, vtmp0x4567); packed_weights += 4;
294 break;
295 }
296 // KC remainder of 2
297 case 2:
298 {
299 uint32x4x2_t vtmp01x0123;
300 vtmp01x0123.val[0] = vdupq_n_u32(0);
301 vtmp01x0123.val[1] = vdupq_n_u32(0);
302 uint32x4x2_t vtmp01x4567;
303 vtmp01x4567.val[0] = vdupq_n_u32(0);
304 vtmp01x4567.val[1] = vdupq_n_u32(0);
305 vtmp01x0123 = vld2q_lane_u32(w0, vtmp01x0123, 0);
306 vtmp01x0123 = vld2q_lane_u32(w1, vtmp01x0123, 1);
307 vtmp01x0123 = vld2q_lane_u32(w2, vtmp01x0123, 2);
308 vtmp01x0123 = vld2q_lane_u32(w3, vtmp01x0123, 3);
309 vtmp01x4567 = vld2q_lane_u32(w4, vtmp01x4567, 0);
310 vtmp01x4567 = vld2q_lane_u32(w5, vtmp01x4567, 1);
311 vtmp01x4567 = vld2q_lane_u32(w6, vtmp01x4567, 2);
312 vst1q_u32(packed_weights, vtmp01x0123.val[0]); packed_weights += 4;
313 vst1q_u32(packed_weights, vtmp01x4567.val[0]); packed_weights += 4;
314 vst1q_u32(packed_weights, vtmp01x0123.val[1]); packed_weights += 4;
315 vst1q_u32(packed_weights, vtmp01x4567.val[1]); packed_weights += 4;
316 break;
317 }
318 // KC remainder of 3
319 case 3:
320 {
321 uint32x4x3_t vtmp012x0123;
322 vtmp012x0123.val[0] = vdupq_n_u32(0);
323 vtmp012x0123.val[1] = vdupq_n_u32(0);
324 vtmp012x0123.val[2] = vdupq_n_u32(0);
325 uint32x4x3_t vtmp012x4567;
326 vtmp012x4567.val[0] = vdupq_n_u32(0);
327 vtmp012x4567.val[1] = vdupq_n_u32(0);
328 vtmp012x4567.val[2] = vdupq_n_u32(0);
329 vtmp012x0123 = vld3q_lane_u32(w0, vtmp012x0123, 0);
330 vtmp012x0123 = vld3q_lane_u32(w1, vtmp012x0123, 1);
331 vtmp012x0123 = vld3q_lane_u32(w2, vtmp012x0123, 2);
332 vtmp012x0123 = vld3q_lane_u32(w3, vtmp012x0123, 3);
333 vtmp012x4567 = vld3q_lane_u32(w4, vtmp012x4567, 0);
334 vtmp012x4567 = vld3q_lane_u32(w5, vtmp012x4567, 1);
335 vtmp012x4567 = vld3q_lane_u32(w6, vtmp012x4567, 2);
336 vst1q_u32(packed_weights, vtmp012x0123.val[0]); packed_weights += 4;
337 vst1q_u32(packed_weights, vtmp012x4567.val[0]); packed_weights += 4;
338 vst1q_u32(packed_weights, vtmp012x0123.val[1]); packed_weights += 4;
339 vst1q_u32(packed_weights, vtmp012x4567.val[1]); packed_weights += 4;
340 vst1q_u32(packed_weights, vtmp012x0123.val[2]); packed_weights += 4;
341 vst1q_u32(packed_weights, vtmp012x4567.val[2]); packed_weights += 4;
342 break;
343 }
344 default:
345 XNN_UNREACHABLE;
346 }
347 }
348 packed_weights = (uint32_t*) ((uintptr_t) packed_weights + extra_bytes);
349 }
350
351 weights += nc * kc;
352 } while (--g != 0);
353 }