Debian Codegraph @

src/x32-packw/gen/x32-packw-x16-gemm-goi-scalar-int-u4.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/scalar.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include "xnnpack/math.h"
16 #include "xnnpack/packw.h"
17
18
19
20 void xnn_x32_packw_gemm_goi_ukernel_x16__scalar_int_u4(
21 size_t g,
22 size_t nc,
23 size_t kc,
24 size_t nr,
25 size_t kr,
26 size_t sr,
27 const uint32_t* weights,
28 const uint32_t* bias,
29 const void* scale,
30 uint32_t* packed_weights,
31 size_t extra_bytes,
32 const void* params)
33 {
34 assert(g != 0);
35 assert(nc != 0);
36 assert(kc != 0);
37 assert(nr == 16);
38 assert(kr == 1);
39 assert(sr == 1);
40 assert(weights != NULL);
41 assert(packed_weights != NULL);
42
43 uint32_t* out = (uint32_t*) packed_weights;
44 const uint32_t* b = (const uint32_t*) bias;
45
46 do {
47 // NC main loop multiple of 16
48 const uint32_t* w0 = (const uint32_t*) weights;
49 size_t n = nc;
50 for (;n >= 16; n -= 16) {
51 if XNN_LIKELY(b != NULL) {
52 out[0] = b[0];
53 out[1] = b[1];
54 out[2] = b[2];
55 out[3] = b[3];
56 out[4] = b[4];
57 out[5] = b[5];
58 out[6] = b[6];
59 out[7] = b[7];
60 out[8] = b[8];
61 out[9] = b[9];
62 out[10] = b[10];
63 out[11] = b[11];
64 out[12] = b[12];
65 out[13] = b[13];
66 out[14] = b[14];
67 out[15] = b[15];
68 b += 16;
69 } else {
70 out[0] = 0;
71 out[1] = 0;
72 out[2] = 0;
73 out[3] = 0;
74 out[4] = 0;
75 out[5] = 0;
76 out[6] = 0;
77 out[7] = 0;
78 out[8] = 0;
79 out[9] = 0;
80 out[10] = 0;
81 out[11] = 0;
82 out[12] = 0;
83 out[13] = 0;
84 out[14] = 0;
85 out[15] = 0;
86 }
87 out += 16;
88
89 const uint32_t* w1 = w0 + kc;
90 const uint32_t* w2 = w1 + kc;
91 const uint32_t* w3 = w2 + kc;
92 const uint32_t* w4 = w3 + kc;
93 const uint32_t* w5 = w4 + kc;
94 const uint32_t* w6 = w5 + kc;
95 const uint32_t* w7 = w6 + kc;
96 const uint32_t* w8 = w7 + kc;
97 const uint32_t* w9 = w8 + kc;
98 const uint32_t* w10 = w9 + kc;
99 const uint32_t* w11 = w10 + kc;
100 const uint32_t* w12 = w11 + kc;
101 const uint32_t* w13 = w12 + kc;
102 const uint32_t* w14 = w13 + kc;
103 const uint32_t* w15 = w14 + kc;
104
105 // KC main loop multiple of 16x4
106 size_t k = kc;
107 for (; k >= 4; k -= 4) {
108 const uint32_t v00 = w0[0];
109 const uint32_t v01 = w0[1];
110 const uint32_t v02 = w0[2];
111 const uint32_t v03 = w0[3];
112 w0 += 4;
113 const uint32_t v10 = w1[0];
114 const uint32_t v11 = w1[1];
115 const uint32_t v12 = w1[2];
116 const uint32_t v13 = w1[3];
117 w1 += 4;
118 const uint32_t v20 = w2[0];
119 const uint32_t v21 = w2[1];
120 const uint32_t v22 = w2[2];
121 const uint32_t v23 = w2[3];
122 w2 += 4;
123 const uint32_t v30 = w3[0];
124 const uint32_t v31 = w3[1];
125 const uint32_t v32 = w3[2];
126 const uint32_t v33 = w3[3];
127 w3 += 4;
128 const uint32_t v40 = w4[0];
129 const uint32_t v41 = w4[1];
130 const uint32_t v42 = w4[2];
131 const uint32_t v43 = w4[3];
132 w4 += 4;
133 const uint32_t v50 = w5[0];
134 const uint32_t v51 = w5[1];
135 const uint32_t v52 = w5[2];
136 const uint32_t v53 = w5[3];
137 w5 += 4;
138 const uint32_t v60 = w6[0];
139 const uint32_t v61 = w6[1];
140 const uint32_t v62 = w6[2];
141 const uint32_t v63 = w6[3];
142 w6 += 4;
143 const uint32_t v70 = w7[0];
144 const uint32_t v71 = w7[1];
145 const uint32_t v72 = w7[2];
146 const uint32_t v73 = w7[3];
147 w7 += 4;
148 const uint32_t v80 = w8[0];
149 const uint32_t v81 = w8[1];
150 const uint32_t v82 = w8[2];
151 const uint32_t v83 = w8[3];
152 w8 += 4;
153 const uint32_t v90 = w9[0];
154 const uint32_t v91 = w9[1];
155 const uint32_t v92 = w9[2];
156 const uint32_t v93 = w9[3];
157 w9 += 4;
158 const uint32_t v100 = w10[0];
159 const uint32_t v101 = w10[1];
160 const uint32_t v102 = w10[2];
161 const uint32_t v103 = w10[3];
162 w10 += 4;
163 const uint32_t v110 = w11[0];
164 const uint32_t v111 = w11[1];
165 const uint32_t v112 = w11[2];
166 const uint32_t v113 = w11[3];
167 w11 += 4;
168 const uint32_t v120 = w12[0];
169 const uint32_t v121 = w12[1];
170 const uint32_t v122 = w12[2];
171 const uint32_t v123 = w12[3];
172 w12 += 4;
173 const uint32_t v130 = w13[0];
174 const uint32_t v131 = w13[1];
175 const uint32_t v132 = w13[2];
176 const uint32_t v133 = w13[3];
177 w13 += 4;
178 const uint32_t v140 = w14[0];
179 const uint32_t v141 = w14[1];
180 const uint32_t v142 = w14[2];
181 const uint32_t v143 = w14[3];
182 w14 += 4;
183 const uint32_t v150 = w15[0];
184 const uint32_t v151 = w15[1];
185 const uint32_t v152 = w15[2];
186 const uint32_t v153 = w15[3];
187 w15 += 4;
188 out[0] = v00;
189 out[1] = v10;
190 out[2] = v20;
191 out[3] = v30;
192 out[4] = v40;
193 out[5] = v50;
194 out[6] = v60;
195 out[7] = v70;
196 out[8] = v80;
197 out[9] = v90;
198 out[10] = v100;
199 out[11] = v110;
200 out[12] = v120;
201 out[13] = v130;
202 out[14] = v140;
203 out[15] = v150;
204 out[16] = v01;
205 out[17] = v11;
206 out[18] = v21;
207 out[19] = v31;
208 out[20] = v41;
209 out[21] = v51;
210 out[22] = v61;
211 out[23] = v71;
212 out[24] = v81;
213 out[25] = v91;
214 out[26] = v101;
215 out[27] = v111;
216 out[28] = v121;
217 out[29] = v131;
218 out[30] = v141;
219 out[31] = v151;
220 out[32] = v02;
221 out[33] = v12;
222 out[34] = v22;
223 out[35] = v32;
224 out[36] = v42;
225 out[37] = v52;
226 out[38] = v62;
227 out[39] = v72;
228 out[40] = v82;
229 out[41] = v92;
230 out[42] = v102;
231 out[43] = v112;
232 out[44] = v122;
233 out[45] = v132;
234 out[46] = v142;
235 out[47] = v152;
236 out[48] = v03;
237 out[49] = v13;
238 out[50] = v23;
239 out[51] = v33;
240 out[52] = v43;
241 out[53] = v53;
242 out[54] = v63;
243 out[55] = v73;
244 out[56] = v83;
245 out[57] = v93;
246 out[58] = v103;
247 out[59] = v113;
248 out[60] = v123;
249 out[61] = v133;
250 out[62] = v143;
251 out[63] = v153;
252 out += 64;
253 }
254
255 // KC remainder
256 for (; k != 0; --k) {
257 const uint32_t v0 = *w0++;
258 out[0] = v0;
259 const uint32_t v1 = *w1++;
260 out[1] = v1;
261 const uint32_t v2 = *w2++;
262 out[2] = v2;
263 const uint32_t v3 = *w3++;
264 out[3] = v3;
265 const uint32_t v4 = *w4++;
266 out[4] = v4;
267 const uint32_t v5 = *w5++;
268 out[5] = v5;
269 const uint32_t v6 = *w6++;
270 out[6] = v6;
271 const uint32_t v7 = *w7++;
272 out[7] = v7;
273 const uint32_t v8 = *w8++;
274 out[8] = v8;
275 const uint32_t v9 = *w9++;
276 out[9] = v9;
277 const uint32_t v10 = *w10++;
278 out[10] = v10;
279 const uint32_t v11 = *w11++;
280 out[11] = v11;
281 const uint32_t v12 = *w12++;
282 out[12] = v12;
283 const uint32_t v13 = *w13++;
284 out[13] = v13;
285 const uint32_t v14 = *w14++;
286 out[14] = v14;
287 const uint32_t v15 = *w15++;
288 out[15] = v15;
289 out += 16;
290 }
291 out = (uint32_t*) ((uintptr_t) out + extra_bytes);
292 w0 = w15;
293 }
294
295 // NC remainder (1..15)
296 if XNN_UNLIKELY(n != 0) {
297 if XNN_LIKELY(b != NULL) {
298 size_t nb = n;
299 do {
300 *out++ = *b++;
301 } while (--nb != 0);
302 } else {
303 size_t nb = n;
304 do {
305 *out++ = 0;
306 } while (--nb != 0);
307 }
308 out += (16 - n);
309
310 // NR remainder has less than 16 rows so last row is not loaded
311 const uint32_t* w1 = w0 + kc;
312 if XNN_UNPREDICTABLE(n < 2) {
313 w1 = w0;
314 }
315 const uint32_t* w2 = w1 + kc;
316 if XNN_UNPREDICTABLE(n <= 2) {
317 w2 = w1;
318 }
319 const uint32_t* w3 = w2 + kc;
320 if XNN_UNPREDICTABLE(n < 4) {
321 w3 = w2;
322 }
323 const uint32_t* w4 = w3 + kc;
324 if XNN_UNPREDICTABLE(n <= 4) {
325 w4 = w3;
326 }
327 const uint32_t* w5 = w4 + kc;
328 if XNN_UNPREDICTABLE(n < 6) {
329 w5 = w4;
330 }
331 const uint32_t* w6 = w5 + kc;
332 if XNN_UNPREDICTABLE(n <= 6) {
333 w6 = w5;
334 }
335 const uint32_t* w7 = w6 + kc;
336 if XNN_UNPREDICTABLE(n < 8) {
337 w7 = w6;
338 }
339 const uint32_t* w8 = w7 + kc;
340 if XNN_UNPREDICTABLE(n <= 8) {
341 w8 = w7;
342 }
343 const uint32_t* w9 = w8 + kc;
344 if XNN_UNPREDICTABLE(n < 10) {
345 w9 = w8;
346 }
347 const uint32_t* w10 = w9 + kc;
348 if XNN_UNPREDICTABLE(n <= 10) {
349 w10 = w9;
350 }
351 const uint32_t* w11 = w10 + kc;
352 if XNN_UNPREDICTABLE(n < 12) {
353 w11 = w10;
354 }
355 const uint32_t* w12 = w11 + kc;
356 if XNN_UNPREDICTABLE(n <= 12) {
357 w12 = w11;
358 }
359 const uint32_t* w13 = w12 + kc;
360 if XNN_UNPREDICTABLE(n < 14) {
361 w13 = w12;
362 }
363 const uint32_t* w14 = w13 + kc;
364 if XNN_UNPREDICTABLE(n <= 14) {
365 w14 = w13;
366 }
367
368 // KC main loop multiple of 16x4
369 size_t k = kc;
370 for (; k >= 4; k -= 4) {
371 const uint32_t v00 = w0[0];
372 const uint32_t v01 = w0[1];
373 const uint32_t v02 = w0[2];
374 const uint32_t v03 = w0[3];
375 w0 += 4;
376 const uint32_t v10 = w1[0];
377 const uint32_t v11 = w1[1];
378 const uint32_t v12 = w1[2];
379 const uint32_t v13 = w1[3];
380 w1 += 4;
381 const uint32_t v20 = w2[0];
382 const uint32_t v21 = w2[1];
383 const uint32_t v22 = w2[2];
384 const uint32_t v23 = w2[3];
385 w2 += 4;
386 const uint32_t v30 = w3[0];
387 const uint32_t v31 = w3[1];
388 const uint32_t v32 = w3[2];
389 const uint32_t v33 = w3[3];
390 w3 += 4;
391 const uint32_t v40 = w4[0];
392 const uint32_t v41 = w4[1];
393 const uint32_t v42 = w4[2];
394 const uint32_t v43 = w4[3];
395 w4 += 4;
396 const uint32_t v50 = w5[0];
397 const uint32_t v51 = w5[1];
398 const uint32_t v52 = w5[2];
399 const uint32_t v53 = w5[3];
400 w5 += 4;
401 const uint32_t v60 = w6[0];
402 const uint32_t v61 = w6[1];
403 const uint32_t v62 = w6[2];
404 const uint32_t v63 = w6[3];
405 w6 += 4;
406 const uint32_t v70 = w7[0];
407 const uint32_t v71 = w7[1];
408 const uint32_t v72 = w7[2];
409 const uint32_t v73 = w7[3];
410 w7 += 4;
411 const uint32_t v80 = w8[0];
412 const uint32_t v81 = w8[1];
413 const uint32_t v82 = w8[2];
414 const uint32_t v83 = w8[3];
415 w8 += 4;
416 const uint32_t v90 = w9[0];
417 const uint32_t v91 = w9[1];
418 const uint32_t v92 = w9[2];
419 const uint32_t v93 = w9[3];
420 w9 += 4;
421 const uint32_t v100 = w10[0];
422 const uint32_t v101 = w10[1];
423 const uint32_t v102 = w10[2];
424 const uint32_t v103 = w10[3];
425 w10 += 4;
426 const uint32_t v110 = w11[0];
427 const uint32_t v111 = w11[1];
428 const uint32_t v112 = w11[2];
429 const uint32_t v113 = w11[3];
430 w11 += 4;
431 const uint32_t v120 = w12[0];
432 const uint32_t v121 = w12[1];
433 const uint32_t v122 = w12[2];
434 const uint32_t v123 = w12[3];
435 w12 += 4;
436 const uint32_t v130 = w13[0];
437 const uint32_t v131 = w13[1];
438 const uint32_t v132 = w13[2];
439 const uint32_t v133 = w13[3];
440 w13 += 4;
441 const uint32_t v140 = w14[0];
442 const uint32_t v141 = w14[1];
443 const uint32_t v142 = w14[2];
444 const uint32_t v143 = w14[3];
445 w14 += 4;
446 out[0] = v00;
447 out[1] = v10;
448 out[2] = v20;
449 out[3] = v30;
450 out[4] = v40;
451 out[5] = v50;
452 out[6] = v60;
453 out[7] = v70;
454 out[8] = v80;
455 out[9] = v90;
456 out[10] = v100;
457 out[11] = v110;
458 out[12] = v120;
459 out[13] = v130;
460 out[14] = v140;
461 out[16] = v01;
462 out[17] = v11;
463 out[18] = v21;
464 out[19] = v31;
465 out[20] = v41;
466 out[21] = v51;
467 out[22] = v61;
468 out[23] = v71;
469 out[24] = v81;
470 out[25] = v91;
471 out[26] = v101;
472 out[27] = v111;
473 out[28] = v121;
474 out[29] = v131;
475 out[30] = v141;
476 out[32] = v02;
477 out[33] = v12;
478 out[34] = v22;
479 out[35] = v32;
480 out[36] = v42;
481 out[37] = v52;
482 out[38] = v62;
483 out[39] = v72;
484 out[40] = v82;
485 out[41] = v92;
486 out[42] = v102;
487 out[43] = v112;
488 out[44] = v122;
489 out[45] = v132;
490 out[46] = v142;
491 out[48] = v03;
492 out[49] = v13;
493 out[50] = v23;
494 out[51] = v33;
495 out[52] = v43;
496 out[53] = v53;
497 out[54] = v63;
498 out[55] = v73;
499 out[56] = v83;
500 out[57] = v93;
501 out[58] = v103;
502 out[59] = v113;
503 out[60] = v123;
504 out[61] = v133;
505 out[62] = v143;
506 out += 64;
507 }
508
509 // KC remainder of 1..3
510 for (; k != 0; --k) {
511 const uint32_t v0 = *w0++;
512 out[0] = v0;
513 const uint32_t v1 = *w1++;
514 out[1] = v1;
515 const uint32_t v2 = *w2++;
516 out[2] = v2;
517 const uint32_t v3 = *w3++;
518 out[3] = v3;
519 const uint32_t v4 = *w4++;
520 out[4] = v4;
521 const uint32_t v5 = *w5++;
522 out[5] = v5;
523 const uint32_t v6 = *w6++;
524 out[6] = v6;
525 const uint32_t v7 = *w7++;
526 out[7] = v7;
527 const uint32_t v8 = *w8++;
528 out[8] = v8;
529 const uint32_t v9 = *w9++;
530 out[9] = v9;
531 const uint32_t v10 = *w10++;
532 out[10] = v10;
533 const uint32_t v11 = *w11++;
534 out[11] = v11;
535 const uint32_t v12 = *w12++;
536 out[12] = v12;
537 const uint32_t v13 = *w13++;
538 out[13] = v13;
539 const uint32_t v14 = *w14++;
540 out[14] = v14;
541 out += 16;
542 }
543 out = (uint32_t*) ((uintptr_t) out + extra_bytes);
544 }
545 weights += nc * kc;
546 } while (--g != 0);
547 }