Debian Codegraph @

src/x32-packw/gen/x32-packw-x16-gemm-goi-scalar-float-u4.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/scalar.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include "xnnpack/math.h"
16 #include "xnnpack/packw.h"
17
18
19
20 void xnn_x32_packw_gemm_goi_ukernel_x16__scalar_float_u4(
21 size_t g,
22 size_t nc,
23 size_t kc,
24 size_t nr,
25 size_t kr,
26 size_t sr,
27 const uint32_t* weights,
28 const uint32_t* bias,
29 const void* scale,
30 uint32_t* packed_weights,
31 size_t extra_bytes,
32 const void* params)
33 {
34 assert(g != 0);
35 assert(nc != 0);
36 assert(kc != 0);
37 assert(nr == 16);
38 assert(kr == 1);
39 assert(sr == 1);
40 assert(weights != NULL);
41 assert(packed_weights != NULL);
42
43 float* out = (float*) packed_weights;
44 const float* b = (const float*) bias;
45
46 do {
47 // NC main loop multiple of 16
48 const float* w0 = (const float*) weights;
49 size_t n = nc;
50 for (;n >= 16; n -= 16) {
51 if XNN_LIKELY(b != NULL) {
52 out[0] = b[0];
53 out[1] = b[1];
54 out[2] = b[2];
55 out[3] = b[3];
56 out[4] = b[4];
57 out[5] = b[5];
58 out[6] = b[6];
59 out[7] = b[7];
60 out[8] = b[8];
61 out[9] = b[9];
62 out[10] = b[10];
63 out[11] = b[11];
64 out[12] = b[12];
65 out[13] = b[13];
66 out[14] = b[14];
67 out[15] = b[15];
68 b += 16;
69 } else {
70 out[0] = 0;
71 out[1] = 0;
72 out[2] = 0;
73 out[3] = 0;
74 out[4] = 0;
75 out[5] = 0;
76 out[6] = 0;
77 out[7] = 0;
78 out[8] = 0;
79 out[9] = 0;
80 out[10] = 0;
81 out[11] = 0;
82 out[12] = 0;
83 out[13] = 0;
84 out[14] = 0;
85 out[15] = 0;
86 }
87 out += 16;
88
89 const float* w1 = w0 + kc;
90 const float* w2 = w1 + kc;
91 const float* w3 = w2 + kc;
92 const float* w4 = w3 + kc;
93 const float* w5 = w4 + kc;
94 const float* w6 = w5 + kc;
95 const float* w7 = w6 + kc;
96 const float* w8 = w7 + kc;
97 const float* w9 = w8 + kc;
98 const float* w10 = w9 + kc;
99 const float* w11 = w10 + kc;
100 const float* w12 = w11 + kc;
101 const float* w13 = w12 + kc;
102 const float* w14 = w13 + kc;
103 const float* w15 = w14 + kc;
104
105 // KC main loop multiple of 16x4
106 size_t k = kc;
107 for (; k >= 4; k -= 4) {
108 const float v00 = w0[0];
109 const float v01 = w0[1];
110 const float v02 = w0[2];
111 const float v03 = w0[3];
112 w0 += 4;
113 const float v10 = w1[0];
114 const float v11 = w1[1];
115 const float v12 = w1[2];
116 const float v13 = w1[3];
117 w1 += 4;
118 const float v20 = w2[0];
119 const float v21 = w2[1];
120 const float v22 = w2[2];
121 const float v23 = w2[3];
122 w2 += 4;
123 const float v30 = w3[0];
124 const float v31 = w3[1];
125 const float v32 = w3[2];
126 const float v33 = w3[3];
127 w3 += 4;
128 const float v40 = w4[0];
129 const float v41 = w4[1];
130 const float v42 = w4[2];
131 const float v43 = w4[3];
132 w4 += 4;
133 const float v50 = w5[0];
134 const float v51 = w5[1];
135 const float v52 = w5[2];
136 const float v53 = w5[3];
137 w5 += 4;
138 const float v60 = w6[0];
139 const float v61 = w6[1];
140 const float v62 = w6[2];
141 const float v63 = w6[3];
142 w6 += 4;
143 const float v70 = w7[0];
144 const float v71 = w7[1];
145 const float v72 = w7[2];
146 const float v73 = w7[3];
147 w7 += 4;
148 const float v80 = w8[0];
149 const float v81 = w8[1];
150 const float v82 = w8[2];
151 const float v83 = w8[3];
152 w8 += 4;
153 const float v90 = w9[0];
154 const float v91 = w9[1];
155 const float v92 = w9[2];
156 const float v93 = w9[3];
157 w9 += 4;
158 const float v100 = w10[0];
159 const float v101 = w10[1];
160 const float v102 = w10[2];
161 const float v103 = w10[3];
162 w10 += 4;
163 const float v110 = w11[0];
164 const float v111 = w11[1];
165 const float v112 = w11[2];
166 const float v113 = w11[3];
167 w11 += 4;
168 const float v120 = w12[0];
169 const float v121 = w12[1];
170 const float v122 = w12[2];
171 const float v123 = w12[3];
172 w12 += 4;
173 const float v130 = w13[0];
174 const float v131 = w13[1];
175 const float v132 = w13[2];
176 const float v133 = w13[3];
177 w13 += 4;
178 const float v140 = w14[0];
179 const float v141 = w14[1];
180 const float v142 = w14[2];
181 const float v143 = w14[3];
182 w14 += 4;
183 const float v150 = w15[0];
184 const float v151 = w15[1];
185 const float v152 = w15[2];
186 const float v153 = w15[3];
187 w15 += 4;
188 out[0] = v00;
189 out[1] = v10;
190 out[2] = v20;
191 out[3] = v30;
192 out[4] = v40;
193 out[5] = v50;
194 out[6] = v60;
195 out[7] = v70;
196 out[8] = v80;
197 out[9] = v90;
198 out[10] = v100;
199 out[11] = v110;
200 out[12] = v120;
201 out[13] = v130;
202 out[14] = v140;
203 out[15] = v150;
204 out[16] = v01;
205 out[17] = v11;
206 out[18] = v21;
207 out[19] = v31;
208 out[20] = v41;
209 out[21] = v51;
210 out[22] = v61;
211 out[23] = v71;
212 out[24] = v81;
213 out[25] = v91;
214 out[26] = v101;
215 out[27] = v111;
216 out[28] = v121;
217 out[29] = v131;
218 out[30] = v141;
219 out[31] = v151;
220 out[32] = v02;
221 out[33] = v12;
222 out[34] = v22;
223 out[35] = v32;
224 out[36] = v42;
225 out[37] = v52;
226 out[38] = v62;
227 out[39] = v72;
228 out[40] = v82;
229 out[41] = v92;
230 out[42] = v102;
231 out[43] = v112;
232 out[44] = v122;
233 out[45] = v132;
234 out[46] = v142;
235 out[47] = v152;
236 out[48] = v03;
237 out[49] = v13;
238 out[50] = v23;
239 out[51] = v33;
240 out[52] = v43;
241 out[53] = v53;
242 out[54] = v63;
243 out[55] = v73;
244 out[56] = v83;
245 out[57] = v93;
246 out[58] = v103;
247 out[59] = v113;
248 out[60] = v123;
249 out[61] = v133;
250 out[62] = v143;
251 out[63] = v153;
252 out += 64;
253 }
254
255 // KC remainder
256 for (; k != 0; --k) {
257 const float v0 = *w0++;
258 out[0] = v0;
259 const float v1 = *w1++;
260 out[1] = v1;
261 const float v2 = *w2++;
262 out[2] = v2;
263 const float v3 = *w3++;
264 out[3] = v3;
265 const float v4 = *w4++;
266 out[4] = v4;
267 const float v5 = *w5++;
268 out[5] = v5;
269 const float v6 = *w6++;
270 out[6] = v6;
271 const float v7 = *w7++;
272 out[7] = v7;
273 const float v8 = *w8++;
274 out[8] = v8;
275 const float v9 = *w9++;
276 out[9] = v9;
277 const float v10 = *w10++;
278 out[10] = v10;
279 const float v11 = *w11++;
280 out[11] = v11;
281 const float v12 = *w12++;
282 out[12] = v12;
283 const float v13 = *w13++;
284 out[13] = v13;
285 const float v14 = *w14++;
286 out[14] = v14;
287 const float v15 = *w15++;
288 out[15] = v15;
289 out += 16;
290 }
291 out = (float*) ((uintptr_t) out + extra_bytes);
292 w0 = w15;
293 }
294
295 // NC remainder (1..15)
296 if XNN_UNLIKELY(n != 0) {
297 if XNN_LIKELY(b != NULL) {
298 size_t nb = n;
299 do {
300 *out++ = *b++;
301 } while (--nb != 0);
302 } else {
303 size_t nb = n;
304 do {
305 *out++ = 0;
306 } while (--nb != 0);
307 }
308 out += (16 - n);
309
310 // NR remainder has less than 16 rows so last row is not loaded
311 const float* w1 = w0 + kc;
312 if XNN_UNPREDICTABLE(n < 2) {
313 w1 = w0;
314 }
315 const float* w2 = w1 + kc;
316 if XNN_UNPREDICTABLE(n <= 2) {
317 w2 = w1;
318 }
319 const float* w3 = w2 + kc;
320 if XNN_UNPREDICTABLE(n < 4) {
321 w3 = w2;
322 }
323 const float* w4 = w3 + kc;
324 if XNN_UNPREDICTABLE(n <= 4) {
325 w4 = w3;
326 }
327 const float* w5 = w4 + kc;
328 if XNN_UNPREDICTABLE(n < 6) {
329 w5 = w4;
330 }
331 const float* w6 = w5 + kc;
332 if XNN_UNPREDICTABLE(n <= 6) {
333 w6 = w5;
334 }
335 const float* w7 = w6 + kc;
336 if XNN_UNPREDICTABLE(n < 8) {
337 w7 = w6;
338 }
339 const float* w8 = w7 + kc;
340 if XNN_UNPREDICTABLE(n <= 8) {
341 w8 = w7;
342 }
343 const float* w9 = w8 + kc;
344 if XNN_UNPREDICTABLE(n < 10) {
345 w9 = w8;
346 }
347 const float* w10 = w9 + kc;
348 if XNN_UNPREDICTABLE(n <= 10) {
349 w10 = w9;
350 }
351 const float* w11 = w10 + kc;
352 if XNN_UNPREDICTABLE(n < 12) {
353 w11 = w10;
354 }
355 const float* w12 = w11 + kc;
356 if XNN_UNPREDICTABLE(n <= 12) {
357 w12 = w11;
358 }
359 const float* w13 = w12 + kc;
360 if XNN_UNPREDICTABLE(n < 14) {
361 w13 = w12;
362 }
363 const float* w14 = w13 + kc;
364 if XNN_UNPREDICTABLE(n <= 14) {
365 w14 = w13;
366 }
367
368 // KC main loop multiple of 16x4
369 size_t k = kc;
370 for (; k >= 4; k -= 4) {
371 const float v00 = w0[0];
372 const float v01 = w0[1];
373 const float v02 = w0[2];
374 const float v03 = w0[3];
375 w0 += 4;
376 const float v10 = w1[0];
377 const float v11 = w1[1];
378 const float v12 = w1[2];
379 const float v13 = w1[3];
380 w1 += 4;
381 const float v20 = w2[0];
382 const float v21 = w2[1];
383 const float v22 = w2[2];
384 const float v23 = w2[3];
385 w2 += 4;
386 const float v30 = w3[0];
387 const float v31 = w3[1];
388 const float v32 = w3[2];
389 const float v33 = w3[3];
390 w3 += 4;
391 const float v40 = w4[0];
392 const float v41 = w4[1];
393 const float v42 = w4[2];
394 const float v43 = w4[3];
395 w4 += 4;
396 const float v50 = w5[0];
397 const float v51 = w5[1];
398 const float v52 = w5[2];
399 const float v53 = w5[3];
400 w5 += 4;
401 const float v60 = w6[0];
402 const float v61 = w6[1];
403 const float v62 = w6[2];
404 const float v63 = w6[3];
405 w6 += 4;
406 const float v70 = w7[0];
407 const float v71 = w7[1];
408 const float v72 = w7[2];
409 const float v73 = w7[3];
410 w7 += 4;
411 const float v80 = w8[0];
412 const float v81 = w8[1];
413 const float v82 = w8[2];
414 const float v83 = w8[3];
415 w8 += 4;
416 const float v90 = w9[0];
417 const float v91 = w9[1];
418 const float v92 = w9[2];
419 const float v93 = w9[3];
420 w9 += 4;
421 const float v100 = w10[0];
422 const float v101 = w10[1];
423 const float v102 = w10[2];
424 const float v103 = w10[3];
425 w10 += 4;
426 const float v110 = w11[0];
427 const float v111 = w11[1];
428 const float v112 = w11[2];
429 const float v113 = w11[3];
430 w11 += 4;
431 const float v120 = w12[0];
432 const float v121 = w12[1];
433 const float v122 = w12[2];
434 const float v123 = w12[3];
435 w12 += 4;
436 const float v130 = w13[0];
437 const float v131 = w13[1];
438 const float v132 = w13[2];
439 const float v133 = w13[3];
440 w13 += 4;
441 const float v140 = w14[0];
442 const float v141 = w14[1];
443 const float v142 = w14[2];
444 const float v143 = w14[3];
445 w14 += 4;
446 out[0] = v00;
447 out[1] = v10;
448 out[2] = v20;
449 out[3] = v30;
450 out[4] = v40;
451 out[5] = v50;
452 out[6] = v60;
453 out[7] = v70;
454 out[8] = v80;
455 out[9] = v90;
456 out[10] = v100;
457 out[11] = v110;
458 out[12] = v120;
459 out[13] = v130;
460 out[14] = v140;
461 out[16] = v01;
462 out[17] = v11;
463 out[18] = v21;
464 out[19] = v31;
465 out[20] = v41;
466 out[21] = v51;
467 out[22] = v61;
468 out[23] = v71;
469 out[24] = v81;
470 out[25] = v91;
471 out[26] = v101;
472 out[27] = v111;
473 out[28] = v121;
474 out[29] = v131;
475 out[30] = v141;
476 out[32] = v02;
477 out[33] = v12;
478 out[34] = v22;
479 out[35] = v32;
480 out[36] = v42;
481 out[37] = v52;
482 out[38] = v62;
483 out[39] = v72;
484 out[40] = v82;
485 out[41] = v92;
486 out[42] = v102;
487 out[43] = v112;
488 out[44] = v122;
489 out[45] = v132;
490 out[46] = v142;
491 out[48] = v03;
492 out[49] = v13;
493 out[50] = v23;
494 out[51] = v33;
495 out[52] = v43;
496 out[53] = v53;
497 out[54] = v63;
498 out[55] = v73;
499 out[56] = v83;
500 out[57] = v93;
501 out[58] = v103;
502 out[59] = v113;
503 out[60] = v123;
504 out[61] = v133;
505 out[62] = v143;
506 out += 64;
507 }
508
509 // KC remainder of 1..3
510 for (; k != 0; --k) {
511 const float v0 = *w0++;
512 out[0] = v0;
513 const float v1 = *w1++;
514 out[1] = v1;
515 const float v2 = *w2++;
516 out[2] = v2;
517 const float v3 = *w3++;
518 out[3] = v3;
519 const float v4 = *w4++;
520 out[4] = v4;
521 const float v5 = *w5++;
522 out[5] = v5;
523 const float v6 = *w6++;
524 out[6] = v6;
525 const float v7 = *w7++;
526 out[7] = v7;
527 const float v8 = *w8++;
528 out[8] = v8;
529 const float v9 = *w9++;
530 out[9] = v9;
531 const float v10 = *w10++;
532 out[10] = v10;
533 const float v11 = *w11++;
534 out[11] = v11;
535 const float v12 = *w12++;
536 out[12] = v12;
537 const float v13 = *w13++;
538 out[13] = v13;
539 const float v14 = *w14++;
540 out[14] = v14;
541 out += 16;
542 }
543 out = (float*) ((uintptr_t) out + extra_bytes);
544 }
545 weights += nc * kc;
546 } while (--g != 0);
547 }