Debian Codegraph @

src/x32-packw/gen/x32-packw-x16-gemm-gio-scalar.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/gio-scalar.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2024 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11 #include <stddef.h>
12 #include <stdint.h>
13
14 #include "xnnpack/packw.h"
15
16
17 void xnn_x32_packw_gemm_gio_ukernel_x16__scalar(
18 size_t g,
19 size_t nc,
20 size_t kc,
21 size_t nr,
22 size_t kr,
23 size_t sr,
24 size_t k_stride,
25 const uint32_t* weights,
26 const uint32_t* bias,
27 const void* scale,
28 uint32_t* packed_weights,
29 size_t extra_bytes,
30 const void* params)
31 {
32 assert(g != 0);
33 assert(nc != 0);
34 assert(kc != 0);
35 assert(nr == 16); // This kernel is for NR=16
36 assert(kr == 1);
37 assert(sr == 1);
38 assert(k_stride != 0);
39 assert(weights != NULL);
40 assert(packed_weights != NULL);
41
42 const float* b = (const float*) bias;
43 float* packed_w = (float*) packed_weights;
44 do {
45 // NC main loop multiple of 16
46 const float* w = (const float*) weights;
47 size_t n = nc;
48
49 for (; n >= 16; n -= 16) {
50 if XNN_LIKELY(b != NULL) {
51 const uint64_t v0 = ((const uint64_t*)b)[0];
52 const uint64_t v1 = ((const uint64_t*)b)[1];
53 const uint64_t v2 = ((const uint64_t*)b)[2];
54 const uint64_t v3 = ((const uint64_t*)b)[3];
55 const uint64_t v4 = ((const uint64_t*)b)[4];
56 const uint64_t v5 = ((const uint64_t*)b)[5];
57 const uint64_t v6 = ((const uint64_t*)b)[6];
58 const uint64_t v7 = ((const uint64_t*)b)[7];
59 ((uint64_t*)packed_w)[0] = v0;
60 ((uint64_t*)packed_w)[1] = v1;
61 ((uint64_t*)packed_w)[2] = v2;
62 ((uint64_t*)packed_w)[3] = v3;
63 ((uint64_t*)packed_w)[4] = v4;
64 ((uint64_t*)packed_w)[5] = v5;
65 ((uint64_t*)packed_w)[6] = v6;
66 ((uint64_t*)packed_w)[7] = v7;
67 b += 16;
68 } else {
69 ((uint64_t*)packed_w)[0] = 0;
70 ((uint64_t*)packed_w)[1] = 0;
71 ((uint64_t*)packed_w)[2] = 0;
72 ((uint64_t*)packed_w)[3] = 0;
73 ((uint64_t*)packed_w)[4] = 0;
74 ((uint64_t*)packed_w)[5] = 0;
75 ((uint64_t*)packed_w)[6] = 0;
76 ((uint64_t*)packed_w)[7] = 0;
77 }
78 packed_w += 16;
79
80 // KC main loop
81 for (size_t k = kc; k > 0; --k) {
82 const uint64_t v0 = ((const uint64_t*)w)[0];
83 const uint64_t v1 = ((const uint64_t*)w)[1];
84 const uint64_t v2 = ((const uint64_t*)w)[2];
85 const uint64_t v3 = ((const uint64_t*)w)[3];
86 const uint64_t v4 = ((const uint64_t*)w)[4];
87 const uint64_t v5 = ((const uint64_t*)w)[5];
88 const uint64_t v6 = ((const uint64_t*)w)[6];
89 const uint64_t v7 = ((const uint64_t*)w)[7];
90 ((uint64_t*)packed_w)[0] = v0;
91 ((uint64_t*)packed_w)[1] = v1;
92 ((uint64_t*)packed_w)[2] = v2;
93 ((uint64_t*)packed_w)[3] = v3;
94 ((uint64_t*)packed_w)[4] = v4;
95 ((uint64_t*)packed_w)[5] = v5;
96 ((uint64_t*)packed_w)[6] = v6;
97 ((uint64_t*)packed_w)[7] = v7;
98 w += k_stride;
99 packed_w += 16;
100 }
101 w = w - kc * k_stride + 16; // Advance to next column of 16 floats
102 }
103
104 // NC remainder (1..15)
105 if XNN_UNLIKELY(n != 0) {
106 assert(n >= 1);
107 assert(n <= 15);
108
109 if XNN_LIKELY(b != NULL) {
110 for (size_t i = 0; i < n; ++i) {
111 packed_w[i] = b[i];
112 }
113 b += n;
114 } else {
115 ((uint64_t*)packed_w)[0] = 0;
116 ((uint64_t*)packed_w)[1] = 0;
117 ((uint64_t*)packed_w)[2] = 0;
118 ((uint64_t*)packed_w)[3] = 0;
119 ((uint64_t*)packed_w)[4] = 0;
120 ((uint64_t*)packed_w)[5] = 0;
121 ((uint64_t*)packed_w)[6] = 0;
122 ((uint64_t*)packed_w)[7] = 0;
123 }
124 packed_w += 16;
125
126 // KC main loop
127 for (size_t k = kc; k > 0; --k) {
128 for (size_t i = 0; i < n; ++i) {
129 packed_w[i] = w[i];
130 }
131 w += k_stride;
132 packed_w += 16;
133 }
134 }
135 weights += nc * kc;
136 } while (--g != 0);
137 }