Debian Codegraph @

src/x32-packw/gen/x32-packw-x32-gemm-gio-scalar.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/gio-scalar.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2024 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11 #include <stddef.h>
12 #include <stdint.h>
13
14 #include "xnnpack/packw.h"
15
16
17 void xnn_x32_packw_gemm_gio_ukernel_x32__scalar(
18 size_t g,
19 size_t nc,
20 size_t kc,
21 size_t nr,
22 size_t kr,
23 size_t sr,
24 size_t k_stride,
25 const uint32_t* weights,
26 const uint32_t* bias,
27 const void* scale,
28 uint32_t* packed_weights,
29 size_t extra_bytes,
30 const void* params)
31 {
32 assert(g != 0);
33 assert(nc != 0);
34 assert(kc != 0);
35 assert(nr == 32); // This kernel is for NR=32
36 assert(kr == 1);
37 assert(sr == 1);
38 assert(k_stride != 0);
39 assert(weights != NULL);
40 assert(packed_weights != NULL);
41
42 const float* b = (const float*) bias;
43 float* packed_w = (float*) packed_weights;
44 do {
45 // NC main loop multiple of 32
46 const float* w = (const float*) weights;
47 size_t n = nc;
48
49 for (; n >= 32; n -= 32) {
50 if XNN_LIKELY(b != NULL) {
51 const uint64_t v0 = ((const uint64_t*)b)[0];
52 const uint64_t v1 = ((const uint64_t*)b)[1];
53 const uint64_t v2 = ((const uint64_t*)b)[2];
54 const uint64_t v3 = ((const uint64_t*)b)[3];
55 const uint64_t v4 = ((const uint64_t*)b)[4];
56 const uint64_t v5 = ((const uint64_t*)b)[5];
57 const uint64_t v6 = ((const uint64_t*)b)[6];
58 const uint64_t v7 = ((const uint64_t*)b)[7];
59 const uint64_t v8 = ((const uint64_t*)b)[8];
60 const uint64_t v9 = ((const uint64_t*)b)[9];
61 const uint64_t v10 = ((const uint64_t*)b)[10];
62 const uint64_t v11 = ((const uint64_t*)b)[11];
63 const uint64_t v12 = ((const uint64_t*)b)[12];
64 const uint64_t v13 = ((const uint64_t*)b)[13];
65 const uint64_t v14 = ((const uint64_t*)b)[14];
66 const uint64_t v15 = ((const uint64_t*)b)[15];
67 ((uint64_t*)packed_w)[0] = v0;
68 ((uint64_t*)packed_w)[1] = v1;
69 ((uint64_t*)packed_w)[2] = v2;
70 ((uint64_t*)packed_w)[3] = v3;
71 ((uint64_t*)packed_w)[4] = v4;
72 ((uint64_t*)packed_w)[5] = v5;
73 ((uint64_t*)packed_w)[6] = v6;
74 ((uint64_t*)packed_w)[7] = v7;
75 ((uint64_t*)packed_w)[8] = v8;
76 ((uint64_t*)packed_w)[9] = v9;
77 ((uint64_t*)packed_w)[10] = v10;
78 ((uint64_t*)packed_w)[11] = v11;
79 ((uint64_t*)packed_w)[12] = v12;
80 ((uint64_t*)packed_w)[13] = v13;
81 ((uint64_t*)packed_w)[14] = v14;
82 ((uint64_t*)packed_w)[15] = v15;
83 b += 32;
84 } else {
85 ((uint64_t*)packed_w)[0] = 0;
86 ((uint64_t*)packed_w)[1] = 0;
87 ((uint64_t*)packed_w)[2] = 0;
88 ((uint64_t*)packed_w)[3] = 0;
89 ((uint64_t*)packed_w)[4] = 0;
90 ((uint64_t*)packed_w)[5] = 0;
91 ((uint64_t*)packed_w)[6] = 0;
92 ((uint64_t*)packed_w)[7] = 0;
93 ((uint64_t*)packed_w)[8] = 0;
94 ((uint64_t*)packed_w)[9] = 0;
95 ((uint64_t*)packed_w)[10] = 0;
96 ((uint64_t*)packed_w)[11] = 0;
97 ((uint64_t*)packed_w)[12] = 0;
98 ((uint64_t*)packed_w)[13] = 0;
99 ((uint64_t*)packed_w)[14] = 0;
100 ((uint64_t*)packed_w)[15] = 0;
101 }
102 packed_w += 32;
103
104 // KC main loop
105 for (size_t k = kc; k > 0; --k) {
106 const uint64_t v0 = ((const uint64_t*)w)[0];
107 const uint64_t v1 = ((const uint64_t*)w)[1];
108 const uint64_t v2 = ((const uint64_t*)w)[2];
109 const uint64_t v3 = ((const uint64_t*)w)[3];
110 const uint64_t v4 = ((const uint64_t*)w)[4];
111 const uint64_t v5 = ((const uint64_t*)w)[5];
112 const uint64_t v6 = ((const uint64_t*)w)[6];
113 const uint64_t v7 = ((const uint64_t*)w)[7];
114 const uint64_t v8 = ((const uint64_t*)w)[8];
115 const uint64_t v9 = ((const uint64_t*)w)[9];
116 const uint64_t v10 = ((const uint64_t*)w)[10];
117 const uint64_t v11 = ((const uint64_t*)w)[11];
118 const uint64_t v12 = ((const uint64_t*)w)[12];
119 const uint64_t v13 = ((const uint64_t*)w)[13];
120 const uint64_t v14 = ((const uint64_t*)w)[14];
121 const uint64_t v15 = ((const uint64_t*)w)[15];
122 ((uint64_t*)packed_w)[0] = v0;
123 ((uint64_t*)packed_w)[1] = v1;
124 ((uint64_t*)packed_w)[2] = v2;
125 ((uint64_t*)packed_w)[3] = v3;
126 ((uint64_t*)packed_w)[4] = v4;
127 ((uint64_t*)packed_w)[5] = v5;
128 ((uint64_t*)packed_w)[6] = v6;
129 ((uint64_t*)packed_w)[7] = v7;
130 ((uint64_t*)packed_w)[8] = v8;
131 ((uint64_t*)packed_w)[9] = v9;
132 ((uint64_t*)packed_w)[10] = v10;
133 ((uint64_t*)packed_w)[11] = v11;
134 ((uint64_t*)packed_w)[12] = v12;
135 ((uint64_t*)packed_w)[13] = v13;
136 ((uint64_t*)packed_w)[14] = v14;
137 ((uint64_t*)packed_w)[15] = v15;
138 w += k_stride;
139 packed_w += 32;
140 }
141 w = w - kc * k_stride + 32; // Advance to next column of 32 floats
142 }
143
144 // NC remainder (1..31)
145 if XNN_UNLIKELY(n != 0) {
146 assert(n >= 1);
147 assert(n <= 31);
148
149 if XNN_LIKELY(b != NULL) {
150 for (size_t i = 0; i < n; ++i) {
151 packed_w[i] = b[i];
152 }
153 b += n;
154 } else {
155 ((uint64_t*)packed_w)[0] = 0;
156 ((uint64_t*)packed_w)[1] = 0;
157 ((uint64_t*)packed_w)[2] = 0;
158 ((uint64_t*)packed_w)[3] = 0;
159 ((uint64_t*)packed_w)[4] = 0;
160 ((uint64_t*)packed_w)[5] = 0;
161 ((uint64_t*)packed_w)[6] = 0;
162 ((uint64_t*)packed_w)[7] = 0;
163 ((uint64_t*)packed_w)[8] = 0;
164 ((uint64_t*)packed_w)[9] = 0;
165 ((uint64_t*)packed_w)[10] = 0;
166 ((uint64_t*)packed_w)[11] = 0;
167 ((uint64_t*)packed_w)[12] = 0;
168 ((uint64_t*)packed_w)[13] = 0;
169 ((uint64_t*)packed_w)[14] = 0;
170 ((uint64_t*)packed_w)[15] = 0;
171 }
172 packed_w += 32;
173
174 // KC main loop
175 for (size_t k = kc; k > 0; --k) {
176 for (size_t i = 0; i < n; ++i) {
177 packed_w[i] = w[i];
178 }
179 w += k_stride;
180 packed_w += 32;
181 }
182 }
183 weights += nc * kc;
184 } while (--g != 0);
185 }