Debian Codegraph @

src/x32-packw/gen/x32-packw-x8-gemm-goi-scalar-float-u4.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/scalar.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include "xnnpack/math.h"
16 #include "xnnpack/packw.h"
17
18
19
20 void xnn_x32_packw_gemm_goi_ukernel_x8__scalar_float_u4(
21 size_t g,
22 size_t nc,
23 size_t kc,
24 size_t nr,
25 size_t kr,
26 size_t sr,
27 const uint32_t* weights,
28 const uint32_t* bias,
29 const void* scale,
30 uint32_t* packed_weights,
31 size_t extra_bytes,
32 const void* params)
33 {
34 assert(g != 0);
35 assert(nc != 0);
36 assert(kc != 0);
37 assert(nr == 8);
38 assert(kr == 1);
39 assert(sr == 1);
40 assert(weights != NULL);
41 assert(packed_weights != NULL);
42
43 float* out = (float*) packed_weights;
44 const float* b = (const float*) bias;
45
46 do {
47 // NC main loop multiple of 8
48 const float* w0 = (const float*) weights;
49 size_t n = nc;
50 for (;n >= 8; n -= 8) {
51 if XNN_LIKELY(b != NULL) {
52 out[0] = b[0];
53 out[1] = b[1];
54 out[2] = b[2];
55 out[3] = b[3];
56 out[4] = b[4];
57 out[5] = b[5];
58 out[6] = b[6];
59 out[7] = b[7];
60 b += 8;
61 } else {
62 out[0] = 0;
63 out[1] = 0;
64 out[2] = 0;
65 out[3] = 0;
66 out[4] = 0;
67 out[5] = 0;
68 out[6] = 0;
69 out[7] = 0;
70 }
71 out += 8;
72
73 const float* w1 = w0 + kc;
74 const float* w2 = w1 + kc;
75 const float* w3 = w2 + kc;
76 const float* w4 = w3 + kc;
77 const float* w5 = w4 + kc;
78 const float* w6 = w5 + kc;
79 const float* w7 = w6 + kc;
80
81 // KC main loop multiple of 8x4
82 size_t k = kc;
83 for (; k >= 4; k -= 4) {
84 const float v00 = w0[0];
85 const float v01 = w0[1];
86 const float v02 = w0[2];
87 const float v03 = w0[3];
88 w0 += 4;
89 const float v10 = w1[0];
90 const float v11 = w1[1];
91 const float v12 = w1[2];
92 const float v13 = w1[3];
93 w1 += 4;
94 const float v20 = w2[0];
95 const float v21 = w2[1];
96 const float v22 = w2[2];
97 const float v23 = w2[3];
98 w2 += 4;
99 const float v30 = w3[0];
100 const float v31 = w3[1];
101 const float v32 = w3[2];
102 const float v33 = w3[3];
103 w3 += 4;
104 const float v40 = w4[0];
105 const float v41 = w4[1];
106 const float v42 = w4[2];
107 const float v43 = w4[3];
108 w4 += 4;
109 const float v50 = w5[0];
110 const float v51 = w5[1];
111 const float v52 = w5[2];
112 const float v53 = w5[3];
113 w5 += 4;
114 const float v60 = w6[0];
115 const float v61 = w6[1];
116 const float v62 = w6[2];
117 const float v63 = w6[3];
118 w6 += 4;
119 const float v70 = w7[0];
120 const float v71 = w7[1];
121 const float v72 = w7[2];
122 const float v73 = w7[3];
123 w7 += 4;
124 out[0] = v00;
125 out[1] = v10;
126 out[2] = v20;
127 out[3] = v30;
128 out[4] = v40;
129 out[5] = v50;
130 out[6] = v60;
131 out[7] = v70;
132 out[8] = v01;
133 out[9] = v11;
134 out[10] = v21;
135 out[11] = v31;
136 out[12] = v41;
137 out[13] = v51;
138 out[14] = v61;
139 out[15] = v71;
140 out[16] = v02;
141 out[17] = v12;
142 out[18] = v22;
143 out[19] = v32;
144 out[20] = v42;
145 out[21] = v52;
146 out[22] = v62;
147 out[23] = v72;
148 out[24] = v03;
149 out[25] = v13;
150 out[26] = v23;
151 out[27] = v33;
152 out[28] = v43;
153 out[29] = v53;
154 out[30] = v63;
155 out[31] = v73;
156 out += 32;
157 }
158
159 // KC remainder
160 for (; k != 0; --k) {
161 const float v0 = *w0++;
162 out[0] = v0;
163 const float v1 = *w1++;
164 out[1] = v1;
165 const float v2 = *w2++;
166 out[2] = v2;
167 const float v3 = *w3++;
168 out[3] = v3;
169 const float v4 = *w4++;
170 out[4] = v4;
171 const float v5 = *w5++;
172 out[5] = v5;
173 const float v6 = *w6++;
174 out[6] = v6;
175 const float v7 = *w7++;
176 out[7] = v7;
177 out += 8;
178 }
179 out = (float*) ((uintptr_t) out + extra_bytes);
180 w0 = w7;
181 }
182
183 // NC remainder (1..7)
184 if XNN_UNLIKELY(n != 0) {
185 if XNN_LIKELY(b != NULL) {
186 size_t nb = n;
187 do {
188 *out++ = *b++;
189 } while (--nb != 0);
190 } else {
191 size_t nb = n;
192 do {
193 *out++ = 0;
194 } while (--nb != 0);
195 }
196 out += (8 - n);
197
198 // NR remainder has less than 8 rows so last row is not loaded
199 const float* w1 = w0 + kc;
200 if XNN_UNPREDICTABLE(n < 2) {
201 w1 = w0;
202 }
203 const float* w2 = w1 + kc;
204 if XNN_UNPREDICTABLE(n <= 2) {
205 w2 = w1;
206 }
207 const float* w3 = w2 + kc;
208 if XNN_UNPREDICTABLE(n < 4) {
209 w3 = w2;
210 }
211 const float* w4 = w3 + kc;
212 if XNN_UNPREDICTABLE(n <= 4) {
213 w4 = w3;
214 }
215 const float* w5 = w4 + kc;
216 if XNN_UNPREDICTABLE(n < 6) {
217 w5 = w4;
218 }
219 const float* w6 = w5 + kc;
220 if XNN_UNPREDICTABLE(n <= 6) {
221 w6 = w5;
222 }
223
224 // KC main loop multiple of 8x4
225 size_t k = kc;
226 for (; k >= 4; k -= 4) {
227 const float v00 = w0[0];
228 const float v01 = w0[1];
229 const float v02 = w0[2];
230 const float v03 = w0[3];
231 w0 += 4;
232 const float v10 = w1[0];
233 const float v11 = w1[1];
234 const float v12 = w1[2];
235 const float v13 = w1[3];
236 w1 += 4;
237 const float v20 = w2[0];
238 const float v21 = w2[1];
239 const float v22 = w2[2];
240 const float v23 = w2[3];
241 w2 += 4;
242 const float v30 = w3[0];
243 const float v31 = w3[1];
244 const float v32 = w3[2];
245 const float v33 = w3[3];
246 w3 += 4;
247 const float v40 = w4[0];
248 const float v41 = w4[1];
249 const float v42 = w4[2];
250 const float v43 = w4[3];
251 w4 += 4;
252 const float v50 = w5[0];
253 const float v51 = w5[1];
254 const float v52 = w5[2];
255 const float v53 = w5[3];
256 w5 += 4;
257 const float v60 = w6[0];
258 const float v61 = w6[1];
259 const float v62 = w6[2];
260 const float v63 = w6[3];
261 w6 += 4;
262 out[0] = v00;
263 out[1] = v10;
264 out[2] = v20;
265 out[3] = v30;
266 out[4] = v40;
267 out[5] = v50;
268 out[6] = v60;
269 out[8] = v01;
270 out[9] = v11;
271 out[10] = v21;
272 out[11] = v31;
273 out[12] = v41;
274 out[13] = v51;
275 out[14] = v61;
276 out[16] = v02;
277 out[17] = v12;
278 out[18] = v22;
279 out[19] = v32;
280 out[20] = v42;
281 out[21] = v52;
282 out[22] = v62;
283 out[24] = v03;
284 out[25] = v13;
285 out[26] = v23;
286 out[27] = v33;
287 out[28] = v43;
288 out[29] = v53;
289 out[30] = v63;
290 out += 32;
291 }
292
293 // KC remainder of 1..3
294 for (; k != 0; --k) {
295 const float v0 = *w0++;
296 out[0] = v0;
297 const float v1 = *w1++;
298 out[1] = v1;
299 const float v2 = *w2++;
300 out[2] = v2;
301 const float v3 = *w3++;
302 out[3] = v3;
303 const float v4 = *w4++;
304 out[4] = v4;
305 const float v5 = *w5++;
306 out[5] = v5;
307 const float v6 = *w6++;
308 out[6] = v6;
309 out += 8;
310 }
311 out = (float*) ((uintptr_t) out + extra_bytes);
312 }
313 weights += nc * kc;
314 } while (--g != 0);
315 }