Debian Codegraph @

src/x32-packw/gen/x32-packw-x4-gemm-goi-scalar-float-u4.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/scalar.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include "xnnpack/math.h"
16 #include "xnnpack/packw.h"
17
18
19
20 void xnn_x32_packw_gemm_goi_ukernel_x4__scalar_float_u4(
21 size_t g,
22 size_t nc,
23 size_t kc,
24 size_t nr,
25 size_t kr,
26 size_t sr,
27 const uint32_t* weights,
28 const uint32_t* bias,
29 const void* scale,
30 uint32_t* packed_weights,
31 size_t extra_bytes,
32 const void* params)
33 {
34 assert(g != 0);
35 assert(nc != 0);
36 assert(kc != 0);
37 assert(nr == 4);
38 assert(kr == 1);
39 assert(sr == 1);
40 assert(weights != NULL);
41 assert(packed_weights != NULL);
42
43 float* out = (float*) packed_weights;
44 const float* b = (const float*) bias;
45
46 do {
47 // NC main loop multiple of 4
48 const float* w0 = (const float*) weights;
49 size_t n = nc;
50 for (;n >= 4; n -= 4) {
51 if XNN_LIKELY(b != NULL) {
52 out[0] = b[0];
53 out[1] = b[1];
54 out[2] = b[2];
55 out[3] = b[3];
56 b += 4;
57 } else {
58 out[0] = 0;
59 out[1] = 0;
60 out[2] = 0;
61 out[3] = 0;
62 }
63 out += 4;
64
65 const float* w1 = w0 + kc;
66 const float* w2 = w1 + kc;
67 const float* w3 = w2 + kc;
68
69 // KC main loop multiple of 4x4
70 size_t k = kc;
71 for (; k >= 4; k -= 4) {
72 const float v00 = w0[0];
73 const float v01 = w0[1];
74 const float v02 = w0[2];
75 const float v03 = w0[3];
76 w0 += 4;
77 const float v10 = w1[0];
78 const float v11 = w1[1];
79 const float v12 = w1[2];
80 const float v13 = w1[3];
81 w1 += 4;
82 const float v20 = w2[0];
83 const float v21 = w2[1];
84 const float v22 = w2[2];
85 const float v23 = w2[3];
86 w2 += 4;
87 const float v30 = w3[0];
88 const float v31 = w3[1];
89 const float v32 = w3[2];
90 const float v33 = w3[3];
91 w3 += 4;
92 out[0] = v00;
93 out[1] = v10;
94 out[2] = v20;
95 out[3] = v30;
96 out[4] = v01;
97 out[5] = v11;
98 out[6] = v21;
99 out[7] = v31;
100 out[8] = v02;
101 out[9] = v12;
102 out[10] = v22;
103 out[11] = v32;
104 out[12] = v03;
105 out[13] = v13;
106 out[14] = v23;
107 out[15] = v33;
108 out += 16;
109 }
110
111 // KC remainder
112 for (; k != 0; --k) {
113 const float v0 = *w0++;
114 out[0] = v0;
115 const float v1 = *w1++;
116 out[1] = v1;
117 const float v2 = *w2++;
118 out[2] = v2;
119 const float v3 = *w3++;
120 out[3] = v3;
121 out += 4;
122 }
123 out = (float*) ((uintptr_t) out + extra_bytes);
124 w0 = w3;
125 }
126
127 // NC remainder (1..3)
128 if XNN_UNLIKELY(n != 0) {
129 if XNN_LIKELY(b != NULL) {
130 size_t nb = n;
131 do {
132 *out++ = *b++;
133 } while (--nb != 0);
134 } else {
135 size_t nb = n;
136 do {
137 *out++ = 0;
138 } while (--nb != 0);
139 }
140 out += (4 - n);
141
142 // NR remainder has less than 4 rows so last row is not loaded
143 const float* w1 = w0 + kc;
144 if XNN_UNPREDICTABLE(n < 2) {
145 w1 = w0;
146 }
147 const float* w2 = w1 + kc;
148 if XNN_UNPREDICTABLE(n <= 2) {
149 w2 = w1;
150 }
151
152 // KC main loop multiple of 4x4
153 size_t k = kc;
154 for (; k >= 4; k -= 4) {
155 const float v00 = w0[0];
156 const float v01 = w0[1];
157 const float v02 = w0[2];
158 const float v03 = w0[3];
159 w0 += 4;
160 const float v10 = w1[0];
161 const float v11 = w1[1];
162 const float v12 = w1[2];
163 const float v13 = w1[3];
164 w1 += 4;
165 const float v20 = w2[0];
166 const float v21 = w2[1];
167 const float v22 = w2[2];
168 const float v23 = w2[3];
169 w2 += 4;
170 out[0] = v00;
171 out[1] = v10;
172 out[2] = v20;
173 out[4] = v01;
174 out[5] = v11;
175 out[6] = v21;
176 out[8] = v02;
177 out[9] = v12;
178 out[10] = v22;
179 out[12] = v03;
180 out[13] = v13;
181 out[14] = v23;
182 out += 16;
183 }
184
185 // KC remainder of 1..3
186 for (; k != 0; --k) {
187 const float v0 = *w0++;
188 out[0] = v0;
189 const float v1 = *w1++;
190 out[1] = v1;
191 const float v2 = *w2++;
192 out[2] = v2;
193 out += 4;
194 }
195 out = (float*) ((uintptr_t) out + extra_bytes);
196 }
197 weights += nc * kc;
198 } while (--g != 0);
199 }