Debian Codegraph @

src/x32-packw/gen/x32-packw-x3-gemm-goi-scalar-int-u4.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/scalar.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include "xnnpack/math.h"
16 #include "xnnpack/packw.h"
17
18
19
20 void xnn_x32_packw_gemm_goi_ukernel_x3__scalar_int_u4(
21 size_t g,
22 size_t nc,
23 size_t kc,
24 size_t nr,
25 size_t kr,
26 size_t sr,
27 const uint32_t* weights,
28 const uint32_t* bias,
29 const void* scale,
30 uint32_t* packed_weights,
31 size_t extra_bytes,
32 const void* params)
33 {
34 assert(g != 0);
35 assert(nc != 0);
36 assert(kc != 0);
37 assert(nr == 3);
38 assert(kr == 1);
39 assert(sr == 1);
40 assert(weights != NULL);
41 assert(packed_weights != NULL);
42
43 uint32_t* out = (uint32_t*) packed_weights;
44 const uint32_t* b = (const uint32_t*) bias;
45
46 do {
47 // NC main loop multiple of 3
48 const uint32_t* w0 = (const uint32_t*) weights;
49 size_t n = nc;
50 for (;n >= 3; n -= 3) {
51 if XNN_LIKELY(b != NULL) {
52 out[0] = b[0];
53 out[1] = b[1];
54 out[2] = b[2];
55 b += 3;
56 } else {
57 out[0] = 0;
58 out[1] = 0;
59 out[2] = 0;
60 }
61 out += 3;
62
63 const uint32_t* w1 = w0 + kc;
64 const uint32_t* w2 = w1 + kc;
65
66 // KC main loop multiple of 3x4
67 size_t k = kc;
68 for (; k >= 4; k -= 4) {
69 const uint32_t v00 = w0[0];
70 const uint32_t v01 = w0[1];
71 const uint32_t v02 = w0[2];
72 const uint32_t v03 = w0[3];
73 w0 += 4;
74 const uint32_t v10 = w1[0];
75 const uint32_t v11 = w1[1];
76 const uint32_t v12 = w1[2];
77 const uint32_t v13 = w1[3];
78 w1 += 4;
79 const uint32_t v20 = w2[0];
80 const uint32_t v21 = w2[1];
81 const uint32_t v22 = w2[2];
82 const uint32_t v23 = w2[3];
83 w2 += 4;
84 out[0] = v00;
85 out[1] = v10;
86 out[2] = v20;
87 out[3] = v01;
88 out[4] = v11;
89 out[5] = v21;
90 out[6] = v02;
91 out[7] = v12;
92 out[8] = v22;
93 out[9] = v03;
94 out[10] = v13;
95 out[11] = v23;
96 out += 12;
97 }
98
99 // KC remainder
100 for (; k != 0; --k) {
101 const uint32_t v0 = *w0++;
102 out[0] = v0;
103 const uint32_t v1 = *w1++;
104 out[1] = v1;
105 const uint32_t v2 = *w2++;
106 out[2] = v2;
107 out += 3;
108 }
109 out = (uint32_t*) ((uintptr_t) out + extra_bytes);
110 w0 = w2;
111 }
112
113 // NC remainder (1..2)
114 if XNN_UNLIKELY(n != 0) {
115 if XNN_LIKELY(b != NULL) {
116 size_t nb = n;
117 do {
118 *out++ = *b++;
119 } while (--nb != 0);
120 } else {
121 size_t nb = n;
122 do {
123 *out++ = 0;
124 } while (--nb != 0);
125 }
126 out += (3 - n);
127
128 // NR remainder has less than 3 rows so last row is not loaded
129 const uint32_t* w1 = w0 + kc;
130 if XNN_UNPREDICTABLE(n < 2) {
131 w1 = w0;
132 }
133
134 // KC main loop multiple of 3x4
135 size_t k = kc;
136 for (; k >= 4; k -= 4) {
137 const uint32_t v00 = w0[0];
138 const uint32_t v01 = w0[1];
139 const uint32_t v02 = w0[2];
140 const uint32_t v03 = w0[3];
141 w0 += 4;
142 const uint32_t v10 = w1[0];
143 const uint32_t v11 = w1[1];
144 const uint32_t v12 = w1[2];
145 const uint32_t v13 = w1[3];
146 w1 += 4;
147 out[0] = v00;
148 out[1] = v10;
149 out[3] = v01;
150 out[4] = v11;
151 out[6] = v02;
152 out[7] = v12;
153 out[9] = v03;
154 out[10] = v13;
155 out += 12;
156 }
157
158 // KC remainder of 1..3
159 for (; k != 0; --k) {
160 const uint32_t v0 = *w0++;
161 out[0] = v0;
162 const uint32_t v1 = *w1++;
163 out[1] = v1;
164 out += 3;
165 }
166 out = (uint32_t*) ((uintptr_t) out + extra_bytes);
167 }
168 weights += nc * kc;
169 } while (--g != 0);
170 }