Debian Codegraph @

src/x32-packw/gen/x32-packw-x2c4-gemm-goi-wasmsimd-u4.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/c4-wasmsimd.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include <wasm_simd128.h>
16
17 #include "xnnpack/packw.h"
18
19
20 void xnn_x32_packw_gemm_goi_ukernel_x2c4__wasmsimd_u4(
21 size_t g,
22 size_t nc,
23 size_t kc,
24 size_t nr,
25 size_t kr,
26 size_t sr,
27 const uint32_t* weights,
28 const uint32_t* bias,
29 const void* scale,
30 uint32_t* packed_weights,
31 size_t extra_bytes,
32 const void* params)
33 {
34 assert(g != 0);
35 assert(nc != 0);
36 assert(kc != 0);
37 assert(nr == 2);
38 assert(kr == 4);
39 assert(sr == 1);
40 assert(weights != NULL);
41 assert(packed_weights != NULL);
42
43 do {
44 // NC main loop multiple of 2
45 const uint32_t* w0 = (const uint32_t*) weights;
46 size_t n = nc;
47
48 for (; n >= 2; n -= 2) {
49 if XNN_LIKELY(bias != NULL) {
50 packed_weights[0] = bias[0];
51 packed_weights[1] = bias[1];
52 bias += 2;
53 } else {
54 packed_weights[0] = 0;
55 packed_weights[1] = 0;
56 }
57 packed_weights += 2;
58
59 const uint32_t* w1 = w0 + kc;
60
61 // KC main loop multiple of 2x4
62 size_t k = kc;
63 for (; k >= 4; k -= 4) {
64 // Read blocks of 2x4
65 // a b c d
66 // e f g h
67 const v128_t v0 = wasm_v128_load(w0);
68 w0 += 4;
69 const v128_t v1 = wasm_v128_load(w1);
70 w1 += 4;
71
72 wasm_v128_store(packed_weights, v0);
73 wasm_v128_store(packed_weights + 4, v1);
74 packed_weights += 8;
75 }
76
77 // KC remainder (1..3)
78 if XNN_UNLIKELY(k != 0) {
79 assert(k >= 1);
80 assert(k <= 3);
81 switch (k) {
82 case 1:
83 {
84 // Read blocks of 2x1
85 // a
86 // e
87 const v128_t v0 = wasm_v128_load32_zero(w0);
88 ++w0;
89 const v128_t v1 = wasm_v128_load32_zero(w1);
90 ++w1;
91 wasm_v128_store(packed_weights, v0);
92 wasm_v128_store(packed_weights + 4, v1);
93 packed_weights += 8;
94 break;
95 }
96 case 2:
97 {
98 // Read blocks of 2x2
99 // a b
100 // e f
101 const v128_t v0 = wasm_v128_load64_zero(w0);
102 w0 += 2;
103 const v128_t v1 = wasm_v128_load64_zero(w1);
104 w1 += 2;
105 wasm_v128_store(packed_weights, v0);
106 wasm_v128_store(packed_weights + 4, v1);
107 packed_weights += 8;
108 break;
109 }
110 case 3:
111 {
112 // Read blocks of 2x3
113 // a b c
114 // e f g
115 v128_t v0 = wasm_v128_load64_zero(w0);
116 v0 = wasm_v128_load32_lane(w0 + 2, v0, 2);
117 w0 += 3;
118 v128_t v1 = wasm_v128_load64_zero(w1);
119 v1 = wasm_v128_load32_lane(w1 + 2, v1, 2);
120 w1 += 3;
121 wasm_v128_store(packed_weights, v0);
122 wasm_v128_store(packed_weights + 4, v1);
123 packed_weights += 8;
124 break;
125 }
126 default:
127 XNN_UNREACHABLE;
128 }
129 }
130 packed_weights = (uint32_t*) ((uintptr_t) packed_weights + extra_bytes);
131 w0 = w1;
132 }
133
134 // NC remainder (1..1)
135 if XNN_UNLIKELY(n != 0) {
136 assert(n >= 1);
137 assert(n <= 1);
138 if XNN_LIKELY(bias != NULL) {
139 size_t nb = n;
140 do {
141 *packed_weights++ = *bias++;
142 } while (--nb != 0);
143 packed_weights += (2 - n);
144 } else {
145 packed_weights[0] = 0;
146 packed_weights[1] = 0;
147 packed_weights += 2;
148 }
149
150 // NR remainder has less than 2 rows so last row is not loaded
151
152
153 // KC main loop multiple of 2x4
154 size_t k = kc;
155 for (; k >= 4; k -= 4) {
156 // Read blocks of 2x4
157 // a b c d
158 // e f g h
159 const v128_t v0 = wasm_v128_load(w0);
160 w0 += 4;
161
162 wasm_v128_store(packed_weights, v0);
163 wasm_v128_store(packed_weights + 4, v0);
164 packed_weights += 8;
165 }
166
167 // KC remainder (1..3)
168 if XNN_UNLIKELY(k != 0) {
169 assert(k >= 1);
170 assert(k <= 3);
171 switch (k) {
172 case 1:
173 {
174 // Read blocks of 1x1
175 // a
176 const v128_t v0 = wasm_v128_load32_zero(w0);
177 ++w0;
178 wasm_v128_store(packed_weights, v0);
179 wasm_v128_store(packed_weights + 4, v0);
180 packed_weights += 8;
181 break;
182 }
183 case 2:
184 {
185 // Read blocks of 1x2
186 // a b
187 const v128_t v0 = wasm_v128_load64_zero(w0);
188 w0 += 2;
189 wasm_v128_store(packed_weights, v0);
190 wasm_v128_store(packed_weights + 4, v0);
191 packed_weights += 8;
192 break;
193 }
194 case 3:
195 {
196 // Read blocks of 1x3
197 // a b c
198 v128_t v0 = wasm_v128_load64_zero(w0);
199 v0 = wasm_v128_load32_lane(w0 + 2, v0, 2);
200 w0 += 3;
201 wasm_v128_store(packed_weights, v0);
202 wasm_v128_store(packed_weights + 4, v0);
203 packed_weights += 8;
204 break;
205 }
206 default:
207 XNN_UNREACHABLE;
208 }
209 }
210 packed_weights = (uint32_t*) ((uintptr_t) packed_weights + extra_bytes);
211 }
212 weights += nc * kc;
213 } while (--g != 0);
214 }