Debian Codegraph @

src/x32-packw/gen/x32-packw-x8-gemm-goi-wasmsimd-u4.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/wasmsimd.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include <wasm_simd128.h>
16
17 #include "xnnpack/common.h"
18 #include "xnnpack/packw.h"
19
20
21 void xnn_x32_packw_gemm_goi_ukernel_x8__wasmsimd_u4(
22 size_t g,
23 size_t nc,
24 size_t kc,
25 size_t nr,
26 size_t kr,
27 size_t sr,
28 const uint32_t* weights,
29 const uint32_t* bias,
30 const void* scale,
31 uint32_t* packed_weights,
32 size_t extra_bytes,
33 const void* params)
34 {
35 assert(g != 0);
36 assert(nc != 0);
37 assert(kc != 0);
38 assert(nr == 8);
39 assert(kr == 1);
40 assert(sr == 1);
41 assert(weights != NULL);
42 assert(packed_weights != NULL);
43
44 do {
45 // NC main loop multiple of 8
46 const uint32_t* w0 = (const uint32_t*) weights;
47
48 size_t n = nc;
49 for (; n >= 8; n -= 8) {
50 if XNN_LIKELY(bias != NULL) {
51 const v128_t vb0123 = wasm_v128_load(bias);
52 const v128_t vb4567 = wasm_v128_load(bias + 4);
53 bias += 8;
54
55 wasm_v128_store(packed_weights, vb0123);
56 wasm_v128_store(packed_weights + 4, vb4567);
57 } else {
58 const v128_t vzero = wasm_i32x4_const_splat(0);
59 wasm_v128_store(packed_weights, vzero);
60 wasm_v128_store(packed_weights + 4, vzero);
61 }
62 packed_weights += 8;
63
64 const uint32_t* w1 = w0 + kc;
65 const uint32_t* w2 = w1 + kc;
66 const uint32_t* w3 = w2 + kc;
67 const uint32_t* w4 = w3 + kc;
68 const uint32_t* w5 = w4 + kc;
69 const uint32_t* w6 = w5 + kc;
70 const uint32_t* w7 = w6 + kc;
71
72 // KC main loop multiple of 8x4
73 size_t k = kc;
74 for (; k >= 4; k -= 4) {
75 const v128_t v0x0123 = wasm_v128_load(w0);
76 w0 += 4;
77 const v128_t v1x0123 = wasm_v128_load(w1);
78 w1 += 4;
79 const v128_t v2x0123 = wasm_v128_load(w2);
80 w2 += 4;
81 const v128_t v3x0123 = wasm_v128_load(w3);
82 w3 += 4;
83 const v128_t v4x0123 = wasm_v128_load(w4);
84 w4 += 4;
85 const v128_t v5x0123 = wasm_v128_load(w5);
86 w5 += 4;
87 const v128_t v6x0123 = wasm_v128_load(w6);
88 w6 += 4;
89 const v128_t v7x0123 = wasm_v128_load(w7);
90 w7 += 4;
91
92 const v128_t v01x0_01x1 = wasm_v32x4_shuffle(v0x0123, v1x0123, 0, 4, 1, 5);
93 const v128_t v23x0_23x1 = wasm_v32x4_shuffle(v2x0123, v3x0123, 0, 4, 1, 5);
94 const v128_t v01x2_01x3 = wasm_v32x4_shuffle(v0x0123, v1x0123, 2, 6, 3, 7);
95 const v128_t v23x2_23x3 = wasm_v32x4_shuffle(v2x0123, v3x0123, 2, 6, 3, 7);
96 const v128_t v45x0_45x1 = wasm_v32x4_shuffle(v4x0123, v5x0123, 0, 4, 1, 5);
97 const v128_t v67x0_67x1 = wasm_v32x4_shuffle(v6x0123, v7x0123, 0, 4, 1, 5);
98 const v128_t v45x2_45x3 = wasm_v32x4_shuffle(v4x0123, v5x0123, 2, 6, 3, 7);
99 const v128_t v67x2_67x3 = wasm_v32x4_shuffle(v6x0123, v7x0123, 2, 6, 3, 7);
100
101 const v128_t v0123x0 = wasm_v64x2_shuffle(v01x0_01x1, v23x0_23x1, 0, 2);
102 const v128_t v0123x1 = wasm_v64x2_shuffle(v01x0_01x1, v23x0_23x1, 1, 3);
103 const v128_t v0123x2 = wasm_v64x2_shuffle(v01x2_01x3, v23x2_23x3, 0, 2);
104 const v128_t v0123x3 = wasm_v64x2_shuffle(v01x2_01x3, v23x2_23x3, 1, 3);
105 const v128_t v4567x0 = wasm_v64x2_shuffle(v45x0_45x1, v67x0_67x1, 0, 2);
106 const v128_t v4567x1 = wasm_v64x2_shuffle(v45x0_45x1, v67x0_67x1, 1, 3);
107 const v128_t v4567x2 = wasm_v64x2_shuffle(v45x2_45x3, v67x2_67x3, 0, 2);
108 const v128_t v4567x3 = wasm_v64x2_shuffle(v45x2_45x3, v67x2_67x3, 1, 3);
109
110 wasm_v128_store(packed_weights, v0123x0);
111 wasm_v128_store(packed_weights + 4, v4567x0);
112 wasm_v128_store(packed_weights + 8, v0123x1);
113 wasm_v128_store(packed_weights + 12, v4567x1);
114 wasm_v128_store(packed_weights + 16, v0123x2);
115 wasm_v128_store(packed_weights + 20, v4567x2);
116 wasm_v128_store(packed_weights + 24, v0123x3);
117 wasm_v128_store(packed_weights + 28, v4567x3);
118 packed_weights += 32;
119 }
120
121 if XNN_UNLIKELY(k != 0) {
122 // KC remainder (1..3)
123 assert(k >= 1);
124 assert(k <= 3);
125 switch (k) {
126 case 1:
127 {
128 v128_t v0123x0 = wasm_v128_load32_zero(w0);
129 w0 += 1;
130 v128_t v4567x0 = wasm_v128_load32_zero(w4);
131 w4 += 1;
132
133 v0123x0 = wasm_v128_load32_lane(w1, v0123x0, 1);
134 w1 += 1;
135 v4567x0 = wasm_v128_load32_lane(w5, v4567x0, 1);
136 w5 += 1;
137 v0123x0 = wasm_v128_load32_lane(w2, v0123x0, 2);
138 w2 += 1;
139 v4567x0 = wasm_v128_load32_lane(w6, v4567x0, 2);
140 w6 += 1;
141 v0123x0 = wasm_v128_load32_lane(w3, v0123x0, 3);
142 w3 += 1;
143 v4567x0 = wasm_v128_load32_lane(w7, v4567x0, 3);
144 w7 += 1;
145
146 wasm_v128_store(packed_weights, v0123x0);
147 wasm_v128_store(packed_weights + 4, v4567x0);
148 packed_weights += 8;
149 break;
150 }
151 case 2:
152 {
153 const v128_t v0x01 = wasm_v128_load64_zero(w0);
154 w0 += 2;
155 const v128_t v1x01 = wasm_v128_load64_zero(w1);
156 w1 += 2;
157 const v128_t v2x01 = wasm_v128_load64_zero(w2);
158 w2 += 2;
159 const v128_t v3x01 = wasm_v128_load64_zero(w3);
160 w3 += 2;
161 const v128_t v4x01 = wasm_v128_load64_zero(w4);
162 w4 += 2;
163 const v128_t v5x01 = wasm_v128_load64_zero(w5);
164 w5 += 2;
165 const v128_t v6x01 = wasm_v128_load64_zero(w6);
166 w6 += 2;
167 const v128_t v7x01 = wasm_v128_load64_zero(w7);
168 w7 += 2;
169
170 const v128_t v01x0_01x1 = wasm_v32x4_shuffle(v0x01, v1x01, 0, 4, 1, 5);
171 const v128_t v23x0_23x1 = wasm_v32x4_shuffle(v2x01, v3x01, 0, 4, 1, 5);
172 const v128_t v45x0_45x1 = wasm_v32x4_shuffle(v4x01, v5x01, 0, 4, 1, 5);
173 const v128_t v67x0_67x1 = wasm_v32x4_shuffle(v6x01, v7x01, 0, 4, 1, 5);
174
175 const v128_t v0123x0 = wasm_v64x2_shuffle(v01x0_01x1, v23x0_23x1, 0, 2);
176 const v128_t v0123x1 = wasm_v64x2_shuffle(v01x0_01x1, v23x0_23x1, 1, 3);
177 const v128_t v4567x0 = wasm_v64x2_shuffle(v45x0_45x1, v67x0_67x1, 0, 2);
178 const v128_t v4567x1 = wasm_v64x2_shuffle(v45x0_45x1, v67x0_67x1, 1, 3);
179
180 wasm_v128_store(packed_weights, v0123x0);
181 wasm_v128_store(packed_weights + 4, v4567x0);
182 wasm_v128_store(packed_weights + 8, v0123x1);
183 wasm_v128_store(packed_weights + 12, v4567x1);
184 packed_weights += 16;
185 break;
186 }
187 case 3:
188 {
189 v128_t v0x012 = wasm_v128_load64_zero(w0);
190 w0 += 2;
191 v128_t v1x012 = wasm_v128_load64_zero(w1);
192 w1 += 2;
193 v128_t v2x012 = wasm_v128_load64_zero(w2);
194 w2 += 2;
195 v128_t v3x012 = wasm_v128_load64_zero(w3);
196 w3 += 2;
197 v128_t v4x012 = wasm_v128_load64_zero(w4);
198 w4 += 2;
199 v128_t v5x012 = wasm_v128_load64_zero(w5);
200 w5 += 2;
201 v128_t v6x012 = wasm_v128_load64_zero(w6);
202 w6 += 2;
203 v128_t v7x012 = wasm_v128_load64_zero(w7);
204 w7 += 2;
205
206 v0x012 = wasm_v128_load32_lane(w0, v0x012, 2);
207 w0 += 1;
208 v1x012 = wasm_v128_load32_lane(w1, v1x012, 2);
209 w1 += 1;
210 v2x012 = wasm_v128_load32_lane(w2, v2x012, 2);
211 w2 += 1;
212 v3x012 = wasm_v128_load32_lane(w3, v3x012, 2);
213 w3 += 1;
214 v4x012 = wasm_v128_load32_lane(w4, v4x012, 2);
215 w4 += 1;
216 v5x012 = wasm_v128_load32_lane(w5, v5x012, 2);
217 w5 += 1;
218 v6x012 = wasm_v128_load32_lane(w6, v6x012, 2);
219 w6 += 1;
220 v7x012 = wasm_v128_load32_lane(w7, v7x012, 2);
221 w7 += 1;
222
223 const v128_t v01x0_01x1 = wasm_v32x4_shuffle(v0x012, v1x012, 0, 4, 1, 5);
224 const v128_t v23x0_23x1 = wasm_v32x4_shuffle(v2x012, v3x012, 0, 4, 1, 5);
225 const v128_t v01x2 = wasm_v32x4_shuffle(v0x012, v1x012, 2, 6, 3, 7);
226 const v128_t v23x2 = wasm_v32x4_shuffle(v2x012, v3x012, 2, 6, 3, 7);
227 const v128_t v45x0_45x1 = wasm_v32x4_shuffle(v4x012, v5x012, 0, 4, 1, 5);
228 const v128_t v67x0_67x1 = wasm_v32x4_shuffle(v6x012, v7x012, 0, 4, 1, 5);
229 const v128_t v45x2 = wasm_v32x4_shuffle(v4x012, v5x012, 2, 6, 3, 7);
230 const v128_t v67x2 = wasm_v32x4_shuffle(v6x012, v7x012, 2, 6, 3, 7);
231
232 const v128_t v0123x0 = wasm_v64x2_shuffle(v01x0_01x1, v23x0_23x1, 0, 2);
233 const v128_t v0123x1 = wasm_v64x2_shuffle(v01x0_01x1, v23x0_23x1, 1, 3);
234 const v128_t v0123x2 = wasm_v64x2_shuffle(v01x2, v23x2, 0, 2);
235 const v128_t v4567x0 = wasm_v64x2_shuffle(v45x0_45x1, v67x0_67x1, 0, 2);
236 const v128_t v4567x1 = wasm_v64x2_shuffle(v45x0_45x1, v67x0_67x1, 1, 3);
237 const v128_t v4567x2 = wasm_v64x2_shuffle(v45x2, v67x2, 0, 2);
238
239 wasm_v128_store(packed_weights, v0123x0);
240 wasm_v128_store(packed_weights + 4, v4567x0);
241 wasm_v128_store(packed_weights + 8, v0123x1);
242 wasm_v128_store(packed_weights + 12, v4567x1);
243 wasm_v128_store(packed_weights + 16, v0123x2);
244 wasm_v128_store(packed_weights + 20, v4567x2);
245 packed_weights += 24;
246 break;
247 }
248 default:
249 XNN_UNREACHABLE;
250 }
251 }
252 packed_weights = (uint32_t*) ((uintptr_t) packed_weights + extra_bytes);
253 w0 = w7;
254 }
255
256 // NC remainder (1..7)
257 if XNN_UNLIKELY(n != 0) {
258 assert(n >= 1);
259 assert(n <= 7);
260 if XNN_LIKELY(bias != NULL) {
261 size_t nb = n;
262 do {
263 *packed_weights++ = *bias++;
264 } while (--nb != 0);
265 packed_weights += (8 - n);
266 } else {
267 const v128_t vzero = wasm_i32x4_const_splat(0);
268 wasm_v128_store(packed_weights, vzero);
269 wasm_v128_store(packed_weights + 4, vzero);
270 packed_weights += 8;
271 }
272
273 const uint32_t* w1 = w0 + kc;
274 if XNN_UNPREDICTABLE(n < 2) {
275 w1 = w0;
276 }
277 const uint32_t* w2 = w1 + kc;
278 if XNN_UNPREDICTABLE(n <= 2) {
279 w2 = w1;
280 }
281 const uint32_t* w3 = w2 + kc;
282 if XNN_UNPREDICTABLE(n < 4) {
283 w3 = w2;
284 }
285 const uint32_t* w4 = w3 + kc;
286 if XNN_UNPREDICTABLE(n <= 4) {
287 w4 = w3;
288 }
289 const uint32_t* w5 = w4 + kc;
290 if XNN_UNPREDICTABLE(n < 6) {
291 w5 = w4;
292 }
293 const uint32_t* w6 = w5 + kc;
294 if XNN_UNPREDICTABLE(n <= 6) {
295 w6 = w5;
296 }
297
298 size_t k = kc;
299 for (; k >= 4; k -= 4) {
300 const v128_t v0x0123 = wasm_v128_load(w0);
301 w0 += 4;
302 const v128_t v1x0123 = wasm_v128_load(w1);
303 w1 += 4;
304 const v128_t v2x0123 = wasm_v128_load(w2);
305 w2 += 4;
306 const v128_t v3x0123 = wasm_v128_load(w3);
307 w3 += 4;
308 const v128_t v4x0123 = wasm_v128_load(w4);
309 w4 += 4;
310 const v128_t v5x0123 = wasm_v128_load(w5);
311 w5 += 4;
312 const v128_t v6x0123 = wasm_v128_load(w6);
313 w6 += 4;
314
315 const v128_t v01x0_01x1 = wasm_v32x4_shuffle(v0x0123, v1x0123, 0, 4, 1, 5);
316 const v128_t v23x0_23x1 = wasm_v32x4_shuffle(v2x0123, v3x0123, 0, 4, 1, 5);
317 const v128_t v01x2_01x3 = wasm_v32x4_shuffle(v0x0123, v1x0123, 2, 6, 3, 7);
318 const v128_t v23x2_23x3 = wasm_v32x4_shuffle(v2x0123, v3x0123, 2, 6, 3, 7);
319 const v128_t v45x0_45x1 = wasm_v32x4_shuffle(v4x0123, v5x0123, 0, 4, 1, 5);
320 const v128_t v67x0_67x1 = wasm_v32x4_shuffle(v6x0123, v6x0123, 0, 4, 1, 5);
321 const v128_t v45x2_45x3 = wasm_v32x4_shuffle(v4x0123, v5x0123, 2, 6, 3, 7);
322 const v128_t v67x2_67x3 = wasm_v32x4_shuffle(v6x0123, v6x0123, 2, 6, 3, 7);
323
324 const v128_t v0123x0 = wasm_v64x2_shuffle(v01x0_01x1, v23x0_23x1, 0, 2);
325 const v128_t v0123x1 = wasm_v64x2_shuffle(v01x0_01x1, v23x0_23x1, 1, 3);
326 const v128_t v0123x2 = wasm_v64x2_shuffle(v01x2_01x3, v23x2_23x3, 0, 2);
327 const v128_t v0123x3 = wasm_v64x2_shuffle(v01x2_01x3, v23x2_23x3, 1, 3);
328 const v128_t v4567x0 = wasm_v64x2_shuffle(v45x0_45x1, v67x0_67x1, 0, 2);
329 const v128_t v4567x1 = wasm_v64x2_shuffle(v45x0_45x1, v67x0_67x1, 1, 3);
330 const v128_t v4567x2 = wasm_v64x2_shuffle(v45x2_45x3, v67x2_67x3, 0, 2);
331 const v128_t v4567x3 = wasm_v64x2_shuffle(v45x2_45x3, v67x2_67x3, 1, 3);
332
333 wasm_v128_store(packed_weights, v0123x0);
334 wasm_v128_store(packed_weights + 4, v4567x0);
335 wasm_v128_store(packed_weights + 8, v0123x1);
336 wasm_v128_store(packed_weights + 12, v4567x1);
337 wasm_v128_store(packed_weights + 16, v0123x2);
338 wasm_v128_store(packed_weights + 20, v4567x2);
339 wasm_v128_store(packed_weights + 24, v0123x3);
340 wasm_v128_store(packed_weights + 28, v4567x3);
341 packed_weights += 32;
342 }
343
344 // KC remainder (1..3)
345 if XNN_UNLIKELY(k != 0) {
346 assert(k >= 1);
347 assert(k <= 3);
348 switch (k) {
349 case 1:
350 {
351 v128_t v0123x0 = wasm_v128_load32_zero(w0);
352 w0 += 1;
353 v128_t v4567x0 = wasm_v128_load32_zero(w4);
354 w4 += 1;
355
356 v0123x0 = wasm_v128_load32_lane(w1, v0123x0, 1);
357 w1 += 1;
358 v4567x0 = wasm_v128_load32_lane(w5, v4567x0, 1);
359 w5 += 1;
360 v0123x0 = wasm_v128_load32_lane(w2, v0123x0, 2);
361 w2 += 1;
362 v4567x0 = wasm_v128_load32_lane(w6, v4567x0, 2);
363 w6 += 1;
364 v0123x0 = wasm_v128_load32_lane(w3, v0123x0, 3);
365 w3 += 1;
366
367 wasm_v128_store(packed_weights, v0123x0);
368 wasm_v128_store(packed_weights + 4, v4567x0);
369 packed_weights += 8;
370 break;
371 }
372 case 2:
373 {
374 const v128_t v0x01 = wasm_v128_load64_zero(w0);
375 w0 += 2;
376 const v128_t v1x01 = wasm_v128_load64_zero(w1);
377 w1 += 2;
378 const v128_t v2x01 = wasm_v128_load64_zero(w2);
379 w2 += 2;
380 const v128_t v3x01 = wasm_v128_load64_zero(w3);
381 w3 += 2;
382 const v128_t v4x01 = wasm_v128_load64_zero(w4);
383 w4 += 2;
384 const v128_t v5x01 = wasm_v128_load64_zero(w5);
385 w5 += 2;
386 const v128_t v6x01 = wasm_v128_load64_zero(w6);
387 w6 += 2;
388
389 const v128_t v01x0_01x1 = wasm_v32x4_shuffle(v0x01, v1x01, 0, 4, 1, 5);
390 const v128_t v23x0_23x1 = wasm_v32x4_shuffle(v2x01, v3x01, 0, 4, 1, 5);
391 const v128_t v45x0_45x1 = wasm_v32x4_shuffle(v4x01, v5x01, 0, 4, 1, 5);
392 const v128_t v67x0_67x1 = wasm_v32x4_shuffle(v6x01, v6x01, 0, 4, 1, 5);
393
394 const v128_t v0123x0 = wasm_v64x2_shuffle(v01x0_01x1, v23x0_23x1, 0, 2);
395 const v128_t v0123x1 = wasm_v64x2_shuffle(v01x0_01x1, v23x0_23x1, 1, 3);
396 const v128_t v4567x0 = wasm_v64x2_shuffle(v45x0_45x1, v67x0_67x1, 0, 2);
397 const v128_t v4567x1 = wasm_v64x2_shuffle(v45x0_45x1, v67x0_67x1, 1, 3);
398
399 wasm_v128_store(packed_weights, v0123x0);
400 wasm_v128_store(packed_weights + 4, v4567x0);
401 wasm_v128_store(packed_weights + 8, v0123x1);
402 wasm_v128_store(packed_weights + 12, v4567x1);
403 packed_weights += 16;
404 break;
405 }
406 case 3:
407 {
408 v128_t v0x012 = wasm_v128_load64_zero(w0);
409 w0 += 2;
410 v128_t v1x012 = wasm_v128_load64_zero(w1);
411 w1 += 2;
412 v128_t v2x012 = wasm_v128_load64_zero(w2);
413 w2 += 2;
414 v128_t v3x012 = wasm_v128_load64_zero(w3);
415 w3 += 2;
416 v128_t v4x012 = wasm_v128_load64_zero(w4);
417 w4 += 2;
418 v128_t v5x012 = wasm_v128_load64_zero(w5);
419 w5 += 2;
420 v128_t v6x012 = wasm_v128_load64_zero(w6);
421 w6 += 2;
422
423 v0x012 = wasm_v128_load32_lane(w0, v0x012, 2);
424 w0 += 1;
425 v1x012 = wasm_v128_load32_lane(w1, v1x012, 2);
426 w1 += 1;
427 v2x012 = wasm_v128_load32_lane(w2, v2x012, 2);
428 w2 += 1;
429 v3x012 = wasm_v128_load32_lane(w3, v3x012, 2);
430 w3 += 1;
431 v4x012 = wasm_v128_load32_lane(w4, v4x012, 2);
432 w4 += 1;
433 v5x012 = wasm_v128_load32_lane(w5, v5x012, 2);
434 w5 += 1;
435 v6x012 = wasm_v128_load32_lane(w6, v6x012, 2);
436 w6 += 1;
437
438 const v128_t v01x0_01x1 = wasm_v32x4_shuffle(v0x012, v1x012, 0, 4, 1, 5);
439 const v128_t v23x0_23x1 = wasm_v32x4_shuffle(v2x012, v3x012, 0, 4, 1, 5);
440 const v128_t v01x2 = wasm_v32x4_shuffle(v0x012, v1x012, 2, 6, 3, 7);
441 const v128_t v23x2 = wasm_v32x4_shuffle(v2x012, v3x012, 2, 6, 3, 7);
442 const v128_t v45x0_45x1 = wasm_v32x4_shuffle(v4x012, v5x012, 0, 4, 1, 5);
443 const v128_t v67x0_67x1 = wasm_v32x4_shuffle(v6x012, v6x012, 0, 4, 1, 5);
444 const v128_t v45x2 = wasm_v32x4_shuffle(v4x012, v5x012, 2, 6, 3, 7);
445 const v128_t v67x2 = wasm_v32x4_shuffle(v6x012, v6x012, 2, 6, 3, 7);
446
447 const v128_t v0123x0 = wasm_v64x2_shuffle(v01x0_01x1, v23x0_23x1, 0, 2);
448 const v128_t v0123x1 = wasm_v64x2_shuffle(v01x0_01x1, v23x0_23x1, 1, 3);
449 const v128_t v0123x2 = wasm_v64x2_shuffle(v01x2, v23x2, 0, 2);
450 const v128_t v4567x0 = wasm_v64x2_shuffle(v45x0_45x1, v67x0_67x1, 0, 2);
451 const v128_t v4567x1 = wasm_v64x2_shuffle(v45x0_45x1, v67x0_67x1, 1, 3);
452 const v128_t v4567x2 = wasm_v64x2_shuffle(v45x2, v67x2, 0, 2);
453
454 wasm_v128_store(packed_weights, v0123x0);
455 wasm_v128_store(packed_weights + 4, v4567x0);
456 wasm_v128_store(packed_weights + 8, v0123x1);
457 wasm_v128_store(packed_weights + 12, v4567x1);
458 wasm_v128_store(packed_weights + 16, v0123x2);
459 wasm_v128_store(packed_weights + 20, v4567x2);
460 packed_weights += 24;
461 break;
462 }
463 default:
464 XNN_UNREACHABLE;
465 }
466 }
467 packed_weights = (uint32_t*) ((uintptr_t) packed_weights + extra_bytes);
468 }
469 weights += nc * kc;
470 } while (--g != 0);
471 }