Debian Codegraph @

src/x32-packw/gen/x32-packw-x8s4-gemm-goi-wasmsimd-u4.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/s4-wasmsimd.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include <wasm_simd128.h>
16
17 #include "xnnpack/common.h"
18 #include "xnnpack/packw.h"
19
20
21 void xnn_x32_packw_gemm_goi_ukernel_x8s4__wasmsimd_u4(
22 size_t g,
23 size_t nc,
24 size_t kc,
25 size_t nr,
26 size_t kr,
27 size_t sr,
28 const uint32_t* weights,
29 const uint32_t* bias,
30 const void* scale,
31 uint32_t* packed_weights,
32 size_t extra_bytes,
33 const void* params)
34 {
35 assert(g != 0);
36 assert(nc != 0);
37 assert(kc != 0);
38 assert(nr == 8);
39 assert(kr == 1);
40 assert(sr == 4);
41 assert(weights != NULL);
42 assert(packed_weights != NULL);
43
44 do {
45 // NC main loop multiple of 8
46 const uint32_t* w0 = (const uint32_t*) weights;
47
48 size_t n = nc;
49 for (; n >= 8; n -= 8) {
50 if XNN_LIKELY(bias != NULL) {
51 const v128_t vb0123 = wasm_v128_load(bias);
52 const v128_t vb4567 = wasm_v128_load(bias + 4);
53 bias += 8;
54
55 wasm_v128_store(packed_weights, vb0123);
56 wasm_v128_store(packed_weights + 4, vb4567);
57 } else {
58 const v128_t vzero = wasm_i32x4_const_splat(0);
59 wasm_v128_store(packed_weights, vzero);
60 wasm_v128_store(packed_weights + 4, vzero);
61 }
62 packed_weights += 8;
63
64 const uint32_t* w1 = w0 + kc;
65 const uint32_t* w2 = w1 + kc;
66 const uint32_t* w3 = w2 + kc;
67 const uint32_t* w4 = w3 + kc;
68 const uint32_t* w5 = w4 + kc;
69 const uint32_t* w6 = w5 + kc;
70 const uint32_t* w7 = w6 + kc;
71
72 // KC main loop multiple of 8x4
73 size_t k = kc;
74 for (; k >= 4; k -= 4) {
75 v128_t v0x0123 = wasm_v128_load(w0);
76 w0 += 4;
77 v128_t v1x0123 = wasm_v128_load(w1);
78 w1 += 4;
79 v128_t v2x0123 = wasm_v128_load(w2);
80 w2 += 4;
81 v128_t v3x0123 = wasm_v128_load(w3);
82 w3 += 4;
83 v128_t v4x0123 = wasm_v128_load(w4);
84 w4 += 4;
85 v128_t v5x0123 = wasm_v128_load(w5);
86 w5 += 4;
87 v128_t v6x0123 = wasm_v128_load(w6);
88 w6 += 4;
89 v128_t v7x0123 = wasm_v128_load(w7);
90 w7 += 4;
91
92 v1x0123 = wasm_v32x4_shuffle(v1x0123, v1x0123, 1, 2, 3, 0);
93 v2x0123 = wasm_v32x4_shuffle(v2x0123, v2x0123, 2, 3, 0, 1);
94 v3x0123 = wasm_v32x4_shuffle(v3x0123, v3x0123, 3, 0, 1, 2);
95 v5x0123 = wasm_v32x4_shuffle(v5x0123, v5x0123, 1, 2, 3, 0);
96 v6x0123 = wasm_v32x4_shuffle(v6x0123, v6x0123, 2, 3, 0, 1);
97 v7x0123 = wasm_v32x4_shuffle(v7x0123, v7x0123, 3, 0, 1, 2);
98
99 const v128_t v01x0_01x1 = wasm_v32x4_shuffle(v0x0123, v1x0123, 0, 4, 1, 5);
100 const v128_t v23x0_23x1 = wasm_v32x4_shuffle(v2x0123, v3x0123, 0, 4, 1, 5);
101 const v128_t v01x2_01x3 = wasm_v32x4_shuffle(v0x0123, v1x0123, 2, 6, 3, 7);
102 const v128_t v23x2_23x3 = wasm_v32x4_shuffle(v2x0123, v3x0123, 2, 6, 3, 7);
103 const v128_t v45x0_45x1 = wasm_v32x4_shuffle(v4x0123, v5x0123, 0, 4, 1, 5);
104 const v128_t v67x0_67x1 = wasm_v32x4_shuffle(v6x0123, v7x0123, 0, 4, 1, 5);
105 const v128_t v45x2_45x3 = wasm_v32x4_shuffle(v4x0123, v5x0123, 2, 6, 3, 7);
106 const v128_t v67x2_67x3 = wasm_v32x4_shuffle(v6x0123, v7x0123, 2, 6, 3, 7);
107
108 const v128_t v0123x0 = wasm_v64x2_shuffle(v01x0_01x1, v23x0_23x1, 0, 2);
109 const v128_t v0123x1 = wasm_v64x2_shuffle(v01x0_01x1, v23x0_23x1, 1, 3);
110 const v128_t v0123x2 = wasm_v64x2_shuffle(v01x2_01x3, v23x2_23x3, 0, 2);
111 const v128_t v0123x3 = wasm_v64x2_shuffle(v01x2_01x3, v23x2_23x3, 1, 3);
112 const v128_t v4567x0 = wasm_v64x2_shuffle(v45x0_45x1, v67x0_67x1, 0, 2);
113 const v128_t v4567x1 = wasm_v64x2_shuffle(v45x0_45x1, v67x0_67x1, 1, 3);
114 const v128_t v4567x2 = wasm_v64x2_shuffle(v45x2_45x3, v67x2_67x3, 0, 2);
115 const v128_t v4567x3 = wasm_v64x2_shuffle(v45x2_45x3, v67x2_67x3, 1, 3);
116
117 wasm_v128_store(packed_weights, v0123x0);
118 wasm_v128_store(packed_weights + 4, v4567x0);
119 wasm_v128_store(packed_weights + 8, v0123x1);
120 wasm_v128_store(packed_weights + 12, v4567x1);
121 wasm_v128_store(packed_weights + 16, v0123x2);
122 wasm_v128_store(packed_weights + 20, v4567x2);
123 wasm_v128_store(packed_weights + 24, v0123x3);
124 wasm_v128_store(packed_weights + 28, v4567x3);
125 packed_weights += 32;
126 }
127
128 // KC remainder (1..3)
129 if XNN_UNLIKELY(k != 0) {
130 assert(k >= 1);
131 assert(k <= 3);
132 v128_t v0x0123 = wasm_i32x4_const_splat(0);
133 v128_t v1x0123 = wasm_i32x4_const_splat(0);
134 v128_t v2x0123 = wasm_i32x4_const_splat(0);
135 v128_t v3x0123 = wasm_i32x4_const_splat(0);
136 v128_t v4x0123 = wasm_i32x4_const_splat(0);
137 v128_t v5x0123 = wasm_i32x4_const_splat(0);
138 v128_t v6x0123 = wasm_i32x4_const_splat(0);
139 v128_t v7x0123 = wasm_i32x4_const_splat(0);
140
141 switch (k) {
142 case 1:
143 v0x0123 = wasm_v128_load32_zero(w0);
144 w0 += 1;
145 v1x0123 = wasm_v128_load32_zero(w1);
146 w1 += 1;
147 v2x0123 = wasm_v128_load32_zero(w2);
148 w2 += 1;
149 v3x0123 = wasm_v128_load32_zero(w3);
150 w3 += 1;
151 v4x0123 = wasm_v128_load32_zero(w4);
152 w4 += 1;
153 v5x0123 = wasm_v128_load32_zero(w5);
154 w5 += 1;
155 v6x0123 = wasm_v128_load32_zero(w6);
156 w6 += 1;
157 v7x0123 = wasm_v128_load32_zero(w7);
158 w7 += 1;
159 break;
160 case 2:
161 v0x0123 = wasm_v128_load64_zero(w0);
162 w0 += 2;
163 v1x0123 = wasm_v128_load64_zero(w1);
164 w1 += 2;
165 v2x0123 = wasm_v128_load64_zero(w2);
166 w2 += 2;
167 v3x0123 = wasm_v128_load64_zero(w3);
168 w3 += 2;
169 v4x0123 = wasm_v128_load64_zero(w4);
170 w4 += 2;
171 v5x0123 = wasm_v128_load64_zero(w5);
172 w5 += 2;
173 v6x0123 = wasm_v128_load64_zero(w6);
174 w6 += 2;
175 v7x0123 = wasm_v128_load64_zero(w7);
176 w7 += 2;
177 break;
178 case 3:
179 v0x0123 = wasm_v128_load64_zero(w0);
180 w0 += 2;
181 v1x0123 = wasm_v128_load64_zero(w1);
182 w1 += 2;
183 v2x0123 = wasm_v128_load64_zero(w2);
184 w2 += 2;
185 v3x0123 = wasm_v128_load64_zero(w3);
186 w3 += 2;
187 v4x0123 = wasm_v128_load64_zero(w4);
188 w4 += 2;
189 v5x0123 = wasm_v128_load64_zero(w5);
190 w5 += 2;
191 v6x0123 = wasm_v128_load64_zero(w6);
192 w6 += 2;
193 v7x0123 = wasm_v128_load64_zero(w7);
194 w7 += 2;
195
196 v0x0123 = wasm_v128_load32_lane(w0, v0x0123, 2);
197 w0 += 1;
198 v1x0123 = wasm_v128_load32_lane(w1, v1x0123, 2);
199 w1 += 1;
200 v2x0123 = wasm_v128_load32_lane(w2, v2x0123, 2);
201 w2 += 1;
202 v3x0123 = wasm_v128_load32_lane(w3, v3x0123, 2);
203 w3 += 1;
204 v4x0123 = wasm_v128_load32_lane(w4, v4x0123, 2);
205 w4 += 1;
206 v5x0123 = wasm_v128_load32_lane(w5, v5x0123, 2);
207 w5 += 1;
208 v6x0123 = wasm_v128_load32_lane(w6, v6x0123, 2);
209 w6 += 1;
210 v7x0123 = wasm_v128_load32_lane(w7, v7x0123, 2);
211 w7 += 1;
212 break;
213 default:
214 XNN_UNREACHABLE;
215 }
216
217 v1x0123 = wasm_v32x4_shuffle(v1x0123, v1x0123, 1, 2, 3, 0);
218 v2x0123 = wasm_v32x4_shuffle(v2x0123, v2x0123, 2, 3, 0, 1);
219 v3x0123 = wasm_v32x4_shuffle(v3x0123, v3x0123, 3, 0, 1, 2);
220 v5x0123 = wasm_v32x4_shuffle(v5x0123, v5x0123, 1, 2, 3, 0);
221 v6x0123 = wasm_v32x4_shuffle(v6x0123, v6x0123, 2, 3, 0, 1);
222 v7x0123 = wasm_v32x4_shuffle(v7x0123, v7x0123, 3, 0, 1, 2);
223
224 const v128_t v01x0_01x1 = wasm_v32x4_shuffle(v0x0123, v1x0123, 0, 4, 1, 5);
225 const v128_t v23x0_23x1 = wasm_v32x4_shuffle(v2x0123, v3x0123, 0, 4, 1, 5);
226 const v128_t v01x2_01x3 = wasm_v32x4_shuffle(v0x0123, v1x0123, 2, 6, 3, 7);
227 const v128_t v23x2_23x3 = wasm_v32x4_shuffle(v2x0123, v3x0123, 2, 6, 3, 7);
228 const v128_t v45x0_45x1 = wasm_v32x4_shuffle(v4x0123, v5x0123, 0, 4, 1, 5);
229 const v128_t v67x0_67x1 = wasm_v32x4_shuffle(v6x0123, v7x0123, 0, 4, 1, 5);
230 const v128_t v45x2_45x3 = wasm_v32x4_shuffle(v4x0123, v5x0123, 2, 6, 3, 7);
231 const v128_t v67x2_67x3 = wasm_v32x4_shuffle(v6x0123, v7x0123, 2, 6, 3, 7);
232
233 const v128_t v0123x0 = wasm_v64x2_shuffle(v01x0_01x1, v23x0_23x1, 0, 2);
234 const v128_t v0123x1 = wasm_v64x2_shuffle(v01x0_01x1, v23x0_23x1, 1, 3);
235 const v128_t v0123x2 = wasm_v64x2_shuffle(v01x2_01x3, v23x2_23x3, 0, 2);
236 const v128_t v0123x3 = wasm_v64x2_shuffle(v01x2_01x3, v23x2_23x3, 1, 3);
237 const v128_t v4567x0 = wasm_v64x2_shuffle(v45x0_45x1, v67x0_67x1, 0, 2);
238 const v128_t v4567x1 = wasm_v64x2_shuffle(v45x0_45x1, v67x0_67x1, 1, 3);
239 const v128_t v4567x2 = wasm_v64x2_shuffle(v45x2_45x3, v67x2_67x3, 0, 2);
240 const v128_t v4567x3 = wasm_v64x2_shuffle(v45x2_45x3, v67x2_67x3, 1, 3);
241
242 wasm_v128_store(packed_weights, v0123x0);
243 wasm_v128_store(packed_weights + 4, v4567x0);
244 wasm_v128_store(packed_weights + 8, v0123x1);
245 wasm_v128_store(packed_weights + 12, v4567x1);
246 wasm_v128_store(packed_weights + 16, v0123x2);
247 wasm_v128_store(packed_weights + 20, v4567x2);
248 wasm_v128_store(packed_weights + 24, v0123x3);
249 wasm_v128_store(packed_weights + 28, v4567x3);
250 packed_weights += 32;
251 }
252 packed_weights = (uint32_t*) ((uintptr_t) packed_weights + extra_bytes);
253 w0 = w7;
254 }
255
256 // NC remainder (1..7)
257 if XNN_UNLIKELY(n != 0) {
258 assert(n >= 1);
259 assert(n <= 7);
260 if XNN_LIKELY(bias != NULL) {
261 size_t nb = n;
262 do {
263 *packed_weights++ = *bias++;
264 } while (--nb != 0);
265 packed_weights += (8 - n);
266 } else {
267 const v128_t vzero = wasm_i32x4_const_splat(0);
268 wasm_v128_store(packed_weights, vzero);
269 wasm_v128_store(packed_weights + 4, vzero);
270 packed_weights += 8;
271 }
272
273 const uint32_t* w1 = w0 + kc;
274 if XNN_UNPREDICTABLE(n < 2) {
275 w1 = w0;
276 }
277 const uint32_t* w2 = w1 + kc;
278 if XNN_UNPREDICTABLE(n <= 2) {
279 w2 = w1;
280 }
281 const uint32_t* w3 = w2 + kc;
282 if XNN_UNPREDICTABLE(n < 4) {
283 w3 = w2;
284 }
285 const uint32_t* w4 = w3 + kc;
286 if XNN_UNPREDICTABLE(n <= 4) {
287 w4 = w3;
288 }
289 const uint32_t* w5 = w4 + kc;
290 if XNN_UNPREDICTABLE(n < 6) {
291 w5 = w4;
292 }
293 const uint32_t* w6 = w5 + kc;
294 if XNN_UNPREDICTABLE(n <= 6) {
295 w6 = w5;
296 }
297
298 size_t k = kc;
299 for (; k >= 4; k -= 4) {
300 v128_t v0x0123 = wasm_v128_load(w0);
301 w0 += 4;
302 v128_t v1x0123 = wasm_v128_load(w1);
303 w1 += 4;
304 v128_t v2x0123 = wasm_v128_load(w2);
305 w2 += 4;
306 v128_t v3x0123 = wasm_v128_load(w3);
307 w3 += 4;
308 v128_t v4x0123 = wasm_v128_load(w4);
309 w4 += 4;
310 v128_t v5x0123 = wasm_v128_load(w5);
311 w5 += 4;
312 v128_t v6x0123 = wasm_v128_load(w6);
313 w6 += 4;
314
315 v1x0123 = wasm_v32x4_shuffle(v1x0123, v1x0123, 1, 2, 3, 0);
316 v2x0123 = wasm_v32x4_shuffle(v2x0123, v2x0123, 2, 3, 0, 1);
317 v3x0123 = wasm_v32x4_shuffle(v3x0123, v3x0123, 3, 0, 1, 2);
318 v5x0123 = wasm_v32x4_shuffle(v5x0123, v5x0123, 1, 2, 3, 0);
319 v6x0123 = wasm_v32x4_shuffle(v6x0123, v6x0123, 2, 3, 0, 1);
320
321 const v128_t v01x0_01x1 = wasm_v32x4_shuffle(v0x0123, v1x0123, 0, 4, 1, 5);
322 const v128_t v23x0_23x1 = wasm_v32x4_shuffle(v2x0123, v3x0123, 0, 4, 1, 5);
323 const v128_t v01x2_01x3 = wasm_v32x4_shuffle(v0x0123, v1x0123, 2, 6, 3, 7);
324 const v128_t v23x2_23x3 = wasm_v32x4_shuffle(v2x0123, v3x0123, 2, 6, 3, 7);
325 const v128_t v45x0_45x1 = wasm_v32x4_shuffle(v4x0123, v5x0123, 0, 4, 1, 5);
326 const v128_t v67x0_67x1 = wasm_v32x4_shuffle(v6x0123, v6x0123, 0, 4, 1, 5);
327 const v128_t v45x2_45x3 = wasm_v32x4_shuffle(v4x0123, v5x0123, 2, 6, 3, 7);
328 const v128_t v67x2_67x3 = wasm_v32x4_shuffle(v6x0123, v6x0123, 2, 6, 3, 7);
329
330 const v128_t v0123x0 = wasm_v64x2_shuffle(v01x0_01x1, v23x0_23x1, 0, 2);
331 const v128_t v0123x1 = wasm_v64x2_shuffle(v01x0_01x1, v23x0_23x1, 1, 3);
332 const v128_t v0123x2 = wasm_v64x2_shuffle(v01x2_01x3, v23x2_23x3, 0, 2);
333 const v128_t v0123x3 = wasm_v64x2_shuffle(v01x2_01x3, v23x2_23x3, 1, 3);
334 const v128_t v4567x0 = wasm_v64x2_shuffle(v45x0_45x1, v67x0_67x1, 0, 2);
335 const v128_t v4567x1 = wasm_v64x2_shuffle(v45x0_45x1, v67x0_67x1, 1, 3);
336 const v128_t v4567x2 = wasm_v64x2_shuffle(v45x2_45x3, v67x2_67x3, 0, 2);
337 const v128_t v4567x3 = wasm_v64x2_shuffle(v45x2_45x3, v67x2_67x3, 1, 3);
338
339 wasm_v128_store(packed_weights, v0123x0);
340 wasm_v128_store(packed_weights + 4, v4567x0);
341 wasm_v128_store(packed_weights + 8, v0123x1);
342 wasm_v128_store(packed_weights + 12, v4567x1);
343 wasm_v128_store(packed_weights + 16, v0123x2);
344 wasm_v128_store(packed_weights + 20, v4567x2);
345 wasm_v128_store(packed_weights + 24, v0123x3);
346 wasm_v128_store(packed_weights + 28, v4567x3);
347 packed_weights += 32;
348 }
349
350 // KC remainder (1..3)
351 if XNN_UNLIKELY(k != 0) {
352 assert(k >= 1);
353 assert(k <= 3);
354 v128_t v0x0123 = wasm_i32x4_const_splat(0);
355 v128_t v1x0123 = wasm_i32x4_const_splat(0);
356 v128_t v2x0123 = wasm_i32x4_const_splat(0);
357 v128_t v3x0123 = wasm_i32x4_const_splat(0);
358 v128_t v4x0123 = wasm_i32x4_const_splat(0);
359 v128_t v5x0123 = wasm_i32x4_const_splat(0);
360 v128_t v6x0123 = wasm_i32x4_const_splat(0);
361
362 switch (k) {
363 case 1:
364 v0x0123 = wasm_v128_load32_zero(w0);
365 w0 += 1;
366 v1x0123 = wasm_v128_load32_zero(w1);
367 w1 += 1;
368 v2x0123 = wasm_v128_load32_zero(w2);
369 w2 += 1;
370 v3x0123 = wasm_v128_load32_zero(w3);
371 w3 += 1;
372 v4x0123 = wasm_v128_load32_zero(w4);
373 w4 += 1;
374 v5x0123 = wasm_v128_load32_zero(w5);
375 w5 += 1;
376 v6x0123 = wasm_v128_load32_zero(w6);
377 w6 += 1;
378 break;
379 case 2:
380 v0x0123 = wasm_v128_load64_zero(w0);
381 w0 += 2;
382 v1x0123 = wasm_v128_load64_zero(w1);
383 w1 += 2;
384 v2x0123 = wasm_v128_load64_zero(w2);
385 w2 += 2;
386 v3x0123 = wasm_v128_load64_zero(w3);
387 w3 += 2;
388 v4x0123 = wasm_v128_load64_zero(w4);
389 w4 += 2;
390 v5x0123 = wasm_v128_load64_zero(w5);
391 w5 += 2;
392 v6x0123 = wasm_v128_load64_zero(w6);
393 w6 += 2;
394 break;
395 case 3:
396 v0x0123 = wasm_v128_load64_zero(w0);
397 w0 += 2;
398 v1x0123 = wasm_v128_load64_zero(w1);
399 w1 += 2;
400 v2x0123 = wasm_v128_load64_zero(w2);
401 w2 += 2;
402 v3x0123 = wasm_v128_load64_zero(w3);
403 w3 += 2;
404 v4x0123 = wasm_v128_load64_zero(w4);
405 w4 += 2;
406 v5x0123 = wasm_v128_load64_zero(w5);
407 w5 += 2;
408 v6x0123 = wasm_v128_load64_zero(w6);
409 w6 += 2;
410
411 v0x0123 = wasm_v128_load32_lane(w0, v0x0123, 2);
412 w0 += 1;
413 v1x0123 = wasm_v128_load32_lane(w1, v1x0123, 2);
414 w1 += 1;
415 v2x0123 = wasm_v128_load32_lane(w2, v2x0123, 2);
416 w2 += 1;
417 v3x0123 = wasm_v128_load32_lane(w3, v3x0123, 2);
418 w3 += 1;
419 v4x0123 = wasm_v128_load32_lane(w4, v4x0123, 2);
420 w4 += 1;
421 v5x0123 = wasm_v128_load32_lane(w5, v5x0123, 2);
422 w5 += 1;
423 v6x0123 = wasm_v128_load32_lane(w6, v6x0123, 2);
424 w6 += 1;
425 break;
426 default:
427 XNN_UNREACHABLE;
428 }
429
430 v1x0123 = wasm_v32x4_shuffle(v1x0123, v1x0123, 1, 2, 3, 0);
431 v2x0123 = wasm_v32x4_shuffle(v2x0123, v2x0123, 2, 3, 0, 1);
432 v3x0123 = wasm_v32x4_shuffle(v3x0123, v3x0123, 3, 0, 1, 2);
433 v5x0123 = wasm_v32x4_shuffle(v5x0123, v5x0123, 1, 2, 3, 0);
434 v6x0123 = wasm_v32x4_shuffle(v6x0123, v6x0123, 2, 3, 0, 1);
435
436 const v128_t v01x0_01x1 = wasm_v32x4_shuffle(v0x0123, v1x0123, 0, 4, 1, 5);
437 const v128_t v23x0_23x1 = wasm_v32x4_shuffle(v2x0123, v3x0123, 0, 4, 1, 5);
438 const v128_t v01x2_01x3 = wasm_v32x4_shuffle(v0x0123, v1x0123, 2, 6, 3, 7);
439 const v128_t v23x2_23x3 = wasm_v32x4_shuffle(v2x0123, v3x0123, 2, 6, 3, 7);
440 const v128_t v45x0_45x1 = wasm_v32x4_shuffle(v4x0123, v5x0123, 0, 4, 1, 5);
441 const v128_t v67x0_67x1 = wasm_v32x4_shuffle(v6x0123, v6x0123, 0, 4, 1, 5);
442 const v128_t v45x2_45x3 = wasm_v32x4_shuffle(v4x0123, v5x0123, 2, 6, 3, 7);
443 const v128_t v67x2_67x3 = wasm_v32x4_shuffle(v6x0123, v6x0123, 2, 6, 3, 7);
444
445 const v128_t v0123x0 = wasm_v64x2_shuffle(v01x0_01x1, v23x0_23x1, 0, 2);
446 const v128_t v0123x1 = wasm_v64x2_shuffle(v01x0_01x1, v23x0_23x1, 1, 3);
447 const v128_t v0123x2 = wasm_v64x2_shuffle(v01x2_01x3, v23x2_23x3, 0, 2);
448 const v128_t v0123x3 = wasm_v64x2_shuffle(v01x2_01x3, v23x2_23x3, 1, 3);
449 const v128_t v4567x0 = wasm_v64x2_shuffle(v45x0_45x1, v67x0_67x1, 0, 2);
450 const v128_t v4567x1 = wasm_v64x2_shuffle(v45x0_45x1, v67x0_67x1, 1, 3);
451 const v128_t v4567x2 = wasm_v64x2_shuffle(v45x2_45x3, v67x2_67x3, 0, 2);
452 const v128_t v4567x3 = wasm_v64x2_shuffle(v45x2_45x3, v67x2_67x3, 1, 3);
453
454 wasm_v128_store(packed_weights, v0123x0);
455 wasm_v128_store(packed_weights + 4, v4567x0);
456 wasm_v128_store(packed_weights + 8, v0123x1);
457 wasm_v128_store(packed_weights + 12, v4567x1);
458 wasm_v128_store(packed_weights + 16, v0123x2);
459 wasm_v128_store(packed_weights + 20, v4567x2);
460 wasm_v128_store(packed_weights + 24, v0123x3);
461 wasm_v128_store(packed_weights + 28, v4567x3);
462 packed_weights += 32;
463 }
464 packed_weights = (uint32_t*) ((uintptr_t) packed_weights + extra_bytes);
465 }
466 weights += nc * kc;
467 } while (--g != 0);
468 }