Debian Codegraph @

src/qd8-f32-qc8w-igemm/gen/qd8-f32-qc8w-igemm-4x8c8-minmax-wasmsdot-u2.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-igemm/c8-wasmdot.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include <wasm_simd128.h>
13
14 #include "xnnpack/gemm.h"
15 #include "xnnpack/math.h"
16
17
18 void xnn_qd8_f32_qc8w_igemm_minmax_ukernel_4x8c8__wasmsdot_u2(
19 size_t mr,
20 size_t nc,
21 size_t kc,
22 size_t ks,
23 const int8_t** restrict a,
24 const void* restrict w,
25 float* restrict c,
26 size_t cm_stride,
27 size_t cn_stride,
28 size_t a_offset,
29 const int8_t* zero,
30 const int8_t* zero_data,
31 const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)],
32 const struct xnn_qd8_quantization_params quantization_params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
33 {
34 assert(mr != 0);
35 assert(mr <= 4);
36 assert(nc != 0);
37 assert(kc != 0);
38 assert(ks != 0);
39 assert(ks % (4 * sizeof(void*)) == 0);
40 assert(a_offset % sizeof(int8_t) == 0);
41 assert(a != NULL);
42 assert(w != NULL);
43 assert(c != NULL);
44
45 kc = round_up_po2(kc, 8 * sizeof(int8_t));
46 float* c0 = c;
47 float* c1 = (float*) ((uintptr_t) c0 + cm_stride);
48 if XNN_UNPREDICTABLE(mr < 2) {
49 c1 = c0;
50 }
51 float* c2 = (float*) ((uintptr_t) c1 + cm_stride);
52 if XNN_UNPREDICTABLE(mr <= 2) {
53 c2 = c1;
54 }
55 float* c3 = (float*) ((uintptr_t) c2 + cm_stride);
56 if XNN_UNPREDICTABLE(mr != 4) {
57 c3 = c2;
58 }
59
60 const v128_t vmin = wasm_v128_load32_splat(&params->scalar.min);
61 const v128_t vmax = wasm_v128_load32_splat(&params->scalar.max);
62 XNN_FORCE_REALIZATION(vmin);
63 XNN_FORCE_REALIZATION(vmax);
64
65 do {
66 const v128_t vinput_zero_point = wasm_v128_load32_splat(&quantization_params->zero_point);
67 v128_t vksum0123 = wasm_v128_load(w); w = (const int32_t*) w + 4;
68 v128_t vsum0x0123 = wasm_i32x4_mul(vksum0123, vinput_zero_point);
69 v128_t vacc0x01 = wasm_u64x2_extend_low_u32x4(vsum0x0123);
70 v128_t vacc0x23 = wasm_u64x2_extend_high_u32x4(vsum0x0123);
71 v128_t vsum1x0123 = wasm_i32x4_mul(vksum0123, vinput_zero_point);
72 v128_t vacc1x01 = wasm_u64x2_extend_low_u32x4(vsum1x0123);
73 v128_t vacc1x23 = wasm_u64x2_extend_high_u32x4(vsum1x0123);
74 v128_t vsum2x0123 = wasm_i32x4_mul(vksum0123, vinput_zero_point);
75 v128_t vacc2x01 = wasm_u64x2_extend_low_u32x4(vsum2x0123);
76 v128_t vacc2x23 = wasm_u64x2_extend_high_u32x4(vsum2x0123);
77 v128_t vsum3x0123 = wasm_i32x4_mul(vksum0123, vinput_zero_point);
78 v128_t vacc3x01 = wasm_u64x2_extend_low_u32x4(vsum3x0123);
79 v128_t vacc3x23 = wasm_u64x2_extend_high_u32x4(vsum3x0123);
80 v128_t vksum4567 = wasm_v128_load(w); w = (const int32_t*) w + 4;
81 v128_t vsum0x4567 = wasm_i32x4_mul(vksum4567, vinput_zero_point);
82 v128_t vacc0x45 = wasm_u64x2_extend_low_u32x4(vsum0x4567);
83 v128_t vacc0x67 = wasm_u64x2_extend_high_u32x4(vsum0x4567);
84 v128_t vsum1x4567 = wasm_i32x4_mul(vksum4567, vinput_zero_point);
85 v128_t vacc1x45 = wasm_u64x2_extend_low_u32x4(vsum1x4567);
86 v128_t vacc1x67 = wasm_u64x2_extend_high_u32x4(vsum1x4567);
87 v128_t vsum2x4567 = wasm_i32x4_mul(vksum4567, vinput_zero_point);
88 v128_t vacc2x45 = wasm_u64x2_extend_low_u32x4(vsum2x4567);
89 v128_t vacc2x67 = wasm_u64x2_extend_high_u32x4(vsum2x4567);
90 v128_t vsum3x4567 = wasm_i32x4_mul(vksum4567, vinput_zero_point);
91 v128_t vacc3x45 = wasm_u64x2_extend_low_u32x4(vsum3x4567);
92 v128_t vacc3x67 = wasm_u64x2_extend_high_u32x4(vsum3x4567);
93
94 size_t p = ks;
95 do {
96 const int8_t* restrict a0 = a[0];
97 if XNN_UNPREDICTABLE(a0 != zero) {
98 a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
99 } else {
100 a0 = zero_data;
101 }
102 const int8_t* restrict a1 = a[1];
103 if XNN_UNPREDICTABLE(a1 != zero) {
104 a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
105 } else {
106 a1 = zero_data;
107 }
108 const int8_t* restrict a2 = a[2];
109 if XNN_UNPREDICTABLE(a2 != zero) {
110 a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
111 } else {
112 a2 = zero_data;
113 }
114 const int8_t* restrict a3 = a[3];
115 if XNN_UNPREDICTABLE(a3 != zero) {
116 a3 = (const int8_t*) ((uintptr_t) a3 + a_offset);
117 } else {
118 a3 = zero_data;
119 }
120 a += 4;
121
122 size_t k = kc;
123 while (k >= 16 * sizeof(int8_t)) {
124 const v128_t va0x01 = wasm_v128_load64_splat(a0);
125 const v128_t va0x23 = wasm_v128_load64_splat((const int8_t*) a0 + 8);
126 a0 += 16;
127 const v128_t va1x01 = wasm_v128_load64_splat(a1);
128 const v128_t va1x23 = wasm_v128_load64_splat((const int8_t*) a1 + 8);
129 a1 += 16;
130 const v128_t va2x01 = wasm_v128_load64_splat(a2);
131 const v128_t va2x23 = wasm_v128_load64_splat((const int8_t*) a2 + 8);
132 a2 += 16;
133 const v128_t va3x01 = wasm_v128_load64_splat(a3);
134 const v128_t va3x23 = wasm_v128_load64_splat((const int8_t*) a3 + 8);
135 a3 += 16;
136
137 const v128_t vb01x01 = wasm_v128_load(w); w = (const int8_t*) w + 16;
138 const v128_t vb23x01 = wasm_v128_load(w); w = (const int8_t*) w + 16;
139 const v128_t vb45x01 = wasm_v128_load(w); w = (const int8_t*) w + 16;
140 const v128_t vb67x01 = wasm_v128_load(w); w = (const int8_t*) w + 16;
141 const v128_t vb01x23 = wasm_v128_load(w); w = (const int8_t*) w + 16;
142 const v128_t vb23x23 = wasm_v128_load(w); w = (const int8_t*) w + 16;
143 const v128_t vb45x23 = wasm_v128_load(w); w = (const int8_t*) w + 16;
144 const v128_t vb67x23 = wasm_v128_load(w); w = (const int8_t*) w + 16;
145
146 vacc0x01 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb01x01, va0x01, vacc0x01);
147 vacc0x23 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb23x01, va0x01, vacc0x23);
148 vacc0x45 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb45x01, va0x01, vacc0x45);
149 vacc0x67 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb67x01, va0x01, vacc0x67);
150 vacc1x01 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb01x01, va1x01, vacc1x01);
151 vacc1x23 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb23x01, va1x01, vacc1x23);
152 vacc1x45 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb45x01, va1x01, vacc1x45);
153 vacc1x67 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb67x01, va1x01, vacc1x67);
154 vacc2x01 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb01x01, va2x01, vacc2x01);
155 vacc2x23 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb23x01, va2x01, vacc2x23);
156 vacc2x45 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb45x01, va2x01, vacc2x45);
157 vacc2x67 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb67x01, va2x01, vacc2x67);
158 vacc3x01 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb01x01, va3x01, vacc3x01);
159 vacc3x23 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb23x01, va3x01, vacc3x23);
160 vacc3x45 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb45x01, va3x01, vacc3x45);
161 vacc3x67 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb67x01, va3x01, vacc3x67);
162 vacc0x01 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb01x23, va0x23, vacc0x01);
163 vacc0x23 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb23x23, va0x23, vacc0x23);
164 vacc0x45 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb45x23, va0x23, vacc0x45);
165 vacc0x67 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb67x23, va0x23, vacc0x67);
166 vacc1x01 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb01x23, va1x23, vacc1x01);
167 vacc1x23 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb23x23, va1x23, vacc1x23);
168 vacc1x45 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb45x23, va1x23, vacc1x45);
169 vacc1x67 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb67x23, va1x23, vacc1x67);
170 vacc2x01 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb01x23, va2x23, vacc2x01);
171 vacc2x23 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb23x23, va2x23, vacc2x23);
172 vacc2x45 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb45x23, va2x23, vacc2x45);
173 vacc2x67 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb67x23, va2x23, vacc2x67);
174 vacc3x01 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb01x23, va3x23, vacc3x01);
175 vacc3x23 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb23x23, va3x23, vacc3x23);
176 vacc3x45 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb45x23, va3x23, vacc3x45);
177 vacc3x67 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb67x23, va3x23, vacc3x67);
178
179 k -= 16 * sizeof(int8_t);
180 }
181
182 if (k != 0) {
183 const v128_t va0x01 = wasm_v128_load64_splat(a0);
184 a0 += 8;
185 const v128_t va1x01 = wasm_v128_load64_splat(a1);
186 a1 += 8;
187 const v128_t va2x01 = wasm_v128_load64_splat(a2);
188 a2 += 8;
189 const v128_t va3x01 = wasm_v128_load64_splat(a3);
190 a3 += 8;
191
192 const v128_t vb01 = wasm_v128_load(w); w = (const int8_t*) w + 16;
193 const v128_t vb23 = wasm_v128_load(w); w = (const int8_t*) w + 16;
194 const v128_t vb45 = wasm_v128_load(w); w = (const int8_t*) w + 16;
195 const v128_t vb67 = wasm_v128_load(w); w = (const int8_t*) w + 16;
196
197 vacc0x01 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb01, va0x01, vacc0x01);
198 vacc0x23 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb23, va0x01, vacc0x23);
199 vacc0x45 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb45, va0x01, vacc0x45);
200 vacc0x67 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb67, va0x01, vacc0x67);
201 vacc1x01 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb01, va1x01, vacc1x01);
202 vacc1x23 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb23, va1x01, vacc1x23);
203 vacc1x45 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb45, va1x01, vacc1x45);
204 vacc1x67 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb67, va1x01, vacc1x67);
205 vacc2x01 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb01, va2x01, vacc2x01);
206 vacc2x23 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb23, va2x01, vacc2x23);
207 vacc2x45 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb45, va2x01, vacc2x45);
208 vacc2x67 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb67, va2x01, vacc2x67);
209 vacc3x01 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb01, va3x01, vacc3x01);
210 vacc3x23 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb23, va3x01, vacc3x23);
211 vacc3x45 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb45, va3x01, vacc3x45);
212 vacc3x67 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb67, va3x01, vacc3x67);
213
214 k -= 8 * sizeof(int8_t);
215 }
216 p -= 4 * sizeof(void*);
217 } while (p != 0);
218
219 v128_t vacc0x0123 = wasm_i32x4_add(wasm_v32x4_shuffle(vacc0x01, vacc0x23, 0, 2, 4, 6), wasm_v32x4_shuffle(vacc0x01, vacc0x23, 1, 3, 5, 7));
220 v128_t vacc0x4567 = wasm_i32x4_add(wasm_v32x4_shuffle(vacc0x45, vacc0x67, 0, 2, 4, 6), wasm_v32x4_shuffle(vacc0x45, vacc0x67, 1, 3, 5, 7));
221 v128_t vacc1x0123 = wasm_i32x4_add(wasm_v32x4_shuffle(vacc1x01, vacc1x23, 0, 2, 4, 6), wasm_v32x4_shuffle(vacc1x01, vacc1x23, 1, 3, 5, 7));
222 v128_t vacc1x4567 = wasm_i32x4_add(wasm_v32x4_shuffle(vacc1x45, vacc1x67, 0, 2, 4, 6), wasm_v32x4_shuffle(vacc1x45, vacc1x67, 1, 3, 5, 7));
223 v128_t vacc2x0123 = wasm_i32x4_add(wasm_v32x4_shuffle(vacc2x01, vacc2x23, 0, 2, 4, 6), wasm_v32x4_shuffle(vacc2x01, vacc2x23, 1, 3, 5, 7));
224 v128_t vacc2x4567 = wasm_i32x4_add(wasm_v32x4_shuffle(vacc2x45, vacc2x67, 0, 2, 4, 6), wasm_v32x4_shuffle(vacc2x45, vacc2x67, 1, 3, 5, 7));
225 v128_t vacc3x0123 = wasm_i32x4_add(wasm_v32x4_shuffle(vacc3x01, vacc3x23, 0, 2, 4, 6), wasm_v32x4_shuffle(vacc3x01, vacc3x23, 1, 3, 5, 7));
226 v128_t vacc3x4567 = wasm_i32x4_add(wasm_v32x4_shuffle(vacc3x45, vacc3x67, 0, 2, 4, 6), wasm_v32x4_shuffle(vacc3x45, vacc3x67, 1, 3, 5, 7));
227
228 vacc0x0123 = wasm_f32x4_convert_i32x4(vacc0x0123);
229 vacc0x4567 = wasm_f32x4_convert_i32x4(vacc0x4567);
230 vacc1x0123 = wasm_f32x4_convert_i32x4(vacc1x0123);
231 vacc1x4567 = wasm_f32x4_convert_i32x4(vacc1x4567);
232 vacc2x0123 = wasm_f32x4_convert_i32x4(vacc2x0123);
233 vacc2x4567 = wasm_f32x4_convert_i32x4(vacc2x4567);
234 vacc3x0123 = wasm_f32x4_convert_i32x4(vacc3x0123);
235 vacc3x4567 = wasm_f32x4_convert_i32x4(vacc3x4567);
236
237 const v128_t vinput_scale = wasm_v128_load32_splat(&quantization_params->inv_scale);
238
239 vacc0x0123 = wasm_f32x4_mul(vacc0x0123, vinput_scale);
240 vacc0x4567 = wasm_f32x4_mul(vacc0x4567, vinput_scale);
241 vacc1x0123 = wasm_f32x4_mul(vacc1x0123, vinput_scale);
242 vacc1x4567 = wasm_f32x4_mul(vacc1x4567, vinput_scale);
243 vacc2x0123 = wasm_f32x4_mul(vacc2x0123, vinput_scale);
244 vacc2x4567 = wasm_f32x4_mul(vacc2x4567, vinput_scale);
245 vacc3x0123 = wasm_f32x4_mul(vacc3x0123, vinput_scale);
246 vacc3x4567 = wasm_f32x4_mul(vacc3x4567, vinput_scale);
247
248 const v128_t vfilter_output_scale0123 = wasm_v128_load(w); w = (const float*) w + 4;
249 const v128_t vfilter_output_scale4567 = wasm_v128_load(w); w = (const float*) w + 4;
250
251 vacc0x0123 = wasm_f32x4_mul(vacc0x0123, vfilter_output_scale0123);
252 vacc0x4567 = wasm_f32x4_mul(vacc0x4567, vfilter_output_scale4567);
253 vacc1x0123 = wasm_f32x4_mul(vacc1x0123, vfilter_output_scale0123);
254 vacc1x4567 = wasm_f32x4_mul(vacc1x4567, vfilter_output_scale4567);
255 vacc2x0123 = wasm_f32x4_mul(vacc2x0123, vfilter_output_scale0123);
256 vacc2x4567 = wasm_f32x4_mul(vacc2x4567, vfilter_output_scale4567);
257 vacc3x0123 = wasm_f32x4_mul(vacc3x0123, vfilter_output_scale0123);
258 vacc3x4567 = wasm_f32x4_mul(vacc3x4567, vfilter_output_scale4567);
259
260 const v128_t vbias0123 = wasm_v128_load(w); w = (const float*) w + 4;
261 const v128_t vbias4567 = wasm_v128_load(w); w = (const float*) w + 4;
262 vacc0x0123 = wasm_f32x4_add(vacc0x0123, vbias0123);
263 vacc0x4567 = wasm_f32x4_add(vacc0x4567, vbias4567);
264 vacc1x0123 = wasm_f32x4_add(vacc1x0123, vbias0123);
265 vacc1x4567 = wasm_f32x4_add(vacc1x4567, vbias4567);
266 vacc2x0123 = wasm_f32x4_add(vacc2x0123, vbias0123);
267 vacc2x4567 = wasm_f32x4_add(vacc2x4567, vbias4567);
268 vacc3x0123 = wasm_f32x4_add(vacc3x0123, vbias0123);
269 vacc3x4567 = wasm_f32x4_add(vacc3x4567, vbias4567);
270
271 vacc0x0123 = wasm_f32x4_pmax(vacc0x0123, vmin);
272 vacc0x4567 = wasm_f32x4_pmax(vacc0x4567, vmin);
273 vacc1x0123 = wasm_f32x4_pmax(vacc1x0123, vmin);
274 vacc1x4567 = wasm_f32x4_pmax(vacc1x4567, vmin);
275 vacc2x0123 = wasm_f32x4_pmax(vacc2x0123, vmin);
276 vacc2x4567 = wasm_f32x4_pmax(vacc2x4567, vmin);
277 vacc3x0123 = wasm_f32x4_pmax(vacc3x0123, vmin);
278 vacc3x4567 = wasm_f32x4_pmax(vacc3x4567, vmin);
279
280 vacc0x0123 = wasm_f32x4_pmin(vacc0x0123, vmax);
281 vacc0x4567 = wasm_f32x4_pmin(vacc0x4567, vmax);
282 vacc1x0123 = wasm_f32x4_pmin(vacc1x0123, vmax);
283 vacc1x4567 = wasm_f32x4_pmin(vacc1x4567, vmax);
284 vacc2x0123 = wasm_f32x4_pmin(vacc2x0123, vmax);
285 vacc2x4567 = wasm_f32x4_pmin(vacc2x4567, vmax);
286 vacc3x0123 = wasm_f32x4_pmin(vacc3x0123, vmax);
287 vacc3x4567 = wasm_f32x4_pmin(vacc3x4567, vmax);
288
289 if XNN_LIKELY(nc >= 8) {
290 wasm_v128_store(c3, vacc3x0123);
291 wasm_v128_store(c3 + 4, vacc3x4567);
292 wasm_v128_store(c2, vacc2x0123);
293 wasm_v128_store(c2 + 4, vacc2x4567);
294 wasm_v128_store(c1, vacc1x0123);
295 wasm_v128_store(c1 + 4, vacc1x4567);
296 wasm_v128_store(c0, vacc0x0123);
297 wasm_v128_store(c0 + 4, vacc0x4567);
298
299 c0 = (float*) ((uintptr_t) c0 + cn_stride);
300 c1 = (float*) ((uintptr_t) c1 + cn_stride);
301 c2 = (float*) ((uintptr_t) c2 + cn_stride);
302 c3 = (float*) ((uintptr_t) c3 + cn_stride);
303
304 a = (const int8_t**restrict) ((uintptr_t) a - ks);
305
306 nc -= 8;
307 } else {
308 if (nc & 4) {
309 wasm_v128_store(c3, vacc3x0123);
310 vacc3x0123 = vacc3x4567;
311 c3 += 4;
312 wasm_v128_store(c2, vacc2x0123);
313 vacc2x0123 = vacc2x4567;
314 c2 += 4;
315 wasm_v128_store(c1, vacc1x0123);
316 vacc1x0123 = vacc1x4567;
317 c1 += 4;
318 wasm_v128_store(c0, vacc0x0123);
319 vacc0x0123 = vacc0x4567;
320 c0 += 4;
321 }
322 if (nc & 2) {
323 wasm_v128_store64_lane(c3, vacc3x0123, 0);
324 vacc3x0123 = wasm_v64x2_shuffle(vacc3x0123, vacc3x0123, 1, 1);
325 c3 += 2;
326 wasm_v128_store64_lane(c2, vacc2x0123, 0);
327 vacc2x0123 = wasm_v64x2_shuffle(vacc2x0123, vacc2x0123, 1, 1);
328 c2 += 2;
329 wasm_v128_store64_lane(c1, vacc1x0123, 0);
330 vacc1x0123 = wasm_v64x2_shuffle(vacc1x0123, vacc1x0123, 1, 1);
331 c1 += 2;
332 wasm_v128_store64_lane(c0, vacc0x0123, 0);
333 vacc0x0123 = wasm_v64x2_shuffle(vacc0x0123, vacc0x0123, 1, 1);
334 c0 += 2;
335 }
336 if (nc & 1) {
337 wasm_v128_store32_lane(c3, vacc3x0123, 0);
338 wasm_v128_store32_lane(c2, vacc2x0123, 0);
339 wasm_v128_store32_lane(c1, vacc1x0123, 0);
340 wasm_v128_store32_lane(c0, vacc0x0123, 0);
341 }
342 nc = 0;
343 }
344 } while (nc != 0);
345 }