Debian Codegraph @

src/qs8-dwconv/gen/qs8-dwconv-8f8m9l16c8s8r-minmax-fp32-wasmsimd-mul16.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-dwconv/multipass-wasmsimd-mul16.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include <wasm_simd128.h>
13
14 #include "xnnpack/dwconv.h"
15 #include "xnnpack/math.h"
16
17
18 void xnn_qs8_dwconv_minmax_fp32_ukernel_8f8m9l16c8s8r__wasmsimd_mul16(
19 size_t channels,
20 size_t output_width,
21 const int8_t** input,
22 const void* weights,
23 int8_t* output,
24 intptr_t input_stride,
25 size_t output_increment,
26 size_t input_offset,
27 const int8_t* zero,
28 size_t kernel_size,
29 int32_t* buffer,
30 const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
31 {
32 assert(channels != 0);
33 assert(output_width != 0);
34 assert(kernel_size > 8);
35
36
37 const v128_t vscale = wasm_v128_load32_splat(&params->fp32_scalar.scale);
38 XNN_FORCE_REALIZATION(vscale);
39
40 const v128_t vmagic_bias = wasm_f32x4_const_splat(12582912.0f);
41 const int32_t output_min_less_zero_point = (int32_t) params->fp32_scalar.output_min - (int32_t) params->fp32_scalar.output_zero_point;
42 const v128_t vmagic_min = wasm_i32x4_splat((int32_t) float_as_uint32(12582912.0f + output_min_less_zero_point));
43 const v128_t vmagic_bias_less_output_zero_point = wasm_i32x4_splat(INT32_C(0x4B400000) - (int32_t) params->fp32_scalar.output_zero_point);
44 const v128_t voutput_max = wasm_i8x16_splat(params->fp32_scalar.output_max);
45 XNN_FORCE_REALIZATION(vmagic_bias);
46 XNN_FORCE_REALIZATION(vmagic_min);
47 XNN_FORCE_REALIZATION(vmagic_bias_less_output_zero_point);
48 XNN_FORCE_REALIZATION(voutput_max);
49
50 do {
51 const void* w = weights;
52
53 // First pass to process 8 inputs.
54 {
55 int32_t* b = buffer;
56 const int8_t* i0 = input[0];
57 assert(i0 != NULL);
58 if XNN_UNPREDICTABLE(i0 != zero) {
59 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
60 }
61 const int8_t* i1 = input[1];
62 assert(i1 != NULL);
63 if XNN_UNPREDICTABLE(i1 != zero) {
64 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
65 }
66 const int8_t* i2 = input[2];
67 assert(i2 != NULL);
68 if XNN_UNPREDICTABLE(i2 != zero) {
69 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
70 }
71 const int8_t* i3 = input[3];
72 assert(i3 != NULL);
73 if XNN_UNPREDICTABLE(i3 != zero) {
74 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
75 }
76 const int8_t* i4 = input[4];
77 assert(i4 != NULL);
78 if XNN_UNPREDICTABLE(i4 != zero) {
79 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
80 }
81 const int8_t* i5 = input[5];
82 assert(i5 != NULL);
83 if XNN_UNPREDICTABLE(i5 != zero) {
84 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
85 }
86 const int8_t* i6 = input[6];
87 assert(i6 != NULL);
88 if XNN_UNPREDICTABLE(i6 != zero) {
89 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
90 }
91 const int8_t* i7 = input[7];
92 assert(i7 != NULL);
93 if XNN_UNPREDICTABLE(i7 != zero) {
94 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
95 }
96 input += 8;
97
98 size_t c = round_up_po2(channels, 8);
99
100 for (; c >= 16; c -= 16) {
101 v128_t vacc0123 = wasm_v128_load(w);
102 v128_t vacc4567 = wasm_v128_load((const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
103 v128_t vacc89AB = wasm_v128_load((const void*) ((uintptr_t) w + 8 * sizeof(int32_t)));
104 v128_t vaccCDEF = wasm_v128_load((const void*) ((uintptr_t) w + 12 * sizeof(int32_t)));
105
106 const v128_t vi0x01234567 = wasm_i16x8_load8x8(i0);
107 v128_t vk0x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 0 * sizeof(int8_t)));
108 const v128_t vi0x89ABCDEF = wasm_i16x8_load8x8(i0 + 8);
109 v128_t vk0x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 8 * sizeof(int8_t)));
110 i0 += 16;
111
112 v128_t vprod01234567 = wasm_i16x8_mul(vi0x01234567, vk0x01234567);
113 v128_t vprod89ABCDEF = wasm_i16x8_mul(vi0x89ABCDEF, vk0x89ABCDEF);
114
115 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
116 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
117 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
118 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
119
120 const v128_t vi1x01234567 = wasm_i16x8_load8x8(i1);
121 v128_t vk1x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 16 * sizeof(int8_t)));
122 const v128_t vi1x89ABCDEF = wasm_i16x8_load8x8(i1 + 8);
123 v128_t vk1x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 24 * sizeof(int8_t)));
124 i1 += 16;
125
126 vprod01234567 = wasm_i16x8_mul(vi1x01234567, vk1x01234567);
127 vprod89ABCDEF = wasm_i16x8_mul(vi1x89ABCDEF, vk1x89ABCDEF);
128
129 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
130 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
131 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
132 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
133
134 const v128_t vi2x01234567 = wasm_i16x8_load8x8(i2);
135 v128_t vk2x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 32 * sizeof(int8_t)));
136 const v128_t vi2x89ABCDEF = wasm_i16x8_load8x8(i2 + 8);
137 v128_t vk2x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 40 * sizeof(int8_t)));
138 i2 += 16;
139
140 vprod01234567 = wasm_i16x8_mul(vi2x01234567, vk2x01234567);
141 vprod89ABCDEF = wasm_i16x8_mul(vi2x89ABCDEF, vk2x89ABCDEF);
142
143 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
144 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
145 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
146 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
147
148 const v128_t vi3x01234567 = wasm_i16x8_load8x8(i3);
149 v128_t vk3x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 48 * sizeof(int8_t)));
150 const v128_t vi3x89ABCDEF = wasm_i16x8_load8x8(i3 + 8);
151 v128_t vk3x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 56 * sizeof(int8_t)));
152 i3 += 16;
153
154 vprod01234567 = wasm_i16x8_mul(vi3x01234567, vk3x01234567);
155 vprod89ABCDEF = wasm_i16x8_mul(vi3x89ABCDEF, vk3x89ABCDEF);
156
157 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
158 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
159 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
160 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
161
162 const v128_t vi4x01234567 = wasm_i16x8_load8x8(i4);
163 v128_t vk4x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 64 * sizeof(int8_t)));
164 const v128_t vi4x89ABCDEF = wasm_i16x8_load8x8(i4 + 8);
165 v128_t vk4x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 72 * sizeof(int8_t)));
166 i4 += 16;
167
168 vprod01234567 = wasm_i16x8_mul(vi4x01234567, vk4x01234567);
169 vprod89ABCDEF = wasm_i16x8_mul(vi4x89ABCDEF, vk4x89ABCDEF);
170
171 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
172 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
173 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
174 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
175
176 const v128_t vi5x01234567 = wasm_i16x8_load8x8(i5);
177 v128_t vk5x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 80 * sizeof(int8_t)));
178 const v128_t vi5x89ABCDEF = wasm_i16x8_load8x8(i5 + 8);
179 v128_t vk5x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 88 * sizeof(int8_t)));
180 i5 += 16;
181
182 vprod01234567 = wasm_i16x8_mul(vi5x01234567, vk5x01234567);
183 vprod89ABCDEF = wasm_i16x8_mul(vi5x89ABCDEF, vk5x89ABCDEF);
184
185 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
186 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
187 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
188 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
189
190 const v128_t vi6x01234567 = wasm_i16x8_load8x8(i6);
191 v128_t vk6x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 96 * sizeof(int8_t)));
192 const v128_t vi6x89ABCDEF = wasm_i16x8_load8x8(i6 + 8);
193 v128_t vk6x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 104 * sizeof(int8_t)));
194 i6 += 16;
195
196 vprod01234567 = wasm_i16x8_mul(vi6x01234567, vk6x01234567);
197 vprod89ABCDEF = wasm_i16x8_mul(vi6x89ABCDEF, vk6x89ABCDEF);
198
199 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
200 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
201 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
202 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
203
204 const v128_t vi7x01234567 = wasm_i16x8_load8x8(i7);
205 v128_t vk7x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 112 * sizeof(int8_t)));
206 const v128_t vi7x89ABCDEF = wasm_i16x8_load8x8(i7 + 8);
207 v128_t vk7x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 120 * sizeof(int8_t)));
208 i7 += 16;
209
210 vprod01234567 = wasm_i16x8_mul(vi7x01234567, vk7x01234567);
211 vprod89ABCDEF = wasm_i16x8_mul(vi7x89ABCDEF, vk7x89ABCDEF);
212
213 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
214 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
215 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
216 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
217
218 w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 128 * sizeof(int8_t));
219
220 wasm_v128_store(b, vacc0123);
221 wasm_v128_store(b + 4, vacc4567);
222 wasm_v128_store(b + 8, vacc89AB);
223 wasm_v128_store(b + 12, vaccCDEF);
224 b += 16;
225 }
226
227 if XNN_UNLIKELY(c != 0) {
228 do {
229 v128_t vacc0123 = wasm_v128_load(w);
230 v128_t vacc4567 = wasm_v128_load((const int32_t*) w + 4);
231
232 const v128_t vi0x01234567 = wasm_i16x8_load8x8(i0);
233 v128_t vk0x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int32_t)));
234 i0 += 8;
235
236 v128_t vprod01234567 = wasm_i16x8_mul(vi0x01234567, vk0x01234567);
237
238 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
239 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
240
241 const v128_t vi1x01234567 = wasm_i16x8_load8x8(i1);
242 v128_t vk1x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 8 * sizeof(int8_t)));
243 i1 += 8;
244
245 vprod01234567 = wasm_i16x8_mul(vi1x01234567, vk1x01234567);
246
247 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
248 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
249
250 const v128_t vi2x01234567 = wasm_i16x8_load8x8(i2);
251 v128_t vk2x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 16 * sizeof(int8_t)));
252 i2 += 8;
253
254 vprod01234567 = wasm_i16x8_mul(vi2x01234567, vk2x01234567);
255
256 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
257 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
258
259 const v128_t vi3x01234567 = wasm_i16x8_load8x8(i3);
260 v128_t vk3x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 24 * sizeof(int8_t)));
261 i3 += 8;
262
263 vprod01234567 = wasm_i16x8_mul(vi3x01234567, vk3x01234567);
264
265 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
266 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
267
268 const v128_t vi4x01234567 = wasm_i16x8_load8x8(i4);
269 v128_t vk4x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 32 * sizeof(int8_t)));
270 i4 += 8;
271
272 vprod01234567 = wasm_i16x8_mul(vi4x01234567, vk4x01234567);
273
274 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
275 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
276
277 const v128_t vi5x01234567 = wasm_i16x8_load8x8(i5);
278 v128_t vk5x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 40 * sizeof(int8_t)));
279 i5 += 8;
280
281 vprod01234567 = wasm_i16x8_mul(vi5x01234567, vk5x01234567);
282
283 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
284 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
285
286 const v128_t vi6x01234567 = wasm_i16x8_load8x8(i6);
287 v128_t vk6x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 48 * sizeof(int8_t)));
288 i6 += 8;
289
290 vprod01234567 = wasm_i16x8_mul(vi6x01234567, vk6x01234567);
291
292 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
293 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
294
295 const v128_t vi7x01234567 = wasm_i16x8_load8x8(i7);
296 v128_t vk7x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 56 * sizeof(int8_t)));
297 i7 += 8;
298
299 vprod01234567 = wasm_i16x8_mul(vi7x01234567, vk7x01234567);
300
301 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
302 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
303
304
305 w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 64 * sizeof(int8_t));
306
307 wasm_v128_store(b, vacc0123);
308 wasm_v128_store(b + 4, vacc4567);
309 b += 8;
310 c -= 8;
311 } while (c != 0);
312 }
313 }
314
315
316 // Middle pass to process 8 inputs in each iteration.
317 for (size_t ks = kernel_size - 8; ks > 9; ks -= 8) {
318 int32_t* b = buffer;
319 const int8_t* i0 = input[0];
320 assert(i0 != NULL);
321 if XNN_UNPREDICTABLE(i0 != zero) {
322 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
323 }
324 const int8_t* i1 = input[1];
325 assert(i1 != NULL);
326 if XNN_UNPREDICTABLE(i1 != zero) {
327 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
328 }
329 const int8_t* i2 = input[2];
330 assert(i2 != NULL);
331 if XNN_UNPREDICTABLE(i2 != zero) {
332 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
333 }
334 const int8_t* i3 = input[3];
335 assert(i3 != NULL);
336 if XNN_UNPREDICTABLE(i3 != zero) {
337 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
338 }
339 const int8_t* i4 = input[4];
340 assert(i4 != NULL);
341 if XNN_UNPREDICTABLE(i4 != zero) {
342 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
343 }
344 const int8_t* i5 = input[5];
345 assert(i5 != NULL);
346 if XNN_UNPREDICTABLE(i5 != zero) {
347 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
348 }
349 const int8_t* i6 = input[6];
350 assert(i6 != NULL);
351 if XNN_UNPREDICTABLE(i6 != zero) {
352 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
353 }
354 const int8_t* i7 = input[7];
355 assert(i7 != NULL);
356 if XNN_UNPREDICTABLE(i7 != zero) {
357 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
358 }
359 input += 8;
360
361 size_t c = round_up_po2(channels, 8);
362
363 for (; c >= 16; c -= 16) {
364 v128_t vacc0123 = wasm_v128_load(b);
365 v128_t vacc4567 = wasm_v128_load(b + 4);
366 v128_t vacc89AB = wasm_v128_load(b + 8);
367 v128_t vaccCDEF = wasm_v128_load(b + 12);
368
369 const v128_t vi0x01234567 = wasm_i16x8_load8x8(i0);
370 v128_t vk0x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 0 * sizeof(int8_t)));
371 const v128_t vi0x89ABCDEF = wasm_i16x8_load8x8(i0 + 8);
372 v128_t vk0x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int8_t)));
373 i0 += 16;
374
375 v128_t vprod01234567 = wasm_i16x8_mul(vi0x01234567, vk0x01234567);
376 v128_t vprod89ABCDEF = wasm_i16x8_mul(vi0x89ABCDEF, vk0x89ABCDEF);
377
378 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
379 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
380 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
381 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
382
383 const v128_t vi1x01234567 = wasm_i16x8_load8x8(i1);
384 v128_t vk1x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int8_t)));
385 const v128_t vi1x89ABCDEF = wasm_i16x8_load8x8(i1 + 8);
386 v128_t vk1x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 24 * sizeof(int8_t)));
387 i1 += 16;
388
389 vprod01234567 = wasm_i16x8_mul(vi1x01234567, vk1x01234567);
390 vprod89ABCDEF = wasm_i16x8_mul(vi1x89ABCDEF, vk1x89ABCDEF);
391
392 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
393 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
394 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
395 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
396
397 const v128_t vi2x01234567 = wasm_i16x8_load8x8(i2);
398 v128_t vk2x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 32 * sizeof(int8_t)));
399 const v128_t vi2x89ABCDEF = wasm_i16x8_load8x8(i2 + 8);
400 v128_t vk2x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 40 * sizeof(int8_t)));
401 i2 += 16;
402
403 vprod01234567 = wasm_i16x8_mul(vi2x01234567, vk2x01234567);
404 vprod89ABCDEF = wasm_i16x8_mul(vi2x89ABCDEF, vk2x89ABCDEF);
405
406 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
407 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
408 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
409 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
410
411 const v128_t vi3x01234567 = wasm_i16x8_load8x8(i3);
412 v128_t vk3x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 48 * sizeof(int8_t)));
413 const v128_t vi3x89ABCDEF = wasm_i16x8_load8x8(i3 + 8);
414 v128_t vk3x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 56 * sizeof(int8_t)));
415 i3 += 16;
416
417 vprod01234567 = wasm_i16x8_mul(vi3x01234567, vk3x01234567);
418 vprod89ABCDEF = wasm_i16x8_mul(vi3x89ABCDEF, vk3x89ABCDEF);
419
420 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
421 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
422 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
423 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
424
425 const v128_t vi4x01234567 = wasm_i16x8_load8x8(i4);
426 v128_t vk4x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 64 * sizeof(int8_t)));
427 const v128_t vi4x89ABCDEF = wasm_i16x8_load8x8(i4 + 8);
428 v128_t vk4x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 72 * sizeof(int8_t)));
429 i4 += 16;
430
431 vprod01234567 = wasm_i16x8_mul(vi4x01234567, vk4x01234567);
432 vprod89ABCDEF = wasm_i16x8_mul(vi4x89ABCDEF, vk4x89ABCDEF);
433
434 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
435 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
436 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
437 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
438
439 const v128_t vi5x01234567 = wasm_i16x8_load8x8(i5);
440 v128_t vk5x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 80 * sizeof(int8_t)));
441 const v128_t vi5x89ABCDEF = wasm_i16x8_load8x8(i5 + 8);
442 v128_t vk5x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 88 * sizeof(int8_t)));
443 i5 += 16;
444
445 vprod01234567 = wasm_i16x8_mul(vi5x01234567, vk5x01234567);
446 vprod89ABCDEF = wasm_i16x8_mul(vi5x89ABCDEF, vk5x89ABCDEF);
447
448 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
449 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
450 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
451 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
452
453 const v128_t vi6x01234567 = wasm_i16x8_load8x8(i6);
454 v128_t vk6x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 96 * sizeof(int8_t)));
455 const v128_t vi6x89ABCDEF = wasm_i16x8_load8x8(i6 + 8);
456 v128_t vk6x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 104 * sizeof(int8_t)));
457 i6 += 16;
458
459 vprod01234567 = wasm_i16x8_mul(vi6x01234567, vk6x01234567);
460 vprod89ABCDEF = wasm_i16x8_mul(vi6x89ABCDEF, vk6x89ABCDEF);
461
462 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
463 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
464 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
465 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
466
467 const v128_t vi7x01234567 = wasm_i16x8_load8x8(i7);
468 v128_t vk7x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 112 * sizeof(int8_t)));
469 const v128_t vi7x89ABCDEF = wasm_i16x8_load8x8(i7 + 8);
470 v128_t vk7x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 120 * sizeof(int8_t)));
471 i7 += 16;
472
473 vprod01234567 = wasm_i16x8_mul(vi7x01234567, vk7x01234567);
474 vprod89ABCDEF = wasm_i16x8_mul(vi7x89ABCDEF, vk7x89ABCDEF);
475
476 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
477 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
478 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
479 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
480
481 w = (const void*) ((uintptr_t) w + 128 * sizeof(int8_t));
482
483 wasm_v128_store(b, vacc0123);
484 wasm_v128_store(b + 4, vacc4567);
485 wasm_v128_store(b + 8, vacc89AB);
486 wasm_v128_store(b + 12, vaccCDEF);
487 b += 16;
488 }
489
490 if XNN_UNLIKELY(c != 0) {
491 do {
492 v128_t vacc0123 = wasm_v128_load(b);
493 v128_t vacc4567 = wasm_v128_load(b + 4);
494
495 const v128_t vi0x01234567 = wasm_i16x8_load8x8(i0);
496 v128_t vk0x01234567 = wasm_i16x8_load8x8(w);
497 i0 += 8;
498
499 v128_t vprod01234567 = wasm_i16x8_mul(vi0x01234567, vk0x01234567);
500
501 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
502 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
503
504 const v128_t vi1x01234567 = wasm_i16x8_load8x8(i1);
505 v128_t vk1x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int8_t)));
506 i1 += 8;
507
508 vprod01234567 = wasm_i16x8_mul(vi1x01234567, vk1x01234567);
509
510 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
511 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
512
513 const v128_t vi2x01234567 = wasm_i16x8_load8x8(i2);
514 v128_t vk2x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int8_t)));
515 i2 += 8;
516
517 vprod01234567 = wasm_i16x8_mul(vi2x01234567, vk2x01234567);
518
519 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
520 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
521
522 const v128_t vi3x01234567 = wasm_i16x8_load8x8(i3);
523 v128_t vk3x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 24 * sizeof(int8_t)));
524 i3 += 8;
525
526 vprod01234567 = wasm_i16x8_mul(vi3x01234567, vk3x01234567);
527
528 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
529 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
530
531 const v128_t vi4x01234567 = wasm_i16x8_load8x8(i4);
532 v128_t vk4x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 32 * sizeof(int8_t)));
533 i4 += 8;
534
535 vprod01234567 = wasm_i16x8_mul(vi4x01234567, vk4x01234567);
536
537 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
538 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
539
540 const v128_t vi5x01234567 = wasm_i16x8_load8x8(i5);
541 v128_t vk5x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 40 * sizeof(int8_t)));
542 i5 += 8;
543
544 vprod01234567 = wasm_i16x8_mul(vi5x01234567, vk5x01234567);
545
546 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
547 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
548
549 const v128_t vi6x01234567 = wasm_i16x8_load8x8(i6);
550 v128_t vk6x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 48 * sizeof(int8_t)));
551 i6 += 8;
552
553 vprod01234567 = wasm_i16x8_mul(vi6x01234567, vk6x01234567);
554
555 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
556 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
557
558 const v128_t vi7x01234567 = wasm_i16x8_load8x8(i7);
559 v128_t vk7x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 56 * sizeof(int8_t)));
560 i7 += 8;
561
562 vprod01234567 = wasm_i16x8_mul(vi7x01234567, vk7x01234567);
563
564 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
565 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
566
567
568 w = (const void*) ((uintptr_t) w + 64 * sizeof(int8_t));
569
570 wasm_v128_store(b, vacc0123);
571 wasm_v128_store(b + 4, vacc4567);
572 b += 8;
573 c -= 8;
574 } while (c != 0);
575 }
576 }
577
578 // Last pass to process up to 9 inputs.
579 {
580 const int32_t* b = buffer;
581 const int8_t* i0 = input[0];
582 assert(i0 != NULL);
583 if XNN_UNPREDICTABLE(i0 != zero) {
584 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
585 }
586 const int8_t* i1 = input[1];
587 assert(i1 != NULL);
588 if XNN_UNPREDICTABLE(i1 != zero) {
589 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
590 }
591 const int8_t* i2 = input[2];
592 assert(i2 != NULL);
593 if XNN_UNPREDICTABLE(i2 != zero) {
594 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
595 }
596 const int8_t* i3 = input[3];
597 assert(i3 != NULL);
598 if XNN_UNPREDICTABLE(i3 != zero) {
599 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
600 }
601 const int8_t* i4 = input[4];
602 assert(i4 != NULL);
603 if XNN_UNPREDICTABLE(i4 != zero) {
604 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
605 }
606 const int8_t* i5 = input[5];
607 assert(i5 != NULL);
608 if XNN_UNPREDICTABLE(i5 != zero) {
609 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
610 }
611 const int8_t* i6 = input[6];
612 assert(i6 != NULL);
613 if XNN_UNPREDICTABLE(i6 != zero) {
614 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
615 }
616 const int8_t* i7 = input[7];
617 assert(i7 != NULL);
618 if XNN_UNPREDICTABLE(i7 != zero) {
619 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
620 }
621 const int8_t* i8 = input[8];
622 assert(i8 != NULL);
623 if XNN_UNPREDICTABLE(i8 != zero) {
624 i8 = (const int8_t*) ((uintptr_t) i8 + input_offset);
625 }
626
627 size_t c = channels;
628
629 for (; c >= 16; c -= 16) {
630 v128_t vacc0123 = wasm_v128_load(b);
631 v128_t vacc4567 = wasm_v128_load(b + 4);
632 v128_t vacc89AB = wasm_v128_load(b + 8);
633 v128_t vaccCDEF = wasm_v128_load(b + 12);
634 b += 16;
635
636 const v128_t vi0x01234567 = wasm_i16x8_load8x8(i0);
637 v128_t vk0x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 0 * sizeof(int8_t)));
638 const v128_t vi0x89ABCDEF = wasm_i16x8_load8x8(i0 + 8);
639 v128_t vk0x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int8_t)));
640 i0 += 16;
641
642 v128_t vprod01234567 = wasm_i16x8_mul(vi0x01234567, vk0x01234567);
643 v128_t vprod89ABCDEF = wasm_i16x8_mul(vi0x89ABCDEF, vk0x89ABCDEF);
644
645 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
646 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
647 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
648 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
649
650 const v128_t vi1x01234567 = wasm_i16x8_load8x8(i1);
651 v128_t vk1x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int8_t)));
652 const v128_t vi1x89ABCDEF = wasm_i16x8_load8x8(i1 + 8);
653 v128_t vk1x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 24 * sizeof(int8_t)));
654 i1 += 16;
655
656 vprod01234567 = wasm_i16x8_mul(vi1x01234567, vk1x01234567);
657 vprod89ABCDEF = wasm_i16x8_mul(vi1x89ABCDEF, vk1x89ABCDEF);
658
659 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
660 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
661 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
662 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
663
664 const v128_t vi2x01234567 = wasm_i16x8_load8x8(i2);
665 v128_t vk2x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 32 * sizeof(int8_t)));
666 const v128_t vi2x89ABCDEF = wasm_i16x8_load8x8(i2 + 8);
667 v128_t vk2x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 40 * sizeof(int8_t)));
668 i2 += 16;
669
670 vprod01234567 = wasm_i16x8_mul(vi2x01234567, vk2x01234567);
671 vprod89ABCDEF = wasm_i16x8_mul(vi2x89ABCDEF, vk2x89ABCDEF);
672
673 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
674 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
675 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
676 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
677
678 const v128_t vi3x01234567 = wasm_i16x8_load8x8(i3);
679 v128_t vk3x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 48 * sizeof(int8_t)));
680 const v128_t vi3x89ABCDEF = wasm_i16x8_load8x8(i3 + 8);
681 v128_t vk3x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 56 * sizeof(int8_t)));
682 i3 += 16;
683
684 vprod01234567 = wasm_i16x8_mul(vi3x01234567, vk3x01234567);
685 vprod89ABCDEF = wasm_i16x8_mul(vi3x89ABCDEF, vk3x89ABCDEF);
686
687 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
688 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
689 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
690 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
691
692 const v128_t vi4x01234567 = wasm_i16x8_load8x8(i4);
693 v128_t vk4x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 64 * sizeof(int8_t)));
694 const v128_t vi4x89ABCDEF = wasm_i16x8_load8x8(i4 + 8);
695 v128_t vk4x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 72 * sizeof(int8_t)));
696 i4 += 16;
697
698 vprod01234567 = wasm_i16x8_mul(vi4x01234567, vk4x01234567);
699 vprod89ABCDEF = wasm_i16x8_mul(vi4x89ABCDEF, vk4x89ABCDEF);
700
701 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
702 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
703 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
704 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
705
706 const v128_t vi5x01234567 = wasm_i16x8_load8x8(i5);
707 v128_t vk5x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 80 * sizeof(int8_t)));
708 const v128_t vi5x89ABCDEF = wasm_i16x8_load8x8(i5 + 8);
709 v128_t vk5x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 88 * sizeof(int8_t)));
710 i5 += 16;
711
712 vprod01234567 = wasm_i16x8_mul(vi5x01234567, vk5x01234567);
713 vprod89ABCDEF = wasm_i16x8_mul(vi5x89ABCDEF, vk5x89ABCDEF);
714
715 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
716 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
717 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
718 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
719
720 const v128_t vi6x01234567 = wasm_i16x8_load8x8(i6);
721 v128_t vk6x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 96 * sizeof(int8_t)));
722 const v128_t vi6x89ABCDEF = wasm_i16x8_load8x8(i6 + 8);
723 v128_t vk6x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 104 * sizeof(int8_t)));
724 i6 += 16;
725
726 vprod01234567 = wasm_i16x8_mul(vi6x01234567, vk6x01234567);
727 vprod89ABCDEF = wasm_i16x8_mul(vi6x89ABCDEF, vk6x89ABCDEF);
728
729 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
730 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
731 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
732 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
733
734 const v128_t vi7x01234567 = wasm_i16x8_load8x8(i7);
735 v128_t vk7x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 112 * sizeof(int8_t)));
736 const v128_t vi7x89ABCDEF = wasm_i16x8_load8x8(i7 + 8);
737 v128_t vk7x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 120 * sizeof(int8_t)));
738 i7 += 16;
739
740 vprod01234567 = wasm_i16x8_mul(vi7x01234567, vk7x01234567);
741 vprod89ABCDEF = wasm_i16x8_mul(vi7x89ABCDEF, vk7x89ABCDEF);
742
743 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
744 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
745 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
746 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
747
748 const v128_t vi8x01234567 = wasm_i16x8_load8x8(i8);
749 v128_t vk8x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 128 * sizeof(int8_t)));
750 const v128_t vi8x89ABCDEF = wasm_i16x8_load8x8(i8 + 8);
751 v128_t vk8x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 136 * sizeof(int8_t)));
752 i8 += 16;
753
754 vprod01234567 = wasm_i16x8_mul(vi8x01234567, vk8x01234567);
755 vprod89ABCDEF = wasm_i16x8_mul(vi8x89ABCDEF, vk8x89ABCDEF);
756
757 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
758 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
759 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
760 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
761
762
763 w = (const void*) ((uintptr_t) w + 144 * sizeof(int8_t));
764
765 vacc0123 = wasm_f32x4_convert_i32x4(vacc0123);
766 vacc4567 = wasm_f32x4_convert_i32x4(vacc4567);
767 vacc89AB = wasm_f32x4_convert_i32x4(vacc89AB);
768 vaccCDEF = wasm_f32x4_convert_i32x4(vaccCDEF);
769
770 vacc0123 = wasm_f32x4_mul(vacc0123, vscale);
771 vacc4567 = wasm_f32x4_mul(vacc4567, vscale);
772 vacc89AB = wasm_f32x4_mul(vacc89AB, vscale);
773 vaccCDEF = wasm_f32x4_mul(vaccCDEF, vscale);
774
775 vacc0123 = wasm_f32x4_add(vacc0123, vmagic_bias);
776 vacc4567 = wasm_f32x4_add(vacc4567, vmagic_bias);
777 vacc89AB = wasm_f32x4_add(vacc89AB, vmagic_bias);
778 vaccCDEF = wasm_f32x4_add(vaccCDEF, vmagic_bias);
779
780 vacc0123 = wasm_i32x4_max(vacc0123, vmagic_min);
781 vacc4567 = wasm_i32x4_max(vacc4567, vmagic_min);
782 vacc89AB = wasm_i32x4_max(vacc89AB, vmagic_min);
783 vaccCDEF = wasm_i32x4_max(vaccCDEF, vmagic_min);
784
785 vacc0123 = wasm_i32x4_sub(vacc0123, vmagic_bias_less_output_zero_point);
786 vacc4567 = wasm_i32x4_sub(vacc4567, vmagic_bias_less_output_zero_point);
787 vacc89AB = wasm_i32x4_sub(vacc89AB, vmagic_bias_less_output_zero_point);
788 vaccCDEF = wasm_i32x4_sub(vaccCDEF, vmagic_bias_less_output_zero_point);
789
790 v128_t vout01234567 = wasm_i16x8_narrow_i32x4(vacc0123, vacc4567);
791 v128_t vout89ABCDEF = wasm_i16x8_narrow_i32x4(vacc89AB, vaccCDEF);
792
793 v128_t vout0123456789ABCDEF = wasm_i8x16_narrow_i16x8(vout01234567, vout89ABCDEF);
794
795 vout0123456789ABCDEF = wasm_i8x16_min(vout0123456789ABCDEF, voutput_max);
796
797 wasm_v128_store(output, vout0123456789ABCDEF);
798 output += 16;
799 }
800
801 if XNN_UNLIKELY(c != 0) {
802 do {
803 v128_t vacc0123 = wasm_v128_load(b);
804 v128_t vacc4567 = wasm_v128_load(b + 4);
805 b += 8;
806
807 const v128_t vi0x01234567 = wasm_i16x8_load8x8(i0);
808 v128_t vk0x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w));
809 i0 += 8;
810
811 v128_t vprod01234567 = wasm_i16x8_mul(vi0x01234567, vk0x01234567);
812
813 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
814 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
815
816 const v128_t vi1x01234567 = wasm_i16x8_load8x8(i1);
817 v128_t vk1x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int8_t)));
818 i1 += 8;
819
820 vprod01234567 = wasm_i16x8_mul(vi1x01234567, vk1x01234567);
821
822 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
823 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
824
825 const v128_t vi2x01234567 = wasm_i16x8_load8x8(i2);
826 v128_t vk2x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int8_t)));
827 i2 += 8;
828
829 vprod01234567 = wasm_i16x8_mul(vi2x01234567, vk2x01234567);
830
831 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
832 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
833
834 const v128_t vi3x01234567 = wasm_i16x8_load8x8(i3);
835 v128_t vk3x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 24 * sizeof(int8_t)));
836 i3 += 8;
837
838 vprod01234567 = wasm_i16x8_mul(vi3x01234567, vk3x01234567);
839
840 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
841 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
842
843 const v128_t vi4x01234567 = wasm_i16x8_load8x8(i4);
844 v128_t vk4x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 32 * sizeof(int8_t)));
845 i4 += 8;
846
847 vprod01234567 = wasm_i16x8_mul(vi4x01234567, vk4x01234567);
848
849 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
850 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
851
852 const v128_t vi5x01234567 = wasm_i16x8_load8x8(i5);
853 v128_t vk5x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 40 * sizeof(int8_t)));
854 i5 += 8;
855
856 vprod01234567 = wasm_i16x8_mul(vi5x01234567, vk5x01234567);
857
858 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
859 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
860
861 const v128_t vi6x01234567 = wasm_i16x8_load8x8(i6);
862 v128_t vk6x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 48 * sizeof(int8_t)));
863 i6 += 8;
864
865 vprod01234567 = wasm_i16x8_mul(vi6x01234567, vk6x01234567);
866
867 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
868 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
869
870 const v128_t vi7x01234567 = wasm_i16x8_load8x8(i7);
871 v128_t vk7x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 56 * sizeof(int8_t)));
872 i7 += 8;
873
874 vprod01234567 = wasm_i16x8_mul(vi7x01234567, vk7x01234567);
875
876 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
877 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
878
879 const v128_t vi8x01234567 = wasm_i16x8_load8x8(i8);
880 v128_t vk8x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 64 * sizeof(int8_t)));
881 i8 += 8;
882
883 vprod01234567 = wasm_i16x8_mul(vi8x01234567, vk8x01234567);
884
885 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
886 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
887
888
889 vacc0123 = wasm_f32x4_convert_i32x4(vacc0123);
890 vacc4567 = wasm_f32x4_convert_i32x4(vacc4567);
891
892 vacc0123 = wasm_f32x4_mul(vacc0123, vscale);
893 vacc4567 = wasm_f32x4_mul(vacc4567, vscale);
894
895 w = (void*) ((uintptr_t) w + 72);
896
897 vacc0123 = wasm_f32x4_add(vacc0123, vmagic_bias);
898 vacc4567 = wasm_f32x4_add(vacc4567, vmagic_bias);
899
900 vacc0123 = wasm_i32x4_max(vacc0123, vmagic_min);
901 vacc4567 = wasm_i32x4_max(vacc4567, vmagic_min);
902
903 vacc0123 = wasm_i32x4_sub(vacc0123, vmagic_bias_less_output_zero_point);
904 vacc4567 = wasm_i32x4_sub(vacc4567, vmagic_bias_less_output_zero_point);
905
906 v128_t vout01234567 = wasm_i16x8_narrow_i32x4(vacc0123, vacc4567);
907 v128_t vout0123456701234567 = wasm_i8x16_narrow_i16x8(vout01234567, vout01234567);
908
909 vout0123456701234567 = wasm_i8x16_min(vout0123456701234567, voutput_max);
910
911 if XNN_LIKELY(c >= 8) {
912 wasm_v128_store64_lane(output, vout0123456701234567, 0);
913 output += 8;
914 c -= 8;
915 } else {
916 if (c & 4) {
917 wasm_v128_store32_lane(output, vout0123456701234567, 0);
918 vout0123456701234567 = wasm_u64x2_shr(vout0123456701234567, 32);
919 output += 4;
920 }
921 if (c & 2) {
922 wasm_v128_store16_lane(output, vout0123456701234567, 0);
923 vout0123456701234567 = wasm_u32x4_shr(vout0123456701234567, 16);
924 output += 2;
925 }
926 if (c & 1) {
927 wasm_v128_store8_lane(output, vout0123456701234567, 0);
928 output += 1;
929 }
930 c = 0;
931 }
932 } while (c != 0);
933 }
934 }
935
936 input = (const int8_t**) ((uintptr_t) input + input_stride);
937 output = (int8_t*) ((uintptr_t) output + output_increment);
938 } while (--output_width != 0);
939 }