Debian Codegraph @

src/qs8-dwconv/gen/qs8-dwconv-8f8m9l16c8s8r-minmax-fp32-wasmsimd-mul16-add16.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-dwconv/multipass-wasmsimd-mul16.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include <wasm_simd128.h>
13
14 #include "xnnpack/dwconv.h"
15 #include "xnnpack/math.h"
16
17
18 void xnn_qs8_dwconv_minmax_fp32_ukernel_8f8m9l16c8s8r__wasmsimd_mul16_add16(
19 size_t channels,
20 size_t output_width,
21 const int8_t** input,
22 const void* weights,
23 int8_t* output,
24 intptr_t input_stride,
25 size_t output_increment,
26 size_t input_offset,
27 const int8_t* zero,
28 size_t kernel_size,
29 int32_t* buffer,
30 const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
31 {
32 assert(channels != 0);
33 assert(output_width != 0);
34 assert(kernel_size > 8);
35
36
37 const v128_t vscale = wasm_v128_load32_splat(&params->fp32_scalar.scale);
38 XNN_FORCE_REALIZATION(vscale);
39
40 const v128_t vmagic_bias = wasm_f32x4_const_splat(12582912.0f);
41 const int32_t output_min_less_zero_point = (int32_t) params->fp32_scalar.output_min - (int32_t) params->fp32_scalar.output_zero_point;
42 const v128_t vmagic_min = wasm_i32x4_splat((int32_t) float_as_uint32(12582912.0f + output_min_less_zero_point));
43 const v128_t vmagic_bias_less_output_zero_point = wasm_i32x4_splat(INT32_C(0x4B400000) - (int32_t) params->fp32_scalar.output_zero_point);
44 const v128_t voutput_max = wasm_i8x16_splat(params->fp32_scalar.output_max);
45 XNN_FORCE_REALIZATION(vmagic_bias);
46 XNN_FORCE_REALIZATION(vmagic_min);
47 XNN_FORCE_REALIZATION(vmagic_bias_less_output_zero_point);
48 XNN_FORCE_REALIZATION(voutput_max);
49
50 do {
51 const void* w = weights;
52
53 // First pass to process 8 inputs.
54 {
55 int32_t* b = buffer;
56 const int8_t* i0 = input[0];
57 assert(i0 != NULL);
58 if XNN_UNPREDICTABLE(i0 != zero) {
59 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
60 }
61 const int8_t* i1 = input[1];
62 assert(i1 != NULL);
63 if XNN_UNPREDICTABLE(i1 != zero) {
64 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
65 }
66 const int8_t* i2 = input[2];
67 assert(i2 != NULL);
68 if XNN_UNPREDICTABLE(i2 != zero) {
69 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
70 }
71 const int8_t* i3 = input[3];
72 assert(i3 != NULL);
73 if XNN_UNPREDICTABLE(i3 != zero) {
74 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
75 }
76 const int8_t* i4 = input[4];
77 assert(i4 != NULL);
78 if XNN_UNPREDICTABLE(i4 != zero) {
79 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
80 }
81 const int8_t* i5 = input[5];
82 assert(i5 != NULL);
83 if XNN_UNPREDICTABLE(i5 != zero) {
84 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
85 }
86 const int8_t* i6 = input[6];
87 assert(i6 != NULL);
88 if XNN_UNPREDICTABLE(i6 != zero) {
89 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
90 }
91 const int8_t* i7 = input[7];
92 assert(i7 != NULL);
93 if XNN_UNPREDICTABLE(i7 != zero) {
94 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
95 }
96 input += 8;
97
98 size_t c = round_up_po2(channels, 8);
99
100 for (; c >= 16; c -= 16) {
101 v128_t vacc0123 = wasm_v128_load(w);
102 v128_t vacc4567 = wasm_v128_load((const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
103 v128_t vacc89AB = wasm_v128_load((const void*) ((uintptr_t) w + 8 * sizeof(int32_t)));
104 v128_t vaccCDEF = wasm_v128_load((const void*) ((uintptr_t) w + 12 * sizeof(int32_t)));
105
106 const v128_t vi0x01234567 = wasm_i16x8_load8x8(i0);
107 v128_t vk0x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 0 * sizeof(int8_t)));
108 const v128_t vi0x89ABCDEF = wasm_i16x8_load8x8(i0 + 8);
109 v128_t vk0x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 8 * sizeof(int8_t)));
110 i0 += 16;
111
112 v128_t vprod01234567 = wasm_i16x8_mul(vi0x01234567, vk0x01234567);
113 v128_t vprod89ABCDEF = wasm_i16x8_mul(vi0x89ABCDEF, vk0x89ABCDEF);
114
115
116 const v128_t vi1x01234567 = wasm_i16x8_load8x8(i1);
117 v128_t vk1x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 16 * sizeof(int8_t)));
118 const v128_t vi1x89ABCDEF = wasm_i16x8_load8x8(i1 + 8);
119 v128_t vk1x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 24 * sizeof(int8_t)));
120 i1 += 16;
121
122 vprod01234567 = wasm_i16x8_add(vprod01234567, wasm_i16x8_mul(vi1x01234567, vk1x01234567));
123 vprod89ABCDEF = wasm_i16x8_add(vprod89ABCDEF, wasm_i16x8_mul(vi1x89ABCDEF, vk1x89ABCDEF));
124
125 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
126 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
127 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
128 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
129
130 const v128_t vi2x01234567 = wasm_i16x8_load8x8(i2);
131 v128_t vk2x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 32 * sizeof(int8_t)));
132 const v128_t vi2x89ABCDEF = wasm_i16x8_load8x8(i2 + 8);
133 v128_t vk2x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 40 * sizeof(int8_t)));
134 i2 += 16;
135
136 vprod01234567 = wasm_i16x8_mul(vi2x01234567, vk2x01234567);
137 vprod89ABCDEF = wasm_i16x8_mul(vi2x89ABCDEF, vk2x89ABCDEF);
138
139
140 const v128_t vi3x01234567 = wasm_i16x8_load8x8(i3);
141 v128_t vk3x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 48 * sizeof(int8_t)));
142 const v128_t vi3x89ABCDEF = wasm_i16x8_load8x8(i3 + 8);
143 v128_t vk3x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 56 * sizeof(int8_t)));
144 i3 += 16;
145
146 vprod01234567 = wasm_i16x8_add(vprod01234567, wasm_i16x8_mul(vi3x01234567, vk3x01234567));
147 vprod89ABCDEF = wasm_i16x8_add(vprod89ABCDEF, wasm_i16x8_mul(vi3x89ABCDEF, vk3x89ABCDEF));
148
149 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
150 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
151 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
152 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
153
154 const v128_t vi4x01234567 = wasm_i16x8_load8x8(i4);
155 v128_t vk4x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 64 * sizeof(int8_t)));
156 const v128_t vi4x89ABCDEF = wasm_i16x8_load8x8(i4 + 8);
157 v128_t vk4x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 72 * sizeof(int8_t)));
158 i4 += 16;
159
160 vprod01234567 = wasm_i16x8_mul(vi4x01234567, vk4x01234567);
161 vprod89ABCDEF = wasm_i16x8_mul(vi4x89ABCDEF, vk4x89ABCDEF);
162
163
164 const v128_t vi5x01234567 = wasm_i16x8_load8x8(i5);
165 v128_t vk5x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 80 * sizeof(int8_t)));
166 const v128_t vi5x89ABCDEF = wasm_i16x8_load8x8(i5 + 8);
167 v128_t vk5x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 88 * sizeof(int8_t)));
168 i5 += 16;
169
170 vprod01234567 = wasm_i16x8_add(vprod01234567, wasm_i16x8_mul(vi5x01234567, vk5x01234567));
171 vprod89ABCDEF = wasm_i16x8_add(vprod89ABCDEF, wasm_i16x8_mul(vi5x89ABCDEF, vk5x89ABCDEF));
172
173 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
174 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
175 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
176 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
177
178 const v128_t vi6x01234567 = wasm_i16x8_load8x8(i6);
179 v128_t vk6x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 96 * sizeof(int8_t)));
180 const v128_t vi6x89ABCDEF = wasm_i16x8_load8x8(i6 + 8);
181 v128_t vk6x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 104 * sizeof(int8_t)));
182 i6 += 16;
183
184 vprod01234567 = wasm_i16x8_mul(vi6x01234567, vk6x01234567);
185 vprod89ABCDEF = wasm_i16x8_mul(vi6x89ABCDEF, vk6x89ABCDEF);
186
187
188 const v128_t vi7x01234567 = wasm_i16x8_load8x8(i7);
189 v128_t vk7x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 112 * sizeof(int8_t)));
190 const v128_t vi7x89ABCDEF = wasm_i16x8_load8x8(i7 + 8);
191 v128_t vk7x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 120 * sizeof(int8_t)));
192 i7 += 16;
193
194 vprod01234567 = wasm_i16x8_add(vprod01234567, wasm_i16x8_mul(vi7x01234567, vk7x01234567));
195 vprod89ABCDEF = wasm_i16x8_add(vprod89ABCDEF, wasm_i16x8_mul(vi7x89ABCDEF, vk7x89ABCDEF));
196
197 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
198 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
199 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
200 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
201
202 w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 128 * sizeof(int8_t));
203
204 wasm_v128_store(b, vacc0123);
205 wasm_v128_store(b + 4, vacc4567);
206 wasm_v128_store(b + 8, vacc89AB);
207 wasm_v128_store(b + 12, vaccCDEF);
208 b += 16;
209 }
210
211 if XNN_UNLIKELY(c != 0) {
212 do {
213 v128_t vacc0123 = wasm_v128_load(w);
214 v128_t vacc4567 = wasm_v128_load((const int32_t*) w + 4);
215
216 const v128_t vi0x01234567 = wasm_i16x8_load8x8(i0);
217 v128_t vk0x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int32_t)));
218 i0 += 8;
219
220 v128_t vprod01234567 = wasm_i16x8_mul(vi0x01234567, vk0x01234567);
221
222
223 const v128_t vi1x01234567 = wasm_i16x8_load8x8(i1);
224 v128_t vk1x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 8 * sizeof(int8_t)));
225 i1 += 8;
226
227 vprod01234567 = wasm_i16x8_add(vprod01234567, wasm_i16x8_mul(vi1x01234567, vk1x01234567));
228
229 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
230 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
231
232 const v128_t vi2x01234567 = wasm_i16x8_load8x8(i2);
233 v128_t vk2x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 16 * sizeof(int8_t)));
234 i2 += 8;
235
236 vprod01234567 = wasm_i16x8_mul(vi2x01234567, vk2x01234567);
237
238
239 const v128_t vi3x01234567 = wasm_i16x8_load8x8(i3);
240 v128_t vk3x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 24 * sizeof(int8_t)));
241 i3 += 8;
242
243 vprod01234567 = wasm_i16x8_add(vprod01234567, wasm_i16x8_mul(vi3x01234567, vk3x01234567));
244
245 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
246 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
247
248 const v128_t vi4x01234567 = wasm_i16x8_load8x8(i4);
249 v128_t vk4x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 32 * sizeof(int8_t)));
250 i4 += 8;
251
252 vprod01234567 = wasm_i16x8_mul(vi4x01234567, vk4x01234567);
253
254
255 const v128_t vi5x01234567 = wasm_i16x8_load8x8(i5);
256 v128_t vk5x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 40 * sizeof(int8_t)));
257 i5 += 8;
258
259 vprod01234567 = wasm_i16x8_add(vprod01234567, wasm_i16x8_mul(vi5x01234567, vk5x01234567));
260
261 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
262 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
263
264 const v128_t vi6x01234567 = wasm_i16x8_load8x8(i6);
265 v128_t vk6x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 48 * sizeof(int8_t)));
266 i6 += 8;
267
268 vprod01234567 = wasm_i16x8_mul(vi6x01234567, vk6x01234567);
269
270
271 const v128_t vi7x01234567 = wasm_i16x8_load8x8(i7);
272 v128_t vk7x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 56 * sizeof(int8_t)));
273 i7 += 8;
274
275 vprod01234567 = wasm_i16x8_add(vprod01234567, wasm_i16x8_mul(vi7x01234567, vk7x01234567));
276
277 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
278 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
279
280
281 w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 64 * sizeof(int8_t));
282
283 wasm_v128_store(b, vacc0123);
284 wasm_v128_store(b + 4, vacc4567);
285 b += 8;
286 c -= 8;
287 } while (c != 0);
288 }
289 }
290
291
292 // Middle pass to process 8 inputs in each iteration.
293 for (size_t ks = kernel_size - 8; ks > 9; ks -= 8) {
294 int32_t* b = buffer;
295 const int8_t* i0 = input[0];
296 assert(i0 != NULL);
297 if XNN_UNPREDICTABLE(i0 != zero) {
298 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
299 }
300 const int8_t* i1 = input[1];
301 assert(i1 != NULL);
302 if XNN_UNPREDICTABLE(i1 != zero) {
303 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
304 }
305 const int8_t* i2 = input[2];
306 assert(i2 != NULL);
307 if XNN_UNPREDICTABLE(i2 != zero) {
308 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
309 }
310 const int8_t* i3 = input[3];
311 assert(i3 != NULL);
312 if XNN_UNPREDICTABLE(i3 != zero) {
313 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
314 }
315 const int8_t* i4 = input[4];
316 assert(i4 != NULL);
317 if XNN_UNPREDICTABLE(i4 != zero) {
318 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
319 }
320 const int8_t* i5 = input[5];
321 assert(i5 != NULL);
322 if XNN_UNPREDICTABLE(i5 != zero) {
323 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
324 }
325 const int8_t* i6 = input[6];
326 assert(i6 != NULL);
327 if XNN_UNPREDICTABLE(i6 != zero) {
328 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
329 }
330 const int8_t* i7 = input[7];
331 assert(i7 != NULL);
332 if XNN_UNPREDICTABLE(i7 != zero) {
333 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
334 }
335 input += 8;
336
337 size_t c = round_up_po2(channels, 8);
338
339 for (; c >= 16; c -= 16) {
340 v128_t vacc0123 = wasm_v128_load(b);
341 v128_t vacc4567 = wasm_v128_load(b + 4);
342 v128_t vacc89AB = wasm_v128_load(b + 8);
343 v128_t vaccCDEF = wasm_v128_load(b + 12);
344
345 const v128_t vi0x01234567 = wasm_i16x8_load8x8(i0);
346 v128_t vk0x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 0 * sizeof(int8_t)));
347 const v128_t vi0x89ABCDEF = wasm_i16x8_load8x8(i0 + 8);
348 v128_t vk0x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int8_t)));
349 i0 += 16;
350
351 v128_t vprod01234567 = wasm_i16x8_mul(vi0x01234567, vk0x01234567);
352 v128_t vprod89ABCDEF = wasm_i16x8_mul(vi0x89ABCDEF, vk0x89ABCDEF);
353
354
355 const v128_t vi1x01234567 = wasm_i16x8_load8x8(i1);
356 v128_t vk1x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int8_t)));
357 const v128_t vi1x89ABCDEF = wasm_i16x8_load8x8(i1 + 8);
358 v128_t vk1x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 24 * sizeof(int8_t)));
359 i1 += 16;
360
361 vprod01234567 = wasm_i16x8_add(vprod01234567, wasm_i16x8_mul(vi1x01234567, vk1x01234567));
362 vprod89ABCDEF = wasm_i16x8_add(vprod89ABCDEF, wasm_i16x8_mul(vi1x89ABCDEF, vk1x89ABCDEF));
363
364 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
365 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
366 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
367 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
368
369 const v128_t vi2x01234567 = wasm_i16x8_load8x8(i2);
370 v128_t vk2x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 32 * sizeof(int8_t)));
371 const v128_t vi2x89ABCDEF = wasm_i16x8_load8x8(i2 + 8);
372 v128_t vk2x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 40 * sizeof(int8_t)));
373 i2 += 16;
374
375 vprod01234567 = wasm_i16x8_mul(vi2x01234567, vk2x01234567);
376 vprod89ABCDEF = wasm_i16x8_mul(vi2x89ABCDEF, vk2x89ABCDEF);
377
378
379 const v128_t vi3x01234567 = wasm_i16x8_load8x8(i3);
380 v128_t vk3x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 48 * sizeof(int8_t)));
381 const v128_t vi3x89ABCDEF = wasm_i16x8_load8x8(i3 + 8);
382 v128_t vk3x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 56 * sizeof(int8_t)));
383 i3 += 16;
384
385 vprod01234567 = wasm_i16x8_add(vprod01234567, wasm_i16x8_mul(vi3x01234567, vk3x01234567));
386 vprod89ABCDEF = wasm_i16x8_add(vprod89ABCDEF, wasm_i16x8_mul(vi3x89ABCDEF, vk3x89ABCDEF));
387
388 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
389 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
390 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
391 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
392
393 const v128_t vi4x01234567 = wasm_i16x8_load8x8(i4);
394 v128_t vk4x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 64 * sizeof(int8_t)));
395 const v128_t vi4x89ABCDEF = wasm_i16x8_load8x8(i4 + 8);
396 v128_t vk4x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 72 * sizeof(int8_t)));
397 i4 += 16;
398
399 vprod01234567 = wasm_i16x8_mul(vi4x01234567, vk4x01234567);
400 vprod89ABCDEF = wasm_i16x8_mul(vi4x89ABCDEF, vk4x89ABCDEF);
401
402
403 const v128_t vi5x01234567 = wasm_i16x8_load8x8(i5);
404 v128_t vk5x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 80 * sizeof(int8_t)));
405 const v128_t vi5x89ABCDEF = wasm_i16x8_load8x8(i5 + 8);
406 v128_t vk5x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 88 * sizeof(int8_t)));
407 i5 += 16;
408
409 vprod01234567 = wasm_i16x8_add(vprod01234567, wasm_i16x8_mul(vi5x01234567, vk5x01234567));
410 vprod89ABCDEF = wasm_i16x8_add(vprod89ABCDEF, wasm_i16x8_mul(vi5x89ABCDEF, vk5x89ABCDEF));
411
412 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
413 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
414 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
415 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
416
417 const v128_t vi6x01234567 = wasm_i16x8_load8x8(i6);
418 v128_t vk6x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 96 * sizeof(int8_t)));
419 const v128_t vi6x89ABCDEF = wasm_i16x8_load8x8(i6 + 8);
420 v128_t vk6x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 104 * sizeof(int8_t)));
421 i6 += 16;
422
423 vprod01234567 = wasm_i16x8_mul(vi6x01234567, vk6x01234567);
424 vprod89ABCDEF = wasm_i16x8_mul(vi6x89ABCDEF, vk6x89ABCDEF);
425
426
427 const v128_t vi7x01234567 = wasm_i16x8_load8x8(i7);
428 v128_t vk7x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 112 * sizeof(int8_t)));
429 const v128_t vi7x89ABCDEF = wasm_i16x8_load8x8(i7 + 8);
430 v128_t vk7x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 120 * sizeof(int8_t)));
431 i7 += 16;
432
433 vprod01234567 = wasm_i16x8_add(vprod01234567, wasm_i16x8_mul(vi7x01234567, vk7x01234567));
434 vprod89ABCDEF = wasm_i16x8_add(vprod89ABCDEF, wasm_i16x8_mul(vi7x89ABCDEF, vk7x89ABCDEF));
435
436 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
437 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
438 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
439 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
440
441 w = (const void*) ((uintptr_t) w + 128 * sizeof(int8_t));
442
443 wasm_v128_store(b, vacc0123);
444 wasm_v128_store(b + 4, vacc4567);
445 wasm_v128_store(b + 8, vacc89AB);
446 wasm_v128_store(b + 12, vaccCDEF);
447 b += 16;
448 }
449
450 if XNN_UNLIKELY(c != 0) {
451 do {
452 v128_t vacc0123 = wasm_v128_load(b);
453 v128_t vacc4567 = wasm_v128_load(b + 4);
454
455 const v128_t vi0x01234567 = wasm_i16x8_load8x8(i0);
456 v128_t vk0x01234567 = wasm_i16x8_load8x8(w);
457 i0 += 8;
458
459 v128_t vprod01234567 = wasm_i16x8_mul(vi0x01234567, vk0x01234567);
460
461
462 const v128_t vi1x01234567 = wasm_i16x8_load8x8(i1);
463 v128_t vk1x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int8_t)));
464 i1 += 8;
465
466 vprod01234567 = wasm_i16x8_add(vprod01234567, wasm_i16x8_mul(vi1x01234567, vk1x01234567));
467
468 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
469 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
470
471 const v128_t vi2x01234567 = wasm_i16x8_load8x8(i2);
472 v128_t vk2x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int8_t)));
473 i2 += 8;
474
475 vprod01234567 = wasm_i16x8_mul(vi2x01234567, vk2x01234567);
476
477
478 const v128_t vi3x01234567 = wasm_i16x8_load8x8(i3);
479 v128_t vk3x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 24 * sizeof(int8_t)));
480 i3 += 8;
481
482 vprod01234567 = wasm_i16x8_add(vprod01234567, wasm_i16x8_mul(vi3x01234567, vk3x01234567));
483
484 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
485 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
486
487 const v128_t vi4x01234567 = wasm_i16x8_load8x8(i4);
488 v128_t vk4x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 32 * sizeof(int8_t)));
489 i4 += 8;
490
491 vprod01234567 = wasm_i16x8_mul(vi4x01234567, vk4x01234567);
492
493
494 const v128_t vi5x01234567 = wasm_i16x8_load8x8(i5);
495 v128_t vk5x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 40 * sizeof(int8_t)));
496 i5 += 8;
497
498 vprod01234567 = wasm_i16x8_add(vprod01234567, wasm_i16x8_mul(vi5x01234567, vk5x01234567));
499
500 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
501 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
502
503 const v128_t vi6x01234567 = wasm_i16x8_load8x8(i6);
504 v128_t vk6x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 48 * sizeof(int8_t)));
505 i6 += 8;
506
507 vprod01234567 = wasm_i16x8_mul(vi6x01234567, vk6x01234567);
508
509
510 const v128_t vi7x01234567 = wasm_i16x8_load8x8(i7);
511 v128_t vk7x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 56 * sizeof(int8_t)));
512 i7 += 8;
513
514 vprod01234567 = wasm_i16x8_add(vprod01234567, wasm_i16x8_mul(vi7x01234567, vk7x01234567));
515
516 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
517 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
518
519
520 w = (const void*) ((uintptr_t) w + 64 * sizeof(int8_t));
521
522 wasm_v128_store(b, vacc0123);
523 wasm_v128_store(b + 4, vacc4567);
524 b += 8;
525 c -= 8;
526 } while (c != 0);
527 }
528 }
529
530 // Last pass to process up to 9 inputs.
531 {
532 const int32_t* b = buffer;
533 const int8_t* i0 = input[0];
534 assert(i0 != NULL);
535 if XNN_UNPREDICTABLE(i0 != zero) {
536 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
537 }
538 const int8_t* i1 = input[1];
539 assert(i1 != NULL);
540 if XNN_UNPREDICTABLE(i1 != zero) {
541 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
542 }
543 const int8_t* i2 = input[2];
544 assert(i2 != NULL);
545 if XNN_UNPREDICTABLE(i2 != zero) {
546 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
547 }
548 const int8_t* i3 = input[3];
549 assert(i3 != NULL);
550 if XNN_UNPREDICTABLE(i3 != zero) {
551 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
552 }
553 const int8_t* i4 = input[4];
554 assert(i4 != NULL);
555 if XNN_UNPREDICTABLE(i4 != zero) {
556 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
557 }
558 const int8_t* i5 = input[5];
559 assert(i5 != NULL);
560 if XNN_UNPREDICTABLE(i5 != zero) {
561 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
562 }
563 const int8_t* i6 = input[6];
564 assert(i6 != NULL);
565 if XNN_UNPREDICTABLE(i6 != zero) {
566 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
567 }
568 const int8_t* i7 = input[7];
569 assert(i7 != NULL);
570 if XNN_UNPREDICTABLE(i7 != zero) {
571 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
572 }
573 const int8_t* i8 = input[8];
574 assert(i8 != NULL);
575 if XNN_UNPREDICTABLE(i8 != zero) {
576 i8 = (const int8_t*) ((uintptr_t) i8 + input_offset);
577 }
578
579 size_t c = channels;
580
581 for (; c >= 16; c -= 16) {
582 v128_t vacc0123 = wasm_v128_load(b);
583 v128_t vacc4567 = wasm_v128_load(b + 4);
584 v128_t vacc89AB = wasm_v128_load(b + 8);
585 v128_t vaccCDEF = wasm_v128_load(b + 12);
586 b += 16;
587
588 const v128_t vi0x01234567 = wasm_i16x8_load8x8(i0);
589 v128_t vk0x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 0 * sizeof(int8_t)));
590 const v128_t vi0x89ABCDEF = wasm_i16x8_load8x8(i0 + 8);
591 v128_t vk0x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int8_t)));
592 i0 += 16;
593
594 v128_t vprod01234567 = wasm_i16x8_mul(vi0x01234567, vk0x01234567);
595 v128_t vprod89ABCDEF = wasm_i16x8_mul(vi0x89ABCDEF, vk0x89ABCDEF);
596
597
598 const v128_t vi1x01234567 = wasm_i16x8_load8x8(i1);
599 v128_t vk1x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int8_t)));
600 const v128_t vi1x89ABCDEF = wasm_i16x8_load8x8(i1 + 8);
601 v128_t vk1x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 24 * sizeof(int8_t)));
602 i1 += 16;
603
604 vprod01234567 = wasm_i16x8_add(vprod01234567, wasm_i16x8_mul(vi1x01234567, vk1x01234567));
605 vprod89ABCDEF = wasm_i16x8_add(vprod89ABCDEF, wasm_i16x8_mul(vi1x89ABCDEF, vk1x89ABCDEF));
606
607 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
608 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
609 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
610 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
611
612 const v128_t vi2x01234567 = wasm_i16x8_load8x8(i2);
613 v128_t vk2x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 32 * sizeof(int8_t)));
614 const v128_t vi2x89ABCDEF = wasm_i16x8_load8x8(i2 + 8);
615 v128_t vk2x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 40 * sizeof(int8_t)));
616 i2 += 16;
617
618 vprod01234567 = wasm_i16x8_mul(vi2x01234567, vk2x01234567);
619 vprod89ABCDEF = wasm_i16x8_mul(vi2x89ABCDEF, vk2x89ABCDEF);
620
621
622 const v128_t vi3x01234567 = wasm_i16x8_load8x8(i3);
623 v128_t vk3x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 48 * sizeof(int8_t)));
624 const v128_t vi3x89ABCDEF = wasm_i16x8_load8x8(i3 + 8);
625 v128_t vk3x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 56 * sizeof(int8_t)));
626 i3 += 16;
627
628 vprod01234567 = wasm_i16x8_add(vprod01234567, wasm_i16x8_mul(vi3x01234567, vk3x01234567));
629 vprod89ABCDEF = wasm_i16x8_add(vprod89ABCDEF, wasm_i16x8_mul(vi3x89ABCDEF, vk3x89ABCDEF));
630
631 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
632 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
633 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
634 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
635
636 const v128_t vi4x01234567 = wasm_i16x8_load8x8(i4);
637 v128_t vk4x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 64 * sizeof(int8_t)));
638 const v128_t vi4x89ABCDEF = wasm_i16x8_load8x8(i4 + 8);
639 v128_t vk4x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 72 * sizeof(int8_t)));
640 i4 += 16;
641
642 vprod01234567 = wasm_i16x8_mul(vi4x01234567, vk4x01234567);
643 vprod89ABCDEF = wasm_i16x8_mul(vi4x89ABCDEF, vk4x89ABCDEF);
644
645
646 const v128_t vi5x01234567 = wasm_i16x8_load8x8(i5);
647 v128_t vk5x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 80 * sizeof(int8_t)));
648 const v128_t vi5x89ABCDEF = wasm_i16x8_load8x8(i5 + 8);
649 v128_t vk5x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 88 * sizeof(int8_t)));
650 i5 += 16;
651
652 vprod01234567 = wasm_i16x8_add(vprod01234567, wasm_i16x8_mul(vi5x01234567, vk5x01234567));
653 vprod89ABCDEF = wasm_i16x8_add(vprod89ABCDEF, wasm_i16x8_mul(vi5x89ABCDEF, vk5x89ABCDEF));
654
655 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
656 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
657 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
658 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
659
660 const v128_t vi6x01234567 = wasm_i16x8_load8x8(i6);
661 v128_t vk6x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 96 * sizeof(int8_t)));
662 const v128_t vi6x89ABCDEF = wasm_i16x8_load8x8(i6 + 8);
663 v128_t vk6x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 104 * sizeof(int8_t)));
664 i6 += 16;
665
666 vprod01234567 = wasm_i16x8_mul(vi6x01234567, vk6x01234567);
667 vprod89ABCDEF = wasm_i16x8_mul(vi6x89ABCDEF, vk6x89ABCDEF);
668
669
670 const v128_t vi7x01234567 = wasm_i16x8_load8x8(i7);
671 v128_t vk7x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 112 * sizeof(int8_t)));
672 const v128_t vi7x89ABCDEF = wasm_i16x8_load8x8(i7 + 8);
673 v128_t vk7x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 120 * sizeof(int8_t)));
674 i7 += 16;
675
676 vprod01234567 = wasm_i16x8_add(vprod01234567, wasm_i16x8_mul(vi7x01234567, vk7x01234567));
677 vprod89ABCDEF = wasm_i16x8_add(vprod89ABCDEF, wasm_i16x8_mul(vi7x89ABCDEF, vk7x89ABCDEF));
678
679 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
680 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
681 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
682 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
683
684 const v128_t vi8x01234567 = wasm_i16x8_load8x8(i8);
685 v128_t vk8x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 128 * sizeof(int8_t)));
686 const v128_t vi8x89ABCDEF = wasm_i16x8_load8x8(i8 + 8);
687 v128_t vk8x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 136 * sizeof(int8_t)));
688 i8 += 16;
689
690 vprod01234567 = wasm_i16x8_mul(vi8x01234567, vk8x01234567);
691 vprod89ABCDEF = wasm_i16x8_mul(vi8x89ABCDEF, vk8x89ABCDEF);
692
693 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
694 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
695 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
696 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
697
698
699 w = (const void*) ((uintptr_t) w + 144 * sizeof(int8_t));
700
701 vacc0123 = wasm_f32x4_convert_i32x4(vacc0123);
702 vacc4567 = wasm_f32x4_convert_i32x4(vacc4567);
703 vacc89AB = wasm_f32x4_convert_i32x4(vacc89AB);
704 vaccCDEF = wasm_f32x4_convert_i32x4(vaccCDEF);
705
706 vacc0123 = wasm_f32x4_mul(vacc0123, vscale);
707 vacc4567 = wasm_f32x4_mul(vacc4567, vscale);
708 vacc89AB = wasm_f32x4_mul(vacc89AB, vscale);
709 vaccCDEF = wasm_f32x4_mul(vaccCDEF, vscale);
710
711 vacc0123 = wasm_f32x4_add(vacc0123, vmagic_bias);
712 vacc4567 = wasm_f32x4_add(vacc4567, vmagic_bias);
713 vacc89AB = wasm_f32x4_add(vacc89AB, vmagic_bias);
714 vaccCDEF = wasm_f32x4_add(vaccCDEF, vmagic_bias);
715
716 vacc0123 = wasm_i32x4_max(vacc0123, vmagic_min);
717 vacc4567 = wasm_i32x4_max(vacc4567, vmagic_min);
718 vacc89AB = wasm_i32x4_max(vacc89AB, vmagic_min);
719 vaccCDEF = wasm_i32x4_max(vaccCDEF, vmagic_min);
720
721 vacc0123 = wasm_i32x4_sub(vacc0123, vmagic_bias_less_output_zero_point);
722 vacc4567 = wasm_i32x4_sub(vacc4567, vmagic_bias_less_output_zero_point);
723 vacc89AB = wasm_i32x4_sub(vacc89AB, vmagic_bias_less_output_zero_point);
724 vaccCDEF = wasm_i32x4_sub(vaccCDEF, vmagic_bias_less_output_zero_point);
725
726 v128_t vout01234567 = wasm_i16x8_narrow_i32x4(vacc0123, vacc4567);
727 v128_t vout89ABCDEF = wasm_i16x8_narrow_i32x4(vacc89AB, vaccCDEF);
728
729 v128_t vout0123456789ABCDEF = wasm_i8x16_narrow_i16x8(vout01234567, vout89ABCDEF);
730
731 vout0123456789ABCDEF = wasm_i8x16_min(vout0123456789ABCDEF, voutput_max);
732
733 wasm_v128_store(output, vout0123456789ABCDEF);
734 output += 16;
735 }
736
737 if XNN_UNLIKELY(c != 0) {
738 do {
739 v128_t vacc0123 = wasm_v128_load(b);
740 v128_t vacc4567 = wasm_v128_load(b + 4);
741 b += 8;
742
743 const v128_t vi0x01234567 = wasm_i16x8_load8x8(i0);
744 v128_t vk0x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w));
745 i0 += 8;
746
747 v128_t vprod01234567 = wasm_i16x8_mul(vi0x01234567, vk0x01234567);
748
749
750 const v128_t vi1x01234567 = wasm_i16x8_load8x8(i1);
751 v128_t vk1x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int8_t)));
752 i1 += 8;
753
754 vprod01234567 = wasm_i16x8_add(vprod01234567, wasm_i16x8_mul(vi1x01234567, vk1x01234567));
755
756 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
757 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
758
759 const v128_t vi2x01234567 = wasm_i16x8_load8x8(i2);
760 v128_t vk2x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int8_t)));
761 i2 += 8;
762
763 vprod01234567 = wasm_i16x8_mul(vi2x01234567, vk2x01234567);
764
765
766 const v128_t vi3x01234567 = wasm_i16x8_load8x8(i3);
767 v128_t vk3x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 24 * sizeof(int8_t)));
768 i3 += 8;
769
770 vprod01234567 = wasm_i16x8_add(vprod01234567, wasm_i16x8_mul(vi3x01234567, vk3x01234567));
771
772 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
773 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
774
775 const v128_t vi4x01234567 = wasm_i16x8_load8x8(i4);
776 v128_t vk4x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 32 * sizeof(int8_t)));
777 i4 += 8;
778
779 vprod01234567 = wasm_i16x8_mul(vi4x01234567, vk4x01234567);
780
781
782 const v128_t vi5x01234567 = wasm_i16x8_load8x8(i5);
783 v128_t vk5x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 40 * sizeof(int8_t)));
784 i5 += 8;
785
786 vprod01234567 = wasm_i16x8_add(vprod01234567, wasm_i16x8_mul(vi5x01234567, vk5x01234567));
787
788 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
789 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
790
791 const v128_t vi6x01234567 = wasm_i16x8_load8x8(i6);
792 v128_t vk6x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 48 * sizeof(int8_t)));
793 i6 += 8;
794
795 vprod01234567 = wasm_i16x8_mul(vi6x01234567, vk6x01234567);
796
797
798 const v128_t vi7x01234567 = wasm_i16x8_load8x8(i7);
799 v128_t vk7x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 56 * sizeof(int8_t)));
800 i7 += 8;
801
802 vprod01234567 = wasm_i16x8_add(vprod01234567, wasm_i16x8_mul(vi7x01234567, vk7x01234567));
803
804 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
805 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
806
807 const v128_t vi8x01234567 = wasm_i16x8_load8x8(i8);
808 v128_t vk8x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 64 * sizeof(int8_t)));
809 i8 += 8;
810
811 vprod01234567 = wasm_i16x8_mul(vi8x01234567, vk8x01234567);
812
813 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
814 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
815
816
817 vacc0123 = wasm_f32x4_convert_i32x4(vacc0123);
818 vacc4567 = wasm_f32x4_convert_i32x4(vacc4567);
819
820 vacc0123 = wasm_f32x4_mul(vacc0123, vscale);
821 vacc4567 = wasm_f32x4_mul(vacc4567, vscale);
822
823 w = (void*) ((uintptr_t) w + 72);
824
825 vacc0123 = wasm_f32x4_add(vacc0123, vmagic_bias);
826 vacc4567 = wasm_f32x4_add(vacc4567, vmagic_bias);
827
828 vacc0123 = wasm_i32x4_max(vacc0123, vmagic_min);
829 vacc4567 = wasm_i32x4_max(vacc4567, vmagic_min);
830
831 vacc0123 = wasm_i32x4_sub(vacc0123, vmagic_bias_less_output_zero_point);
832 vacc4567 = wasm_i32x4_sub(vacc4567, vmagic_bias_less_output_zero_point);
833
834 v128_t vout01234567 = wasm_i16x8_narrow_i32x4(vacc0123, vacc4567);
835 v128_t vout0123456701234567 = wasm_i8x16_narrow_i16x8(vout01234567, vout01234567);
836
837 vout0123456701234567 = wasm_i8x16_min(vout0123456701234567, voutput_max);
838
839 if XNN_LIKELY(c >= 8) {
840 wasm_v128_store64_lane(output, vout0123456701234567, 0);
841 output += 8;
842 c -= 8;
843 } else {
844 if (c & 4) {
845 wasm_v128_store32_lane(output, vout0123456701234567, 0);
846 vout0123456701234567 = wasm_u64x2_shr(vout0123456701234567, 32);
847 output += 4;
848 }
849 if (c & 2) {
850 wasm_v128_store16_lane(output, vout0123456701234567, 0);
851 vout0123456701234567 = wasm_u32x4_shr(vout0123456701234567, 16);
852 output += 2;
853 }
854 if (c & 1) {
855 wasm_v128_store8_lane(output, vout0123456701234567, 0);
856 output += 1;
857 }
858 c = 0;
859 }
860 } while (c != 0);
861 }
862 }
863
864 input = (const int8_t**) ((uintptr_t) input + input_stride);
865 output = (int8_t*) ((uintptr_t) output + output_increment);
866 } while (--output_width != 0);
867 }