Debian Codegraph @

src/qs8-dwconv/gen/qs8-dwconv-8f8m9l8c8s8r-minmax-fp32-wasmsimd-mul16.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-dwconv/multipass-wasmsimd-mul16.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include <wasm_simd128.h>
13
14 #include "xnnpack/dwconv.h"
15 #include "xnnpack/math.h"
16
17
18 void xnn_qs8_dwconv_minmax_fp32_ukernel_8f8m9l8c8s8r__wasmsimd_mul16(
19 size_t channels,
20 size_t output_width,
21 const int8_t** input,
22 const void* weights,
23 int8_t* output,
24 intptr_t input_stride,
25 size_t output_increment,
26 size_t input_offset,
27 const int8_t* zero,
28 size_t kernel_size,
29 int32_t* buffer,
30 const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
31 {
32 assert(channels != 0);
33 assert(output_width != 0);
34 assert(kernel_size > 8);
35
36
37 const v128_t vscale = wasm_v128_load32_splat(&params->fp32_scalar.scale);
38 XNN_FORCE_REALIZATION(vscale);
39
40 const v128_t vmagic_bias = wasm_f32x4_const_splat(12582912.0f);
41 const int32_t output_min_less_zero_point = (int32_t) params->fp32_scalar.output_min - (int32_t) params->fp32_scalar.output_zero_point;
42 const v128_t vmagic_min = wasm_i32x4_splat((int32_t) float_as_uint32(12582912.0f + output_min_less_zero_point));
43 const v128_t vmagic_bias_less_output_zero_point = wasm_i32x4_splat(INT32_C(0x4B400000) - (int32_t) params->fp32_scalar.output_zero_point);
44 const v128_t voutput_max = wasm_i8x16_splat(params->fp32_scalar.output_max);
45 XNN_FORCE_REALIZATION(vmagic_bias);
46 XNN_FORCE_REALIZATION(vmagic_min);
47 XNN_FORCE_REALIZATION(vmagic_bias_less_output_zero_point);
48 XNN_FORCE_REALIZATION(voutput_max);
49
50 do {
51 const void* w = weights;
52
53 // First pass to process 8 inputs.
54 {
55 int32_t* b = buffer;
56 const int8_t* i0 = input[0];
57 assert(i0 != NULL);
58 if XNN_UNPREDICTABLE(i0 != zero) {
59 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
60 }
61 const int8_t* i1 = input[1];
62 assert(i1 != NULL);
63 if XNN_UNPREDICTABLE(i1 != zero) {
64 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
65 }
66 const int8_t* i2 = input[2];
67 assert(i2 != NULL);
68 if XNN_UNPREDICTABLE(i2 != zero) {
69 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
70 }
71 const int8_t* i3 = input[3];
72 assert(i3 != NULL);
73 if XNN_UNPREDICTABLE(i3 != zero) {
74 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
75 }
76 const int8_t* i4 = input[4];
77 assert(i4 != NULL);
78 if XNN_UNPREDICTABLE(i4 != zero) {
79 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
80 }
81 const int8_t* i5 = input[5];
82 assert(i5 != NULL);
83 if XNN_UNPREDICTABLE(i5 != zero) {
84 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
85 }
86 const int8_t* i6 = input[6];
87 assert(i6 != NULL);
88 if XNN_UNPREDICTABLE(i6 != zero) {
89 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
90 }
91 const int8_t* i7 = input[7];
92 assert(i7 != NULL);
93 if XNN_UNPREDICTABLE(i7 != zero) {
94 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
95 }
96 input += 8;
97
98 size_t c = round_up_po2(channels, 8);
99
100 for (; c >= 8; c -= 8) {
101 v128_t vacc0123 = wasm_v128_load(w);
102 v128_t vacc4567 = wasm_v128_load((const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
103
104 const v128_t vi0x01234567 = wasm_i16x8_load8x8(i0);
105 v128_t vk0x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 0 * sizeof(int8_t)));
106 i0 += 8;
107
108 v128_t vprod01234567 = wasm_i16x8_mul(vi0x01234567, vk0x01234567);
109
110 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
111 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
112
113 const v128_t vi1x01234567 = wasm_i16x8_load8x8(i1);
114 v128_t vk1x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 8 * sizeof(int8_t)));
115 i1 += 8;
116
117 vprod01234567 = wasm_i16x8_mul(vi1x01234567, vk1x01234567);
118
119 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
120 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
121
122 const v128_t vi2x01234567 = wasm_i16x8_load8x8(i2);
123 v128_t vk2x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 16 * sizeof(int8_t)));
124 i2 += 8;
125
126 vprod01234567 = wasm_i16x8_mul(vi2x01234567, vk2x01234567);
127
128 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
129 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
130
131 const v128_t vi3x01234567 = wasm_i16x8_load8x8(i3);
132 v128_t vk3x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 24 * sizeof(int8_t)));
133 i3 += 8;
134
135 vprod01234567 = wasm_i16x8_mul(vi3x01234567, vk3x01234567);
136
137 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
138 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
139
140 const v128_t vi4x01234567 = wasm_i16x8_load8x8(i4);
141 v128_t vk4x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 32 * sizeof(int8_t)));
142 i4 += 8;
143
144 vprod01234567 = wasm_i16x8_mul(vi4x01234567, vk4x01234567);
145
146 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
147 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
148
149 const v128_t vi5x01234567 = wasm_i16x8_load8x8(i5);
150 v128_t vk5x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 40 * sizeof(int8_t)));
151 i5 += 8;
152
153 vprod01234567 = wasm_i16x8_mul(vi5x01234567, vk5x01234567);
154
155 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
156 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
157
158 const v128_t vi6x01234567 = wasm_i16x8_load8x8(i6);
159 v128_t vk6x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 48 * sizeof(int8_t)));
160 i6 += 8;
161
162 vprod01234567 = wasm_i16x8_mul(vi6x01234567, vk6x01234567);
163
164 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
165 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
166
167 const v128_t vi7x01234567 = wasm_i16x8_load8x8(i7);
168 v128_t vk7x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 56 * sizeof(int8_t)));
169 i7 += 8;
170
171 vprod01234567 = wasm_i16x8_mul(vi7x01234567, vk7x01234567);
172
173 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
174 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
175
176 w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 64 * sizeof(int8_t));
177
178 wasm_v128_store(b, vacc0123);
179 wasm_v128_store(b + 4, vacc4567);
180 b += 8;
181 }
182
183 assert(c == 0);
184 }
185
186
187 // Middle pass to process 8 inputs in each iteration.
188 for (size_t ks = kernel_size - 8; ks > 9; ks -= 8) {
189 int32_t* b = buffer;
190 const int8_t* i0 = input[0];
191 assert(i0 != NULL);
192 if XNN_UNPREDICTABLE(i0 != zero) {
193 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
194 }
195 const int8_t* i1 = input[1];
196 assert(i1 != NULL);
197 if XNN_UNPREDICTABLE(i1 != zero) {
198 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
199 }
200 const int8_t* i2 = input[2];
201 assert(i2 != NULL);
202 if XNN_UNPREDICTABLE(i2 != zero) {
203 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
204 }
205 const int8_t* i3 = input[3];
206 assert(i3 != NULL);
207 if XNN_UNPREDICTABLE(i3 != zero) {
208 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
209 }
210 const int8_t* i4 = input[4];
211 assert(i4 != NULL);
212 if XNN_UNPREDICTABLE(i4 != zero) {
213 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
214 }
215 const int8_t* i5 = input[5];
216 assert(i5 != NULL);
217 if XNN_UNPREDICTABLE(i5 != zero) {
218 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
219 }
220 const int8_t* i6 = input[6];
221 assert(i6 != NULL);
222 if XNN_UNPREDICTABLE(i6 != zero) {
223 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
224 }
225 const int8_t* i7 = input[7];
226 assert(i7 != NULL);
227 if XNN_UNPREDICTABLE(i7 != zero) {
228 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
229 }
230 input += 8;
231
232 size_t c = round_up_po2(channels, 8);
233
234 for (; c >= 8; c -= 8) {
235 v128_t vacc0123 = wasm_v128_load(b);
236 v128_t vacc4567 = wasm_v128_load(b + 4);
237
238 const v128_t vi0x01234567 = wasm_i16x8_load8x8(i0);
239 v128_t vk0x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 0 * sizeof(int8_t)));
240 i0 += 8;
241
242 v128_t vprod01234567 = wasm_i16x8_mul(vi0x01234567, vk0x01234567);
243
244 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
245 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
246
247 const v128_t vi1x01234567 = wasm_i16x8_load8x8(i1);
248 v128_t vk1x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int8_t)));
249 i1 += 8;
250
251 vprod01234567 = wasm_i16x8_mul(vi1x01234567, vk1x01234567);
252
253 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
254 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
255
256 const v128_t vi2x01234567 = wasm_i16x8_load8x8(i2);
257 v128_t vk2x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int8_t)));
258 i2 += 8;
259
260 vprod01234567 = wasm_i16x8_mul(vi2x01234567, vk2x01234567);
261
262 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
263 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
264
265 const v128_t vi3x01234567 = wasm_i16x8_load8x8(i3);
266 v128_t vk3x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 24 * sizeof(int8_t)));
267 i3 += 8;
268
269 vprod01234567 = wasm_i16x8_mul(vi3x01234567, vk3x01234567);
270
271 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
272 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
273
274 const v128_t vi4x01234567 = wasm_i16x8_load8x8(i4);
275 v128_t vk4x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 32 * sizeof(int8_t)));
276 i4 += 8;
277
278 vprod01234567 = wasm_i16x8_mul(vi4x01234567, vk4x01234567);
279
280 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
281 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
282
283 const v128_t vi5x01234567 = wasm_i16x8_load8x8(i5);
284 v128_t vk5x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 40 * sizeof(int8_t)));
285 i5 += 8;
286
287 vprod01234567 = wasm_i16x8_mul(vi5x01234567, vk5x01234567);
288
289 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
290 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
291
292 const v128_t vi6x01234567 = wasm_i16x8_load8x8(i6);
293 v128_t vk6x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 48 * sizeof(int8_t)));
294 i6 += 8;
295
296 vprod01234567 = wasm_i16x8_mul(vi6x01234567, vk6x01234567);
297
298 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
299 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
300
301 const v128_t vi7x01234567 = wasm_i16x8_load8x8(i7);
302 v128_t vk7x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 56 * sizeof(int8_t)));
303 i7 += 8;
304
305 vprod01234567 = wasm_i16x8_mul(vi7x01234567, vk7x01234567);
306
307 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
308 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
309
310 w = (const void*) ((uintptr_t) w + 64 * sizeof(int8_t));
311
312 wasm_v128_store(b, vacc0123);
313 wasm_v128_store(b + 4, vacc4567);
314 b += 8;
315 }
316
317 assert(c == 0);
318 }
319
320 // Last pass to process up to 9 inputs.
321 {
322 const int32_t* b = buffer;
323 const int8_t* i0 = input[0];
324 assert(i0 != NULL);
325 if XNN_UNPREDICTABLE(i0 != zero) {
326 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
327 }
328 const int8_t* i1 = input[1];
329 assert(i1 != NULL);
330 if XNN_UNPREDICTABLE(i1 != zero) {
331 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
332 }
333 const int8_t* i2 = input[2];
334 assert(i2 != NULL);
335 if XNN_UNPREDICTABLE(i2 != zero) {
336 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
337 }
338 const int8_t* i3 = input[3];
339 assert(i3 != NULL);
340 if XNN_UNPREDICTABLE(i3 != zero) {
341 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
342 }
343 const int8_t* i4 = input[4];
344 assert(i4 != NULL);
345 if XNN_UNPREDICTABLE(i4 != zero) {
346 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
347 }
348 const int8_t* i5 = input[5];
349 assert(i5 != NULL);
350 if XNN_UNPREDICTABLE(i5 != zero) {
351 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
352 }
353 const int8_t* i6 = input[6];
354 assert(i6 != NULL);
355 if XNN_UNPREDICTABLE(i6 != zero) {
356 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
357 }
358 const int8_t* i7 = input[7];
359 assert(i7 != NULL);
360 if XNN_UNPREDICTABLE(i7 != zero) {
361 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
362 }
363 const int8_t* i8 = input[8];
364 assert(i8 != NULL);
365 if XNN_UNPREDICTABLE(i8 != zero) {
366 i8 = (const int8_t*) ((uintptr_t) i8 + input_offset);
367 }
368
369 size_t c = channels;
370
371 for (; c >= 8; c -= 8) {
372 v128_t vacc0123 = wasm_v128_load(b);
373 v128_t vacc4567 = wasm_v128_load(b + 4);
374 b += 8;
375
376 const v128_t vi0x01234567 = wasm_i16x8_load8x8(i0);
377 v128_t vk0x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 0 * sizeof(int8_t)));
378 i0 += 8;
379
380 v128_t vprod01234567 = wasm_i16x8_mul(vi0x01234567, vk0x01234567);
381
382 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
383 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
384
385 const v128_t vi1x01234567 = wasm_i16x8_load8x8(i1);
386 v128_t vk1x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int8_t)));
387 i1 += 8;
388
389 vprod01234567 = wasm_i16x8_mul(vi1x01234567, vk1x01234567);
390
391 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
392 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
393
394 const v128_t vi2x01234567 = wasm_i16x8_load8x8(i2);
395 v128_t vk2x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int8_t)));
396 i2 += 8;
397
398 vprod01234567 = wasm_i16x8_mul(vi2x01234567, vk2x01234567);
399
400 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
401 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
402
403 const v128_t vi3x01234567 = wasm_i16x8_load8x8(i3);
404 v128_t vk3x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 24 * sizeof(int8_t)));
405 i3 += 8;
406
407 vprod01234567 = wasm_i16x8_mul(vi3x01234567, vk3x01234567);
408
409 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
410 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
411
412 const v128_t vi4x01234567 = wasm_i16x8_load8x8(i4);
413 v128_t vk4x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 32 * sizeof(int8_t)));
414 i4 += 8;
415
416 vprod01234567 = wasm_i16x8_mul(vi4x01234567, vk4x01234567);
417
418 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
419 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
420
421 const v128_t vi5x01234567 = wasm_i16x8_load8x8(i5);
422 v128_t vk5x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 40 * sizeof(int8_t)));
423 i5 += 8;
424
425 vprod01234567 = wasm_i16x8_mul(vi5x01234567, vk5x01234567);
426
427 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
428 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
429
430 const v128_t vi6x01234567 = wasm_i16x8_load8x8(i6);
431 v128_t vk6x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 48 * sizeof(int8_t)));
432 i6 += 8;
433
434 vprod01234567 = wasm_i16x8_mul(vi6x01234567, vk6x01234567);
435
436 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
437 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
438
439 const v128_t vi7x01234567 = wasm_i16x8_load8x8(i7);
440 v128_t vk7x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 56 * sizeof(int8_t)));
441 i7 += 8;
442
443 vprod01234567 = wasm_i16x8_mul(vi7x01234567, vk7x01234567);
444
445 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
446 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
447
448 const v128_t vi8x01234567 = wasm_i16x8_load8x8(i8);
449 v128_t vk8x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 64 * sizeof(int8_t)));
450 i8 += 8;
451
452 vprod01234567 = wasm_i16x8_mul(vi8x01234567, vk8x01234567);
453
454 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
455 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
456
457
458 w = (const void*) ((uintptr_t) w + 72 * sizeof(int8_t));
459
460 vacc0123 = wasm_f32x4_convert_i32x4(vacc0123);
461 vacc4567 = wasm_f32x4_convert_i32x4(vacc4567);
462
463 vacc0123 = wasm_f32x4_mul(vacc0123, vscale);
464 vacc4567 = wasm_f32x4_mul(vacc4567, vscale);
465
466 vacc0123 = wasm_f32x4_add(vacc0123, vmagic_bias);
467 vacc4567 = wasm_f32x4_add(vacc4567, vmagic_bias);
468
469 vacc0123 = wasm_i32x4_max(vacc0123, vmagic_min);
470 vacc4567 = wasm_i32x4_max(vacc4567, vmagic_min);
471
472 vacc0123 = wasm_i32x4_sub(vacc0123, vmagic_bias_less_output_zero_point);
473 vacc4567 = wasm_i32x4_sub(vacc4567, vmagic_bias_less_output_zero_point);
474
475 v128_t vout01234567 = wasm_i16x8_narrow_i32x4(vacc0123, vacc4567);
476
477 v128_t vout0123456701234567 = wasm_i8x16_narrow_i16x8(vout01234567, vout01234567);
478
479 vout0123456701234567 = wasm_i8x16_min(vout0123456701234567, voutput_max);
480
481 wasm_v128_store64_lane(output, vout0123456701234567, 0);
482 output += 8;
483 }
484
485 if XNN_UNLIKELY(c != 0) {
486 {
487 v128_t vacc0123 = wasm_v128_load(b);
488 v128_t vacc4567 = wasm_v128_load(b + 4);
489
490 const v128_t vi0x01234567 = wasm_i16x8_load8x8(i0);
491 v128_t vk0x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w));
492
493 v128_t vprod01234567 = wasm_i16x8_mul(vi0x01234567, vk0x01234567);
494
495 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
496 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
497
498 const v128_t vi1x01234567 = wasm_i16x8_load8x8(i1);
499 v128_t vk1x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int8_t)));
500
501 vprod01234567 = wasm_i16x8_mul(vi1x01234567, vk1x01234567);
502
503 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
504 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
505
506 const v128_t vi2x01234567 = wasm_i16x8_load8x8(i2);
507 v128_t vk2x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int8_t)));
508
509 vprod01234567 = wasm_i16x8_mul(vi2x01234567, vk2x01234567);
510
511 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
512 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
513
514 const v128_t vi3x01234567 = wasm_i16x8_load8x8(i3);
515 v128_t vk3x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 24 * sizeof(int8_t)));
516
517 vprod01234567 = wasm_i16x8_mul(vi3x01234567, vk3x01234567);
518
519 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
520 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
521
522 const v128_t vi4x01234567 = wasm_i16x8_load8x8(i4);
523 v128_t vk4x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 32 * sizeof(int8_t)));
524
525 vprod01234567 = wasm_i16x8_mul(vi4x01234567, vk4x01234567);
526
527 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
528 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
529
530 const v128_t vi5x01234567 = wasm_i16x8_load8x8(i5);
531 v128_t vk5x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 40 * sizeof(int8_t)));
532
533 vprod01234567 = wasm_i16x8_mul(vi5x01234567, vk5x01234567);
534
535 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
536 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
537
538 const v128_t vi6x01234567 = wasm_i16x8_load8x8(i6);
539 v128_t vk6x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 48 * sizeof(int8_t)));
540
541 vprod01234567 = wasm_i16x8_mul(vi6x01234567, vk6x01234567);
542
543 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
544 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
545
546 const v128_t vi7x01234567 = wasm_i16x8_load8x8(i7);
547 v128_t vk7x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 56 * sizeof(int8_t)));
548
549 vprod01234567 = wasm_i16x8_mul(vi7x01234567, vk7x01234567);
550
551 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
552 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
553
554 const v128_t vi8x01234567 = wasm_i16x8_load8x8(i8);
555 v128_t vk8x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 64 * sizeof(int8_t)));
556
557 vprod01234567 = wasm_i16x8_mul(vi8x01234567, vk8x01234567);
558
559 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
560 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
561
562
563 vacc0123 = wasm_f32x4_convert_i32x4(vacc0123);
564 vacc4567 = wasm_f32x4_convert_i32x4(vacc4567);
565
566 vacc0123 = wasm_f32x4_mul(vacc0123, vscale);
567 vacc4567 = wasm_f32x4_mul(vacc4567, vscale);
568
569
570 vacc0123 = wasm_f32x4_add(vacc0123, vmagic_bias);
571 vacc4567 = wasm_f32x4_add(vacc4567, vmagic_bias);
572
573 vacc0123 = wasm_i32x4_max(vacc0123, vmagic_min);
574 vacc4567 = wasm_i32x4_max(vacc4567, vmagic_min);
575
576 vacc0123 = wasm_i32x4_sub(vacc0123, vmagic_bias_less_output_zero_point);
577 vacc4567 = wasm_i32x4_sub(vacc4567, vmagic_bias_less_output_zero_point);
578
579 v128_t vout01234567 = wasm_i16x8_narrow_i32x4(vacc0123, vacc4567);
580 v128_t vout0123456701234567 = wasm_i8x16_narrow_i16x8(vout01234567, vout01234567);
581
582 vout0123456701234567 = wasm_i8x16_min(vout0123456701234567, voutput_max);
583
584 if (c & 4) {
585 wasm_v128_store32_lane(output, vout0123456701234567, 0);
586 vout0123456701234567 = wasm_u64x2_shr(vout0123456701234567, 32);
587 output += 4;
588 }
589 if (c & 2) {
590 wasm_v128_store16_lane(output, vout0123456701234567, 0);
591 vout0123456701234567 = wasm_u32x4_shr(vout0123456701234567, 16);
592 output += 2;
593 }
594 if (c & 1) {
595 wasm_v128_store8_lane(output, vout0123456701234567, 0);
596 output += 1;
597 }
598 }
599 }
600 }
601
602 input = (const int8_t**) ((uintptr_t) input + input_stride);
603 output = (int8_t*) ((uintptr_t) output + output_increment);
604 } while (--output_width != 0);
605 }