Debian Codegraph @

src/qs8-dwconv/gen/qs8-dwconv-6f6m7l16c8s8r-minmax-fp32-wasmsimd-mul16-add16.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-dwconv/multipass-wasmsimd-mul16.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include <wasm_simd128.h>
13
14 #include "xnnpack/dwconv.h"
15 #include "xnnpack/math.h"
16
17
18 void xnn_qs8_dwconv_minmax_fp32_ukernel_6f6m7l16c8s8r__wasmsimd_mul16_add16(
19 size_t channels,
20 size_t output_width,
21 const int8_t** input,
22 const void* weights,
23 int8_t* output,
24 intptr_t input_stride,
25 size_t output_increment,
26 size_t input_offset,
27 const int8_t* zero,
28 size_t kernel_size,
29 int32_t* buffer,
30 const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
31 {
32 assert(channels != 0);
33 assert(output_width != 0);
34 assert(kernel_size > 6);
35
36
37 const v128_t vscale = wasm_v128_load32_splat(&params->fp32_scalar.scale);
38 XNN_FORCE_REALIZATION(vscale);
39
40 const v128_t vmagic_bias = wasm_f32x4_const_splat(12582912.0f);
41 const int32_t output_min_less_zero_point = (int32_t) params->fp32_scalar.output_min - (int32_t) params->fp32_scalar.output_zero_point;
42 const v128_t vmagic_min = wasm_i32x4_splat((int32_t) float_as_uint32(12582912.0f + output_min_less_zero_point));
43 const v128_t vmagic_bias_less_output_zero_point = wasm_i32x4_splat(INT32_C(0x4B400000) - (int32_t) params->fp32_scalar.output_zero_point);
44 const v128_t voutput_max = wasm_i8x16_splat(params->fp32_scalar.output_max);
45 XNN_FORCE_REALIZATION(vmagic_bias);
46 XNN_FORCE_REALIZATION(vmagic_min);
47 XNN_FORCE_REALIZATION(vmagic_bias_less_output_zero_point);
48 XNN_FORCE_REALIZATION(voutput_max);
49
50 do {
51 const void* w = weights;
52
53 // First pass to process 6 inputs.
54 {
55 int32_t* b = buffer;
56 const int8_t* i0 = input[0];
57 assert(i0 != NULL);
58 if XNN_UNPREDICTABLE(i0 != zero) {
59 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
60 }
61 const int8_t* i1 = input[1];
62 assert(i1 != NULL);
63 if XNN_UNPREDICTABLE(i1 != zero) {
64 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
65 }
66 const int8_t* i2 = input[2];
67 assert(i2 != NULL);
68 if XNN_UNPREDICTABLE(i2 != zero) {
69 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
70 }
71 const int8_t* i3 = input[3];
72 assert(i3 != NULL);
73 if XNN_UNPREDICTABLE(i3 != zero) {
74 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
75 }
76 const int8_t* i4 = input[4];
77 assert(i4 != NULL);
78 if XNN_UNPREDICTABLE(i4 != zero) {
79 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
80 }
81 const int8_t* i5 = input[5];
82 assert(i5 != NULL);
83 if XNN_UNPREDICTABLE(i5 != zero) {
84 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
85 }
86 input += 6;
87
88 size_t c = round_up_po2(channels, 8);
89
90 for (; c >= 16; c -= 16) {
91 v128_t vacc0123 = wasm_v128_load(w);
92 v128_t vacc4567 = wasm_v128_load((const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
93 v128_t vacc89AB = wasm_v128_load((const void*) ((uintptr_t) w + 8 * sizeof(int32_t)));
94 v128_t vaccCDEF = wasm_v128_load((const void*) ((uintptr_t) w + 12 * sizeof(int32_t)));
95
96 const v128_t vi0x01234567 = wasm_i16x8_load8x8(i0);
97 v128_t vk0x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 0 * sizeof(int8_t)));
98 const v128_t vi0x89ABCDEF = wasm_i16x8_load8x8(i0 + 8);
99 v128_t vk0x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 8 * sizeof(int8_t)));
100 i0 += 16;
101
102 v128_t vprod01234567 = wasm_i16x8_mul(vi0x01234567, vk0x01234567);
103 v128_t vprod89ABCDEF = wasm_i16x8_mul(vi0x89ABCDEF, vk0x89ABCDEF);
104
105
106 const v128_t vi1x01234567 = wasm_i16x8_load8x8(i1);
107 v128_t vk1x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 16 * sizeof(int8_t)));
108 const v128_t vi1x89ABCDEF = wasm_i16x8_load8x8(i1 + 8);
109 v128_t vk1x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 24 * sizeof(int8_t)));
110 i1 += 16;
111
112 vprod01234567 = wasm_i16x8_add(vprod01234567, wasm_i16x8_mul(vi1x01234567, vk1x01234567));
113 vprod89ABCDEF = wasm_i16x8_add(vprod89ABCDEF, wasm_i16x8_mul(vi1x89ABCDEF, vk1x89ABCDEF));
114
115 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
116 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
117 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
118 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
119
120 const v128_t vi2x01234567 = wasm_i16x8_load8x8(i2);
121 v128_t vk2x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 32 * sizeof(int8_t)));
122 const v128_t vi2x89ABCDEF = wasm_i16x8_load8x8(i2 + 8);
123 v128_t vk2x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 40 * sizeof(int8_t)));
124 i2 += 16;
125
126 vprod01234567 = wasm_i16x8_mul(vi2x01234567, vk2x01234567);
127 vprod89ABCDEF = wasm_i16x8_mul(vi2x89ABCDEF, vk2x89ABCDEF);
128
129
130 const v128_t vi3x01234567 = wasm_i16x8_load8x8(i3);
131 v128_t vk3x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 48 * sizeof(int8_t)));
132 const v128_t vi3x89ABCDEF = wasm_i16x8_load8x8(i3 + 8);
133 v128_t vk3x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 56 * sizeof(int8_t)));
134 i3 += 16;
135
136 vprod01234567 = wasm_i16x8_add(vprod01234567, wasm_i16x8_mul(vi3x01234567, vk3x01234567));
137 vprod89ABCDEF = wasm_i16x8_add(vprod89ABCDEF, wasm_i16x8_mul(vi3x89ABCDEF, vk3x89ABCDEF));
138
139 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
140 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
141 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
142 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
143
144 const v128_t vi4x01234567 = wasm_i16x8_load8x8(i4);
145 v128_t vk4x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 64 * sizeof(int8_t)));
146 const v128_t vi4x89ABCDEF = wasm_i16x8_load8x8(i4 + 8);
147 v128_t vk4x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 72 * sizeof(int8_t)));
148 i4 += 16;
149
150 vprod01234567 = wasm_i16x8_mul(vi4x01234567, vk4x01234567);
151 vprod89ABCDEF = wasm_i16x8_mul(vi4x89ABCDEF, vk4x89ABCDEF);
152
153
154 const v128_t vi5x01234567 = wasm_i16x8_load8x8(i5);
155 v128_t vk5x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 80 * sizeof(int8_t)));
156 const v128_t vi5x89ABCDEF = wasm_i16x8_load8x8(i5 + 8);
157 v128_t vk5x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 88 * sizeof(int8_t)));
158 i5 += 16;
159
160 vprod01234567 = wasm_i16x8_add(vprod01234567, wasm_i16x8_mul(vi5x01234567, vk5x01234567));
161 vprod89ABCDEF = wasm_i16x8_add(vprod89ABCDEF, wasm_i16x8_mul(vi5x89ABCDEF, vk5x89ABCDEF));
162
163 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
164 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
165 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
166 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
167
168 w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 96 * sizeof(int8_t));
169
170 wasm_v128_store(b, vacc0123);
171 wasm_v128_store(b + 4, vacc4567);
172 wasm_v128_store(b + 8, vacc89AB);
173 wasm_v128_store(b + 12, vaccCDEF);
174 b += 16;
175 }
176
177 if XNN_UNLIKELY(c != 0) {
178 do {
179 v128_t vacc0123 = wasm_v128_load(w);
180 v128_t vacc4567 = wasm_v128_load((const int32_t*) w + 4);
181
182 const v128_t vi0x01234567 = wasm_i16x8_load8x8(i0);
183 v128_t vk0x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int32_t)));
184 i0 += 8;
185
186 v128_t vprod01234567 = wasm_i16x8_mul(vi0x01234567, vk0x01234567);
187
188
189 const v128_t vi1x01234567 = wasm_i16x8_load8x8(i1);
190 v128_t vk1x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 8 * sizeof(int8_t)));
191 i1 += 8;
192
193 vprod01234567 = wasm_i16x8_add(vprod01234567, wasm_i16x8_mul(vi1x01234567, vk1x01234567));
194
195 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
196 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
197
198 const v128_t vi2x01234567 = wasm_i16x8_load8x8(i2);
199 v128_t vk2x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 16 * sizeof(int8_t)));
200 i2 += 8;
201
202 vprod01234567 = wasm_i16x8_mul(vi2x01234567, vk2x01234567);
203
204
205 const v128_t vi3x01234567 = wasm_i16x8_load8x8(i3);
206 v128_t vk3x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 24 * sizeof(int8_t)));
207 i3 += 8;
208
209 vprod01234567 = wasm_i16x8_add(vprod01234567, wasm_i16x8_mul(vi3x01234567, vk3x01234567));
210
211 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
212 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
213
214 const v128_t vi4x01234567 = wasm_i16x8_load8x8(i4);
215 v128_t vk4x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 32 * sizeof(int8_t)));
216 i4 += 8;
217
218 vprod01234567 = wasm_i16x8_mul(vi4x01234567, vk4x01234567);
219
220
221 const v128_t vi5x01234567 = wasm_i16x8_load8x8(i5);
222 v128_t vk5x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 40 * sizeof(int8_t)));
223 i5 += 8;
224
225 vprod01234567 = wasm_i16x8_add(vprod01234567, wasm_i16x8_mul(vi5x01234567, vk5x01234567));
226
227 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
228 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
229
230
231 w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 48 * sizeof(int8_t));
232
233 wasm_v128_store(b, vacc0123);
234 wasm_v128_store(b + 4, vacc4567);
235 b += 8;
236 c -= 8;
237 } while (c != 0);
238 }
239 }
240
241
242 // Middle pass to process 6 inputs in each iteration.
243 for (size_t ks = kernel_size - 6; ks > 7; ks -= 6) {
244 int32_t* b = buffer;
245 const int8_t* i0 = input[0];
246 assert(i0 != NULL);
247 if XNN_UNPREDICTABLE(i0 != zero) {
248 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
249 }
250 const int8_t* i1 = input[1];
251 assert(i1 != NULL);
252 if XNN_UNPREDICTABLE(i1 != zero) {
253 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
254 }
255 const int8_t* i2 = input[2];
256 assert(i2 != NULL);
257 if XNN_UNPREDICTABLE(i2 != zero) {
258 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
259 }
260 const int8_t* i3 = input[3];
261 assert(i3 != NULL);
262 if XNN_UNPREDICTABLE(i3 != zero) {
263 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
264 }
265 const int8_t* i4 = input[4];
266 assert(i4 != NULL);
267 if XNN_UNPREDICTABLE(i4 != zero) {
268 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
269 }
270 const int8_t* i5 = input[5];
271 assert(i5 != NULL);
272 if XNN_UNPREDICTABLE(i5 != zero) {
273 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
274 }
275 input += 6;
276
277 size_t c = round_up_po2(channels, 8);
278
279 for (; c >= 16; c -= 16) {
280 v128_t vacc0123 = wasm_v128_load(b);
281 v128_t vacc4567 = wasm_v128_load(b + 4);
282 v128_t vacc89AB = wasm_v128_load(b + 8);
283 v128_t vaccCDEF = wasm_v128_load(b + 12);
284
285 const v128_t vi0x01234567 = wasm_i16x8_load8x8(i0);
286 v128_t vk0x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 0 * sizeof(int8_t)));
287 const v128_t vi0x89ABCDEF = wasm_i16x8_load8x8(i0 + 8);
288 v128_t vk0x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int8_t)));
289 i0 += 16;
290
291 v128_t vprod01234567 = wasm_i16x8_mul(vi0x01234567, vk0x01234567);
292 v128_t vprod89ABCDEF = wasm_i16x8_mul(vi0x89ABCDEF, vk0x89ABCDEF);
293
294
295 const v128_t vi1x01234567 = wasm_i16x8_load8x8(i1);
296 v128_t vk1x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int8_t)));
297 const v128_t vi1x89ABCDEF = wasm_i16x8_load8x8(i1 + 8);
298 v128_t vk1x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 24 * sizeof(int8_t)));
299 i1 += 16;
300
301 vprod01234567 = wasm_i16x8_add(vprod01234567, wasm_i16x8_mul(vi1x01234567, vk1x01234567));
302 vprod89ABCDEF = wasm_i16x8_add(vprod89ABCDEF, wasm_i16x8_mul(vi1x89ABCDEF, vk1x89ABCDEF));
303
304 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
305 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
306 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
307 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
308
309 const v128_t vi2x01234567 = wasm_i16x8_load8x8(i2);
310 v128_t vk2x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 32 * sizeof(int8_t)));
311 const v128_t vi2x89ABCDEF = wasm_i16x8_load8x8(i2 + 8);
312 v128_t vk2x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 40 * sizeof(int8_t)));
313 i2 += 16;
314
315 vprod01234567 = wasm_i16x8_mul(vi2x01234567, vk2x01234567);
316 vprod89ABCDEF = wasm_i16x8_mul(vi2x89ABCDEF, vk2x89ABCDEF);
317
318
319 const v128_t vi3x01234567 = wasm_i16x8_load8x8(i3);
320 v128_t vk3x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 48 * sizeof(int8_t)));
321 const v128_t vi3x89ABCDEF = wasm_i16x8_load8x8(i3 + 8);
322 v128_t vk3x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 56 * sizeof(int8_t)));
323 i3 += 16;
324
325 vprod01234567 = wasm_i16x8_add(vprod01234567, wasm_i16x8_mul(vi3x01234567, vk3x01234567));
326 vprod89ABCDEF = wasm_i16x8_add(vprod89ABCDEF, wasm_i16x8_mul(vi3x89ABCDEF, vk3x89ABCDEF));
327
328 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
329 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
330 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
331 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
332
333 const v128_t vi4x01234567 = wasm_i16x8_load8x8(i4);
334 v128_t vk4x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 64 * sizeof(int8_t)));
335 const v128_t vi4x89ABCDEF = wasm_i16x8_load8x8(i4 + 8);
336 v128_t vk4x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 72 * sizeof(int8_t)));
337 i4 += 16;
338
339 vprod01234567 = wasm_i16x8_mul(vi4x01234567, vk4x01234567);
340 vprod89ABCDEF = wasm_i16x8_mul(vi4x89ABCDEF, vk4x89ABCDEF);
341
342
343 const v128_t vi5x01234567 = wasm_i16x8_load8x8(i5);
344 v128_t vk5x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 80 * sizeof(int8_t)));
345 const v128_t vi5x89ABCDEF = wasm_i16x8_load8x8(i5 + 8);
346 v128_t vk5x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 88 * sizeof(int8_t)));
347 i5 += 16;
348
349 vprod01234567 = wasm_i16x8_add(vprod01234567, wasm_i16x8_mul(vi5x01234567, vk5x01234567));
350 vprod89ABCDEF = wasm_i16x8_add(vprod89ABCDEF, wasm_i16x8_mul(vi5x89ABCDEF, vk5x89ABCDEF));
351
352 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
353 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
354 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
355 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
356
357 w = (const void*) ((uintptr_t) w + 96 * sizeof(int8_t));
358
359 wasm_v128_store(b, vacc0123);
360 wasm_v128_store(b + 4, vacc4567);
361 wasm_v128_store(b + 8, vacc89AB);
362 wasm_v128_store(b + 12, vaccCDEF);
363 b += 16;
364 }
365
366 if XNN_UNLIKELY(c != 0) {
367 do {
368 v128_t vacc0123 = wasm_v128_load(b);
369 v128_t vacc4567 = wasm_v128_load(b + 4);
370
371 const v128_t vi0x01234567 = wasm_i16x8_load8x8(i0);
372 v128_t vk0x01234567 = wasm_i16x8_load8x8(w);
373 i0 += 8;
374
375 v128_t vprod01234567 = wasm_i16x8_mul(vi0x01234567, vk0x01234567);
376
377
378 const v128_t vi1x01234567 = wasm_i16x8_load8x8(i1);
379 v128_t vk1x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int8_t)));
380 i1 += 8;
381
382 vprod01234567 = wasm_i16x8_add(vprod01234567, wasm_i16x8_mul(vi1x01234567, vk1x01234567));
383
384 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
385 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
386
387 const v128_t vi2x01234567 = wasm_i16x8_load8x8(i2);
388 v128_t vk2x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int8_t)));
389 i2 += 8;
390
391 vprod01234567 = wasm_i16x8_mul(vi2x01234567, vk2x01234567);
392
393
394 const v128_t vi3x01234567 = wasm_i16x8_load8x8(i3);
395 v128_t vk3x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 24 * sizeof(int8_t)));
396 i3 += 8;
397
398 vprod01234567 = wasm_i16x8_add(vprod01234567, wasm_i16x8_mul(vi3x01234567, vk3x01234567));
399
400 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
401 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
402
403 const v128_t vi4x01234567 = wasm_i16x8_load8x8(i4);
404 v128_t vk4x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 32 * sizeof(int8_t)));
405 i4 += 8;
406
407 vprod01234567 = wasm_i16x8_mul(vi4x01234567, vk4x01234567);
408
409
410 const v128_t vi5x01234567 = wasm_i16x8_load8x8(i5);
411 v128_t vk5x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 40 * sizeof(int8_t)));
412 i5 += 8;
413
414 vprod01234567 = wasm_i16x8_add(vprod01234567, wasm_i16x8_mul(vi5x01234567, vk5x01234567));
415
416 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
417 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
418
419
420 w = (const void*) ((uintptr_t) w + 48 * sizeof(int8_t));
421
422 wasm_v128_store(b, vacc0123);
423 wasm_v128_store(b + 4, vacc4567);
424 b += 8;
425 c -= 8;
426 } while (c != 0);
427 }
428 }
429
430 // Last pass to process up to 7 inputs.
431 {
432 const int32_t* b = buffer;
433 const int8_t* i0 = input[0];
434 assert(i0 != NULL);
435 if XNN_UNPREDICTABLE(i0 != zero) {
436 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
437 }
438 const int8_t* i1 = input[1];
439 assert(i1 != NULL);
440 if XNN_UNPREDICTABLE(i1 != zero) {
441 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
442 }
443 const int8_t* i2 = input[2];
444 assert(i2 != NULL);
445 if XNN_UNPREDICTABLE(i2 != zero) {
446 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
447 }
448 const int8_t* i3 = input[3];
449 assert(i3 != NULL);
450 if XNN_UNPREDICTABLE(i3 != zero) {
451 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
452 }
453 const int8_t* i4 = input[4];
454 assert(i4 != NULL);
455 if XNN_UNPREDICTABLE(i4 != zero) {
456 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
457 }
458 const int8_t* i5 = input[5];
459 assert(i5 != NULL);
460 if XNN_UNPREDICTABLE(i5 != zero) {
461 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
462 }
463 const int8_t* i6 = input[6];
464 assert(i6 != NULL);
465 if XNN_UNPREDICTABLE(i6 != zero) {
466 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
467 }
468
469 size_t c = channels;
470
471 for (; c >= 16; c -= 16) {
472 v128_t vacc0123 = wasm_v128_load(b);
473 v128_t vacc4567 = wasm_v128_load(b + 4);
474 v128_t vacc89AB = wasm_v128_load(b + 8);
475 v128_t vaccCDEF = wasm_v128_load(b + 12);
476 b += 16;
477
478 const v128_t vi0x01234567 = wasm_i16x8_load8x8(i0);
479 v128_t vk0x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 0 * sizeof(int8_t)));
480 const v128_t vi0x89ABCDEF = wasm_i16x8_load8x8(i0 + 8);
481 v128_t vk0x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int8_t)));
482 i0 += 16;
483
484 v128_t vprod01234567 = wasm_i16x8_mul(vi0x01234567, vk0x01234567);
485 v128_t vprod89ABCDEF = wasm_i16x8_mul(vi0x89ABCDEF, vk0x89ABCDEF);
486
487
488 const v128_t vi1x01234567 = wasm_i16x8_load8x8(i1);
489 v128_t vk1x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int8_t)));
490 const v128_t vi1x89ABCDEF = wasm_i16x8_load8x8(i1 + 8);
491 v128_t vk1x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 24 * sizeof(int8_t)));
492 i1 += 16;
493
494 vprod01234567 = wasm_i16x8_add(vprod01234567, wasm_i16x8_mul(vi1x01234567, vk1x01234567));
495 vprod89ABCDEF = wasm_i16x8_add(vprod89ABCDEF, wasm_i16x8_mul(vi1x89ABCDEF, vk1x89ABCDEF));
496
497 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
498 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
499 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
500 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
501
502 const v128_t vi2x01234567 = wasm_i16x8_load8x8(i2);
503 v128_t vk2x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 32 * sizeof(int8_t)));
504 const v128_t vi2x89ABCDEF = wasm_i16x8_load8x8(i2 + 8);
505 v128_t vk2x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 40 * sizeof(int8_t)));
506 i2 += 16;
507
508 vprod01234567 = wasm_i16x8_mul(vi2x01234567, vk2x01234567);
509 vprod89ABCDEF = wasm_i16x8_mul(vi2x89ABCDEF, vk2x89ABCDEF);
510
511
512 const v128_t vi3x01234567 = wasm_i16x8_load8x8(i3);
513 v128_t vk3x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 48 * sizeof(int8_t)));
514 const v128_t vi3x89ABCDEF = wasm_i16x8_load8x8(i3 + 8);
515 v128_t vk3x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 56 * sizeof(int8_t)));
516 i3 += 16;
517
518 vprod01234567 = wasm_i16x8_add(vprod01234567, wasm_i16x8_mul(vi3x01234567, vk3x01234567));
519 vprod89ABCDEF = wasm_i16x8_add(vprod89ABCDEF, wasm_i16x8_mul(vi3x89ABCDEF, vk3x89ABCDEF));
520
521 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
522 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
523 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
524 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
525
526 const v128_t vi4x01234567 = wasm_i16x8_load8x8(i4);
527 v128_t vk4x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 64 * sizeof(int8_t)));
528 const v128_t vi4x89ABCDEF = wasm_i16x8_load8x8(i4 + 8);
529 v128_t vk4x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 72 * sizeof(int8_t)));
530 i4 += 16;
531
532 vprod01234567 = wasm_i16x8_mul(vi4x01234567, vk4x01234567);
533 vprod89ABCDEF = wasm_i16x8_mul(vi4x89ABCDEF, vk4x89ABCDEF);
534
535
536 const v128_t vi5x01234567 = wasm_i16x8_load8x8(i5);
537 v128_t vk5x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 80 * sizeof(int8_t)));
538 const v128_t vi5x89ABCDEF = wasm_i16x8_load8x8(i5 + 8);
539 v128_t vk5x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 88 * sizeof(int8_t)));
540 i5 += 16;
541
542 vprod01234567 = wasm_i16x8_add(vprod01234567, wasm_i16x8_mul(vi5x01234567, vk5x01234567));
543 vprod89ABCDEF = wasm_i16x8_add(vprod89ABCDEF, wasm_i16x8_mul(vi5x89ABCDEF, vk5x89ABCDEF));
544
545 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
546 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
547 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
548 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
549
550 const v128_t vi6x01234567 = wasm_i16x8_load8x8(i6);
551 v128_t vk6x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 96 * sizeof(int8_t)));
552 const v128_t vi6x89ABCDEF = wasm_i16x8_load8x8(i6 + 8);
553 v128_t vk6x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 104 * sizeof(int8_t)));
554 i6 += 16;
555
556 vprod01234567 = wasm_i16x8_mul(vi6x01234567, vk6x01234567);
557 vprod89ABCDEF = wasm_i16x8_mul(vi6x89ABCDEF, vk6x89ABCDEF);
558
559 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
560 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
561 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
562 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
563
564
565 w = (const void*) ((uintptr_t) w + 112 * sizeof(int8_t));
566
567 vacc0123 = wasm_f32x4_convert_i32x4(vacc0123);
568 vacc4567 = wasm_f32x4_convert_i32x4(vacc4567);
569 vacc89AB = wasm_f32x4_convert_i32x4(vacc89AB);
570 vaccCDEF = wasm_f32x4_convert_i32x4(vaccCDEF);
571
572 vacc0123 = wasm_f32x4_mul(vacc0123, vscale);
573 vacc4567 = wasm_f32x4_mul(vacc4567, vscale);
574 vacc89AB = wasm_f32x4_mul(vacc89AB, vscale);
575 vaccCDEF = wasm_f32x4_mul(vaccCDEF, vscale);
576
577 vacc0123 = wasm_f32x4_add(vacc0123, vmagic_bias);
578 vacc4567 = wasm_f32x4_add(vacc4567, vmagic_bias);
579 vacc89AB = wasm_f32x4_add(vacc89AB, vmagic_bias);
580 vaccCDEF = wasm_f32x4_add(vaccCDEF, vmagic_bias);
581
582 vacc0123 = wasm_i32x4_max(vacc0123, vmagic_min);
583 vacc4567 = wasm_i32x4_max(vacc4567, vmagic_min);
584 vacc89AB = wasm_i32x4_max(vacc89AB, vmagic_min);
585 vaccCDEF = wasm_i32x4_max(vaccCDEF, vmagic_min);
586
587 vacc0123 = wasm_i32x4_sub(vacc0123, vmagic_bias_less_output_zero_point);
588 vacc4567 = wasm_i32x4_sub(vacc4567, vmagic_bias_less_output_zero_point);
589 vacc89AB = wasm_i32x4_sub(vacc89AB, vmagic_bias_less_output_zero_point);
590 vaccCDEF = wasm_i32x4_sub(vaccCDEF, vmagic_bias_less_output_zero_point);
591
592 v128_t vout01234567 = wasm_i16x8_narrow_i32x4(vacc0123, vacc4567);
593 v128_t vout89ABCDEF = wasm_i16x8_narrow_i32x4(vacc89AB, vaccCDEF);
594
595 v128_t vout0123456789ABCDEF = wasm_i8x16_narrow_i16x8(vout01234567, vout89ABCDEF);
596
597 vout0123456789ABCDEF = wasm_i8x16_min(vout0123456789ABCDEF, voutput_max);
598
599 wasm_v128_store(output, vout0123456789ABCDEF);
600 output += 16;
601 }
602
603 if XNN_UNLIKELY(c != 0) {
604 do {
605 v128_t vacc0123 = wasm_v128_load(b);
606 v128_t vacc4567 = wasm_v128_load(b + 4);
607 b += 8;
608
609 const v128_t vi0x01234567 = wasm_i16x8_load8x8(i0);
610 v128_t vk0x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w));
611 i0 += 8;
612
613 v128_t vprod01234567 = wasm_i16x8_mul(vi0x01234567, vk0x01234567);
614
615
616 const v128_t vi1x01234567 = wasm_i16x8_load8x8(i1);
617 v128_t vk1x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int8_t)));
618 i1 += 8;
619
620 vprod01234567 = wasm_i16x8_add(vprod01234567, wasm_i16x8_mul(vi1x01234567, vk1x01234567));
621
622 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
623 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
624
625 const v128_t vi2x01234567 = wasm_i16x8_load8x8(i2);
626 v128_t vk2x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int8_t)));
627 i2 += 8;
628
629 vprod01234567 = wasm_i16x8_mul(vi2x01234567, vk2x01234567);
630
631
632 const v128_t vi3x01234567 = wasm_i16x8_load8x8(i3);
633 v128_t vk3x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 24 * sizeof(int8_t)));
634 i3 += 8;
635
636 vprod01234567 = wasm_i16x8_add(vprod01234567, wasm_i16x8_mul(vi3x01234567, vk3x01234567));
637
638 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
639 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
640
641 const v128_t vi4x01234567 = wasm_i16x8_load8x8(i4);
642 v128_t vk4x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 32 * sizeof(int8_t)));
643 i4 += 8;
644
645 vprod01234567 = wasm_i16x8_mul(vi4x01234567, vk4x01234567);
646
647
648 const v128_t vi5x01234567 = wasm_i16x8_load8x8(i5);
649 v128_t vk5x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 40 * sizeof(int8_t)));
650 i5 += 8;
651
652 vprod01234567 = wasm_i16x8_add(vprod01234567, wasm_i16x8_mul(vi5x01234567, vk5x01234567));
653
654 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
655 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
656
657 const v128_t vi6x01234567 = wasm_i16x8_load8x8(i6);
658 v128_t vk6x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 48 * sizeof(int8_t)));
659 i6 += 8;
660
661 vprod01234567 = wasm_i16x8_mul(vi6x01234567, vk6x01234567);
662
663 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
664 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
665
666
667 vacc0123 = wasm_f32x4_convert_i32x4(vacc0123);
668 vacc4567 = wasm_f32x4_convert_i32x4(vacc4567);
669
670 vacc0123 = wasm_f32x4_mul(vacc0123, vscale);
671 vacc4567 = wasm_f32x4_mul(vacc4567, vscale);
672
673 w = (void*) ((uintptr_t) w + 56);
674
675 vacc0123 = wasm_f32x4_add(vacc0123, vmagic_bias);
676 vacc4567 = wasm_f32x4_add(vacc4567, vmagic_bias);
677
678 vacc0123 = wasm_i32x4_max(vacc0123, vmagic_min);
679 vacc4567 = wasm_i32x4_max(vacc4567, vmagic_min);
680
681 vacc0123 = wasm_i32x4_sub(vacc0123, vmagic_bias_less_output_zero_point);
682 vacc4567 = wasm_i32x4_sub(vacc4567, vmagic_bias_less_output_zero_point);
683
684 v128_t vout01234567 = wasm_i16x8_narrow_i32x4(vacc0123, vacc4567);
685 v128_t vout0123456701234567 = wasm_i8x16_narrow_i16x8(vout01234567, vout01234567);
686
687 vout0123456701234567 = wasm_i8x16_min(vout0123456701234567, voutput_max);
688
689 if XNN_LIKELY(c >= 8) {
690 wasm_v128_store64_lane(output, vout0123456701234567, 0);
691 output += 8;
692 c -= 8;
693 } else {
694 if (c & 4) {
695 wasm_v128_store32_lane(output, vout0123456701234567, 0);
696 vout0123456701234567 = wasm_u64x2_shr(vout0123456701234567, 32);
697 output += 4;
698 }
699 if (c & 2) {
700 wasm_v128_store16_lane(output, vout0123456701234567, 0);
701 vout0123456701234567 = wasm_u32x4_shr(vout0123456701234567, 16);
702 output += 2;
703 }
704 if (c & 1) {
705 wasm_v128_store8_lane(output, vout0123456701234567, 0);
706 output += 1;
707 }
708 c = 0;
709 }
710 } while (c != 0);
711 }
712 }
713
714 input = (const int8_t**) ((uintptr_t) input + input_stride);
715 output = (int8_t*) ((uintptr_t) output + output_increment);
716 } while (--output_width != 0);
717 }