Debian Codegraph @

src/qs8-dwconv/gen/qs8-dwconv-5f5m5l16c8s8r-minmax-fp32-wasmsimd-mul16.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-dwconv/multipass-wasmsimd-mul16.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include <wasm_simd128.h>
13
14 #include "xnnpack/dwconv.h"
15 #include "xnnpack/math.h"
16
17
18 void xnn_qs8_dwconv_minmax_fp32_ukernel_5f5m5l16c8s8r__wasmsimd_mul16(
19 size_t channels,
20 size_t output_width,
21 const int8_t** input,
22 const void* weights,
23 int8_t* output,
24 intptr_t input_stride,
25 size_t output_increment,
26 size_t input_offset,
27 const int8_t* zero,
28 size_t kernel_size,
29 int32_t* buffer,
30 const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
31 {
32 assert(channels != 0);
33 assert(output_width != 0);
34 assert(kernel_size > 5);
35
36
37 const v128_t vscale = wasm_v128_load32_splat(&params->fp32_scalar.scale);
38 XNN_FORCE_REALIZATION(vscale);
39
40 const v128_t vmagic_bias = wasm_f32x4_const_splat(12582912.0f);
41 const int32_t output_min_less_zero_point = (int32_t) params->fp32_scalar.output_min - (int32_t) params->fp32_scalar.output_zero_point;
42 const v128_t vmagic_min = wasm_i32x4_splat((int32_t) float_as_uint32(12582912.0f + output_min_less_zero_point));
43 const v128_t vmagic_bias_less_output_zero_point = wasm_i32x4_splat(INT32_C(0x4B400000) - (int32_t) params->fp32_scalar.output_zero_point);
44 const v128_t voutput_max = wasm_i8x16_splat(params->fp32_scalar.output_max);
45 XNN_FORCE_REALIZATION(vmagic_bias);
46 XNN_FORCE_REALIZATION(vmagic_min);
47 XNN_FORCE_REALIZATION(vmagic_bias_less_output_zero_point);
48 XNN_FORCE_REALIZATION(voutput_max);
49
50 do {
51 const void* w = weights;
52
53 // First pass to process 5 inputs.
54 {
55 int32_t* b = buffer;
56 const int8_t* i0 = input[0];
57 assert(i0 != NULL);
58 if XNN_UNPREDICTABLE(i0 != zero) {
59 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
60 }
61 const int8_t* i1 = input[1];
62 assert(i1 != NULL);
63 if XNN_UNPREDICTABLE(i1 != zero) {
64 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
65 }
66 const int8_t* i2 = input[2];
67 assert(i2 != NULL);
68 if XNN_UNPREDICTABLE(i2 != zero) {
69 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
70 }
71 const int8_t* i3 = input[3];
72 assert(i3 != NULL);
73 if XNN_UNPREDICTABLE(i3 != zero) {
74 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
75 }
76 const int8_t* i4 = input[4];
77 assert(i4 != NULL);
78 if XNN_UNPREDICTABLE(i4 != zero) {
79 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
80 }
81 input += 5;
82
83 size_t c = round_up_po2(channels, 8);
84
85 for (; c >= 16; c -= 16) {
86 v128_t vacc0123 = wasm_v128_load(w);
87 v128_t vacc4567 = wasm_v128_load((const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
88 v128_t vacc89AB = wasm_v128_load((const void*) ((uintptr_t) w + 8 * sizeof(int32_t)));
89 v128_t vaccCDEF = wasm_v128_load((const void*) ((uintptr_t) w + 12 * sizeof(int32_t)));
90
91 const v128_t vi0x01234567 = wasm_i16x8_load8x8(i0);
92 v128_t vk0x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 0 * sizeof(int8_t)));
93 const v128_t vi0x89ABCDEF = wasm_i16x8_load8x8(i0 + 8);
94 v128_t vk0x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 8 * sizeof(int8_t)));
95 i0 += 16;
96
97 v128_t vprod01234567 = wasm_i16x8_mul(vi0x01234567, vk0x01234567);
98 v128_t vprod89ABCDEF = wasm_i16x8_mul(vi0x89ABCDEF, vk0x89ABCDEF);
99
100 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
101 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
102 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
103 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
104
105 const v128_t vi1x01234567 = wasm_i16x8_load8x8(i1);
106 v128_t vk1x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 16 * sizeof(int8_t)));
107 const v128_t vi1x89ABCDEF = wasm_i16x8_load8x8(i1 + 8);
108 v128_t vk1x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 24 * sizeof(int8_t)));
109 i1 += 16;
110
111 vprod01234567 = wasm_i16x8_mul(vi1x01234567, vk1x01234567);
112 vprod89ABCDEF = wasm_i16x8_mul(vi1x89ABCDEF, vk1x89ABCDEF);
113
114 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
115 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
116 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
117 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
118
119 const v128_t vi2x01234567 = wasm_i16x8_load8x8(i2);
120 v128_t vk2x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 32 * sizeof(int8_t)));
121 const v128_t vi2x89ABCDEF = wasm_i16x8_load8x8(i2 + 8);
122 v128_t vk2x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 40 * sizeof(int8_t)));
123 i2 += 16;
124
125 vprod01234567 = wasm_i16x8_mul(vi2x01234567, vk2x01234567);
126 vprod89ABCDEF = wasm_i16x8_mul(vi2x89ABCDEF, vk2x89ABCDEF);
127
128 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
129 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
130 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
131 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
132
133 const v128_t vi3x01234567 = wasm_i16x8_load8x8(i3);
134 v128_t vk3x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 48 * sizeof(int8_t)));
135 const v128_t vi3x89ABCDEF = wasm_i16x8_load8x8(i3 + 8);
136 v128_t vk3x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 56 * sizeof(int8_t)));
137 i3 += 16;
138
139 vprod01234567 = wasm_i16x8_mul(vi3x01234567, vk3x01234567);
140 vprod89ABCDEF = wasm_i16x8_mul(vi3x89ABCDEF, vk3x89ABCDEF);
141
142 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
143 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
144 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
145 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
146
147 const v128_t vi4x01234567 = wasm_i16x8_load8x8(i4);
148 v128_t vk4x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 64 * sizeof(int8_t)));
149 const v128_t vi4x89ABCDEF = wasm_i16x8_load8x8(i4 + 8);
150 v128_t vk4x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 72 * sizeof(int8_t)));
151 i4 += 16;
152
153 vprod01234567 = wasm_i16x8_mul(vi4x01234567, vk4x01234567);
154 vprod89ABCDEF = wasm_i16x8_mul(vi4x89ABCDEF, vk4x89ABCDEF);
155
156 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
157 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
158 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
159 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
160
161 w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 80 * sizeof(int8_t));
162
163 wasm_v128_store(b, vacc0123);
164 wasm_v128_store(b + 4, vacc4567);
165 wasm_v128_store(b + 8, vacc89AB);
166 wasm_v128_store(b + 12, vaccCDEF);
167 b += 16;
168 }
169
170 if XNN_UNLIKELY(c != 0) {
171 do {
172 v128_t vacc0123 = wasm_v128_load(w);
173 v128_t vacc4567 = wasm_v128_load((const int32_t*) w + 4);
174
175 const v128_t vi0x01234567 = wasm_i16x8_load8x8(i0);
176 v128_t vk0x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int32_t)));
177 i0 += 8;
178
179 v128_t vprod01234567 = wasm_i16x8_mul(vi0x01234567, vk0x01234567);
180
181 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
182 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
183
184 const v128_t vi1x01234567 = wasm_i16x8_load8x8(i1);
185 v128_t vk1x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 8 * sizeof(int8_t)));
186 i1 += 8;
187
188 vprod01234567 = wasm_i16x8_mul(vi1x01234567, vk1x01234567);
189
190 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
191 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
192
193 const v128_t vi2x01234567 = wasm_i16x8_load8x8(i2);
194 v128_t vk2x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 16 * sizeof(int8_t)));
195 i2 += 8;
196
197 vprod01234567 = wasm_i16x8_mul(vi2x01234567, vk2x01234567);
198
199 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
200 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
201
202 const v128_t vi3x01234567 = wasm_i16x8_load8x8(i3);
203 v128_t vk3x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 24 * sizeof(int8_t)));
204 i3 += 8;
205
206 vprod01234567 = wasm_i16x8_mul(vi3x01234567, vk3x01234567);
207
208 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
209 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
210
211 const v128_t vi4x01234567 = wasm_i16x8_load8x8(i4);
212 v128_t vk4x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 32 * sizeof(int8_t)));
213 i4 += 8;
214
215 vprod01234567 = wasm_i16x8_mul(vi4x01234567, vk4x01234567);
216
217 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
218 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
219
220
221 w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 40 * sizeof(int8_t));
222
223 wasm_v128_store(b, vacc0123);
224 wasm_v128_store(b + 4, vacc4567);
225 b += 8;
226 c -= 8;
227 } while (c != 0);
228 }
229 }
230
231
232 // Middle pass to process 5 inputs in each iteration.
233 for (size_t ks = kernel_size - 5; ks > 5; ks -= 5) {
234 int32_t* b = buffer;
235 const int8_t* i0 = input[0];
236 assert(i0 != NULL);
237 if XNN_UNPREDICTABLE(i0 != zero) {
238 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
239 }
240 const int8_t* i1 = input[1];
241 assert(i1 != NULL);
242 if XNN_UNPREDICTABLE(i1 != zero) {
243 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
244 }
245 const int8_t* i2 = input[2];
246 assert(i2 != NULL);
247 if XNN_UNPREDICTABLE(i2 != zero) {
248 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
249 }
250 const int8_t* i3 = input[3];
251 assert(i3 != NULL);
252 if XNN_UNPREDICTABLE(i3 != zero) {
253 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
254 }
255 const int8_t* i4 = input[4];
256 assert(i4 != NULL);
257 if XNN_UNPREDICTABLE(i4 != zero) {
258 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
259 }
260 input += 5;
261
262 size_t c = round_up_po2(channels, 8);
263
264 for (; c >= 16; c -= 16) {
265 v128_t vacc0123 = wasm_v128_load(b);
266 v128_t vacc4567 = wasm_v128_load(b + 4);
267 v128_t vacc89AB = wasm_v128_load(b + 8);
268 v128_t vaccCDEF = wasm_v128_load(b + 12);
269
270 const v128_t vi0x01234567 = wasm_i16x8_load8x8(i0);
271 v128_t vk0x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 0 * sizeof(int8_t)));
272 const v128_t vi0x89ABCDEF = wasm_i16x8_load8x8(i0 + 8);
273 v128_t vk0x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int8_t)));
274 i0 += 16;
275
276 v128_t vprod01234567 = wasm_i16x8_mul(vi0x01234567, vk0x01234567);
277 v128_t vprod89ABCDEF = wasm_i16x8_mul(vi0x89ABCDEF, vk0x89ABCDEF);
278
279 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
280 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
281 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
282 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
283
284 const v128_t vi1x01234567 = wasm_i16x8_load8x8(i1);
285 v128_t vk1x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int8_t)));
286 const v128_t vi1x89ABCDEF = wasm_i16x8_load8x8(i1 + 8);
287 v128_t vk1x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 24 * sizeof(int8_t)));
288 i1 += 16;
289
290 vprod01234567 = wasm_i16x8_mul(vi1x01234567, vk1x01234567);
291 vprod89ABCDEF = wasm_i16x8_mul(vi1x89ABCDEF, vk1x89ABCDEF);
292
293 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
294 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
295 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
296 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
297
298 const v128_t vi2x01234567 = wasm_i16x8_load8x8(i2);
299 v128_t vk2x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 32 * sizeof(int8_t)));
300 const v128_t vi2x89ABCDEF = wasm_i16x8_load8x8(i2 + 8);
301 v128_t vk2x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 40 * sizeof(int8_t)));
302 i2 += 16;
303
304 vprod01234567 = wasm_i16x8_mul(vi2x01234567, vk2x01234567);
305 vprod89ABCDEF = wasm_i16x8_mul(vi2x89ABCDEF, vk2x89ABCDEF);
306
307 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
308 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
309 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
310 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
311
312 const v128_t vi3x01234567 = wasm_i16x8_load8x8(i3);
313 v128_t vk3x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 48 * sizeof(int8_t)));
314 const v128_t vi3x89ABCDEF = wasm_i16x8_load8x8(i3 + 8);
315 v128_t vk3x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 56 * sizeof(int8_t)));
316 i3 += 16;
317
318 vprod01234567 = wasm_i16x8_mul(vi3x01234567, vk3x01234567);
319 vprod89ABCDEF = wasm_i16x8_mul(vi3x89ABCDEF, vk3x89ABCDEF);
320
321 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
322 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
323 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
324 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
325
326 const v128_t vi4x01234567 = wasm_i16x8_load8x8(i4);
327 v128_t vk4x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 64 * sizeof(int8_t)));
328 const v128_t vi4x89ABCDEF = wasm_i16x8_load8x8(i4 + 8);
329 v128_t vk4x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 72 * sizeof(int8_t)));
330 i4 += 16;
331
332 vprod01234567 = wasm_i16x8_mul(vi4x01234567, vk4x01234567);
333 vprod89ABCDEF = wasm_i16x8_mul(vi4x89ABCDEF, vk4x89ABCDEF);
334
335 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
336 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
337 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
338 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
339
340 w = (const void*) ((uintptr_t) w + 80 * sizeof(int8_t));
341
342 wasm_v128_store(b, vacc0123);
343 wasm_v128_store(b + 4, vacc4567);
344 wasm_v128_store(b + 8, vacc89AB);
345 wasm_v128_store(b + 12, vaccCDEF);
346 b += 16;
347 }
348
349 if XNN_UNLIKELY(c != 0) {
350 do {
351 v128_t vacc0123 = wasm_v128_load(b);
352 v128_t vacc4567 = wasm_v128_load(b + 4);
353
354 const v128_t vi0x01234567 = wasm_i16x8_load8x8(i0);
355 v128_t vk0x01234567 = wasm_i16x8_load8x8(w);
356 i0 += 8;
357
358 v128_t vprod01234567 = wasm_i16x8_mul(vi0x01234567, vk0x01234567);
359
360 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
361 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
362
363 const v128_t vi1x01234567 = wasm_i16x8_load8x8(i1);
364 v128_t vk1x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int8_t)));
365 i1 += 8;
366
367 vprod01234567 = wasm_i16x8_mul(vi1x01234567, vk1x01234567);
368
369 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
370 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
371
372 const v128_t vi2x01234567 = wasm_i16x8_load8x8(i2);
373 v128_t vk2x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int8_t)));
374 i2 += 8;
375
376 vprod01234567 = wasm_i16x8_mul(vi2x01234567, vk2x01234567);
377
378 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
379 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
380
381 const v128_t vi3x01234567 = wasm_i16x8_load8x8(i3);
382 v128_t vk3x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 24 * sizeof(int8_t)));
383 i3 += 8;
384
385 vprod01234567 = wasm_i16x8_mul(vi3x01234567, vk3x01234567);
386
387 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
388 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
389
390 const v128_t vi4x01234567 = wasm_i16x8_load8x8(i4);
391 v128_t vk4x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 32 * sizeof(int8_t)));
392 i4 += 8;
393
394 vprod01234567 = wasm_i16x8_mul(vi4x01234567, vk4x01234567);
395
396 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
397 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
398
399
400 w = (const void*) ((uintptr_t) w + 40 * sizeof(int8_t));
401
402 wasm_v128_store(b, vacc0123);
403 wasm_v128_store(b + 4, vacc4567);
404 b += 8;
405 c -= 8;
406 } while (c != 0);
407 }
408 }
409
410 // Last pass to process up to 5 inputs.
411 {
412 const int32_t* b = buffer;
413 const int8_t* i0 = input[0];
414 assert(i0 != NULL);
415 if XNN_UNPREDICTABLE(i0 != zero) {
416 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
417 }
418 const int8_t* i1 = input[1];
419 assert(i1 != NULL);
420 if XNN_UNPREDICTABLE(i1 != zero) {
421 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
422 }
423 const int8_t* i2 = input[2];
424 assert(i2 != NULL);
425 if XNN_UNPREDICTABLE(i2 != zero) {
426 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
427 }
428 const int8_t* i3 = input[3];
429 assert(i3 != NULL);
430 if XNN_UNPREDICTABLE(i3 != zero) {
431 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
432 }
433 const int8_t* i4 = input[4];
434 assert(i4 != NULL);
435 if XNN_UNPREDICTABLE(i4 != zero) {
436 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
437 }
438
439 size_t c = channels;
440
441 for (; c >= 16; c -= 16) {
442 v128_t vacc0123 = wasm_v128_load(b);
443 v128_t vacc4567 = wasm_v128_load(b + 4);
444 v128_t vacc89AB = wasm_v128_load(b + 8);
445 v128_t vaccCDEF = wasm_v128_load(b + 12);
446 b += 16;
447
448 const v128_t vi0x01234567 = wasm_i16x8_load8x8(i0);
449 v128_t vk0x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 0 * sizeof(int8_t)));
450 const v128_t vi0x89ABCDEF = wasm_i16x8_load8x8(i0 + 8);
451 v128_t vk0x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int8_t)));
452 i0 += 16;
453
454 v128_t vprod01234567 = wasm_i16x8_mul(vi0x01234567, vk0x01234567);
455 v128_t vprod89ABCDEF = wasm_i16x8_mul(vi0x89ABCDEF, vk0x89ABCDEF);
456
457 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
458 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
459 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
460 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
461
462 const v128_t vi1x01234567 = wasm_i16x8_load8x8(i1);
463 v128_t vk1x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int8_t)));
464 const v128_t vi1x89ABCDEF = wasm_i16x8_load8x8(i1 + 8);
465 v128_t vk1x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 24 * sizeof(int8_t)));
466 i1 += 16;
467
468 vprod01234567 = wasm_i16x8_mul(vi1x01234567, vk1x01234567);
469 vprod89ABCDEF = wasm_i16x8_mul(vi1x89ABCDEF, vk1x89ABCDEF);
470
471 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
472 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
473 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
474 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
475
476 const v128_t vi2x01234567 = wasm_i16x8_load8x8(i2);
477 v128_t vk2x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 32 * sizeof(int8_t)));
478 const v128_t vi2x89ABCDEF = wasm_i16x8_load8x8(i2 + 8);
479 v128_t vk2x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 40 * sizeof(int8_t)));
480 i2 += 16;
481
482 vprod01234567 = wasm_i16x8_mul(vi2x01234567, vk2x01234567);
483 vprod89ABCDEF = wasm_i16x8_mul(vi2x89ABCDEF, vk2x89ABCDEF);
484
485 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
486 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
487 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
488 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
489
490 const v128_t vi3x01234567 = wasm_i16x8_load8x8(i3);
491 v128_t vk3x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 48 * sizeof(int8_t)));
492 const v128_t vi3x89ABCDEF = wasm_i16x8_load8x8(i3 + 8);
493 v128_t vk3x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 56 * sizeof(int8_t)));
494 i3 += 16;
495
496 vprod01234567 = wasm_i16x8_mul(vi3x01234567, vk3x01234567);
497 vprod89ABCDEF = wasm_i16x8_mul(vi3x89ABCDEF, vk3x89ABCDEF);
498
499 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
500 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
501 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
502 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
503
504 const v128_t vi4x01234567 = wasm_i16x8_load8x8(i4);
505 v128_t vk4x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 64 * sizeof(int8_t)));
506 const v128_t vi4x89ABCDEF = wasm_i16x8_load8x8(i4 + 8);
507 v128_t vk4x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 72 * sizeof(int8_t)));
508 i4 += 16;
509
510 vprod01234567 = wasm_i16x8_mul(vi4x01234567, vk4x01234567);
511 vprod89ABCDEF = wasm_i16x8_mul(vi4x89ABCDEF, vk4x89ABCDEF);
512
513 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
514 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
515 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
516 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
517
518
519 w = (const void*) ((uintptr_t) w + 80 * sizeof(int8_t));
520
521 vacc0123 = wasm_f32x4_convert_i32x4(vacc0123);
522 vacc4567 = wasm_f32x4_convert_i32x4(vacc4567);
523 vacc89AB = wasm_f32x4_convert_i32x4(vacc89AB);
524 vaccCDEF = wasm_f32x4_convert_i32x4(vaccCDEF);
525
526 vacc0123 = wasm_f32x4_mul(vacc0123, vscale);
527 vacc4567 = wasm_f32x4_mul(vacc4567, vscale);
528 vacc89AB = wasm_f32x4_mul(vacc89AB, vscale);
529 vaccCDEF = wasm_f32x4_mul(vaccCDEF, vscale);
530
531 vacc0123 = wasm_f32x4_add(vacc0123, vmagic_bias);
532 vacc4567 = wasm_f32x4_add(vacc4567, vmagic_bias);
533 vacc89AB = wasm_f32x4_add(vacc89AB, vmagic_bias);
534 vaccCDEF = wasm_f32x4_add(vaccCDEF, vmagic_bias);
535
536 vacc0123 = wasm_i32x4_max(vacc0123, vmagic_min);
537 vacc4567 = wasm_i32x4_max(vacc4567, vmagic_min);
538 vacc89AB = wasm_i32x4_max(vacc89AB, vmagic_min);
539 vaccCDEF = wasm_i32x4_max(vaccCDEF, vmagic_min);
540
541 vacc0123 = wasm_i32x4_sub(vacc0123, vmagic_bias_less_output_zero_point);
542 vacc4567 = wasm_i32x4_sub(vacc4567, vmagic_bias_less_output_zero_point);
543 vacc89AB = wasm_i32x4_sub(vacc89AB, vmagic_bias_less_output_zero_point);
544 vaccCDEF = wasm_i32x4_sub(vaccCDEF, vmagic_bias_less_output_zero_point);
545
546 v128_t vout01234567 = wasm_i16x8_narrow_i32x4(vacc0123, vacc4567);
547 v128_t vout89ABCDEF = wasm_i16x8_narrow_i32x4(vacc89AB, vaccCDEF);
548
549 v128_t vout0123456789ABCDEF = wasm_i8x16_narrow_i16x8(vout01234567, vout89ABCDEF);
550
551 vout0123456789ABCDEF = wasm_i8x16_min(vout0123456789ABCDEF, voutput_max);
552
553 wasm_v128_store(output, vout0123456789ABCDEF);
554 output += 16;
555 }
556
557 if XNN_UNLIKELY(c != 0) {
558 do {
559 v128_t vacc0123 = wasm_v128_load(b);
560 v128_t vacc4567 = wasm_v128_load(b + 4);
561 b += 8;
562
563 const v128_t vi0x01234567 = wasm_i16x8_load8x8(i0);
564 v128_t vk0x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w));
565 i0 += 8;
566
567 v128_t vprod01234567 = wasm_i16x8_mul(vi0x01234567, vk0x01234567);
568
569 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
570 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
571
572 const v128_t vi1x01234567 = wasm_i16x8_load8x8(i1);
573 v128_t vk1x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int8_t)));
574 i1 += 8;
575
576 vprod01234567 = wasm_i16x8_mul(vi1x01234567, vk1x01234567);
577
578 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
579 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
580
581 const v128_t vi2x01234567 = wasm_i16x8_load8x8(i2);
582 v128_t vk2x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int8_t)));
583 i2 += 8;
584
585 vprod01234567 = wasm_i16x8_mul(vi2x01234567, vk2x01234567);
586
587 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
588 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
589
590 const v128_t vi3x01234567 = wasm_i16x8_load8x8(i3);
591 v128_t vk3x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 24 * sizeof(int8_t)));
592 i3 += 8;
593
594 vprod01234567 = wasm_i16x8_mul(vi3x01234567, vk3x01234567);
595
596 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
597 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
598
599 const v128_t vi4x01234567 = wasm_i16x8_load8x8(i4);
600 v128_t vk4x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 32 * sizeof(int8_t)));
601 i4 += 8;
602
603 vprod01234567 = wasm_i16x8_mul(vi4x01234567, vk4x01234567);
604
605 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
606 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
607
608
609 vacc0123 = wasm_f32x4_convert_i32x4(vacc0123);
610 vacc4567 = wasm_f32x4_convert_i32x4(vacc4567);
611
612 vacc0123 = wasm_f32x4_mul(vacc0123, vscale);
613 vacc4567 = wasm_f32x4_mul(vacc4567, vscale);
614
615 w = (void*) ((uintptr_t) w + 40);
616
617 vacc0123 = wasm_f32x4_add(vacc0123, vmagic_bias);
618 vacc4567 = wasm_f32x4_add(vacc4567, vmagic_bias);
619
620 vacc0123 = wasm_i32x4_max(vacc0123, vmagic_min);
621 vacc4567 = wasm_i32x4_max(vacc4567, vmagic_min);
622
623 vacc0123 = wasm_i32x4_sub(vacc0123, vmagic_bias_less_output_zero_point);
624 vacc4567 = wasm_i32x4_sub(vacc4567, vmagic_bias_less_output_zero_point);
625
626 v128_t vout01234567 = wasm_i16x8_narrow_i32x4(vacc0123, vacc4567);
627 v128_t vout0123456701234567 = wasm_i8x16_narrow_i16x8(vout01234567, vout01234567);
628
629 vout0123456701234567 = wasm_i8x16_min(vout0123456701234567, voutput_max);
630
631 if XNN_LIKELY(c >= 8) {
632 wasm_v128_store64_lane(output, vout0123456701234567, 0);
633 output += 8;
634 c -= 8;
635 } else {
636 if (c & 4) {
637 wasm_v128_store32_lane(output, vout0123456701234567, 0);
638 vout0123456701234567 = wasm_u64x2_shr(vout0123456701234567, 32);
639 output += 4;
640 }
641 if (c & 2) {
642 wasm_v128_store16_lane(output, vout0123456701234567, 0);
643 vout0123456701234567 = wasm_u32x4_shr(vout0123456701234567, 16);
644 output += 2;
645 }
646 if (c & 1) {
647 wasm_v128_store8_lane(output, vout0123456701234567, 0);
648 output += 1;
649 }
650 c = 0;
651 }
652 } while (c != 0);
653 }
654 }
655
656 input = (const int8_t**) ((uintptr_t) input + input_stride);
657 output = (int8_t*) ((uintptr_t) output + output_increment);
658 } while (--output_width != 0);
659 }