Debian Codegraph @

src/qs8-dwconv/gen/qs8-dwconv-6f6m7l16c8s8r-minmax-fp32-wasmsimd-mul16.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-dwconv/multipass-wasmsimd-mul16.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include <wasm_simd128.h>
13
14 #include "xnnpack/dwconv.h"
15 #include "xnnpack/math.h"
16
17
18 void xnn_qs8_dwconv_minmax_fp32_ukernel_6f6m7l16c8s8r__wasmsimd_mul16(
19 size_t channels,
20 size_t output_width,
21 const int8_t** input,
22 const void* weights,
23 int8_t* output,
24 intptr_t input_stride,
25 size_t output_increment,
26 size_t input_offset,
27 const int8_t* zero,
28 size_t kernel_size,
29 int32_t* buffer,
30 const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
31 {
32 assert(channels != 0);
33 assert(output_width != 0);
34 assert(kernel_size > 6);
35
36
37 const v128_t vscale = wasm_v128_load32_splat(&params->fp32_scalar.scale);
38 XNN_FORCE_REALIZATION(vscale);
39
40 const v128_t vmagic_bias = wasm_f32x4_const_splat(12582912.0f);
41 const int32_t output_min_less_zero_point = (int32_t) params->fp32_scalar.output_min - (int32_t) params->fp32_scalar.output_zero_point;
42 const v128_t vmagic_min = wasm_i32x4_splat((int32_t) float_as_uint32(12582912.0f + output_min_less_zero_point));
43 const v128_t vmagic_bias_less_output_zero_point = wasm_i32x4_splat(INT32_C(0x4B400000) - (int32_t) params->fp32_scalar.output_zero_point);
44 const v128_t voutput_max = wasm_i8x16_splat(params->fp32_scalar.output_max);
45 XNN_FORCE_REALIZATION(vmagic_bias);
46 XNN_FORCE_REALIZATION(vmagic_min);
47 XNN_FORCE_REALIZATION(vmagic_bias_less_output_zero_point);
48 XNN_FORCE_REALIZATION(voutput_max);
49
50 do {
51 const void* w = weights;
52
53 // First pass to process 6 inputs.
54 {
55 int32_t* b = buffer;
56 const int8_t* i0 = input[0];
57 assert(i0 != NULL);
58 if XNN_UNPREDICTABLE(i0 != zero) {
59 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
60 }
61 const int8_t* i1 = input[1];
62 assert(i1 != NULL);
63 if XNN_UNPREDICTABLE(i1 != zero) {
64 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
65 }
66 const int8_t* i2 = input[2];
67 assert(i2 != NULL);
68 if XNN_UNPREDICTABLE(i2 != zero) {
69 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
70 }
71 const int8_t* i3 = input[3];
72 assert(i3 != NULL);
73 if XNN_UNPREDICTABLE(i3 != zero) {
74 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
75 }
76 const int8_t* i4 = input[4];
77 assert(i4 != NULL);
78 if XNN_UNPREDICTABLE(i4 != zero) {
79 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
80 }
81 const int8_t* i5 = input[5];
82 assert(i5 != NULL);
83 if XNN_UNPREDICTABLE(i5 != zero) {
84 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
85 }
86 input += 6;
87
88 size_t c = round_up_po2(channels, 8);
89
90 for (; c >= 16; c -= 16) {
91 v128_t vacc0123 = wasm_v128_load(w);
92 v128_t vacc4567 = wasm_v128_load((const void*) ((uintptr_t) w + 4 * sizeof(int32_t)));
93 v128_t vacc89AB = wasm_v128_load((const void*) ((uintptr_t) w + 8 * sizeof(int32_t)));
94 v128_t vaccCDEF = wasm_v128_load((const void*) ((uintptr_t) w + 12 * sizeof(int32_t)));
95
96 const v128_t vi0x01234567 = wasm_i16x8_load8x8(i0);
97 v128_t vk0x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 0 * sizeof(int8_t)));
98 const v128_t vi0x89ABCDEF = wasm_i16x8_load8x8(i0 + 8);
99 v128_t vk0x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 8 * sizeof(int8_t)));
100 i0 += 16;
101
102 v128_t vprod01234567 = wasm_i16x8_mul(vi0x01234567, vk0x01234567);
103 v128_t vprod89ABCDEF = wasm_i16x8_mul(vi0x89ABCDEF, vk0x89ABCDEF);
104
105 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
106 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
107 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
108 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
109
110 const v128_t vi1x01234567 = wasm_i16x8_load8x8(i1);
111 v128_t vk1x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 16 * sizeof(int8_t)));
112 const v128_t vi1x89ABCDEF = wasm_i16x8_load8x8(i1 + 8);
113 v128_t vk1x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 24 * sizeof(int8_t)));
114 i1 += 16;
115
116 vprod01234567 = wasm_i16x8_mul(vi1x01234567, vk1x01234567);
117 vprod89ABCDEF = wasm_i16x8_mul(vi1x89ABCDEF, vk1x89ABCDEF);
118
119 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
120 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
121 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
122 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
123
124 const v128_t vi2x01234567 = wasm_i16x8_load8x8(i2);
125 v128_t vk2x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 32 * sizeof(int8_t)));
126 const v128_t vi2x89ABCDEF = wasm_i16x8_load8x8(i2 + 8);
127 v128_t vk2x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 40 * sizeof(int8_t)));
128 i2 += 16;
129
130 vprod01234567 = wasm_i16x8_mul(vi2x01234567, vk2x01234567);
131 vprod89ABCDEF = wasm_i16x8_mul(vi2x89ABCDEF, vk2x89ABCDEF);
132
133 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
134 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
135 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
136 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
137
138 const v128_t vi3x01234567 = wasm_i16x8_load8x8(i3);
139 v128_t vk3x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 48 * sizeof(int8_t)));
140 const v128_t vi3x89ABCDEF = wasm_i16x8_load8x8(i3 + 8);
141 v128_t vk3x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 56 * sizeof(int8_t)));
142 i3 += 16;
143
144 vprod01234567 = wasm_i16x8_mul(vi3x01234567, vk3x01234567);
145 vprod89ABCDEF = wasm_i16x8_mul(vi3x89ABCDEF, vk3x89ABCDEF);
146
147 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
148 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
149 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
150 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
151
152 const v128_t vi4x01234567 = wasm_i16x8_load8x8(i4);
153 v128_t vk4x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 64 * sizeof(int8_t)));
154 const v128_t vi4x89ABCDEF = wasm_i16x8_load8x8(i4 + 8);
155 v128_t vk4x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 72 * sizeof(int8_t)));
156 i4 += 16;
157
158 vprod01234567 = wasm_i16x8_mul(vi4x01234567, vk4x01234567);
159 vprod89ABCDEF = wasm_i16x8_mul(vi4x89ABCDEF, vk4x89ABCDEF);
160
161 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
162 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
163 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
164 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
165
166 const v128_t vi5x01234567 = wasm_i16x8_load8x8(i5);
167 v128_t vk5x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 80 * sizeof(int8_t)));
168 const v128_t vi5x89ABCDEF = wasm_i16x8_load8x8(i5 + 8);
169 v128_t vk5x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 88 * sizeof(int8_t)));
170 i5 += 16;
171
172 vprod01234567 = wasm_i16x8_mul(vi5x01234567, vk5x01234567);
173 vprod89ABCDEF = wasm_i16x8_mul(vi5x89ABCDEF, vk5x89ABCDEF);
174
175 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
176 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
177 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
178 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
179
180 w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 96 * sizeof(int8_t));
181
182 wasm_v128_store(b, vacc0123);
183 wasm_v128_store(b + 4, vacc4567);
184 wasm_v128_store(b + 8, vacc89AB);
185 wasm_v128_store(b + 12, vaccCDEF);
186 b += 16;
187 }
188
189 if XNN_UNLIKELY(c != 0) {
190 do {
191 v128_t vacc0123 = wasm_v128_load(w);
192 v128_t vacc4567 = wasm_v128_load((const int32_t*) w + 4);
193
194 const v128_t vi0x01234567 = wasm_i16x8_load8x8(i0);
195 v128_t vk0x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int32_t)));
196 i0 += 8;
197
198 v128_t vprod01234567 = wasm_i16x8_mul(vi0x01234567, vk0x01234567);
199
200 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
201 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
202
203 const v128_t vi1x01234567 = wasm_i16x8_load8x8(i1);
204 v128_t vk1x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 8 * sizeof(int8_t)));
205 i1 += 8;
206
207 vprod01234567 = wasm_i16x8_mul(vi1x01234567, vk1x01234567);
208
209 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
210 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
211
212 const v128_t vi2x01234567 = wasm_i16x8_load8x8(i2);
213 v128_t vk2x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 16 * sizeof(int8_t)));
214 i2 += 8;
215
216 vprod01234567 = wasm_i16x8_mul(vi2x01234567, vk2x01234567);
217
218 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
219 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
220
221 const v128_t vi3x01234567 = wasm_i16x8_load8x8(i3);
222 v128_t vk3x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 24 * sizeof(int8_t)));
223 i3 += 8;
224
225 vprod01234567 = wasm_i16x8_mul(vi3x01234567, vk3x01234567);
226
227 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
228 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
229
230 const v128_t vi4x01234567 = wasm_i16x8_load8x8(i4);
231 v128_t vk4x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 32 * sizeof(int8_t)));
232 i4 += 8;
233
234 vprod01234567 = wasm_i16x8_mul(vi4x01234567, vk4x01234567);
235
236 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
237 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
238
239 const v128_t vi5x01234567 = wasm_i16x8_load8x8(i5);
240 v128_t vk5x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 40 * sizeof(int8_t)));
241 i5 += 8;
242
243 vprod01234567 = wasm_i16x8_mul(vi5x01234567, vk5x01234567);
244
245 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
246 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
247
248
249 w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 48 * sizeof(int8_t));
250
251 wasm_v128_store(b, vacc0123);
252 wasm_v128_store(b + 4, vacc4567);
253 b += 8;
254 c -= 8;
255 } while (c != 0);
256 }
257 }
258
259
260 // Middle pass to process 6 inputs in each iteration.
261 for (size_t ks = kernel_size - 6; ks > 7; ks -= 6) {
262 int32_t* b = buffer;
263 const int8_t* i0 = input[0];
264 assert(i0 != NULL);
265 if XNN_UNPREDICTABLE(i0 != zero) {
266 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
267 }
268 const int8_t* i1 = input[1];
269 assert(i1 != NULL);
270 if XNN_UNPREDICTABLE(i1 != zero) {
271 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
272 }
273 const int8_t* i2 = input[2];
274 assert(i2 != NULL);
275 if XNN_UNPREDICTABLE(i2 != zero) {
276 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
277 }
278 const int8_t* i3 = input[3];
279 assert(i3 != NULL);
280 if XNN_UNPREDICTABLE(i3 != zero) {
281 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
282 }
283 const int8_t* i4 = input[4];
284 assert(i4 != NULL);
285 if XNN_UNPREDICTABLE(i4 != zero) {
286 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
287 }
288 const int8_t* i5 = input[5];
289 assert(i5 != NULL);
290 if XNN_UNPREDICTABLE(i5 != zero) {
291 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
292 }
293 input += 6;
294
295 size_t c = round_up_po2(channels, 8);
296
297 for (; c >= 16; c -= 16) {
298 v128_t vacc0123 = wasm_v128_load(b);
299 v128_t vacc4567 = wasm_v128_load(b + 4);
300 v128_t vacc89AB = wasm_v128_load(b + 8);
301 v128_t vaccCDEF = wasm_v128_load(b + 12);
302
303 const v128_t vi0x01234567 = wasm_i16x8_load8x8(i0);
304 v128_t vk0x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 0 * sizeof(int8_t)));
305 const v128_t vi0x89ABCDEF = wasm_i16x8_load8x8(i0 + 8);
306 v128_t vk0x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int8_t)));
307 i0 += 16;
308
309 v128_t vprod01234567 = wasm_i16x8_mul(vi0x01234567, vk0x01234567);
310 v128_t vprod89ABCDEF = wasm_i16x8_mul(vi0x89ABCDEF, vk0x89ABCDEF);
311
312 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
313 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
314 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
315 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
316
317 const v128_t vi1x01234567 = wasm_i16x8_load8x8(i1);
318 v128_t vk1x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int8_t)));
319 const v128_t vi1x89ABCDEF = wasm_i16x8_load8x8(i1 + 8);
320 v128_t vk1x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 24 * sizeof(int8_t)));
321 i1 += 16;
322
323 vprod01234567 = wasm_i16x8_mul(vi1x01234567, vk1x01234567);
324 vprod89ABCDEF = wasm_i16x8_mul(vi1x89ABCDEF, vk1x89ABCDEF);
325
326 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
327 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
328 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
329 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
330
331 const v128_t vi2x01234567 = wasm_i16x8_load8x8(i2);
332 v128_t vk2x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 32 * sizeof(int8_t)));
333 const v128_t vi2x89ABCDEF = wasm_i16x8_load8x8(i2 + 8);
334 v128_t vk2x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 40 * sizeof(int8_t)));
335 i2 += 16;
336
337 vprod01234567 = wasm_i16x8_mul(vi2x01234567, vk2x01234567);
338 vprod89ABCDEF = wasm_i16x8_mul(vi2x89ABCDEF, vk2x89ABCDEF);
339
340 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
341 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
342 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
343 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
344
345 const v128_t vi3x01234567 = wasm_i16x8_load8x8(i3);
346 v128_t vk3x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 48 * sizeof(int8_t)));
347 const v128_t vi3x89ABCDEF = wasm_i16x8_load8x8(i3 + 8);
348 v128_t vk3x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 56 * sizeof(int8_t)));
349 i3 += 16;
350
351 vprod01234567 = wasm_i16x8_mul(vi3x01234567, vk3x01234567);
352 vprod89ABCDEF = wasm_i16x8_mul(vi3x89ABCDEF, vk3x89ABCDEF);
353
354 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
355 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
356 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
357 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
358
359 const v128_t vi4x01234567 = wasm_i16x8_load8x8(i4);
360 v128_t vk4x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 64 * sizeof(int8_t)));
361 const v128_t vi4x89ABCDEF = wasm_i16x8_load8x8(i4 + 8);
362 v128_t vk4x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 72 * sizeof(int8_t)));
363 i4 += 16;
364
365 vprod01234567 = wasm_i16x8_mul(vi4x01234567, vk4x01234567);
366 vprod89ABCDEF = wasm_i16x8_mul(vi4x89ABCDEF, vk4x89ABCDEF);
367
368 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
369 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
370 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
371 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
372
373 const v128_t vi5x01234567 = wasm_i16x8_load8x8(i5);
374 v128_t vk5x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 80 * sizeof(int8_t)));
375 const v128_t vi5x89ABCDEF = wasm_i16x8_load8x8(i5 + 8);
376 v128_t vk5x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 88 * sizeof(int8_t)));
377 i5 += 16;
378
379 vprod01234567 = wasm_i16x8_mul(vi5x01234567, vk5x01234567);
380 vprod89ABCDEF = wasm_i16x8_mul(vi5x89ABCDEF, vk5x89ABCDEF);
381
382 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
383 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
384 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
385 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
386
387 w = (const void*) ((uintptr_t) w + 96 * sizeof(int8_t));
388
389 wasm_v128_store(b, vacc0123);
390 wasm_v128_store(b + 4, vacc4567);
391 wasm_v128_store(b + 8, vacc89AB);
392 wasm_v128_store(b + 12, vaccCDEF);
393 b += 16;
394 }
395
396 if XNN_UNLIKELY(c != 0) {
397 do {
398 v128_t vacc0123 = wasm_v128_load(b);
399 v128_t vacc4567 = wasm_v128_load(b + 4);
400
401 const v128_t vi0x01234567 = wasm_i16x8_load8x8(i0);
402 v128_t vk0x01234567 = wasm_i16x8_load8x8(w);
403 i0 += 8;
404
405 v128_t vprod01234567 = wasm_i16x8_mul(vi0x01234567, vk0x01234567);
406
407 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
408 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
409
410 const v128_t vi1x01234567 = wasm_i16x8_load8x8(i1);
411 v128_t vk1x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int8_t)));
412 i1 += 8;
413
414 vprod01234567 = wasm_i16x8_mul(vi1x01234567, vk1x01234567);
415
416 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
417 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
418
419 const v128_t vi2x01234567 = wasm_i16x8_load8x8(i2);
420 v128_t vk2x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int8_t)));
421 i2 += 8;
422
423 vprod01234567 = wasm_i16x8_mul(vi2x01234567, vk2x01234567);
424
425 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
426 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
427
428 const v128_t vi3x01234567 = wasm_i16x8_load8x8(i3);
429 v128_t vk3x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 24 * sizeof(int8_t)));
430 i3 += 8;
431
432 vprod01234567 = wasm_i16x8_mul(vi3x01234567, vk3x01234567);
433
434 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
435 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
436
437 const v128_t vi4x01234567 = wasm_i16x8_load8x8(i4);
438 v128_t vk4x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 32 * sizeof(int8_t)));
439 i4 += 8;
440
441 vprod01234567 = wasm_i16x8_mul(vi4x01234567, vk4x01234567);
442
443 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
444 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
445
446 const v128_t vi5x01234567 = wasm_i16x8_load8x8(i5);
447 v128_t vk5x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 40 * sizeof(int8_t)));
448 i5 += 8;
449
450 vprod01234567 = wasm_i16x8_mul(vi5x01234567, vk5x01234567);
451
452 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
453 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
454
455
456 w = (const void*) ((uintptr_t) w + 48 * sizeof(int8_t));
457
458 wasm_v128_store(b, vacc0123);
459 wasm_v128_store(b + 4, vacc4567);
460 b += 8;
461 c -= 8;
462 } while (c != 0);
463 }
464 }
465
466 // Last pass to process up to 7 inputs.
467 {
468 const int32_t* b = buffer;
469 const int8_t* i0 = input[0];
470 assert(i0 != NULL);
471 if XNN_UNPREDICTABLE(i0 != zero) {
472 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
473 }
474 const int8_t* i1 = input[1];
475 assert(i1 != NULL);
476 if XNN_UNPREDICTABLE(i1 != zero) {
477 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
478 }
479 const int8_t* i2 = input[2];
480 assert(i2 != NULL);
481 if XNN_UNPREDICTABLE(i2 != zero) {
482 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
483 }
484 const int8_t* i3 = input[3];
485 assert(i3 != NULL);
486 if XNN_UNPREDICTABLE(i3 != zero) {
487 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
488 }
489 const int8_t* i4 = input[4];
490 assert(i4 != NULL);
491 if XNN_UNPREDICTABLE(i4 != zero) {
492 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
493 }
494 const int8_t* i5 = input[5];
495 assert(i5 != NULL);
496 if XNN_UNPREDICTABLE(i5 != zero) {
497 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
498 }
499 const int8_t* i6 = input[6];
500 assert(i6 != NULL);
501 if XNN_UNPREDICTABLE(i6 != zero) {
502 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
503 }
504
505 size_t c = channels;
506
507 for (; c >= 16; c -= 16) {
508 v128_t vacc0123 = wasm_v128_load(b);
509 v128_t vacc4567 = wasm_v128_load(b + 4);
510 v128_t vacc89AB = wasm_v128_load(b + 8);
511 v128_t vaccCDEF = wasm_v128_load(b + 12);
512 b += 16;
513
514 const v128_t vi0x01234567 = wasm_i16x8_load8x8(i0);
515 v128_t vk0x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 0 * sizeof(int8_t)));
516 const v128_t vi0x89ABCDEF = wasm_i16x8_load8x8(i0 + 8);
517 v128_t vk0x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int8_t)));
518 i0 += 16;
519
520 v128_t vprod01234567 = wasm_i16x8_mul(vi0x01234567, vk0x01234567);
521 v128_t vprod89ABCDEF = wasm_i16x8_mul(vi0x89ABCDEF, vk0x89ABCDEF);
522
523 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
524 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
525 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
526 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
527
528 const v128_t vi1x01234567 = wasm_i16x8_load8x8(i1);
529 v128_t vk1x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int8_t)));
530 const v128_t vi1x89ABCDEF = wasm_i16x8_load8x8(i1 + 8);
531 v128_t vk1x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 24 * sizeof(int8_t)));
532 i1 += 16;
533
534 vprod01234567 = wasm_i16x8_mul(vi1x01234567, vk1x01234567);
535 vprod89ABCDEF = wasm_i16x8_mul(vi1x89ABCDEF, vk1x89ABCDEF);
536
537 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
538 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
539 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
540 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
541
542 const v128_t vi2x01234567 = wasm_i16x8_load8x8(i2);
543 v128_t vk2x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 32 * sizeof(int8_t)));
544 const v128_t vi2x89ABCDEF = wasm_i16x8_load8x8(i2 + 8);
545 v128_t vk2x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 40 * sizeof(int8_t)));
546 i2 += 16;
547
548 vprod01234567 = wasm_i16x8_mul(vi2x01234567, vk2x01234567);
549 vprod89ABCDEF = wasm_i16x8_mul(vi2x89ABCDEF, vk2x89ABCDEF);
550
551 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
552 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
553 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
554 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
555
556 const v128_t vi3x01234567 = wasm_i16x8_load8x8(i3);
557 v128_t vk3x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 48 * sizeof(int8_t)));
558 const v128_t vi3x89ABCDEF = wasm_i16x8_load8x8(i3 + 8);
559 v128_t vk3x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 56 * sizeof(int8_t)));
560 i3 += 16;
561
562 vprod01234567 = wasm_i16x8_mul(vi3x01234567, vk3x01234567);
563 vprod89ABCDEF = wasm_i16x8_mul(vi3x89ABCDEF, vk3x89ABCDEF);
564
565 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
566 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
567 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
568 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
569
570 const v128_t vi4x01234567 = wasm_i16x8_load8x8(i4);
571 v128_t vk4x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 64 * sizeof(int8_t)));
572 const v128_t vi4x89ABCDEF = wasm_i16x8_load8x8(i4 + 8);
573 v128_t vk4x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 72 * sizeof(int8_t)));
574 i4 += 16;
575
576 vprod01234567 = wasm_i16x8_mul(vi4x01234567, vk4x01234567);
577 vprod89ABCDEF = wasm_i16x8_mul(vi4x89ABCDEF, vk4x89ABCDEF);
578
579 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
580 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
581 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
582 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
583
584 const v128_t vi5x01234567 = wasm_i16x8_load8x8(i5);
585 v128_t vk5x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 80 * sizeof(int8_t)));
586 const v128_t vi5x89ABCDEF = wasm_i16x8_load8x8(i5 + 8);
587 v128_t vk5x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 88 * sizeof(int8_t)));
588 i5 += 16;
589
590 vprod01234567 = wasm_i16x8_mul(vi5x01234567, vk5x01234567);
591 vprod89ABCDEF = wasm_i16x8_mul(vi5x89ABCDEF, vk5x89ABCDEF);
592
593 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
594 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
595 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
596 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
597
598 const v128_t vi6x01234567 = wasm_i16x8_load8x8(i6);
599 v128_t vk6x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 96 * sizeof(int8_t)));
600 const v128_t vi6x89ABCDEF = wasm_i16x8_load8x8(i6 + 8);
601 v128_t vk6x89ABCDEF = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 104 * sizeof(int8_t)));
602 i6 += 16;
603
604 vprod01234567 = wasm_i16x8_mul(vi6x01234567, vk6x01234567);
605 vprod89ABCDEF = wasm_i16x8_mul(vi6x89ABCDEF, vk6x89ABCDEF);
606
607 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
608 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
609 vacc89AB = wasm_i32x4_add(vacc89AB, wasm_i32x4_extend_low_i16x8(vprod89ABCDEF));
610 vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_i32x4_extend_high_i16x8(vprod89ABCDEF));
611
612
613 w = (const void*) ((uintptr_t) w + 112 * sizeof(int8_t));
614
615 vacc0123 = wasm_f32x4_convert_i32x4(vacc0123);
616 vacc4567 = wasm_f32x4_convert_i32x4(vacc4567);
617 vacc89AB = wasm_f32x4_convert_i32x4(vacc89AB);
618 vaccCDEF = wasm_f32x4_convert_i32x4(vaccCDEF);
619
620 vacc0123 = wasm_f32x4_mul(vacc0123, vscale);
621 vacc4567 = wasm_f32x4_mul(vacc4567, vscale);
622 vacc89AB = wasm_f32x4_mul(vacc89AB, vscale);
623 vaccCDEF = wasm_f32x4_mul(vaccCDEF, vscale);
624
625 vacc0123 = wasm_f32x4_add(vacc0123, vmagic_bias);
626 vacc4567 = wasm_f32x4_add(vacc4567, vmagic_bias);
627 vacc89AB = wasm_f32x4_add(vacc89AB, vmagic_bias);
628 vaccCDEF = wasm_f32x4_add(vaccCDEF, vmagic_bias);
629
630 vacc0123 = wasm_i32x4_max(vacc0123, vmagic_min);
631 vacc4567 = wasm_i32x4_max(vacc4567, vmagic_min);
632 vacc89AB = wasm_i32x4_max(vacc89AB, vmagic_min);
633 vaccCDEF = wasm_i32x4_max(vaccCDEF, vmagic_min);
634
635 vacc0123 = wasm_i32x4_sub(vacc0123, vmagic_bias_less_output_zero_point);
636 vacc4567 = wasm_i32x4_sub(vacc4567, vmagic_bias_less_output_zero_point);
637 vacc89AB = wasm_i32x4_sub(vacc89AB, vmagic_bias_less_output_zero_point);
638 vaccCDEF = wasm_i32x4_sub(vaccCDEF, vmagic_bias_less_output_zero_point);
639
640 v128_t vout01234567 = wasm_i16x8_narrow_i32x4(vacc0123, vacc4567);
641 v128_t vout89ABCDEF = wasm_i16x8_narrow_i32x4(vacc89AB, vaccCDEF);
642
643 v128_t vout0123456789ABCDEF = wasm_i8x16_narrow_i16x8(vout01234567, vout89ABCDEF);
644
645 vout0123456789ABCDEF = wasm_i8x16_min(vout0123456789ABCDEF, voutput_max);
646
647 wasm_v128_store(output, vout0123456789ABCDEF);
648 output += 16;
649 }
650
651 if XNN_UNLIKELY(c != 0) {
652 do {
653 v128_t vacc0123 = wasm_v128_load(b);
654 v128_t vacc4567 = wasm_v128_load(b + 4);
655 b += 8;
656
657 const v128_t vi0x01234567 = wasm_i16x8_load8x8(i0);
658 v128_t vk0x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w));
659 i0 += 8;
660
661 v128_t vprod01234567 = wasm_i16x8_mul(vi0x01234567, vk0x01234567);
662
663 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
664 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
665
666 const v128_t vi1x01234567 = wasm_i16x8_load8x8(i1);
667 v128_t vk1x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 8 * sizeof(int8_t)));
668 i1 += 8;
669
670 vprod01234567 = wasm_i16x8_mul(vi1x01234567, vk1x01234567);
671
672 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
673 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
674
675 const v128_t vi2x01234567 = wasm_i16x8_load8x8(i2);
676 v128_t vk2x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 16 * sizeof(int8_t)));
677 i2 += 8;
678
679 vprod01234567 = wasm_i16x8_mul(vi2x01234567, vk2x01234567);
680
681 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
682 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
683
684 const v128_t vi3x01234567 = wasm_i16x8_load8x8(i3);
685 v128_t vk3x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 24 * sizeof(int8_t)));
686 i3 += 8;
687
688 vprod01234567 = wasm_i16x8_mul(vi3x01234567, vk3x01234567);
689
690 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
691 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
692
693 const v128_t vi4x01234567 = wasm_i16x8_load8x8(i4);
694 v128_t vk4x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 32 * sizeof(int8_t)));
695 i4 += 8;
696
697 vprod01234567 = wasm_i16x8_mul(vi4x01234567, vk4x01234567);
698
699 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
700 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
701
702 const v128_t vi5x01234567 = wasm_i16x8_load8x8(i5);
703 v128_t vk5x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 40 * sizeof(int8_t)));
704 i5 += 8;
705
706 vprod01234567 = wasm_i16x8_mul(vi5x01234567, vk5x01234567);
707
708 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
709 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
710
711 const v128_t vi6x01234567 = wasm_i16x8_load8x8(i6);
712 v128_t vk6x01234567 = wasm_i16x8_load8x8((const void*) ((uintptr_t) w + 48 * sizeof(int8_t)));
713 i6 += 8;
714
715 vprod01234567 = wasm_i16x8_mul(vi6x01234567, vk6x01234567);
716
717 vacc0123 = wasm_i32x4_add(vacc0123, wasm_i32x4_extend_low_i16x8(vprod01234567));
718 vacc4567 = wasm_i32x4_add(vacc4567, wasm_i32x4_extend_high_i16x8(vprod01234567));
719
720
721 vacc0123 = wasm_f32x4_convert_i32x4(vacc0123);
722 vacc4567 = wasm_f32x4_convert_i32x4(vacc4567);
723
724 vacc0123 = wasm_f32x4_mul(vacc0123, vscale);
725 vacc4567 = wasm_f32x4_mul(vacc4567, vscale);
726
727 w = (void*) ((uintptr_t) w + 56);
728
729 vacc0123 = wasm_f32x4_add(vacc0123, vmagic_bias);
730 vacc4567 = wasm_f32x4_add(vacc4567, vmagic_bias);
731
732 vacc0123 = wasm_i32x4_max(vacc0123, vmagic_min);
733 vacc4567 = wasm_i32x4_max(vacc4567, vmagic_min);
734
735 vacc0123 = wasm_i32x4_sub(vacc0123, vmagic_bias_less_output_zero_point);
736 vacc4567 = wasm_i32x4_sub(vacc4567, vmagic_bias_less_output_zero_point);
737
738 v128_t vout01234567 = wasm_i16x8_narrow_i32x4(vacc0123, vacc4567);
739 v128_t vout0123456701234567 = wasm_i8x16_narrow_i16x8(vout01234567, vout01234567);
740
741 vout0123456701234567 = wasm_i8x16_min(vout0123456701234567, voutput_max);
742
743 if XNN_LIKELY(c >= 8) {
744 wasm_v128_store64_lane(output, vout0123456701234567, 0);
745 output += 8;
746 c -= 8;
747 } else {
748 if (c & 4) {
749 wasm_v128_store32_lane(output, vout0123456701234567, 0);
750 vout0123456701234567 = wasm_u64x2_shr(vout0123456701234567, 32);
751 output += 4;
752 }
753 if (c & 2) {
754 wasm_v128_store16_lane(output, vout0123456701234567, 0);
755 vout0123456701234567 = wasm_u32x4_shr(vout0123456701234567, 16);
756 output += 2;
757 }
758 if (c & 1) {
759 wasm_v128_store8_lane(output, vout0123456701234567, 0);
760 output += 1;
761 }
762 c = 0;
763 }
764 } while (c != 0);
765 }
766 }
767
768 input = (const int8_t**) ((uintptr_t) input + input_stride);
769 output = (int8_t*) ((uintptr_t) output + output_increment);
770 } while (--output_width != 0);
771 }