Debian Codegraph @

src/qs8-dwconv/gen/qs8-dwconv-8f8m9l16c8s8r-minmax-rndnu-neon-mul8-ld64.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-dwconv/multipass-neon-mul8.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12
13 #include <arm_neon.h>
14
15 #include "xnnpack/dwconv.h"
16 #include "xnnpack/math.h"
17
18
19 void xnn_qs8_dwconv_minmax_rndnu_ukernel_8f8m9l16c8s8r__neon_mul8_ld64(
20 size_t channels,
21 size_t output_width,
22 const int8_t** input,
23 const void* weights,
24 int8_t* output,
25 intptr_t input_stride,
26 size_t output_increment,
27 size_t input_offset,
28 const int8_t* zero,
29 size_t kernel_size,
30 int32_t* buffer,
31 const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
32 {
33 assert(channels != 0);
34 assert(output_width != 0);
35 assert(kernel_size > 8);
36
37 const int32x4_t vright_pre_shift = vld1q_dup_s32(&params->rndnu_neon.right_pre_shift);
38 const int32x4_t vmultiplier = vld1q_dup_s32(&params->rndnu_neon.multiplier);
39 const int32x4_t vright_post_shift = vld1q_dup_s32(&params->rndnu_neon.right_post_shift);
40 const int16x8_t voutput_zero_point = vld1q_dup_s16(&params->rndnu_neon.output_zero_point);
41 const int8x16_t voutput_min = vld1q_dup_s8(&params->rndnu_neon.output_min);
42 const int8x16_t voutput_max = vld1q_dup_s8(&params->rndnu_neon.output_max);
43 do {
44 const void* w = weights;
45
46 // First pass to process 8 inputs.
47 {
48 int32_t* b = buffer;
49 const int8_t* i0 = input[0];
50 assert(i0 != NULL);
51 if XNN_UNPREDICTABLE(i0 != zero) {
52 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
53 }
54 const int8_t* i1 = input[1];
55 assert(i1 != NULL);
56 if XNN_UNPREDICTABLE(i1 != zero) {
57 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
58 }
59 const int8_t* i2 = input[2];
60 assert(i2 != NULL);
61 if XNN_UNPREDICTABLE(i2 != zero) {
62 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
63 }
64 const int8_t* i3 = input[3];
65 assert(i3 != NULL);
66 if XNN_UNPREDICTABLE(i3 != zero) {
67 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
68 }
69 const int8_t* i4 = input[4];
70 assert(i4 != NULL);
71 if XNN_UNPREDICTABLE(i4 != zero) {
72 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
73 }
74 const int8_t* i5 = input[5];
75 assert(i5 != NULL);
76 if XNN_UNPREDICTABLE(i5 != zero) {
77 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
78 }
79 const int8_t* i6 = input[6];
80 assert(i6 != NULL);
81 if XNN_UNPREDICTABLE(i6 != zero) {
82 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
83 }
84 const int8_t* i7 = input[7];
85 assert(i7 != NULL);
86 if XNN_UNPREDICTABLE(i7 != zero) {
87 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
88 }
89 input += 8;
90
91 size_t c = round_up_po2(channels, 8);
92
93 for (; c >= 16; c -= 16) {
94 int32x4_t vacc0123 = vld1q_s32(w); w = (const int32_t*) w + 4;
95 int32x4_t vacc4567 = vld1q_s32(w); w = (const int32_t*) w + 4;
96 int32x4_t vacc89AB = vld1q_s32(w); w = (const int32_t*) w + 4;
97 int32x4_t vaccCDEF = vld1q_s32(w); w = (const int32_t*) w + 4;
98
99 const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
100 const int8x8_t vk0x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
101 const int8x8_t vi0x89ABCDEF = vld1_s8(i0); i0 += 8;
102 const int8x8_t vk0x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
103
104 int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
105 int16x8_t vprod89ABCDEF = vmull_s8(vi0x89ABCDEF, vk0x89ABCDEF);
106
107 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
108 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
109 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
110 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
111
112 const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
113 const int8x8_t vk1x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
114 const int8x8_t vi1x89ABCDEF = vld1_s8(i1); i1 += 8;
115 const int8x8_t vk1x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
116
117 vprod01234567 = vmull_s8(vi1x01234567, vk1x01234567);
118 vprod89ABCDEF = vmull_s8(vi1x89ABCDEF, vk1x89ABCDEF);
119
120 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
121 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
122 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
123 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
124
125 const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
126 const int8x8_t vk2x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
127 const int8x8_t vi2x89ABCDEF = vld1_s8(i2); i2 += 8;
128 const int8x8_t vk2x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
129
130 vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
131 vprod89ABCDEF = vmull_s8(vi2x89ABCDEF, vk2x89ABCDEF);
132
133 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
134 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
135 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
136 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
137
138 const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
139 const int8x8_t vk3x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
140 const int8x8_t vi3x89ABCDEF = vld1_s8(i3); i3 += 8;
141 const int8x8_t vk3x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
142
143 vprod01234567 = vmull_s8(vi3x01234567, vk3x01234567);
144 vprod89ABCDEF = vmull_s8(vi3x89ABCDEF, vk3x89ABCDEF);
145
146 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
147 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
148 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
149 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
150
151 const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
152 const int8x8_t vk4x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
153 const int8x8_t vi4x89ABCDEF = vld1_s8(i4); i4 += 8;
154 const int8x8_t vk4x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
155
156 vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
157 vprod89ABCDEF = vmull_s8(vi4x89ABCDEF, vk4x89ABCDEF);
158
159 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
160 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
161 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
162 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
163
164 const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
165 const int8x8_t vk5x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
166 const int8x8_t vi5x89ABCDEF = vld1_s8(i5); i5 += 8;
167 const int8x8_t vk5x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
168
169 vprod01234567 = vmull_s8(vi5x01234567, vk5x01234567);
170 vprod89ABCDEF = vmull_s8(vi5x89ABCDEF, vk5x89ABCDEF);
171
172 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
173 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
174 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
175 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
176
177 const int8x8_t vi6x01234567 = vld1_s8(i6); i6 += 8;
178 const int8x8_t vk6x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
179 const int8x8_t vi6x89ABCDEF = vld1_s8(i6); i6 += 8;
180 const int8x8_t vk6x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
181
182 vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
183 vprod89ABCDEF = vmull_s8(vi6x89ABCDEF, vk6x89ABCDEF);
184
185 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
186 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
187 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
188 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
189
190 const int8x8_t vi7x01234567 = vld1_s8(i7); i7 += 8;
191 const int8x8_t vk7x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
192 const int8x8_t vi7x89ABCDEF = vld1_s8(i7); i7 += 8;
193 const int8x8_t vk7x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
194
195 vprod01234567 = vmull_s8(vi7x01234567, vk7x01234567);
196 vprod89ABCDEF = vmull_s8(vi7x89ABCDEF, vk7x89ABCDEF);
197
198 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
199 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
200 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
201 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
202
203 vst1q_s32(b, vacc0123); b += 4;
204 vst1q_s32(b, vacc4567); b += 4;
205 vst1q_s32(b, vacc89AB); b += 4;
206 vst1q_s32(b, vaccCDEF); b += 4;
207 }
208
209 if XNN_UNLIKELY(c != 0) {
210 do {
211 int32x4_t vacc0123 = vld1q_s32(w); w = (const int32_t*) w + 4;
212 int32x4_t vacc4567 = vld1q_s32(w); w = (const int32_t*) w + 4;
213
214 const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
215 const int8x8_t vk0x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
216
217 int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
218
219 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
220 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
221
222 const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
223 const int8x8_t vk1x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
224
225 vprod01234567 = vmull_s8(vi1x01234567, vk1x01234567);
226
227 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
228 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
229
230 const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
231 const int8x8_t vk2x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
232
233 vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
234
235 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
236 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
237
238 const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
239 const int8x8_t vk3x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
240
241 vprod01234567 = vmull_s8(vi3x01234567, vk3x01234567);
242
243 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
244 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
245
246 const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
247 const int8x8_t vk4x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
248
249 vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
250
251 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
252 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
253
254 const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
255 const int8x8_t vk5x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
256
257 vprod01234567 = vmull_s8(vi5x01234567, vk5x01234567);
258
259 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
260 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
261
262 const int8x8_t vi6x01234567 = vld1_s8(i6); i6 += 8;
263 const int8x8_t vk6x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
264
265 vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
266
267 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
268 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
269
270 const int8x8_t vi7x01234567 = vld1_s8(i7); i7 += 8;
271 const int8x8_t vk7x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
272
273 vprod01234567 = vmull_s8(vi7x01234567, vk7x01234567);
274
275 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
276 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
277
278 vst1q_s32(b, vacc0123); b += 4;
279 vst1q_s32(b, vacc4567); b += 4;
280 c -= 8;
281 } while (c != 0);
282 }
283 }
284
285 // Middle pass to process 8 inputs in each iteration.
286 for (size_t ks = kernel_size - 8; ks > 9; ks -= 8) {
287 int32_t* b = buffer;
288
289 const int8_t* i0 = input[0];
290 assert(i0 != NULL);
291 if XNN_UNPREDICTABLE(i0 != zero) {
292 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
293 }
294 const int8_t* i1 = input[1];
295 assert(i1 != NULL);
296 if XNN_UNPREDICTABLE(i1 != zero) {
297 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
298 }
299 const int8_t* i2 = input[2];
300 assert(i2 != NULL);
301 if XNN_UNPREDICTABLE(i2 != zero) {
302 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
303 }
304 const int8_t* i3 = input[3];
305 assert(i3 != NULL);
306 if XNN_UNPREDICTABLE(i3 != zero) {
307 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
308 }
309 const int8_t* i4 = input[4];
310 assert(i4 != NULL);
311 if XNN_UNPREDICTABLE(i4 != zero) {
312 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
313 }
314 const int8_t* i5 = input[5];
315 assert(i5 != NULL);
316 if XNN_UNPREDICTABLE(i5 != zero) {
317 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
318 }
319 const int8_t* i6 = input[6];
320 assert(i6 != NULL);
321 if XNN_UNPREDICTABLE(i6 != zero) {
322 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
323 }
324 const int8_t* i7 = input[7];
325 assert(i7 != NULL);
326 if XNN_UNPREDICTABLE(i7 != zero) {
327 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
328 }
329 input += 8;
330
331 size_t c = round_up_po2(channels, 8);
332
333 for (; c >= 16; c -= 16) {
334 int32x4_t vacc0123 = vld1q_s32(b);
335 int32x4_t vacc4567 = vld1q_s32(b + 4);
336 int32x4_t vacc89AB = vld1q_s32(b + 8);
337 int32x4_t vaccCDEF = vld1q_s32(b + 12);
338
339 const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
340 const int8x8_t vk0x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
341 const int8x8_t vi0x89ABCDEF = vld1_s8(i0); i0 += 8;
342 const int8x8_t vk0x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
343
344 int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
345 int16x8_t vprod89ABCDEF = vmull_s8(vi0x89ABCDEF, vk0x89ABCDEF);
346
347 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
348 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
349 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
350 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
351
352 const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
353 const int8x8_t vk1x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
354 const int8x8_t vi1x89ABCDEF = vld1_s8(i1); i1 += 8;
355 const int8x8_t vk1x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
356
357 vprod01234567 = vmull_s8(vi1x01234567, vk1x01234567);
358 vprod89ABCDEF = vmull_s8(vi1x89ABCDEF, vk1x89ABCDEF);
359
360 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
361 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
362 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
363 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
364
365 const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
366 const int8x8_t vk2x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
367 const int8x8_t vi2x89ABCDEF = vld1_s8(i2); i2 += 8;
368 const int8x8_t vk2x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
369
370 vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
371 vprod89ABCDEF = vmull_s8(vi2x89ABCDEF, vk2x89ABCDEF);
372
373 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
374 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
375 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
376 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
377
378 const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
379 const int8x8_t vk3x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
380 const int8x8_t vi3x89ABCDEF = vld1_s8(i3); i3 += 8;
381 const int8x8_t vk3x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
382
383 vprod01234567 = vmull_s8(vi3x01234567, vk3x01234567);
384 vprod89ABCDEF = vmull_s8(vi3x89ABCDEF, vk3x89ABCDEF);
385
386 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
387 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
388 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
389 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
390
391 const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
392 const int8x8_t vk4x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
393 const int8x8_t vi4x89ABCDEF = vld1_s8(i4); i4 += 8;
394 const int8x8_t vk4x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
395
396 vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
397 vprod89ABCDEF = vmull_s8(vi4x89ABCDEF, vk4x89ABCDEF);
398
399 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
400 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
401 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
402 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
403
404 const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
405 const int8x8_t vk5x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
406 const int8x8_t vi5x89ABCDEF = vld1_s8(i5); i5 += 8;
407 const int8x8_t vk5x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
408
409 vprod01234567 = vmull_s8(vi5x01234567, vk5x01234567);
410 vprod89ABCDEF = vmull_s8(vi5x89ABCDEF, vk5x89ABCDEF);
411
412 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
413 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
414 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
415 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
416
417 const int8x8_t vi6x01234567 = vld1_s8(i6); i6 += 8;
418 const int8x8_t vk6x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
419 const int8x8_t vi6x89ABCDEF = vld1_s8(i6); i6 += 8;
420 const int8x8_t vk6x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
421
422 vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
423 vprod89ABCDEF = vmull_s8(vi6x89ABCDEF, vk6x89ABCDEF);
424
425 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
426 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
427 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
428 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
429
430 const int8x8_t vi7x01234567 = vld1_s8(i7); i7 += 8;
431 const int8x8_t vk7x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
432 const int8x8_t vi7x89ABCDEF = vld1_s8(i7); i7 += 8;
433 const int8x8_t vk7x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
434
435 vprod01234567 = vmull_s8(vi7x01234567, vk7x01234567);
436 vprod89ABCDEF = vmull_s8(vi7x89ABCDEF, vk7x89ABCDEF);
437
438 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
439 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
440 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
441 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
442
443 vst1q_s32(b, vacc0123); b += 4;
444 vst1q_s32(b, vacc4567); b += 4;
445 vst1q_s32(b, vacc89AB); b += 4;
446 vst1q_s32(b, vaccCDEF); b += 4;
447 }
448
449 if XNN_UNLIKELY(c != 0) {
450 do {
451 int32x4_t vacc0123 = vld1q_s32(b);
452 int32x4_t vacc4567 = vld1q_s32(b + 4);
453
454 const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
455 const int8x8_t vk0x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
456
457 int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
458
459 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
460 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
461
462 const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
463 const int8x8_t vk1x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
464
465 vprod01234567 = vmull_s8(vi1x01234567, vk1x01234567);
466
467 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
468 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
469
470 const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
471 const int8x8_t vk2x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
472
473 vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
474
475 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
476 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
477
478 const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
479 const int8x8_t vk3x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
480
481 vprod01234567 = vmull_s8(vi3x01234567, vk3x01234567);
482
483 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
484 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
485
486 const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
487 const int8x8_t vk4x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
488
489 vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
490
491 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
492 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
493
494 const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
495 const int8x8_t vk5x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
496
497 vprod01234567 = vmull_s8(vi5x01234567, vk5x01234567);
498
499 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
500 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
501
502 const int8x8_t vi6x01234567 = vld1_s8(i6); i6 += 8;
503 const int8x8_t vk6x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
504
505 vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
506
507 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
508 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
509
510 const int8x8_t vi7x01234567 = vld1_s8(i7); i7 += 8;
511 const int8x8_t vk7x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
512
513 vprod01234567 = vmull_s8(vi7x01234567, vk7x01234567);
514
515 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
516 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
517
518 vst1q_s32(b, vacc0123); b += 4;
519 vst1q_s32(b, vacc4567); b += 4;
520 c -= 8;
521 } while (c != 0);
522 }
523 }
524
525 // Last pass to process up to 9 inputs.
526 {
527 const int32_t* b = buffer;
528 const int8_t* i0 = input[0];
529 assert(i0 != NULL);
530 if XNN_UNPREDICTABLE(i0 != zero) {
531 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
532 }
533 const int8_t* i1 = input[1];
534 assert(i1 != NULL);
535 if XNN_UNPREDICTABLE(i1 != zero) {
536 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
537 }
538 const int8_t* i2 = input[2];
539 assert(i2 != NULL);
540 if XNN_UNPREDICTABLE(i2 != zero) {
541 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
542 }
543 const int8_t* i3 = input[3];
544 assert(i3 != NULL);
545 if XNN_UNPREDICTABLE(i3 != zero) {
546 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
547 }
548 const int8_t* i4 = input[4];
549 assert(i4 != NULL);
550 if XNN_UNPREDICTABLE(i4 != zero) {
551 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
552 }
553 const int8_t* i5 = input[5];
554 assert(i5 != NULL);
555 if XNN_UNPREDICTABLE(i5 != zero) {
556 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
557 }
558 const int8_t* i6 = input[6];
559 assert(i6 != NULL);
560 if XNN_UNPREDICTABLE(i6 != zero) {
561 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
562 }
563 const int8_t* i7 = input[7];
564 assert(i7 != NULL);
565 if XNN_UNPREDICTABLE(i7 != zero) {
566 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
567 }
568 const int8_t* i8 = input[8];
569 assert(i8 != NULL);
570 if XNN_UNPREDICTABLE(i8 != zero) {
571 i8 = (const int8_t*) ((uintptr_t) i8 + input_offset);
572 }
573
574 size_t c = channels;
575 for (; c >= 16; c -= 16) {
576 int32x4_t vacc0123 = vld1q_s32(b); b += 4;
577 int32x4_t vacc4567 = vld1q_s32(b); b += 4;
578 int32x4_t vacc89AB = vld1q_s32(b); b += 4;
579 int32x4_t vaccCDEF = vld1q_s32(b); b += 4;
580
581 const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
582 const int8x8_t vk0x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
583 const int8x8_t vi0x89ABCDEF = vld1_s8(i0); i0 += 8;
584 const int8x8_t vk0x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
585
586 int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
587 int16x8_t vprod89ABCDEF = vmull_s8(vi0x89ABCDEF, vk0x89ABCDEF);
588
589 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
590 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
591 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
592 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
593
594 const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
595 const int8x8_t vk1x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
596 const int8x8_t vi1x89ABCDEF = vld1_s8(i1); i1 += 8;
597 const int8x8_t vk1x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
598
599 vprod01234567 = vmull_s8(vi1x01234567, vk1x01234567);
600 vprod89ABCDEF = vmull_s8(vi1x89ABCDEF, vk1x89ABCDEF);
601
602 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
603 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
604 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
605 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
606
607 const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
608 const int8x8_t vk2x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
609 const int8x8_t vi2x89ABCDEF = vld1_s8(i2); i2 += 8;
610 const int8x8_t vk2x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
611
612 vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
613 vprod89ABCDEF = vmull_s8(vi2x89ABCDEF, vk2x89ABCDEF);
614
615 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
616 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
617 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
618 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
619
620 const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
621 const int8x8_t vk3x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
622 const int8x8_t vi3x89ABCDEF = vld1_s8(i3); i3 += 8;
623 const int8x8_t vk3x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
624
625 vprod01234567 = vmull_s8(vi3x01234567, vk3x01234567);
626 vprod89ABCDEF = vmull_s8(vi3x89ABCDEF, vk3x89ABCDEF);
627
628 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
629 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
630 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
631 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
632
633 const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
634 const int8x8_t vk4x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
635 const int8x8_t vi4x89ABCDEF = vld1_s8(i4); i4 += 8;
636 const int8x8_t vk4x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
637
638 vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
639 vprod89ABCDEF = vmull_s8(vi4x89ABCDEF, vk4x89ABCDEF);
640
641 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
642 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
643 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
644 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
645
646 const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
647 const int8x8_t vk5x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
648 const int8x8_t vi5x89ABCDEF = vld1_s8(i5); i5 += 8;
649 const int8x8_t vk5x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
650
651 vprod01234567 = vmull_s8(vi5x01234567, vk5x01234567);
652 vprod89ABCDEF = vmull_s8(vi5x89ABCDEF, vk5x89ABCDEF);
653
654 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
655 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
656 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
657 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
658
659 const int8x8_t vi6x01234567 = vld1_s8(i6); i6 += 8;
660 const int8x8_t vk6x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
661 const int8x8_t vi6x89ABCDEF = vld1_s8(i6); i6 += 8;
662 const int8x8_t vk6x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
663
664 vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
665 vprod89ABCDEF = vmull_s8(vi6x89ABCDEF, vk6x89ABCDEF);
666
667 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
668 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
669 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
670 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
671
672 const int8x8_t vi7x01234567 = vld1_s8(i7); i7 += 8;
673 const int8x8_t vk7x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
674 const int8x8_t vi7x89ABCDEF = vld1_s8(i7); i7 += 8;
675 const int8x8_t vk7x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
676
677 vprod01234567 = vmull_s8(vi7x01234567, vk7x01234567);
678 vprod89ABCDEF = vmull_s8(vi7x89ABCDEF, vk7x89ABCDEF);
679
680 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
681 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
682 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
683 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
684
685 const int8x8_t vi8x01234567 = vld1_s8(i8); i8 += 8;
686 const int8x8_t vk8x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
687 const int8x8_t vi8x89ABCDEF = vld1_s8(i8); i8 += 8;
688 const int8x8_t vk8x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
689
690 vprod01234567 = vmull_s8(vi8x01234567, vk8x01234567);
691 vprod89ABCDEF = vmull_s8(vi8x89ABCDEF, vk8x89ABCDEF);
692
693 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
694 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
695 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
696 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
697
698 vacc0123 = vqshlq_s32(vacc0123, vright_pre_shift);
699 vacc4567 = vqshlq_s32(vacc4567, vright_pre_shift);
700 vacc89AB = vqshlq_s32(vacc89AB, vright_pre_shift);
701 vaccCDEF = vqshlq_s32(vaccCDEF, vright_pre_shift);
702
703 vacc0123 = vqdmulhq_s32(vacc0123, vmultiplier);
704 vacc4567 = vqdmulhq_s32(vacc4567, vmultiplier);
705 vacc89AB = vqdmulhq_s32(vacc89AB, vmultiplier);
706 vaccCDEF = vqdmulhq_s32(vaccCDEF, vmultiplier);
707
708 vacc0123 = vrshlq_s32(vacc0123, vright_post_shift);
709 vacc4567 = vrshlq_s32(vacc4567, vright_post_shift);
710 vacc89AB = vrshlq_s32(vacc89AB, vright_post_shift);
711 vaccCDEF = vrshlq_s32(vaccCDEF, vright_post_shift);
712
713 #if XNN_ARCH_ARM64
714 int16x8_t vacc01234567 = vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567);
715 int16x8_t vacc89ABCDEF = vqmovn_high_s32(vqmovn_s32(vacc89AB), vaccCDEF);
716
717 vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
718 vacc89ABCDEF = vqaddq_s16(vacc89ABCDEF, voutput_zero_point);
719
720 int8x16_t vout0123456789ABCDEF = vqmovn_high_s16(vqmovn_s16(vacc01234567), vacc89ABCDEF);
721 #else // !XNN_ARCH_ARM64
722 int16x8_t vacc01234567 = vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567));
723 int16x8_t vacc89ABCDEF = vcombine_s16(vqmovn_s32(vacc89AB), vqmovn_s32(vaccCDEF));
724
725 vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
726 vacc89ABCDEF = vqaddq_s16(vacc89ABCDEF, voutput_zero_point);
727
728 int8x16_t vout0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc01234567), vqmovn_s16(vacc89ABCDEF));
729 #endif // !XNN_ARCH_ARM64
730
731 vout0123456789ABCDEF = vmaxq_s8(vout0123456789ABCDEF, voutput_min);
732
733 vout0123456789ABCDEF = vminq_s8(vout0123456789ABCDEF, voutput_max);
734
735 vst1q_s8(output, vout0123456789ABCDEF); output += 16;
736 }
737
738
739 if XNN_UNLIKELY(c != 0) {
740 do {
741 int32x4_t vacc0123 = vld1q_s32(b); b += 4;
742 int32x4_t vacc4567 = vld1q_s32(b); b += 4;
743
744 const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
745 const int8x8_t vk0x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
746
747 int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
748
749 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
750 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
751 const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
752 const int8x8_t vk1x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
753
754 vprod01234567 = vmull_s8(vi1x01234567, vk1x01234567);
755
756 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
757 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
758 const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
759 const int8x8_t vk2x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
760
761 vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
762
763 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
764 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
765 const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
766 const int8x8_t vk3x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
767
768 vprod01234567 = vmull_s8(vi3x01234567, vk3x01234567);
769
770 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
771 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
772 const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
773 const int8x8_t vk4x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
774
775 vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
776
777 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
778 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
779 const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
780 const int8x8_t vk5x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
781
782 vprod01234567 = vmull_s8(vi5x01234567, vk5x01234567);
783
784 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
785 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
786 const int8x8_t vi6x01234567 = vld1_s8(i6); i6 += 8;
787 const int8x8_t vk6x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
788
789 vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
790
791 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
792 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
793 const int8x8_t vi7x01234567 = vld1_s8(i7); i7 += 8;
794 const int8x8_t vk7x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
795
796 vprod01234567 = vmull_s8(vi7x01234567, vk7x01234567);
797
798 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
799 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
800 const int8x8_t vi8x01234567 = vld1_s8(i8); i8 += 8;
801 const int8x8_t vk8x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
802
803 vprod01234567 = vmull_s8(vi8x01234567, vk8x01234567);
804
805 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
806 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
807
808 vacc0123 = vqshlq_s32(vacc0123, vright_pre_shift);
809 vacc4567 = vqshlq_s32(vacc4567, vright_pre_shift);
810
811 vacc0123 = vqdmulhq_s32(vacc0123, vmultiplier);
812 vacc4567 = vqdmulhq_s32(vacc4567, vmultiplier);
813
814 vacc0123 = vrshlq_s32(vacc0123, vright_post_shift);
815 vacc4567 = vrshlq_s32(vacc4567, vright_post_shift);
816
817 #if XNN_ARCH_ARM64
818 int16x8_t vacc01234567 = vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567);
819 #else
820 int16x8_t vacc01234567 = vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567));
821 #endif
822 vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
823
824 int8x8_t vout01234567 = vqmovn_s16(vacc01234567);
825 vout01234567 = vmax_s8(vout01234567, vget_low_s8(voutput_min));
826 vout01234567 = vmin_s8(vout01234567, vget_low_s8(voutput_max));
827
828 if XNN_LIKELY(c >= 8) {
829 vst1_s8(output, vout01234567); output += 8;
830 c -= 8;
831 } else {
832 if (c & 4) {
833 vst1_lane_u32((void*) output, vreinterpret_u32_s8(vout01234567), 0); output += 4;
834 vout01234567 = vext_s8(vout01234567, vout01234567, 4);
835 }
836 if (c & 2) {
837 vst1_lane_u16((void*) output, vreinterpret_u16_s8(vout01234567), 0); output += 2;
838 vout01234567 = vext_s8(vout01234567, vout01234567, 2);
839 }
840 if (c & 1) {
841 vst1_lane_s8(output, vout01234567, 0); output += 1;
842 }
843 c = 0;
844 }
845
846 } while (c != 0);
847 }
848 }
849
850 input = (const int8_t**) ((uintptr_t) input + input_stride);
851 output = (int8_t*) ((uintptr_t) output + output_increment);
852 } while (--output_width != 0);
853 }