Debian Codegraph @

src/qs8-dwconv/gen/qs8-dwconv-5f5m5l16c8s8r-minmax-rndnu-neon-mla8-ld128.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-dwconv/multipass-neon-mul8.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12
13 #include <arm_neon.h>
14
15 #include "xnnpack/dwconv.h"
16 #include "xnnpack/math.h"
17
18
19 void xnn_qs8_dwconv_minmax_rndnu_ukernel_5f5m5l16c8s8r__neon_mla8_ld128(
20 size_t channels,
21 size_t output_width,
22 const int8_t** input,
23 const void* weights,
24 int8_t* output,
25 intptr_t input_stride,
26 size_t output_increment,
27 size_t input_offset,
28 const int8_t* zero,
29 size_t kernel_size,
30 int32_t* buffer,
31 const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
32 {
33 assert(channels != 0);
34 assert(output_width != 0);
35 assert(kernel_size > 5);
36
37 const int32x4_t vright_pre_shift = vld1q_dup_s32(&params->rndnu_neon.right_pre_shift);
38 const int32x4_t vmultiplier = vld1q_dup_s32(&params->rndnu_neon.multiplier);
39 const int32x4_t vright_post_shift = vld1q_dup_s32(&params->rndnu_neon.right_post_shift);
40 const int16x8_t voutput_zero_point = vld1q_dup_s16(&params->rndnu_neon.output_zero_point);
41 const int8x16_t voutput_min = vld1q_dup_s8(&params->rndnu_neon.output_min);
42 const int8x16_t voutput_max = vld1q_dup_s8(&params->rndnu_neon.output_max);
43 do {
44 const void* w = weights;
45
46 // First pass to process 5 inputs.
47 {
48 int32_t* b = buffer;
49 const int8_t* i0 = input[0];
50 assert(i0 != NULL);
51 if XNN_UNPREDICTABLE(i0 != zero) {
52 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
53 }
54 const int8_t* i1 = input[1];
55 assert(i1 != NULL);
56 if XNN_UNPREDICTABLE(i1 != zero) {
57 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
58 }
59 const int8_t* i2 = input[2];
60 assert(i2 != NULL);
61 if XNN_UNPREDICTABLE(i2 != zero) {
62 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
63 }
64 const int8_t* i3 = input[3];
65 assert(i3 != NULL);
66 if XNN_UNPREDICTABLE(i3 != zero) {
67 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
68 }
69 const int8_t* i4 = input[4];
70 assert(i4 != NULL);
71 if XNN_UNPREDICTABLE(i4 != zero) {
72 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
73 }
74 input += 5;
75
76 size_t c = round_up_po2(channels, 8);
77
78 for (; c >= 16; c -= 16) {
79 int32x4_t vacc0123 = vld1q_s32(w); w = (const int32_t*) w + 4;
80 int32x4_t vacc4567 = vld1q_s32(w); w = (const int32_t*) w + 4;
81 int32x4_t vacc89AB = vld1q_s32(w); w = (const int32_t*) w + 4;
82 int32x4_t vaccCDEF = vld1q_s32(w); w = (const int32_t*) w + 4;
83
84 const int8x16_t vi0x0123456789ABCDEF = vld1q_s8(i0); i0 += 16;
85 const int8x16_t vk0x0123456789ABCDEF = vld1q_s8(w); w = (const int8_t*) w + 16;
86
87 int16x8_t vprod01234567 = vmull_s8(vget_low_s8(vi0x0123456789ABCDEF), vget_low_s8(vk0x0123456789ABCDEF));
88 int16x8_t vprod89ABCDEF = vmull_s8(vget_high_s8(vi0x0123456789ABCDEF), vget_high_s8(vk0x0123456789ABCDEF));
89
90
91 const int8x16_t vi1x0123456789ABCDEF = vld1q_s8(i1); i1 += 16;
92 const int8x16_t vk1x0123456789ABCDEF = vld1q_s8(w); w = (const int8_t*) w + 16;
93
94 vprod01234567 = vmlal_s8(vprod01234567, vget_low_s8(vi1x0123456789ABCDEF), vget_low_s8(vk1x0123456789ABCDEF));
95 vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vget_high_s8(vi1x0123456789ABCDEF), vget_high_s8(vk1x0123456789ABCDEF));
96
97 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
98 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
99 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
100 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
101
102 const int8x16_t vi2x0123456789ABCDEF = vld1q_s8(i2); i2 += 16;
103 const int8x16_t vk2x0123456789ABCDEF = vld1q_s8(w); w = (const int8_t*) w + 16;
104
105 vprod01234567 = vmull_s8(vget_low_s8(vi2x0123456789ABCDEF), vget_low_s8(vk2x0123456789ABCDEF));
106 vprod89ABCDEF = vmull_s8(vget_high_s8(vi2x0123456789ABCDEF), vget_high_s8(vk2x0123456789ABCDEF));
107
108
109 const int8x16_t vi3x0123456789ABCDEF = vld1q_s8(i3); i3 += 16;
110 const int8x16_t vk3x0123456789ABCDEF = vld1q_s8(w); w = (const int8_t*) w + 16;
111
112 vprod01234567 = vmlal_s8(vprod01234567, vget_low_s8(vi3x0123456789ABCDEF), vget_low_s8(vk3x0123456789ABCDEF));
113 vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vget_high_s8(vi3x0123456789ABCDEF), vget_high_s8(vk3x0123456789ABCDEF));
114
115 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
116 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
117 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
118 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
119
120 const int8x16_t vi4x0123456789ABCDEF = vld1q_s8(i4); i4 += 16;
121 const int8x16_t vk4x0123456789ABCDEF = vld1q_s8(w); w = (const int8_t*) w + 16;
122
123 vprod01234567 = vmull_s8(vget_low_s8(vi4x0123456789ABCDEF), vget_low_s8(vk4x0123456789ABCDEF));
124 vprod89ABCDEF = vmull_s8(vget_high_s8(vi4x0123456789ABCDEF), vget_high_s8(vk4x0123456789ABCDEF));
125
126 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
127 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
128 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
129 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
130
131 vst1q_s32(b, vacc0123); b += 4;
132 vst1q_s32(b, vacc4567); b += 4;
133 vst1q_s32(b, vacc89AB); b += 4;
134 vst1q_s32(b, vaccCDEF); b += 4;
135 }
136
137 if XNN_UNLIKELY(c != 0) {
138 do {
139 int32x4_t vacc0123 = vld1q_s32(w); w = (const int32_t*) w + 4;
140 int32x4_t vacc4567 = vld1q_s32(w); w = (const int32_t*) w + 4;
141
142 const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
143 const int8x8_t vk0x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
144
145 int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
146
147
148 const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
149 const int8x8_t vk1x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
150
151 vprod01234567 = vmlal_s8(vprod01234567, vi1x01234567, vk1x01234567);
152
153 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
154 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
155
156 const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
157 const int8x8_t vk2x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
158
159 vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
160
161
162 const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
163 const int8x8_t vk3x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
164
165 vprod01234567 = vmlal_s8(vprod01234567, vi3x01234567, vk3x01234567);
166
167 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
168 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
169
170 const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
171 const int8x8_t vk4x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
172
173 vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
174
175 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
176 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
177
178 vst1q_s32(b, vacc0123); b += 4;
179 vst1q_s32(b, vacc4567); b += 4;
180 c -= 8;
181 } while (c != 0);
182 }
183 }
184
185 // Middle pass to process 5 inputs in each iteration.
186 for (size_t ks = kernel_size - 5; ks > 5; ks -= 5) {
187 int32_t* b = buffer;
188
189 const int8_t* i0 = input[0];
190 assert(i0 != NULL);
191 if XNN_UNPREDICTABLE(i0 != zero) {
192 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
193 }
194 const int8_t* i1 = input[1];
195 assert(i1 != NULL);
196 if XNN_UNPREDICTABLE(i1 != zero) {
197 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
198 }
199 const int8_t* i2 = input[2];
200 assert(i2 != NULL);
201 if XNN_UNPREDICTABLE(i2 != zero) {
202 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
203 }
204 const int8_t* i3 = input[3];
205 assert(i3 != NULL);
206 if XNN_UNPREDICTABLE(i3 != zero) {
207 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
208 }
209 const int8_t* i4 = input[4];
210 assert(i4 != NULL);
211 if XNN_UNPREDICTABLE(i4 != zero) {
212 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
213 }
214 input += 5;
215
216 size_t c = round_up_po2(channels, 8);
217
218 for (; c >= 16; c -= 16) {
219 int32x4_t vacc0123 = vld1q_s32(b);
220 int32x4_t vacc4567 = vld1q_s32(b + 4);
221 int32x4_t vacc89AB = vld1q_s32(b + 8);
222 int32x4_t vaccCDEF = vld1q_s32(b + 12);
223
224 const int8x16_t vi0x0123456789ABCDEF = vld1q_s8(i0); i0 += 16;
225 const int8x16_t vk0x0123456789ABCDEF = vld1q_s8(w); w = (const int8_t*) w + 16;
226
227 int16x8_t vprod01234567 = vmull_s8(vget_low_s8(vi0x0123456789ABCDEF), vget_low_s8(vk0x0123456789ABCDEF));
228 int16x8_t vprod89ABCDEF = vmull_s8(vget_high_s8(vi0x0123456789ABCDEF), vget_high_s8(vk0x0123456789ABCDEF));
229
230
231 const int8x16_t vi1x0123456789ABCDEF = vld1q_s8(i1); i1 += 16;
232 const int8x16_t vk1x0123456789ABCDEF = vld1q_s8(w); w = (const int8_t*) w + 16;
233
234 vprod01234567 = vmlal_s8(vprod01234567, vget_low_s8(vi1x0123456789ABCDEF), vget_low_s8(vk1x0123456789ABCDEF));
235 vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vget_high_s8(vi1x0123456789ABCDEF), vget_high_s8(vk1x0123456789ABCDEF));
236
237 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
238 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
239 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
240 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
241
242 const int8x16_t vi2x0123456789ABCDEF = vld1q_s8(i2); i2 += 16;
243 const int8x16_t vk2x0123456789ABCDEF = vld1q_s8(w); w = (const int8_t*) w + 16;
244
245 vprod01234567 = vmull_s8(vget_low_s8(vi2x0123456789ABCDEF), vget_low_s8(vk2x0123456789ABCDEF));
246 vprod89ABCDEF = vmull_s8(vget_high_s8(vi2x0123456789ABCDEF), vget_high_s8(vk2x0123456789ABCDEF));
247
248
249 const int8x16_t vi3x0123456789ABCDEF = vld1q_s8(i3); i3 += 16;
250 const int8x16_t vk3x0123456789ABCDEF = vld1q_s8(w); w = (const int8_t*) w + 16;
251
252 vprod01234567 = vmlal_s8(vprod01234567, vget_low_s8(vi3x0123456789ABCDEF), vget_low_s8(vk3x0123456789ABCDEF));
253 vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vget_high_s8(vi3x0123456789ABCDEF), vget_high_s8(vk3x0123456789ABCDEF));
254
255 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
256 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
257 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
258 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
259
260 const int8x16_t vi4x0123456789ABCDEF = vld1q_s8(i4); i4 += 16;
261 const int8x16_t vk4x0123456789ABCDEF = vld1q_s8(w); w = (const int8_t*) w + 16;
262
263 vprod01234567 = vmull_s8(vget_low_s8(vi4x0123456789ABCDEF), vget_low_s8(vk4x0123456789ABCDEF));
264 vprod89ABCDEF = vmull_s8(vget_high_s8(vi4x0123456789ABCDEF), vget_high_s8(vk4x0123456789ABCDEF));
265
266 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
267 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
268 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
269 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
270
271 vst1q_s32(b, vacc0123); b += 4;
272 vst1q_s32(b, vacc4567); b += 4;
273 vst1q_s32(b, vacc89AB); b += 4;
274 vst1q_s32(b, vaccCDEF); b += 4;
275 }
276
277 if XNN_UNLIKELY(c != 0) {
278 do {
279 int32x4_t vacc0123 = vld1q_s32(b);
280 int32x4_t vacc4567 = vld1q_s32(b + 4);
281
282 const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
283 const int8x8_t vk0x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
284
285 int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
286
287
288 const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
289 const int8x8_t vk1x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
290
291 vprod01234567 = vmlal_s8(vprod01234567, vi1x01234567, vk1x01234567);
292
293 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
294 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
295
296 const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
297 const int8x8_t vk2x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
298
299 vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
300
301
302 const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
303 const int8x8_t vk3x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
304
305 vprod01234567 = vmlal_s8(vprod01234567, vi3x01234567, vk3x01234567);
306
307 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
308 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
309
310 const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
311 const int8x8_t vk4x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
312
313 vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
314
315 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
316 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
317
318 vst1q_s32(b, vacc0123); b += 4;
319 vst1q_s32(b, vacc4567); b += 4;
320 c -= 8;
321 } while (c != 0);
322 }
323 }
324
325 // Last pass to process up to 5 inputs.
326 {
327 const int32_t* b = buffer;
328 const int8_t* i0 = input[0];
329 assert(i0 != NULL);
330 if XNN_UNPREDICTABLE(i0 != zero) {
331 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
332 }
333 const int8_t* i1 = input[1];
334 assert(i1 != NULL);
335 if XNN_UNPREDICTABLE(i1 != zero) {
336 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
337 }
338 const int8_t* i2 = input[2];
339 assert(i2 != NULL);
340 if XNN_UNPREDICTABLE(i2 != zero) {
341 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
342 }
343 const int8_t* i3 = input[3];
344 assert(i3 != NULL);
345 if XNN_UNPREDICTABLE(i3 != zero) {
346 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
347 }
348 const int8_t* i4 = input[4];
349 assert(i4 != NULL);
350 if XNN_UNPREDICTABLE(i4 != zero) {
351 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
352 }
353
354 size_t c = channels;
355 for (; c >= 16; c -= 16) {
356 int32x4_t vacc0123 = vld1q_s32(b); b += 4;
357 int32x4_t vacc4567 = vld1q_s32(b); b += 4;
358 int32x4_t vacc89AB = vld1q_s32(b); b += 4;
359 int32x4_t vaccCDEF = vld1q_s32(b); b += 4;
360
361 const int8x16_t vi0x0123456789ABCDEF = vld1q_s8(i0); i0 += 16;
362 const int8x16_t vk0x0123456789ABCDEF = vld1q_s8(w); w = (const int8_t*) w + 16;
363
364 int16x8_t vprod01234567 = vmull_s8(vget_low_s8(vi0x0123456789ABCDEF), vget_low_s8(vk0x0123456789ABCDEF));
365 int16x8_t vprod89ABCDEF = vmull_s8(vget_high_s8(vi0x0123456789ABCDEF), vget_high_s8(vk0x0123456789ABCDEF));
366
367
368 const int8x16_t vi1x0123456789ABCDEF = vld1q_s8(i1); i1 += 16;
369 const int8x16_t vk1x0123456789ABCDEF = vld1q_s8(w); w = (const int8_t*) w + 16;
370
371 vprod01234567 = vmlal_s8(vprod01234567, vget_low_s8(vi1x0123456789ABCDEF), vget_low_s8(vk1x0123456789ABCDEF));
372 vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vget_high_s8(vi1x0123456789ABCDEF), vget_high_s8(vk1x0123456789ABCDEF));
373
374 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
375 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
376 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
377 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
378
379 const int8x16_t vi2x0123456789ABCDEF = vld1q_s8(i2); i2 += 16;
380 const int8x16_t vk2x0123456789ABCDEF = vld1q_s8(w); w = (const int8_t*) w + 16;
381
382 vprod01234567 = vmull_s8(vget_low_s8(vi2x0123456789ABCDEF), vget_low_s8(vk2x0123456789ABCDEF));
383 vprod89ABCDEF = vmull_s8(vget_high_s8(vi2x0123456789ABCDEF), vget_high_s8(vk2x0123456789ABCDEF));
384
385
386 const int8x16_t vi3x0123456789ABCDEF = vld1q_s8(i3); i3 += 16;
387 const int8x16_t vk3x0123456789ABCDEF = vld1q_s8(w); w = (const int8_t*) w + 16;
388
389 vprod01234567 = vmlal_s8(vprod01234567, vget_low_s8(vi3x0123456789ABCDEF), vget_low_s8(vk3x0123456789ABCDEF));
390 vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vget_high_s8(vi3x0123456789ABCDEF), vget_high_s8(vk3x0123456789ABCDEF));
391
392 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
393 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
394 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
395 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
396
397 const int8x16_t vi4x0123456789ABCDEF = vld1q_s8(i4); i4 += 16;
398 const int8x16_t vk4x0123456789ABCDEF = vld1q_s8(w); w = (const int8_t*) w + 16;
399
400 vprod01234567 = vmull_s8(vget_low_s8(vi4x0123456789ABCDEF), vget_low_s8(vk4x0123456789ABCDEF));
401 vprod89ABCDEF = vmull_s8(vget_high_s8(vi4x0123456789ABCDEF), vget_high_s8(vk4x0123456789ABCDEF));
402
403 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
404 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
405 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
406 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
407
408 vacc0123 = vqshlq_s32(vacc0123, vright_pre_shift);
409 vacc4567 = vqshlq_s32(vacc4567, vright_pre_shift);
410 vacc89AB = vqshlq_s32(vacc89AB, vright_pre_shift);
411 vaccCDEF = vqshlq_s32(vaccCDEF, vright_pre_shift);
412
413 vacc0123 = vqdmulhq_s32(vacc0123, vmultiplier);
414 vacc4567 = vqdmulhq_s32(vacc4567, vmultiplier);
415 vacc89AB = vqdmulhq_s32(vacc89AB, vmultiplier);
416 vaccCDEF = vqdmulhq_s32(vaccCDEF, vmultiplier);
417
418 vacc0123 = vrshlq_s32(vacc0123, vright_post_shift);
419 vacc4567 = vrshlq_s32(vacc4567, vright_post_shift);
420 vacc89AB = vrshlq_s32(vacc89AB, vright_post_shift);
421 vaccCDEF = vrshlq_s32(vaccCDEF, vright_post_shift);
422
423 #if XNN_ARCH_ARM64
424 int16x8_t vacc01234567 = vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567);
425 int16x8_t vacc89ABCDEF = vqmovn_high_s32(vqmovn_s32(vacc89AB), vaccCDEF);
426
427 vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
428 vacc89ABCDEF = vqaddq_s16(vacc89ABCDEF, voutput_zero_point);
429
430 int8x16_t vout0123456789ABCDEF = vqmovn_high_s16(vqmovn_s16(vacc01234567), vacc89ABCDEF);
431 #else // !XNN_ARCH_ARM64
432 int16x8_t vacc01234567 = vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567));
433 int16x8_t vacc89ABCDEF = vcombine_s16(vqmovn_s32(vacc89AB), vqmovn_s32(vaccCDEF));
434
435 vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
436 vacc89ABCDEF = vqaddq_s16(vacc89ABCDEF, voutput_zero_point);
437
438 int8x16_t vout0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc01234567), vqmovn_s16(vacc89ABCDEF));
439 #endif // !XNN_ARCH_ARM64
440
441 vout0123456789ABCDEF = vmaxq_s8(vout0123456789ABCDEF, voutput_min);
442
443 vout0123456789ABCDEF = vminq_s8(vout0123456789ABCDEF, voutput_max);
444
445 vst1q_s8(output, vout0123456789ABCDEF); output += 16;
446 }
447
448
449 if XNN_UNLIKELY(c != 0) {
450 do {
451 int32x4_t vacc0123 = vld1q_s32(b); b += 4;
452 int32x4_t vacc4567 = vld1q_s32(b); b += 4;
453
454 const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
455 const int8x8_t vk0x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
456
457 int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
458
459 const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
460 const int8x8_t vk1x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
461
462 vprod01234567 = vmlal_s8(vprod01234567, vi1x01234567, vk1x01234567);
463
464 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
465 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
466 const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
467 const int8x8_t vk2x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
468
469 vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
470
471 const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
472 const int8x8_t vk3x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
473
474 vprod01234567 = vmlal_s8(vprod01234567, vi3x01234567, vk3x01234567);
475
476 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
477 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
478 const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
479 const int8x8_t vk4x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
480
481 vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
482
483 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
484 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
485
486 vacc0123 = vqshlq_s32(vacc0123, vright_pre_shift);
487 vacc4567 = vqshlq_s32(vacc4567, vright_pre_shift);
488
489 vacc0123 = vqdmulhq_s32(vacc0123, vmultiplier);
490 vacc4567 = vqdmulhq_s32(vacc4567, vmultiplier);
491
492 vacc0123 = vrshlq_s32(vacc0123, vright_post_shift);
493 vacc4567 = vrshlq_s32(vacc4567, vright_post_shift);
494
495 #if XNN_ARCH_ARM64
496 int16x8_t vacc01234567 = vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567);
497 #else
498 int16x8_t vacc01234567 = vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567));
499 #endif
500 vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
501
502 int8x8_t vout01234567 = vqmovn_s16(vacc01234567);
503 vout01234567 = vmax_s8(vout01234567, vget_low_s8(voutput_min));
504 vout01234567 = vmin_s8(vout01234567, vget_low_s8(voutput_max));
505
506 if XNN_LIKELY(c >= 8) {
507 vst1_s8(output, vout01234567); output += 8;
508 c -= 8;
509 } else {
510 if (c & 4) {
511 vst1_lane_u32((void*) output, vreinterpret_u32_s8(vout01234567), 0); output += 4;
512 vout01234567 = vext_s8(vout01234567, vout01234567, 4);
513 }
514 if (c & 2) {
515 vst1_lane_u16((void*) output, vreinterpret_u16_s8(vout01234567), 0); output += 2;
516 vout01234567 = vext_s8(vout01234567, vout01234567, 2);
517 }
518 if (c & 1) {
519 vst1_lane_s8(output, vout01234567, 0); output += 1;
520 }
521 c = 0;
522 }
523
524 } while (c != 0);
525 }
526 }
527
528 input = (const int8_t**) ((uintptr_t) input + input_stride);
529 output = (int8_t*) ((uintptr_t) output + output_increment);
530 } while (--output_width != 0);
531 }