Debian Codegraph @

src/qs8-dwconv/gen/qs8-dwconv-5f5m5l16c8s8r-minmax-rndnu-neon-mla8-ld64.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-dwconv/multipass-neon-mul8.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12
13 #include <arm_neon.h>
14
15 #include "xnnpack/dwconv.h"
16 #include "xnnpack/math.h"
17
18
19 void xnn_qs8_dwconv_minmax_rndnu_ukernel_5f5m5l16c8s8r__neon_mla8_ld64(
20 size_t channels,
21 size_t output_width,
22 const int8_t** input,
23 const void* weights,
24 int8_t* output,
25 intptr_t input_stride,
26 size_t output_increment,
27 size_t input_offset,
28 const int8_t* zero,
29 size_t kernel_size,
30 int32_t* buffer,
31 const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
32 {
33 assert(channels != 0);
34 assert(output_width != 0);
35 assert(kernel_size > 5);
36
37 const int32x4_t vright_pre_shift = vld1q_dup_s32(&params->rndnu_neon.right_pre_shift);
38 const int32x4_t vmultiplier = vld1q_dup_s32(&params->rndnu_neon.multiplier);
39 const int32x4_t vright_post_shift = vld1q_dup_s32(&params->rndnu_neon.right_post_shift);
40 const int16x8_t voutput_zero_point = vld1q_dup_s16(&params->rndnu_neon.output_zero_point);
41 const int8x16_t voutput_min = vld1q_dup_s8(&params->rndnu_neon.output_min);
42 const int8x16_t voutput_max = vld1q_dup_s8(&params->rndnu_neon.output_max);
43 do {
44 const void* w = weights;
45
46 // First pass to process 5 inputs.
47 {
48 int32_t* b = buffer;
49 const int8_t* i0 = input[0];
50 assert(i0 != NULL);
51 if XNN_UNPREDICTABLE(i0 != zero) {
52 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
53 }
54 const int8_t* i1 = input[1];
55 assert(i1 != NULL);
56 if XNN_UNPREDICTABLE(i1 != zero) {
57 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
58 }
59 const int8_t* i2 = input[2];
60 assert(i2 != NULL);
61 if XNN_UNPREDICTABLE(i2 != zero) {
62 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
63 }
64 const int8_t* i3 = input[3];
65 assert(i3 != NULL);
66 if XNN_UNPREDICTABLE(i3 != zero) {
67 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
68 }
69 const int8_t* i4 = input[4];
70 assert(i4 != NULL);
71 if XNN_UNPREDICTABLE(i4 != zero) {
72 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
73 }
74 input += 5;
75
76 size_t c = round_up_po2(channels, 8);
77
78 for (; c >= 16; c -= 16) {
79 int32x4_t vacc0123 = vld1q_s32(w); w = (const int32_t*) w + 4;
80 int32x4_t vacc4567 = vld1q_s32(w); w = (const int32_t*) w + 4;
81 int32x4_t vacc89AB = vld1q_s32(w); w = (const int32_t*) w + 4;
82 int32x4_t vaccCDEF = vld1q_s32(w); w = (const int32_t*) w + 4;
83
84 const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
85 const int8x8_t vk0x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
86 const int8x8_t vi0x89ABCDEF = vld1_s8(i0); i0 += 8;
87 const int8x8_t vk0x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
88
89 int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
90 int16x8_t vprod89ABCDEF = vmull_s8(vi0x89ABCDEF, vk0x89ABCDEF);
91
92
93 const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
94 const int8x8_t vk1x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
95 const int8x8_t vi1x89ABCDEF = vld1_s8(i1); i1 += 8;
96 const int8x8_t vk1x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
97
98 vprod01234567 = vmlal_s8(vprod01234567, vi1x01234567, vk1x01234567);
99 vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi1x89ABCDEF, vk1x89ABCDEF);
100
101 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
102 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
103 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
104 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
105
106 const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
107 const int8x8_t vk2x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
108 const int8x8_t vi2x89ABCDEF = vld1_s8(i2); i2 += 8;
109 const int8x8_t vk2x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
110
111 vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
112 vprod89ABCDEF = vmull_s8(vi2x89ABCDEF, vk2x89ABCDEF);
113
114
115 const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
116 const int8x8_t vk3x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
117 const int8x8_t vi3x89ABCDEF = vld1_s8(i3); i3 += 8;
118 const int8x8_t vk3x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
119
120 vprod01234567 = vmlal_s8(vprod01234567, vi3x01234567, vk3x01234567);
121 vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi3x89ABCDEF, vk3x89ABCDEF);
122
123 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
124 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
125 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
126 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
127
128 const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
129 const int8x8_t vk4x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
130 const int8x8_t vi4x89ABCDEF = vld1_s8(i4); i4 += 8;
131 const int8x8_t vk4x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
132
133 vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
134 vprod89ABCDEF = vmull_s8(vi4x89ABCDEF, vk4x89ABCDEF);
135
136 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
137 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
138 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
139 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
140
141 vst1q_s32(b, vacc0123); b += 4;
142 vst1q_s32(b, vacc4567); b += 4;
143 vst1q_s32(b, vacc89AB); b += 4;
144 vst1q_s32(b, vaccCDEF); b += 4;
145 }
146
147 if XNN_UNLIKELY(c != 0) {
148 do {
149 int32x4_t vacc0123 = vld1q_s32(w); w = (const int32_t*) w + 4;
150 int32x4_t vacc4567 = vld1q_s32(w); w = (const int32_t*) w + 4;
151
152 const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
153 const int8x8_t vk0x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
154
155 int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
156
157
158 const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
159 const int8x8_t vk1x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
160
161 vprod01234567 = vmlal_s8(vprod01234567, vi1x01234567, vk1x01234567);
162
163 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
164 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
165
166 const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
167 const int8x8_t vk2x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
168
169 vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
170
171
172 const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
173 const int8x8_t vk3x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
174
175 vprod01234567 = vmlal_s8(vprod01234567, vi3x01234567, vk3x01234567);
176
177 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
178 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
179
180 const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
181 const int8x8_t vk4x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
182
183 vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
184
185 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
186 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
187
188 vst1q_s32(b, vacc0123); b += 4;
189 vst1q_s32(b, vacc4567); b += 4;
190 c -= 8;
191 } while (c != 0);
192 }
193 }
194
195 // Middle pass to process 5 inputs in each iteration.
196 for (size_t ks = kernel_size - 5; ks > 5; ks -= 5) {
197 int32_t* b = buffer;
198
199 const int8_t* i0 = input[0];
200 assert(i0 != NULL);
201 if XNN_UNPREDICTABLE(i0 != zero) {
202 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
203 }
204 const int8_t* i1 = input[1];
205 assert(i1 != NULL);
206 if XNN_UNPREDICTABLE(i1 != zero) {
207 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
208 }
209 const int8_t* i2 = input[2];
210 assert(i2 != NULL);
211 if XNN_UNPREDICTABLE(i2 != zero) {
212 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
213 }
214 const int8_t* i3 = input[3];
215 assert(i3 != NULL);
216 if XNN_UNPREDICTABLE(i3 != zero) {
217 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
218 }
219 const int8_t* i4 = input[4];
220 assert(i4 != NULL);
221 if XNN_UNPREDICTABLE(i4 != zero) {
222 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
223 }
224 input += 5;
225
226 size_t c = round_up_po2(channels, 8);
227
228 for (; c >= 16; c -= 16) {
229 int32x4_t vacc0123 = vld1q_s32(b);
230 int32x4_t vacc4567 = vld1q_s32(b + 4);
231 int32x4_t vacc89AB = vld1q_s32(b + 8);
232 int32x4_t vaccCDEF = vld1q_s32(b + 12);
233
234 const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
235 const int8x8_t vk0x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
236 const int8x8_t vi0x89ABCDEF = vld1_s8(i0); i0 += 8;
237 const int8x8_t vk0x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
238
239 int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
240 int16x8_t vprod89ABCDEF = vmull_s8(vi0x89ABCDEF, vk0x89ABCDEF);
241
242
243 const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
244 const int8x8_t vk1x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
245 const int8x8_t vi1x89ABCDEF = vld1_s8(i1); i1 += 8;
246 const int8x8_t vk1x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
247
248 vprod01234567 = vmlal_s8(vprod01234567, vi1x01234567, vk1x01234567);
249 vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi1x89ABCDEF, vk1x89ABCDEF);
250
251 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
252 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
253 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
254 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
255
256 const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
257 const int8x8_t vk2x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
258 const int8x8_t vi2x89ABCDEF = vld1_s8(i2); i2 += 8;
259 const int8x8_t vk2x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
260
261 vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
262 vprod89ABCDEF = vmull_s8(vi2x89ABCDEF, vk2x89ABCDEF);
263
264
265 const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
266 const int8x8_t vk3x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
267 const int8x8_t vi3x89ABCDEF = vld1_s8(i3); i3 += 8;
268 const int8x8_t vk3x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
269
270 vprod01234567 = vmlal_s8(vprod01234567, vi3x01234567, vk3x01234567);
271 vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi3x89ABCDEF, vk3x89ABCDEF);
272
273 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
274 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
275 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
276 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
277
278 const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
279 const int8x8_t vk4x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
280 const int8x8_t vi4x89ABCDEF = vld1_s8(i4); i4 += 8;
281 const int8x8_t vk4x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
282
283 vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
284 vprod89ABCDEF = vmull_s8(vi4x89ABCDEF, vk4x89ABCDEF);
285
286 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
287 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
288 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
289 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
290
291 vst1q_s32(b, vacc0123); b += 4;
292 vst1q_s32(b, vacc4567); b += 4;
293 vst1q_s32(b, vacc89AB); b += 4;
294 vst1q_s32(b, vaccCDEF); b += 4;
295 }
296
297 if XNN_UNLIKELY(c != 0) {
298 do {
299 int32x4_t vacc0123 = vld1q_s32(b);
300 int32x4_t vacc4567 = vld1q_s32(b + 4);
301
302 const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
303 const int8x8_t vk0x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
304
305 int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
306
307
308 const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
309 const int8x8_t vk1x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
310
311 vprod01234567 = vmlal_s8(vprod01234567, vi1x01234567, vk1x01234567);
312
313 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
314 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
315
316 const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
317 const int8x8_t vk2x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
318
319 vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
320
321
322 const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
323 const int8x8_t vk3x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
324
325 vprod01234567 = vmlal_s8(vprod01234567, vi3x01234567, vk3x01234567);
326
327 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
328 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
329
330 const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
331 const int8x8_t vk4x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
332
333 vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
334
335 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
336 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
337
338 vst1q_s32(b, vacc0123); b += 4;
339 vst1q_s32(b, vacc4567); b += 4;
340 c -= 8;
341 } while (c != 0);
342 }
343 }
344
345 // Last pass to process up to 5 inputs.
346 {
347 const int32_t* b = buffer;
348 const int8_t* i0 = input[0];
349 assert(i0 != NULL);
350 if XNN_UNPREDICTABLE(i0 != zero) {
351 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
352 }
353 const int8_t* i1 = input[1];
354 assert(i1 != NULL);
355 if XNN_UNPREDICTABLE(i1 != zero) {
356 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
357 }
358 const int8_t* i2 = input[2];
359 assert(i2 != NULL);
360 if XNN_UNPREDICTABLE(i2 != zero) {
361 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
362 }
363 const int8_t* i3 = input[3];
364 assert(i3 != NULL);
365 if XNN_UNPREDICTABLE(i3 != zero) {
366 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
367 }
368 const int8_t* i4 = input[4];
369 assert(i4 != NULL);
370 if XNN_UNPREDICTABLE(i4 != zero) {
371 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
372 }
373
374 size_t c = channels;
375 for (; c >= 16; c -= 16) {
376 int32x4_t vacc0123 = vld1q_s32(b); b += 4;
377 int32x4_t vacc4567 = vld1q_s32(b); b += 4;
378 int32x4_t vacc89AB = vld1q_s32(b); b += 4;
379 int32x4_t vaccCDEF = vld1q_s32(b); b += 4;
380
381 const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
382 const int8x8_t vk0x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
383 const int8x8_t vi0x89ABCDEF = vld1_s8(i0); i0 += 8;
384 const int8x8_t vk0x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
385
386 int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
387 int16x8_t vprod89ABCDEF = vmull_s8(vi0x89ABCDEF, vk0x89ABCDEF);
388
389
390 const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
391 const int8x8_t vk1x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
392 const int8x8_t vi1x89ABCDEF = vld1_s8(i1); i1 += 8;
393 const int8x8_t vk1x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
394
395 vprod01234567 = vmlal_s8(vprod01234567, vi1x01234567, vk1x01234567);
396 vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi1x89ABCDEF, vk1x89ABCDEF);
397
398 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
399 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
400 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
401 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
402
403 const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
404 const int8x8_t vk2x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
405 const int8x8_t vi2x89ABCDEF = vld1_s8(i2); i2 += 8;
406 const int8x8_t vk2x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
407
408 vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
409 vprod89ABCDEF = vmull_s8(vi2x89ABCDEF, vk2x89ABCDEF);
410
411
412 const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
413 const int8x8_t vk3x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
414 const int8x8_t vi3x89ABCDEF = vld1_s8(i3); i3 += 8;
415 const int8x8_t vk3x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
416
417 vprod01234567 = vmlal_s8(vprod01234567, vi3x01234567, vk3x01234567);
418 vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi3x89ABCDEF, vk3x89ABCDEF);
419
420 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
421 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
422 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
423 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
424
425 const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
426 const int8x8_t vk4x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
427 const int8x8_t vi4x89ABCDEF = vld1_s8(i4); i4 += 8;
428 const int8x8_t vk4x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
429
430 vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
431 vprod89ABCDEF = vmull_s8(vi4x89ABCDEF, vk4x89ABCDEF);
432
433 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
434 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
435 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
436 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
437
438 vacc0123 = vqshlq_s32(vacc0123, vright_pre_shift);
439 vacc4567 = vqshlq_s32(vacc4567, vright_pre_shift);
440 vacc89AB = vqshlq_s32(vacc89AB, vright_pre_shift);
441 vaccCDEF = vqshlq_s32(vaccCDEF, vright_pre_shift);
442
443 vacc0123 = vqdmulhq_s32(vacc0123, vmultiplier);
444 vacc4567 = vqdmulhq_s32(vacc4567, vmultiplier);
445 vacc89AB = vqdmulhq_s32(vacc89AB, vmultiplier);
446 vaccCDEF = vqdmulhq_s32(vaccCDEF, vmultiplier);
447
448 vacc0123 = vrshlq_s32(vacc0123, vright_post_shift);
449 vacc4567 = vrshlq_s32(vacc4567, vright_post_shift);
450 vacc89AB = vrshlq_s32(vacc89AB, vright_post_shift);
451 vaccCDEF = vrshlq_s32(vaccCDEF, vright_post_shift);
452
453 #if XNN_ARCH_ARM64
454 int16x8_t vacc01234567 = vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567);
455 int16x8_t vacc89ABCDEF = vqmovn_high_s32(vqmovn_s32(vacc89AB), vaccCDEF);
456
457 vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
458 vacc89ABCDEF = vqaddq_s16(vacc89ABCDEF, voutput_zero_point);
459
460 int8x16_t vout0123456789ABCDEF = vqmovn_high_s16(vqmovn_s16(vacc01234567), vacc89ABCDEF);
461 #else // !XNN_ARCH_ARM64
462 int16x8_t vacc01234567 = vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567));
463 int16x8_t vacc89ABCDEF = vcombine_s16(vqmovn_s32(vacc89AB), vqmovn_s32(vaccCDEF));
464
465 vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
466 vacc89ABCDEF = vqaddq_s16(vacc89ABCDEF, voutput_zero_point);
467
468 int8x16_t vout0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc01234567), vqmovn_s16(vacc89ABCDEF));
469 #endif // !XNN_ARCH_ARM64
470
471 vout0123456789ABCDEF = vmaxq_s8(vout0123456789ABCDEF, voutput_min);
472
473 vout0123456789ABCDEF = vminq_s8(vout0123456789ABCDEF, voutput_max);
474
475 vst1q_s8(output, vout0123456789ABCDEF); output += 16;
476 }
477
478
479 if XNN_UNLIKELY(c != 0) {
480 do {
481 int32x4_t vacc0123 = vld1q_s32(b); b += 4;
482 int32x4_t vacc4567 = vld1q_s32(b); b += 4;
483
484 const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
485 const int8x8_t vk0x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
486
487 int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
488
489 const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
490 const int8x8_t vk1x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
491
492 vprod01234567 = vmlal_s8(vprod01234567, vi1x01234567, vk1x01234567);
493
494 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
495 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
496 const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
497 const int8x8_t vk2x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
498
499 vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
500
501 const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
502 const int8x8_t vk3x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
503
504 vprod01234567 = vmlal_s8(vprod01234567, vi3x01234567, vk3x01234567);
505
506 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
507 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
508 const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
509 const int8x8_t vk4x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
510
511 vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
512
513 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
514 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
515
516 vacc0123 = vqshlq_s32(vacc0123, vright_pre_shift);
517 vacc4567 = vqshlq_s32(vacc4567, vright_pre_shift);
518
519 vacc0123 = vqdmulhq_s32(vacc0123, vmultiplier);
520 vacc4567 = vqdmulhq_s32(vacc4567, vmultiplier);
521
522 vacc0123 = vrshlq_s32(vacc0123, vright_post_shift);
523 vacc4567 = vrshlq_s32(vacc4567, vright_post_shift);
524
525 #if XNN_ARCH_ARM64
526 int16x8_t vacc01234567 = vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567);
527 #else
528 int16x8_t vacc01234567 = vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567));
529 #endif
530 vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
531
532 int8x8_t vout01234567 = vqmovn_s16(vacc01234567);
533 vout01234567 = vmax_s8(vout01234567, vget_low_s8(voutput_min));
534 vout01234567 = vmin_s8(vout01234567, vget_low_s8(voutput_max));
535
536 if XNN_LIKELY(c >= 8) {
537 vst1_s8(output, vout01234567); output += 8;
538 c -= 8;
539 } else {
540 if (c & 4) {
541 vst1_lane_u32((void*) output, vreinterpret_u32_s8(vout01234567), 0); output += 4;
542 vout01234567 = vext_s8(vout01234567, vout01234567, 4);
543 }
544 if (c & 2) {
545 vst1_lane_u16((void*) output, vreinterpret_u16_s8(vout01234567), 0); output += 2;
546 vout01234567 = vext_s8(vout01234567, vout01234567, 2);
547 }
548 if (c & 1) {
549 vst1_lane_s8(output, vout01234567, 0); output += 1;
550 }
551 c = 0;
552 }
553
554 } while (c != 0);
555 }
556 }
557
558 input = (const int8_t**) ((uintptr_t) input + input_stride);
559 output = (int8_t*) ((uintptr_t) output + output_increment);
560 } while (--output_width != 0);
561 }