Debian Codegraph @

src/qs8-dwconv/gen/qs8-dwconv-8f8m9l16c8s8r-minmax-rndnu-neon-mul16.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-dwconv/multipass-neon-mul16.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12
13 #include <arm_neon.h>
14
15 #include "xnnpack/dwconv.h"
16 #include "xnnpack/math.h"
17
18
19 void xnn_qs8_dwconv_minmax_rndnu_ukernel_8f8m9l16c8s8r__neon_mul16(
20 size_t channels,
21 size_t output_width,
22 const int8_t** input,
23 const void* weights,
24 int8_t* output,
25 intptr_t input_stride,
26 size_t output_increment,
27 size_t input_offset,
28 const int8_t* zero,
29 size_t kernel_size,
30 int32_t* buffer,
31 const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
32 {
33 assert(channels != 0);
34 assert(output_width != 0);
35 assert(kernel_size > 8);
36
37 const int32x4_t vright_pre_shift = vld1q_dup_s32(&params->rndnu_neon.right_pre_shift);
38 const int32x4_t vmultiplier = vld1q_dup_s32(&params->rndnu_neon.multiplier);
39 const int32x4_t vright_post_shift = vld1q_dup_s32(&params->rndnu_neon.right_post_shift);
40 const int16x8_t voutput_zero_point = vld1q_dup_s16(&params->rndnu_neon.output_zero_point);
41 const int8x16_t voutput_min = vld1q_dup_s8(&params->rndnu_neon.output_min);
42 const int8x16_t voutput_max = vld1q_dup_s8(&params->rndnu_neon.output_max);
43
44 do {
45 const void* w = weights;
46
47 // First pass to process 8 inputs.
48 {
49 int32_t* b = buffer;
50 const int8_t* i0 = input[0];
51 assert(i0 != NULL);
52 if XNN_UNPREDICTABLE(i0 != zero) {
53 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
54 }
55 const int8_t* i1 = input[1];
56 assert(i1 != NULL);
57 if XNN_UNPREDICTABLE(i1 != zero) {
58 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
59 }
60 const int8_t* i2 = input[2];
61 assert(i2 != NULL);
62 if XNN_UNPREDICTABLE(i2 != zero) {
63 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
64 }
65 const int8_t* i3 = input[3];
66 assert(i3 != NULL);
67 if XNN_UNPREDICTABLE(i3 != zero) {
68 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
69 }
70 const int8_t* i4 = input[4];
71 assert(i4 != NULL);
72 if XNN_UNPREDICTABLE(i4 != zero) {
73 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
74 }
75 const int8_t* i5 = input[5];
76 assert(i5 != NULL);
77 if XNN_UNPREDICTABLE(i5 != zero) {
78 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
79 }
80 const int8_t* i6 = input[6];
81 assert(i6 != NULL);
82 if XNN_UNPREDICTABLE(i6 != zero) {
83 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
84 }
85 const int8_t* i7 = input[7];
86 assert(i7 != NULL);
87 if XNN_UNPREDICTABLE(i7 != zero) {
88 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
89 }
90 input += 8;
91
92 size_t c = round_up_po2(channels, 8);
93
94 for (; c >= 16; c -= 16) {
95 int32x4_t vacc0123 = vld1q_s32(w); w = (const int32_t*) w + 4;
96 int32x4_t vacc4567 = vld1q_s32(w); w = (const int32_t*) w + 4;
97 int32x4_t vacc89AB = vld1q_s32(w); w = (const int32_t*) w + 4;
98 int32x4_t vaccCDEF = vld1q_s32(w); w = (const int32_t*) w + 4;
99
100 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0)); i0 += 8;
101 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
102 const int16x8_t vi0x89ABCDEF = vmovl_s8(vld1_s8(i0)); i0 += 8;
103 const int16x8_t vk0x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
104
105 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
106 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
107 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi0x89ABCDEF), vget_low_s16(vk0x89ABCDEF));
108 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi0x89ABCDEF), vget_high_s16(vk0x89ABCDEF));
109
110 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1)); i1 += 8;
111 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
112 const int16x8_t vi1x89ABCDEF = vmovl_s8(vld1_s8(i1)); i1 += 8;
113 const int16x8_t vk1x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
114
115 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
116 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
117 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi1x89ABCDEF), vget_low_s16(vk1x89ABCDEF));
118 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi1x89ABCDEF), vget_high_s16(vk1x89ABCDEF));
119
120 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2)); i2 += 8;
121 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
122 const int16x8_t vi2x89ABCDEF = vmovl_s8(vld1_s8(i2)); i2 += 8;
123 const int16x8_t vk2x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
124
125 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
126 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
127 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi2x89ABCDEF), vget_low_s16(vk2x89ABCDEF));
128 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi2x89ABCDEF), vget_high_s16(vk2x89ABCDEF));
129
130 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3)); i3 += 8;
131 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
132 const int16x8_t vi3x89ABCDEF = vmovl_s8(vld1_s8(i3)); i3 += 8;
133 const int16x8_t vk3x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
134
135 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
136 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
137 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi3x89ABCDEF), vget_low_s16(vk3x89ABCDEF));
138 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi3x89ABCDEF), vget_high_s16(vk3x89ABCDEF));
139
140 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4)); i4 += 8;
141 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
142 const int16x8_t vi4x89ABCDEF = vmovl_s8(vld1_s8(i4)); i4 += 8;
143 const int16x8_t vk4x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
144
145 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
146 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
147 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi4x89ABCDEF), vget_low_s16(vk4x89ABCDEF));
148 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi4x89ABCDEF), vget_high_s16(vk4x89ABCDEF));
149
150 const int16x8_t vi5x01234567 = vmovl_s8(vld1_s8(i5)); i5 += 8;
151 const int16x8_t vk5x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
152 const int16x8_t vi5x89ABCDEF = vmovl_s8(vld1_s8(i5)); i5 += 8;
153 const int16x8_t vk5x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
154
155 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi5x01234567), vget_low_s16(vk5x01234567));
156 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi5x01234567), vget_high_s16(vk5x01234567));
157 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi5x89ABCDEF), vget_low_s16(vk5x89ABCDEF));
158 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi5x89ABCDEF), vget_high_s16(vk5x89ABCDEF));
159
160 const int16x8_t vi6x01234567 = vmovl_s8(vld1_s8(i6)); i6 += 8;
161 const int16x8_t vk6x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
162 const int16x8_t vi6x89ABCDEF = vmovl_s8(vld1_s8(i6)); i6 += 8;
163 const int16x8_t vk6x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
164
165 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi6x01234567), vget_low_s16(vk6x01234567));
166 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi6x01234567), vget_high_s16(vk6x01234567));
167 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi6x89ABCDEF), vget_low_s16(vk6x89ABCDEF));
168 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi6x89ABCDEF), vget_high_s16(vk6x89ABCDEF));
169
170 const int16x8_t vi7x01234567 = vmovl_s8(vld1_s8(i7)); i7 += 8;
171 const int16x8_t vk7x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
172 const int16x8_t vi7x89ABCDEF = vmovl_s8(vld1_s8(i7)); i7 += 8;
173 const int16x8_t vk7x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
174
175 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi7x01234567), vget_low_s16(vk7x01234567));
176 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi7x01234567), vget_high_s16(vk7x01234567));
177 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi7x89ABCDEF), vget_low_s16(vk7x89ABCDEF));
178 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi7x89ABCDEF), vget_high_s16(vk7x89ABCDEF));
179
180 vst1q_s32(b, vacc0123); b += 4;
181 vst1q_s32(b, vacc4567); b += 4;
182 vst1q_s32(b, vacc89AB); b += 4;
183 vst1q_s32(b, vaccCDEF); b += 4;
184 }
185
186 if XNN_UNLIKELY(c != 0) {
187 do {
188 int32x4_t vacc0123 = vld1q_s32(w); w = (const int32_t*) w + 4;
189 int32x4_t vacc4567 = vld1q_s32(w); w = (const int32_t*) w + 4;
190
191 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0)); i0 += 8;
192 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
193
194 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
195 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
196
197 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1)); i1 += 8;
198 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
199
200 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
201 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
202
203 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2)); i2 += 8;
204 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
205
206 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
207 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
208
209 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3)); i3 += 8;
210 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
211
212 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
213 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
214
215 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4)); i4 += 8;
216 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
217
218 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
219 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
220
221 const int16x8_t vi5x01234567 = vmovl_s8(vld1_s8(i5)); i5 += 8;
222 const int16x8_t vk5x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
223
224 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi5x01234567), vget_low_s16(vk5x01234567));
225 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi5x01234567), vget_high_s16(vk5x01234567));
226
227 const int16x8_t vi6x01234567 = vmovl_s8(vld1_s8(i6)); i6 += 8;
228 const int16x8_t vk6x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
229
230 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi6x01234567), vget_low_s16(vk6x01234567));
231 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi6x01234567), vget_high_s16(vk6x01234567));
232
233 const int16x8_t vi7x01234567 = vmovl_s8(vld1_s8(i7)); i7 += 8;
234 const int16x8_t vk7x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
235
236 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi7x01234567), vget_low_s16(vk7x01234567));
237 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi7x01234567), vget_high_s16(vk7x01234567));
238
239 vst1q_s32(b, vacc0123); b += 4;
240 vst1q_s32(b, vacc4567); b += 4;
241 c -= 8;
242 } while (c != 0);
243 }
244 }
245
246 // Middle pass to process 8 inputs in each iteration.
247 for (size_t ks = kernel_size - 8; ks > 9; ks -= 8) {
248 int32_t* b = buffer;
249
250 const int8_t* i0 = input[0];
251 assert(i0 != NULL);
252 if XNN_UNPREDICTABLE(i0 != zero) {
253 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
254 }
255 const int8_t* i1 = input[1];
256 assert(i1 != NULL);
257 if XNN_UNPREDICTABLE(i1 != zero) {
258 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
259 }
260 const int8_t* i2 = input[2];
261 assert(i2 != NULL);
262 if XNN_UNPREDICTABLE(i2 != zero) {
263 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
264 }
265 const int8_t* i3 = input[3];
266 assert(i3 != NULL);
267 if XNN_UNPREDICTABLE(i3 != zero) {
268 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
269 }
270 const int8_t* i4 = input[4];
271 assert(i4 != NULL);
272 if XNN_UNPREDICTABLE(i4 != zero) {
273 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
274 }
275 const int8_t* i5 = input[5];
276 assert(i5 != NULL);
277 if XNN_UNPREDICTABLE(i5 != zero) {
278 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
279 }
280 const int8_t* i6 = input[6];
281 assert(i6 != NULL);
282 if XNN_UNPREDICTABLE(i6 != zero) {
283 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
284 }
285 const int8_t* i7 = input[7];
286 assert(i7 != NULL);
287 if XNN_UNPREDICTABLE(i7 != zero) {
288 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
289 }
290 input += 8;
291
292 size_t c = round_up_po2(channels, 8);
293
294 for (; c >= 16; c -= 16) {
295 int32x4_t vacc0123 = vld1q_s32(b);
296 int32x4_t vacc4567 = vld1q_s32(b + 4);
297 int32x4_t vacc89AB = vld1q_s32(b + 8);
298 int32x4_t vaccCDEF = vld1q_s32(b + 12);
299
300 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0)); i0 += 8;
301 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
302 const int16x8_t vi0x89ABCDEF = vmovl_s8(vld1_s8(i0)); i0 += 8;
303 const int16x8_t vk0x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
304
305 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
306 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
307 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi0x89ABCDEF), vget_low_s16(vk0x89ABCDEF));
308 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi0x89ABCDEF), vget_high_s16(vk0x89ABCDEF));
309
310 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1)); i1 += 8;
311 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
312 const int16x8_t vi1x89ABCDEF = vmovl_s8(vld1_s8(i1)); i1 += 8;
313 const int16x8_t vk1x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
314
315 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
316 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
317 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi1x89ABCDEF), vget_low_s16(vk1x89ABCDEF));
318 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi1x89ABCDEF), vget_high_s16(vk1x89ABCDEF));
319
320 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2)); i2 += 8;
321 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
322 const int16x8_t vi2x89ABCDEF = vmovl_s8(vld1_s8(i2)); i2 += 8;
323 const int16x8_t vk2x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
324
325 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
326 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
327 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi2x89ABCDEF), vget_low_s16(vk2x89ABCDEF));
328 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi2x89ABCDEF), vget_high_s16(vk2x89ABCDEF));
329
330 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3)); i3 += 8;
331 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
332 const int16x8_t vi3x89ABCDEF = vmovl_s8(vld1_s8(i3)); i3 += 8;
333 const int16x8_t vk3x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
334
335 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
336 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
337 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi3x89ABCDEF), vget_low_s16(vk3x89ABCDEF));
338 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi3x89ABCDEF), vget_high_s16(vk3x89ABCDEF));
339
340 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4)); i4 += 8;
341 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
342 const int16x8_t vi4x89ABCDEF = vmovl_s8(vld1_s8(i4)); i4 += 8;
343 const int16x8_t vk4x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
344
345 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
346 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
347 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi4x89ABCDEF), vget_low_s16(vk4x89ABCDEF));
348 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi4x89ABCDEF), vget_high_s16(vk4x89ABCDEF));
349
350 const int16x8_t vi5x01234567 = vmovl_s8(vld1_s8(i5)); i5 += 8;
351 const int16x8_t vk5x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
352 const int16x8_t vi5x89ABCDEF = vmovl_s8(vld1_s8(i5)); i5 += 8;
353 const int16x8_t vk5x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
354
355 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi5x01234567), vget_low_s16(vk5x01234567));
356 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi5x01234567), vget_high_s16(vk5x01234567));
357 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi5x89ABCDEF), vget_low_s16(vk5x89ABCDEF));
358 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi5x89ABCDEF), vget_high_s16(vk5x89ABCDEF));
359
360 const int16x8_t vi6x01234567 = vmovl_s8(vld1_s8(i6)); i6 += 8;
361 const int16x8_t vk6x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
362 const int16x8_t vi6x89ABCDEF = vmovl_s8(vld1_s8(i6)); i6 += 8;
363 const int16x8_t vk6x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
364
365 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi6x01234567), vget_low_s16(vk6x01234567));
366 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi6x01234567), vget_high_s16(vk6x01234567));
367 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi6x89ABCDEF), vget_low_s16(vk6x89ABCDEF));
368 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi6x89ABCDEF), vget_high_s16(vk6x89ABCDEF));
369
370 const int16x8_t vi7x01234567 = vmovl_s8(vld1_s8(i7)); i7 += 8;
371 const int16x8_t vk7x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
372 const int16x8_t vi7x89ABCDEF = vmovl_s8(vld1_s8(i7)); i7 += 8;
373 const int16x8_t vk7x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
374
375 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi7x01234567), vget_low_s16(vk7x01234567));
376 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi7x01234567), vget_high_s16(vk7x01234567));
377 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi7x89ABCDEF), vget_low_s16(vk7x89ABCDEF));
378 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi7x89ABCDEF), vget_high_s16(vk7x89ABCDEF));
379
380 vst1q_s32(b, vacc0123); b += 4;
381 vst1q_s32(b, vacc4567); b += 4;
382 vst1q_s32(b, vacc89AB); b += 4;
383 vst1q_s32(b, vaccCDEF); b += 4;
384 }
385
386 if XNN_UNLIKELY(c != 0) {
387 do {
388 int32x4_t vacc0123 = vld1q_s32(b);
389 int32x4_t vacc4567 = vld1q_s32(b + 4);
390
391 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0)); i0 += 8;
392 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
393
394 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
395 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
396
397 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1)); i1 += 8;
398 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
399
400 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
401 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
402
403 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2)); i2 += 8;
404 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
405
406 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
407 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
408
409 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3)); i3 += 8;
410 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
411
412 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
413 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
414
415 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4)); i4 += 8;
416 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
417
418 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
419 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
420
421 const int16x8_t vi5x01234567 = vmovl_s8(vld1_s8(i5)); i5 += 8;
422 const int16x8_t vk5x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
423
424 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi5x01234567), vget_low_s16(vk5x01234567));
425 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi5x01234567), vget_high_s16(vk5x01234567));
426
427 const int16x8_t vi6x01234567 = vmovl_s8(vld1_s8(i6)); i6 += 8;
428 const int16x8_t vk6x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
429
430 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi6x01234567), vget_low_s16(vk6x01234567));
431 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi6x01234567), vget_high_s16(vk6x01234567));
432
433 const int16x8_t vi7x01234567 = vmovl_s8(vld1_s8(i7)); i7 += 8;
434 const int16x8_t vk7x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
435
436 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi7x01234567), vget_low_s16(vk7x01234567));
437 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi7x01234567), vget_high_s16(vk7x01234567));
438
439 vst1q_s32(b, vacc0123); b += 4;
440 vst1q_s32(b, vacc4567); b += 4;
441 c -= 8;
442 } while (c != 0);
443 }
444 }
445
446 // Last pass to process up to 9 inputs.
447 {
448 const int32_t* b = buffer;
449
450 const int8_t* i0 = input[0];
451 assert(i0 != NULL);
452 if XNN_UNPREDICTABLE(i0 != zero) {
453 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
454 }
455 const int8_t* i1 = input[1];
456 assert(i1 != NULL);
457 if XNN_UNPREDICTABLE(i1 != zero) {
458 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
459 }
460 const int8_t* i2 = input[2];
461 assert(i2 != NULL);
462 if XNN_UNPREDICTABLE(i2 != zero) {
463 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
464 }
465 const int8_t* i3 = input[3];
466 assert(i3 != NULL);
467 if XNN_UNPREDICTABLE(i3 != zero) {
468 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
469 }
470 const int8_t* i4 = input[4];
471 assert(i4 != NULL);
472 if XNN_UNPREDICTABLE(i4 != zero) {
473 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
474 }
475 const int8_t* i5 = input[5];
476 assert(i5 != NULL);
477 if XNN_UNPREDICTABLE(i5 != zero) {
478 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
479 }
480 const int8_t* i6 = input[6];
481 assert(i6 != NULL);
482 if XNN_UNPREDICTABLE(i6 != zero) {
483 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
484 }
485 const int8_t* i7 = input[7];
486 assert(i7 != NULL);
487 if XNN_UNPREDICTABLE(i7 != zero) {
488 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
489 }
490 const int8_t* i8 = input[8];
491 assert(i8 != NULL);
492 if XNN_UNPREDICTABLE(i8 != zero) {
493 i8 = (const int8_t*) ((uintptr_t) i8 + input_offset);
494 }
495
496 size_t c = channels;
497 for (; c >= 16; c -= 16) {
498 int32x4_t vacc0123 = vld1q_s32(b); b += 4;
499 int32x4_t vacc4567 = vld1q_s32(b); b += 4;
500 int32x4_t vacc89AB = vld1q_s32(b); b += 4;
501 int32x4_t vaccCDEF = vld1q_s32(b); b += 4;
502
503 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0)); i0 += 8;
504 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
505 const int16x8_t vi0x89ABCDEF = vmovl_s8(vld1_s8(i0)); i0 += 8;
506 const int16x8_t vk0x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
507
508 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
509 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
510 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi0x89ABCDEF), vget_low_s16(vk0x89ABCDEF));
511 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi0x89ABCDEF), vget_high_s16(vk0x89ABCDEF));
512
513 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1)); i1 += 8;
514 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
515 const int16x8_t vi1x89ABCDEF = vmovl_s8(vld1_s8(i1)); i1 += 8;
516 const int16x8_t vk1x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
517
518 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
519 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
520 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi1x89ABCDEF), vget_low_s16(vk1x89ABCDEF));
521 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi1x89ABCDEF), vget_high_s16(vk1x89ABCDEF));
522
523 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2)); i2 += 8;
524 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
525 const int16x8_t vi2x89ABCDEF = vmovl_s8(vld1_s8(i2)); i2 += 8;
526 const int16x8_t vk2x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
527
528 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
529 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
530 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi2x89ABCDEF), vget_low_s16(vk2x89ABCDEF));
531 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi2x89ABCDEF), vget_high_s16(vk2x89ABCDEF));
532
533 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3)); i3 += 8;
534 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
535 const int16x8_t vi3x89ABCDEF = vmovl_s8(vld1_s8(i3)); i3 += 8;
536 const int16x8_t vk3x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
537
538 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
539 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
540 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi3x89ABCDEF), vget_low_s16(vk3x89ABCDEF));
541 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi3x89ABCDEF), vget_high_s16(vk3x89ABCDEF));
542
543 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4)); i4 += 8;
544 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
545 const int16x8_t vi4x89ABCDEF = vmovl_s8(vld1_s8(i4)); i4 += 8;
546 const int16x8_t vk4x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
547
548 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
549 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
550 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi4x89ABCDEF), vget_low_s16(vk4x89ABCDEF));
551 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi4x89ABCDEF), vget_high_s16(vk4x89ABCDEF));
552
553 const int16x8_t vi5x01234567 = vmovl_s8(vld1_s8(i5)); i5 += 8;
554 const int16x8_t vk5x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
555 const int16x8_t vi5x89ABCDEF = vmovl_s8(vld1_s8(i5)); i5 += 8;
556 const int16x8_t vk5x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
557
558 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi5x01234567), vget_low_s16(vk5x01234567));
559 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi5x01234567), vget_high_s16(vk5x01234567));
560 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi5x89ABCDEF), vget_low_s16(vk5x89ABCDEF));
561 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi5x89ABCDEF), vget_high_s16(vk5x89ABCDEF));
562
563 const int16x8_t vi6x01234567 = vmovl_s8(vld1_s8(i6)); i6 += 8;
564 const int16x8_t vk6x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
565 const int16x8_t vi6x89ABCDEF = vmovl_s8(vld1_s8(i6)); i6 += 8;
566 const int16x8_t vk6x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
567
568 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi6x01234567), vget_low_s16(vk6x01234567));
569 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi6x01234567), vget_high_s16(vk6x01234567));
570 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi6x89ABCDEF), vget_low_s16(vk6x89ABCDEF));
571 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi6x89ABCDEF), vget_high_s16(vk6x89ABCDEF));
572
573 const int16x8_t vi7x01234567 = vmovl_s8(vld1_s8(i7)); i7 += 8;
574 const int16x8_t vk7x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
575 const int16x8_t vi7x89ABCDEF = vmovl_s8(vld1_s8(i7)); i7 += 8;
576 const int16x8_t vk7x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
577
578 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi7x01234567), vget_low_s16(vk7x01234567));
579 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi7x01234567), vget_high_s16(vk7x01234567));
580 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi7x89ABCDEF), vget_low_s16(vk7x89ABCDEF));
581 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi7x89ABCDEF), vget_high_s16(vk7x89ABCDEF));
582
583 const int16x8_t vi8x01234567 = vmovl_s8(vld1_s8(i8)); i8 += 8;
584 const int16x8_t vk8x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
585 const int16x8_t vi8x89ABCDEF = vmovl_s8(vld1_s8(i8)); i8 += 8;
586 const int16x8_t vk8x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
587
588 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi8x01234567), vget_low_s16(vk8x01234567));
589 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi8x01234567), vget_high_s16(vk8x01234567));
590 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi8x89ABCDEF), vget_low_s16(vk8x89ABCDEF));
591 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi8x89ABCDEF), vget_high_s16(vk8x89ABCDEF));
592
593 vacc0123 = vqshlq_s32(vacc0123, vright_pre_shift);
594 vacc4567 = vqshlq_s32(vacc4567, vright_pre_shift);
595 vacc89AB = vqshlq_s32(vacc89AB, vright_pre_shift);
596 vaccCDEF = vqshlq_s32(vaccCDEF, vright_pre_shift);
597
598 vacc0123 = vqdmulhq_s32(vacc0123, vmultiplier);
599 vacc4567 = vqdmulhq_s32(vacc4567, vmultiplier);
600 vacc89AB = vqdmulhq_s32(vacc89AB, vmultiplier);
601 vaccCDEF = vqdmulhq_s32(vaccCDEF, vmultiplier);
602
603 vacc0123 = vrshlq_s32(vacc0123, vright_post_shift);
604 vacc4567 = vrshlq_s32(vacc4567, vright_post_shift);
605 vacc89AB = vrshlq_s32(vacc89AB, vright_post_shift);
606 vaccCDEF = vrshlq_s32(vaccCDEF, vright_post_shift);
607
608 #if XNN_ARCH_ARM64
609 int16x8_t vacc01234567 = vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567);
610 int16x8_t vacc89ABCDEF = vqmovn_high_s32(vqmovn_s32(vacc89AB), vaccCDEF);
611
612 vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
613 vacc89ABCDEF = vqaddq_s16(vacc89ABCDEF, voutput_zero_point);
614
615 int8x16_t vout0123456789ABCDEF = vqmovn_high_s16(vqmovn_s16(vacc01234567), vacc89ABCDEF);
616 #else // !XNN_ARCH_ARM64
617 int16x8_t vacc01234567 = vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567));
618 int16x8_t vacc89ABCDEF = vcombine_s16(vqmovn_s32(vacc89AB), vqmovn_s32(vaccCDEF));
619
620 vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
621 vacc89ABCDEF = vqaddq_s16(vacc89ABCDEF, voutput_zero_point);
622
623 int8x16_t vout0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc01234567), vqmovn_s16(vacc89ABCDEF));
624 #endif // !XNN_ARCH_ARM64
625
626 vout0123456789ABCDEF = vmaxq_s8(vout0123456789ABCDEF, voutput_min);
627
628 vout0123456789ABCDEF = vminq_s8(vout0123456789ABCDEF, voutput_max);
629
630 vst1q_s8(output, vout0123456789ABCDEF); output += 16;
631 }
632
633 if XNN_UNLIKELY(c != 0) {
634 do {
635 int32x4_t vacc0123 = vld1q_s32(b); b += 4;
636 int32x4_t vacc4567 = vld1q_s32(b); b += 4;
637
638 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0)); i0 += 8;
639 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
640
641 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
642 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
643
644 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1)); i1 += 8;
645 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
646
647 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
648 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
649
650 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2)); i2 += 8;
651 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
652
653 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
654 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
655
656 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3)); i3 += 8;
657 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
658
659 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
660 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
661
662 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4)); i4 += 8;
663 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
664
665 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
666 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
667
668 const int16x8_t vi5x01234567 = vmovl_s8(vld1_s8(i5)); i5 += 8;
669 const int16x8_t vk5x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
670
671 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi5x01234567), vget_low_s16(vk5x01234567));
672 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi5x01234567), vget_high_s16(vk5x01234567));
673
674 const int16x8_t vi6x01234567 = vmovl_s8(vld1_s8(i6)); i6 += 8;
675 const int16x8_t vk6x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
676
677 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi6x01234567), vget_low_s16(vk6x01234567));
678 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi6x01234567), vget_high_s16(vk6x01234567));
679
680 const int16x8_t vi7x01234567 = vmovl_s8(vld1_s8(i7)); i7 += 8;
681 const int16x8_t vk7x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
682
683 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi7x01234567), vget_low_s16(vk7x01234567));
684 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi7x01234567), vget_high_s16(vk7x01234567));
685
686 const int16x8_t vi8x01234567 = vmovl_s8(vld1_s8(i8)); i8 += 8;
687 const int16x8_t vk8x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
688
689 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi8x01234567), vget_low_s16(vk8x01234567));
690 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi8x01234567), vget_high_s16(vk8x01234567));
691
692 vacc0123 = vqshlq_s32(vacc0123, vright_pre_shift);
693 vacc4567 = vqshlq_s32(vacc4567, vright_pre_shift);
694
695 vacc0123 = vqdmulhq_s32(vacc0123, vmultiplier);
696 vacc4567 = vqdmulhq_s32(vacc4567, vmultiplier);
697
698 vacc0123 = vrshlq_s32(vacc0123, vright_post_shift);
699 vacc4567 = vrshlq_s32(vacc4567, vright_post_shift);
700
701 #if XNN_ARCH_ARM64
702 int16x8_t vacc01234567 = vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567);
703 #else
704 int16x8_t vacc01234567 = vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567));
705 #endif
706 vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
707
708 int8x8_t vout01234567 = vqmovn_s16(vacc01234567);
709 vout01234567 = vmax_s8(vout01234567, vget_low_s8(voutput_min));
710 vout01234567 = vmin_s8(vout01234567, vget_low_s8(voutput_max));
711
712 if XNN_LIKELY(c >= 8) {
713 vst1_s8(output, vout01234567); output += 8;
714 c -= 8;
715 } else {
716 if (c & 4) {
717 vst1_lane_u32((void*) output, vreinterpret_u32_s8(vout01234567), 0); output += 4;
718 vout01234567 = vext_s8(vout01234567, vout01234567, 4);
719 }
720 if (c & 2) {
721 vst1_lane_u16((void*) output, vreinterpret_u16_s8(vout01234567), 0); output += 2;
722 vout01234567 = vext_s8(vout01234567, vout01234567, 2);
723 }
724 if (c & 1) {
725 vst1_lane_s8(output, vout01234567, 0); output += 1;
726 }
727 c = 0;
728 }
729 } while (c != 0);
730 }
731 }
732
733 input = (const int8_t**) ((uintptr_t) input + input_stride);
734 output = (int8_t*) ((uintptr_t) output + output_increment);
735 } while (--output_width != 0);
736 }