Debian Codegraph @

src/qs8-dwconv/gen/qs8-dwconv-8f8m9l8c8s8r-minmax-rndnu-neon-mul16.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-dwconv/multipass-neon-mul16.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12
13 #include <arm_neon.h>
14
15 #include "xnnpack/dwconv.h"
16 #include "xnnpack/math.h"
17
18
19 void xnn_qs8_dwconv_minmax_rndnu_ukernel_8f8m9l8c8s8r__neon_mul16(
20 size_t channels,
21 size_t output_width,
22 const int8_t** input,
23 const void* weights,
24 int8_t* output,
25 intptr_t input_stride,
26 size_t output_increment,
27 size_t input_offset,
28 const int8_t* zero,
29 size_t kernel_size,
30 int32_t* buffer,
31 const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
32 {
33 assert(channels != 0);
34 assert(output_width != 0);
35 assert(kernel_size > 8);
36
37 const int32x4_t vright_pre_shift = vld1q_dup_s32(&params->rndnu_neon.right_pre_shift);
38 const int32x4_t vmultiplier = vld1q_dup_s32(&params->rndnu_neon.multiplier);
39 const int32x4_t vright_post_shift = vld1q_dup_s32(&params->rndnu_neon.right_post_shift);
40 const int16x8_t voutput_zero_point = vld1q_dup_s16(&params->rndnu_neon.output_zero_point);
41 const int8x8_t voutput_min = vld1_dup_s8(&params->rndnu_neon.output_min);
42 const int8x8_t voutput_max = vld1_dup_s8(&params->rndnu_neon.output_max);
43
44 do {
45 const void* w = weights;
46
47 // First pass to process 8 inputs.
48 {
49 int32_t* b = buffer;
50 const int8_t* i0 = input[0];
51 assert(i0 != NULL);
52 if XNN_UNPREDICTABLE(i0 != zero) {
53 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
54 }
55 const int8_t* i1 = input[1];
56 assert(i1 != NULL);
57 if XNN_UNPREDICTABLE(i1 != zero) {
58 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
59 }
60 const int8_t* i2 = input[2];
61 assert(i2 != NULL);
62 if XNN_UNPREDICTABLE(i2 != zero) {
63 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
64 }
65 const int8_t* i3 = input[3];
66 assert(i3 != NULL);
67 if XNN_UNPREDICTABLE(i3 != zero) {
68 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
69 }
70 const int8_t* i4 = input[4];
71 assert(i4 != NULL);
72 if XNN_UNPREDICTABLE(i4 != zero) {
73 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
74 }
75 const int8_t* i5 = input[5];
76 assert(i5 != NULL);
77 if XNN_UNPREDICTABLE(i5 != zero) {
78 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
79 }
80 const int8_t* i6 = input[6];
81 assert(i6 != NULL);
82 if XNN_UNPREDICTABLE(i6 != zero) {
83 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
84 }
85 const int8_t* i7 = input[7];
86 assert(i7 != NULL);
87 if XNN_UNPREDICTABLE(i7 != zero) {
88 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
89 }
90 input += 8;
91
92 size_t c = round_up_po2(channels, 8);
93
94 for (; c >= 8; c -= 8) {
95 int32x4_t vacc0123 = vld1q_s32(w); w = (const int32_t*) w + 4;
96 int32x4_t vacc4567 = vld1q_s32(w); w = (const int32_t*) w + 4;
97
98 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0)); i0 += 8;
99 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
100
101 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
102 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
103
104 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1)); i1 += 8;
105 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
106
107 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
108 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
109
110 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2)); i2 += 8;
111 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
112
113 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
114 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
115
116 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3)); i3 += 8;
117 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
118
119 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
120 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
121
122 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4)); i4 += 8;
123 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
124
125 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
126 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
127
128 const int16x8_t vi5x01234567 = vmovl_s8(vld1_s8(i5)); i5 += 8;
129 const int16x8_t vk5x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
130
131 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi5x01234567), vget_low_s16(vk5x01234567));
132 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi5x01234567), vget_high_s16(vk5x01234567));
133
134 const int16x8_t vi6x01234567 = vmovl_s8(vld1_s8(i6)); i6 += 8;
135 const int16x8_t vk6x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
136
137 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi6x01234567), vget_low_s16(vk6x01234567));
138 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi6x01234567), vget_high_s16(vk6x01234567));
139
140 const int16x8_t vi7x01234567 = vmovl_s8(vld1_s8(i7)); i7 += 8;
141 const int16x8_t vk7x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
142
143 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi7x01234567), vget_low_s16(vk7x01234567));
144 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi7x01234567), vget_high_s16(vk7x01234567));
145
146 vst1q_s32(b, vacc0123); b += 4;
147 vst1q_s32(b, vacc4567); b += 4;
148 }
149
150 assert(c == 0);
151 }
152
153 // Middle pass to process 8 inputs in each iteration.
154 for (size_t ks = kernel_size - 8; ks > 9; ks -= 8) {
155 int32_t* b = buffer;
156
157 const int8_t* i0 = input[0];
158 assert(i0 != NULL);
159 if XNN_UNPREDICTABLE(i0 != zero) {
160 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
161 }
162 const int8_t* i1 = input[1];
163 assert(i1 != NULL);
164 if XNN_UNPREDICTABLE(i1 != zero) {
165 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
166 }
167 const int8_t* i2 = input[2];
168 assert(i2 != NULL);
169 if XNN_UNPREDICTABLE(i2 != zero) {
170 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
171 }
172 const int8_t* i3 = input[3];
173 assert(i3 != NULL);
174 if XNN_UNPREDICTABLE(i3 != zero) {
175 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
176 }
177 const int8_t* i4 = input[4];
178 assert(i4 != NULL);
179 if XNN_UNPREDICTABLE(i4 != zero) {
180 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
181 }
182 const int8_t* i5 = input[5];
183 assert(i5 != NULL);
184 if XNN_UNPREDICTABLE(i5 != zero) {
185 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
186 }
187 const int8_t* i6 = input[6];
188 assert(i6 != NULL);
189 if XNN_UNPREDICTABLE(i6 != zero) {
190 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
191 }
192 const int8_t* i7 = input[7];
193 assert(i7 != NULL);
194 if XNN_UNPREDICTABLE(i7 != zero) {
195 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
196 }
197 input += 8;
198
199 size_t c = round_up_po2(channels, 8);
200
201 for (; c >= 8; c -= 8) {
202 int32x4_t vacc0123 = vld1q_s32(b);
203 int32x4_t vacc4567 = vld1q_s32(b + 4);
204
205 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0)); i0 += 8;
206 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
207
208 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
209 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
210
211 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1)); i1 += 8;
212 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
213
214 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
215 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
216
217 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2)); i2 += 8;
218 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
219
220 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
221 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
222
223 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3)); i3 += 8;
224 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
225
226 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
227 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
228
229 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4)); i4 += 8;
230 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
231
232 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
233 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
234
235 const int16x8_t vi5x01234567 = vmovl_s8(vld1_s8(i5)); i5 += 8;
236 const int16x8_t vk5x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
237
238 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi5x01234567), vget_low_s16(vk5x01234567));
239 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi5x01234567), vget_high_s16(vk5x01234567));
240
241 const int16x8_t vi6x01234567 = vmovl_s8(vld1_s8(i6)); i6 += 8;
242 const int16x8_t vk6x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
243
244 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi6x01234567), vget_low_s16(vk6x01234567));
245 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi6x01234567), vget_high_s16(vk6x01234567));
246
247 const int16x8_t vi7x01234567 = vmovl_s8(vld1_s8(i7)); i7 += 8;
248 const int16x8_t vk7x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
249
250 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi7x01234567), vget_low_s16(vk7x01234567));
251 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi7x01234567), vget_high_s16(vk7x01234567));
252
253 vst1q_s32(b, vacc0123); b += 4;
254 vst1q_s32(b, vacc4567); b += 4;
255 }
256
257 assert(c == 0);
258 }
259
260 // Last pass to process up to 9 inputs.
261 {
262 const int32_t* b = buffer;
263
264 const int8_t* i0 = input[0];
265 assert(i0 != NULL);
266 if XNN_UNPREDICTABLE(i0 != zero) {
267 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
268 }
269 const int8_t* i1 = input[1];
270 assert(i1 != NULL);
271 if XNN_UNPREDICTABLE(i1 != zero) {
272 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
273 }
274 const int8_t* i2 = input[2];
275 assert(i2 != NULL);
276 if XNN_UNPREDICTABLE(i2 != zero) {
277 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
278 }
279 const int8_t* i3 = input[3];
280 assert(i3 != NULL);
281 if XNN_UNPREDICTABLE(i3 != zero) {
282 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
283 }
284 const int8_t* i4 = input[4];
285 assert(i4 != NULL);
286 if XNN_UNPREDICTABLE(i4 != zero) {
287 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
288 }
289 const int8_t* i5 = input[5];
290 assert(i5 != NULL);
291 if XNN_UNPREDICTABLE(i5 != zero) {
292 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
293 }
294 const int8_t* i6 = input[6];
295 assert(i6 != NULL);
296 if XNN_UNPREDICTABLE(i6 != zero) {
297 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
298 }
299 const int8_t* i7 = input[7];
300 assert(i7 != NULL);
301 if XNN_UNPREDICTABLE(i7 != zero) {
302 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
303 }
304 const int8_t* i8 = input[8];
305 assert(i8 != NULL);
306 if XNN_UNPREDICTABLE(i8 != zero) {
307 i8 = (const int8_t*) ((uintptr_t) i8 + input_offset);
308 }
309
310 size_t c = channels;
311 for (; c >= 8; c -= 8) {
312 int32x4_t vacc0123 = vld1q_s32(b); b += 4;
313 int32x4_t vacc4567 = vld1q_s32(b); b += 4;
314
315 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0)); i0 += 8;
316 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
317
318 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
319 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
320
321 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1)); i1 += 8;
322 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
323
324 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
325 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
326
327 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2)); i2 += 8;
328 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
329
330 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
331 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
332
333 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3)); i3 += 8;
334 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
335
336 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
337 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
338
339 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4)); i4 += 8;
340 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
341
342 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
343 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
344
345 const int16x8_t vi5x01234567 = vmovl_s8(vld1_s8(i5)); i5 += 8;
346 const int16x8_t vk5x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
347
348 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi5x01234567), vget_low_s16(vk5x01234567));
349 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi5x01234567), vget_high_s16(vk5x01234567));
350
351 const int16x8_t vi6x01234567 = vmovl_s8(vld1_s8(i6)); i6 += 8;
352 const int16x8_t vk6x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
353
354 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi6x01234567), vget_low_s16(vk6x01234567));
355 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi6x01234567), vget_high_s16(vk6x01234567));
356
357 const int16x8_t vi7x01234567 = vmovl_s8(vld1_s8(i7)); i7 += 8;
358 const int16x8_t vk7x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
359
360 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi7x01234567), vget_low_s16(vk7x01234567));
361 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi7x01234567), vget_high_s16(vk7x01234567));
362
363 const int16x8_t vi8x01234567 = vmovl_s8(vld1_s8(i8)); i8 += 8;
364 const int16x8_t vk8x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
365
366 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi8x01234567), vget_low_s16(vk8x01234567));
367 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi8x01234567), vget_high_s16(vk8x01234567));
368
369 vacc0123 = vqshlq_s32(vacc0123, vright_pre_shift);
370 vacc4567 = vqshlq_s32(vacc4567, vright_pre_shift);
371
372 vacc0123 = vqdmulhq_s32(vacc0123, vmultiplier);
373 vacc4567 = vqdmulhq_s32(vacc4567, vmultiplier);
374
375 vacc0123 = vrshlq_s32(vacc0123, vright_post_shift);
376 vacc4567 = vrshlq_s32(vacc4567, vright_post_shift);
377
378 #if XNN_ARCH_ARM64
379 int16x8_t vacc01234567 = vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567);
380
381 vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
382
383 int8x8_t vout01234567 = vqmovn_s16(vacc01234567);
384 #else // !XNN_ARCH_ARM64
385 int16x8_t vacc01234567 = vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567));
386
387 vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
388
389 int8x8_t vout01234567 = vqmovn_s16(vacc01234567);
390 #endif // !XNN_ARCH_ARM64
391
392 vout01234567 = vmax_s8(vout01234567, voutput_min);
393
394 vout01234567 = vmin_s8(vout01234567, voutput_max);
395
396 vst1_s8(output, vout01234567); output += 8;
397 }
398
399 if XNN_UNLIKELY(c != 0) {
400 {
401 int32x4_t vacc0123 = vld1q_s32(b); b += 4;
402 int32x4_t vacc4567 = vld1q_s32(b); b += 4;
403
404 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0));
405 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
406
407 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
408 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
409
410 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1));
411 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
412
413 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
414 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
415
416 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2));
417 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
418
419 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
420 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
421
422 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3));
423 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
424
425 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
426 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
427
428 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4));
429 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
430
431 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
432 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
433
434 const int16x8_t vi5x01234567 = vmovl_s8(vld1_s8(i5));
435 const int16x8_t vk5x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
436
437 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi5x01234567), vget_low_s16(vk5x01234567));
438 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi5x01234567), vget_high_s16(vk5x01234567));
439
440 const int16x8_t vi6x01234567 = vmovl_s8(vld1_s8(i6));
441 const int16x8_t vk6x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
442
443 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi6x01234567), vget_low_s16(vk6x01234567));
444 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi6x01234567), vget_high_s16(vk6x01234567));
445
446 const int16x8_t vi7x01234567 = vmovl_s8(vld1_s8(i7));
447 const int16x8_t vk7x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
448
449 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi7x01234567), vget_low_s16(vk7x01234567));
450 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi7x01234567), vget_high_s16(vk7x01234567));
451
452 const int16x8_t vi8x01234567 = vmovl_s8(vld1_s8(i8));
453 const int16x8_t vk8x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
454
455 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi8x01234567), vget_low_s16(vk8x01234567));
456 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi8x01234567), vget_high_s16(vk8x01234567));
457
458 vacc0123 = vqshlq_s32(vacc0123, vright_pre_shift);
459 vacc4567 = vqshlq_s32(vacc4567, vright_pre_shift);
460
461 vacc0123 = vqdmulhq_s32(vacc0123, vmultiplier);
462 vacc4567 = vqdmulhq_s32(vacc4567, vmultiplier);
463
464 vacc0123 = vrshlq_s32(vacc0123, vright_post_shift);
465 vacc4567 = vrshlq_s32(vacc4567, vright_post_shift);
466
467 #if XNN_ARCH_ARM64
468 int16x8_t vacc01234567 = vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567);
469 #else
470 int16x8_t vacc01234567 = vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567));
471 #endif
472 vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
473
474 int8x8_t vout01234567 = vqmovn_s16(vacc01234567);
475 vout01234567 = vmax_s8(vout01234567, voutput_min);
476 vout01234567 = vmin_s8(vout01234567, voutput_max);
477
478 if (c & 4) {
479 vst1_lane_u32((void*) output, vreinterpret_u32_s8(vout01234567), 0); output += 4;
480 vout01234567 = vext_s8(vout01234567, vout01234567, 4);
481 }
482 if (c & 2) {
483 vst1_lane_u16((void*) output, vreinterpret_u16_s8(vout01234567), 0); output += 2;
484 vout01234567 = vext_s8(vout01234567, vout01234567, 2);
485 }
486 if (c & 1) {
487 vst1_lane_s8(output, vout01234567, 0); output += 1;
488 }
489 }
490 }
491 }
492
493 input = (const int8_t**) ((uintptr_t) input + input_stride);
494 output = (int8_t*) ((uintptr_t) output + output_increment);
495 } while (--output_width != 0);
496 }