Debian Codegraph @

src/qs8-dwconv/gen/qs8-dwconv-8f8m9l8c8s8r-minmax-fp32-neonv8-mul16.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-dwconv/multipass-neon-mul16.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12
13 #include <arm_neon.h>
14
15 #include "xnnpack/dwconv.h"
16 #include "xnnpack/intrinsics-polyfill.h"
17 #include "xnnpack/math.h"
18
19
20 void xnn_qs8_dwconv_minmax_fp32_ukernel_8f8m9l8c8s8r__neonv8_mul16(
21 size_t channels,
22 size_t output_width,
23 const int8_t** input,
24 const void* weights,
25 int8_t* output,
26 intptr_t input_stride,
27 size_t output_increment,
28 size_t input_offset,
29 const int8_t* zero,
30 size_t kernel_size,
31 int32_t* buffer,
32 const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
33 {
34 assert(channels != 0);
35 assert(output_width != 0);
36 assert(kernel_size > 8);
37
38 const float32x4_t vscale = vld1q_dup_f32(&params->fp32_neonv8.scale);
39 const int16x8_t voutput_zero_point = vld1q_dup_s16(&params->fp32_neonv8.output_zero_point);
40 const int8x8_t voutput_min = vld1_dup_s8(&params->fp32_neonv8.output_min);
41 const int8x8_t voutput_max = vld1_dup_s8(&params->fp32_neonv8.output_max);
42
43 do {
44 const void* w = weights;
45
46 // First pass to process 8 inputs.
47 {
48 int32_t* b = buffer;
49 const int8_t* i0 = input[0];
50 assert(i0 != NULL);
51 if XNN_UNPREDICTABLE(i0 != zero) {
52 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
53 }
54 const int8_t* i1 = input[1];
55 assert(i1 != NULL);
56 if XNN_UNPREDICTABLE(i1 != zero) {
57 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
58 }
59 const int8_t* i2 = input[2];
60 assert(i2 != NULL);
61 if XNN_UNPREDICTABLE(i2 != zero) {
62 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
63 }
64 const int8_t* i3 = input[3];
65 assert(i3 != NULL);
66 if XNN_UNPREDICTABLE(i3 != zero) {
67 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
68 }
69 const int8_t* i4 = input[4];
70 assert(i4 != NULL);
71 if XNN_UNPREDICTABLE(i4 != zero) {
72 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
73 }
74 const int8_t* i5 = input[5];
75 assert(i5 != NULL);
76 if XNN_UNPREDICTABLE(i5 != zero) {
77 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
78 }
79 const int8_t* i6 = input[6];
80 assert(i6 != NULL);
81 if XNN_UNPREDICTABLE(i6 != zero) {
82 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
83 }
84 const int8_t* i7 = input[7];
85 assert(i7 != NULL);
86 if XNN_UNPREDICTABLE(i7 != zero) {
87 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
88 }
89 input += 8;
90
91 size_t c = round_up_po2(channels, 8);
92
93 for (; c >= 8; c -= 8) {
94 int32x4_t vacc0123 = vld1q_s32(w); w = (const int32_t*) w + 4;
95 int32x4_t vacc4567 = vld1q_s32(w); w = (const int32_t*) w + 4;
96
97 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0)); i0 += 8;
98 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
99
100 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
101 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
102
103 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1)); i1 += 8;
104 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
105
106 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
107 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
108
109 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2)); i2 += 8;
110 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
111
112 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
113 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
114
115 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3)); i3 += 8;
116 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
117
118 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
119 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
120
121 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4)); i4 += 8;
122 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
123
124 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
125 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
126
127 const int16x8_t vi5x01234567 = vmovl_s8(vld1_s8(i5)); i5 += 8;
128 const int16x8_t vk5x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
129
130 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi5x01234567), vget_low_s16(vk5x01234567));
131 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi5x01234567), vget_high_s16(vk5x01234567));
132
133 const int16x8_t vi6x01234567 = vmovl_s8(vld1_s8(i6)); i6 += 8;
134 const int16x8_t vk6x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
135
136 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi6x01234567), vget_low_s16(vk6x01234567));
137 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi6x01234567), vget_high_s16(vk6x01234567));
138
139 const int16x8_t vi7x01234567 = vmovl_s8(vld1_s8(i7)); i7 += 8;
140 const int16x8_t vk7x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
141
142 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi7x01234567), vget_low_s16(vk7x01234567));
143 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi7x01234567), vget_high_s16(vk7x01234567));
144
145 vst1q_s32(b, vacc0123); b += 4;
146 vst1q_s32(b, vacc4567); b += 4;
147 }
148
149 assert(c == 0);
150 }
151
152 // Middle pass to process 8 inputs in each iteration.
153 for (size_t ks = kernel_size - 8; ks > 9; ks -= 8) {
154 int32_t* b = buffer;
155
156 const int8_t* i0 = input[0];
157 assert(i0 != NULL);
158 if XNN_UNPREDICTABLE(i0 != zero) {
159 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
160 }
161 const int8_t* i1 = input[1];
162 assert(i1 != NULL);
163 if XNN_UNPREDICTABLE(i1 != zero) {
164 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
165 }
166 const int8_t* i2 = input[2];
167 assert(i2 != NULL);
168 if XNN_UNPREDICTABLE(i2 != zero) {
169 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
170 }
171 const int8_t* i3 = input[3];
172 assert(i3 != NULL);
173 if XNN_UNPREDICTABLE(i3 != zero) {
174 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
175 }
176 const int8_t* i4 = input[4];
177 assert(i4 != NULL);
178 if XNN_UNPREDICTABLE(i4 != zero) {
179 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
180 }
181 const int8_t* i5 = input[5];
182 assert(i5 != NULL);
183 if XNN_UNPREDICTABLE(i5 != zero) {
184 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
185 }
186 const int8_t* i6 = input[6];
187 assert(i6 != NULL);
188 if XNN_UNPREDICTABLE(i6 != zero) {
189 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
190 }
191 const int8_t* i7 = input[7];
192 assert(i7 != NULL);
193 if XNN_UNPREDICTABLE(i7 != zero) {
194 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
195 }
196 input += 8;
197
198 size_t c = round_up_po2(channels, 8);
199
200 for (; c >= 8; c -= 8) {
201 int32x4_t vacc0123 = vld1q_s32(b);
202 int32x4_t vacc4567 = vld1q_s32(b + 4);
203
204 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0)); i0 += 8;
205 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
206
207 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
208 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
209
210 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1)); i1 += 8;
211 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
212
213 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
214 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
215
216 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2)); i2 += 8;
217 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
218
219 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
220 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
221
222 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3)); i3 += 8;
223 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
224
225 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
226 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
227
228 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4)); i4 += 8;
229 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
230
231 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
232 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
233
234 const int16x8_t vi5x01234567 = vmovl_s8(vld1_s8(i5)); i5 += 8;
235 const int16x8_t vk5x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
236
237 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi5x01234567), vget_low_s16(vk5x01234567));
238 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi5x01234567), vget_high_s16(vk5x01234567));
239
240 const int16x8_t vi6x01234567 = vmovl_s8(vld1_s8(i6)); i6 += 8;
241 const int16x8_t vk6x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
242
243 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi6x01234567), vget_low_s16(vk6x01234567));
244 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi6x01234567), vget_high_s16(vk6x01234567));
245
246 const int16x8_t vi7x01234567 = vmovl_s8(vld1_s8(i7)); i7 += 8;
247 const int16x8_t vk7x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
248
249 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi7x01234567), vget_low_s16(vk7x01234567));
250 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi7x01234567), vget_high_s16(vk7x01234567));
251
252 vst1q_s32(b, vacc0123); b += 4;
253 vst1q_s32(b, vacc4567); b += 4;
254 }
255
256 assert(c == 0);
257 }
258
259 // Last pass to process up to 9 inputs.
260 {
261 const int32_t* b = buffer;
262
263 const int8_t* i0 = input[0];
264 assert(i0 != NULL);
265 if XNN_UNPREDICTABLE(i0 != zero) {
266 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
267 }
268 const int8_t* i1 = input[1];
269 assert(i1 != NULL);
270 if XNN_UNPREDICTABLE(i1 != zero) {
271 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
272 }
273 const int8_t* i2 = input[2];
274 assert(i2 != NULL);
275 if XNN_UNPREDICTABLE(i2 != zero) {
276 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
277 }
278 const int8_t* i3 = input[3];
279 assert(i3 != NULL);
280 if XNN_UNPREDICTABLE(i3 != zero) {
281 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
282 }
283 const int8_t* i4 = input[4];
284 assert(i4 != NULL);
285 if XNN_UNPREDICTABLE(i4 != zero) {
286 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
287 }
288 const int8_t* i5 = input[5];
289 assert(i5 != NULL);
290 if XNN_UNPREDICTABLE(i5 != zero) {
291 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
292 }
293 const int8_t* i6 = input[6];
294 assert(i6 != NULL);
295 if XNN_UNPREDICTABLE(i6 != zero) {
296 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
297 }
298 const int8_t* i7 = input[7];
299 assert(i7 != NULL);
300 if XNN_UNPREDICTABLE(i7 != zero) {
301 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
302 }
303 const int8_t* i8 = input[8];
304 assert(i8 != NULL);
305 if XNN_UNPREDICTABLE(i8 != zero) {
306 i8 = (const int8_t*) ((uintptr_t) i8 + input_offset);
307 }
308
309 size_t c = channels;
310 for (; c >= 8; c -= 8) {
311 int32x4_t vacc0123 = vld1q_s32(b); b += 4;
312 int32x4_t vacc4567 = vld1q_s32(b); b += 4;
313
314 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0)); i0 += 8;
315 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
316
317 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
318 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
319
320 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1)); i1 += 8;
321 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
322
323 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
324 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
325
326 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2)); i2 += 8;
327 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
328
329 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
330 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
331
332 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3)); i3 += 8;
333 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
334
335 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
336 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
337
338 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4)); i4 += 8;
339 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
340
341 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
342 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
343
344 const int16x8_t vi5x01234567 = vmovl_s8(vld1_s8(i5)); i5 += 8;
345 const int16x8_t vk5x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
346
347 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi5x01234567), vget_low_s16(vk5x01234567));
348 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi5x01234567), vget_high_s16(vk5x01234567));
349
350 const int16x8_t vi6x01234567 = vmovl_s8(vld1_s8(i6)); i6 += 8;
351 const int16x8_t vk6x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
352
353 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi6x01234567), vget_low_s16(vk6x01234567));
354 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi6x01234567), vget_high_s16(vk6x01234567));
355
356 const int16x8_t vi7x01234567 = vmovl_s8(vld1_s8(i7)); i7 += 8;
357 const int16x8_t vk7x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
358
359 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi7x01234567), vget_low_s16(vk7x01234567));
360 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi7x01234567), vget_high_s16(vk7x01234567));
361
362 const int16x8_t vi8x01234567 = vmovl_s8(vld1_s8(i8)); i8 += 8;
363 const int16x8_t vk8x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
364
365 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi8x01234567), vget_low_s16(vk8x01234567));
366 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi8x01234567), vget_high_s16(vk8x01234567));
367
368 float32x4_t vfpacc0123 = vcvtq_f32_s32(vacc0123);
369 float32x4_t vfpacc4567 = vcvtq_f32_s32(vacc4567);
370
371 vfpacc0123 = vmulq_f32(vfpacc0123, vscale);
372 vfpacc4567 = vmulq_f32(vfpacc4567, vscale);
373
374 vacc0123 = vcvtnq_s32_f32(vfpacc0123);
375 vacc4567 = vcvtnq_s32_f32(vfpacc4567);
376
377 #if XNN_ARCH_ARM64
378 int16x8_t vacc01234567 = vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567);
379
380 vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
381
382 int8x8_t vout01234567 = vqmovn_s16(vacc01234567);
383 #else // !XNN_ARCH_ARM64
384 int16x8_t vacc01234567 = vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567));
385
386 vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
387
388 int8x8_t vout01234567 = vqmovn_s16(vacc01234567);
389 #endif // !XNN_ARCH_ARM64
390
391 vout01234567 = vmax_s8(vout01234567, voutput_min);
392
393 vout01234567 = vmin_s8(vout01234567, voutput_max);
394
395 vst1_s8(output, vout01234567); output += 8;
396 }
397
398 if XNN_UNLIKELY(c != 0) {
399 {
400 int32x4_t vacc0123 = vld1q_s32(b); b += 4;
401 int32x4_t vacc4567 = vld1q_s32(b); b += 4;
402
403 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0));
404 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
405
406 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
407 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
408
409 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1));
410 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
411
412 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
413 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
414
415 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2));
416 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
417
418 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
419 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
420
421 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3));
422 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
423
424 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
425 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
426
427 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4));
428 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
429
430 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
431 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
432
433 const int16x8_t vi5x01234567 = vmovl_s8(vld1_s8(i5));
434 const int16x8_t vk5x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
435
436 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi5x01234567), vget_low_s16(vk5x01234567));
437 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi5x01234567), vget_high_s16(vk5x01234567));
438
439 const int16x8_t vi6x01234567 = vmovl_s8(vld1_s8(i6));
440 const int16x8_t vk6x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
441
442 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi6x01234567), vget_low_s16(vk6x01234567));
443 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi6x01234567), vget_high_s16(vk6x01234567));
444
445 const int16x8_t vi7x01234567 = vmovl_s8(vld1_s8(i7));
446 const int16x8_t vk7x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
447
448 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi7x01234567), vget_low_s16(vk7x01234567));
449 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi7x01234567), vget_high_s16(vk7x01234567));
450
451 const int16x8_t vi8x01234567 = vmovl_s8(vld1_s8(i8));
452 const int16x8_t vk8x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
453
454 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi8x01234567), vget_low_s16(vk8x01234567));
455 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi8x01234567), vget_high_s16(vk8x01234567));
456
457 float32x4_t vfpacc0123 = vcvtq_f32_s32(vacc0123);
458 float32x4_t vfpacc4567 = vcvtq_f32_s32(vacc4567);
459
460 vfpacc0123 = vmulq_f32(vfpacc0123, vscale);
461 vfpacc4567 = vmulq_f32(vfpacc4567, vscale);
462
463 vacc0123 = vcvtnq_s32_f32(vfpacc0123);
464 vacc4567 = vcvtnq_s32_f32(vfpacc4567);
465
466 #if XNN_ARCH_ARM64
467 int16x8_t vacc01234567 = vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567);
468 #else
469 int16x8_t vacc01234567 = vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567));
470 #endif
471 vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
472
473 int8x8_t vout01234567 = vqmovn_s16(vacc01234567);
474 vout01234567 = vmax_s8(vout01234567, voutput_min);
475 vout01234567 = vmin_s8(vout01234567, voutput_max);
476
477 if (c & 4) {
478 vst1_lane_u32((void*) output, vreinterpret_u32_s8(vout01234567), 0); output += 4;
479 vout01234567 = vext_s8(vout01234567, vout01234567, 4);
480 }
481 if (c & 2) {
482 vst1_lane_u16((void*) output, vreinterpret_u16_s8(vout01234567), 0); output += 2;
483 vout01234567 = vext_s8(vout01234567, vout01234567, 2);
484 }
485 if (c & 1) {
486 vst1_lane_s8(output, vout01234567, 0); output += 1;
487 }
488 }
489 }
490 }
491
492 input = (const int8_t**) ((uintptr_t) input + input_stride);
493 output = (int8_t*) ((uintptr_t) output + output_increment);
494 } while (--output_width != 0);
495 }