Debian Codegraph @

src/qs8-dwconv/gen/qs8-dwconv-8f8m9l8c8s8r-minmax-fp32-neon-mul16.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-dwconv/multipass-neon-mul16.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12
13 #include <arm_neon.h>
14
15 #include "xnnpack/dwconv.h"
16 #include "xnnpack/math.h"
17
18
19 void xnn_qs8_dwconv_minmax_fp32_ukernel_8f8m9l8c8s8r__neon_mul16(
20 size_t channels,
21 size_t output_width,
22 const int8_t** input,
23 const void* weights,
24 int8_t* output,
25 intptr_t input_stride,
26 size_t output_increment,
27 size_t input_offset,
28 const int8_t* zero,
29 size_t kernel_size,
30 int32_t* buffer,
31 const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
32 {
33 assert(channels != 0);
34 assert(output_width != 0);
35 assert(kernel_size > 8);
36
37 const float32x4_t vscale = vld1q_dup_f32(&params->fp32_neon.scale);
38 const float32x4_t vmagic_bias = vld1q_dup_f32(&params->fp32_neon.magic_bias);
39 const int32x4_t vmagic_bias_less_output_zero_point = vld1q_dup_s32(&params->fp32_neon.magic_bias_less_output_zero_point);
40 const int8x8_t voutput_min = vld1_dup_s8(&params->fp32_neon.output_min);
41 const int8x8_t voutput_max = vld1_dup_s8(&params->fp32_neon.output_max);
42
43 do {
44 const void* w = weights;
45
46 // First pass to process 8 inputs.
47 {
48 int32_t* b = buffer;
49 const int8_t* i0 = input[0];
50 assert(i0 != NULL);
51 if XNN_UNPREDICTABLE(i0 != zero) {
52 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
53 }
54 const int8_t* i1 = input[1];
55 assert(i1 != NULL);
56 if XNN_UNPREDICTABLE(i1 != zero) {
57 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
58 }
59 const int8_t* i2 = input[2];
60 assert(i2 != NULL);
61 if XNN_UNPREDICTABLE(i2 != zero) {
62 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
63 }
64 const int8_t* i3 = input[3];
65 assert(i3 != NULL);
66 if XNN_UNPREDICTABLE(i3 != zero) {
67 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
68 }
69 const int8_t* i4 = input[4];
70 assert(i4 != NULL);
71 if XNN_UNPREDICTABLE(i4 != zero) {
72 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
73 }
74 const int8_t* i5 = input[5];
75 assert(i5 != NULL);
76 if XNN_UNPREDICTABLE(i5 != zero) {
77 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
78 }
79 const int8_t* i6 = input[6];
80 assert(i6 != NULL);
81 if XNN_UNPREDICTABLE(i6 != zero) {
82 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
83 }
84 const int8_t* i7 = input[7];
85 assert(i7 != NULL);
86 if XNN_UNPREDICTABLE(i7 != zero) {
87 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
88 }
89 input += 8;
90
91 size_t c = round_up_po2(channels, 8);
92
93 for (; c >= 8; c -= 8) {
94 int32x4_t vacc0123 = vld1q_s32(w); w = (const int32_t*) w + 4;
95 int32x4_t vacc4567 = vld1q_s32(w); w = (const int32_t*) w + 4;
96
97 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0)); i0 += 8;
98 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
99
100 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
101 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
102
103 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1)); i1 += 8;
104 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
105
106 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
107 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
108
109 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2)); i2 += 8;
110 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
111
112 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
113 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
114
115 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3)); i3 += 8;
116 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
117
118 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
119 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
120
121 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4)); i4 += 8;
122 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
123
124 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
125 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
126
127 const int16x8_t vi5x01234567 = vmovl_s8(vld1_s8(i5)); i5 += 8;
128 const int16x8_t vk5x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
129
130 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi5x01234567), vget_low_s16(vk5x01234567));
131 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi5x01234567), vget_high_s16(vk5x01234567));
132
133 const int16x8_t vi6x01234567 = vmovl_s8(vld1_s8(i6)); i6 += 8;
134 const int16x8_t vk6x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
135
136 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi6x01234567), vget_low_s16(vk6x01234567));
137 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi6x01234567), vget_high_s16(vk6x01234567));
138
139 const int16x8_t vi7x01234567 = vmovl_s8(vld1_s8(i7)); i7 += 8;
140 const int16x8_t vk7x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
141
142 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi7x01234567), vget_low_s16(vk7x01234567));
143 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi7x01234567), vget_high_s16(vk7x01234567));
144
145 vst1q_s32(b, vacc0123); b += 4;
146 vst1q_s32(b, vacc4567); b += 4;
147 }
148
149 assert(c == 0);
150 }
151
152 // Middle pass to process 8 inputs in each iteration.
153 for (size_t ks = kernel_size - 8; ks > 9; ks -= 8) {
154 int32_t* b = buffer;
155
156 const int8_t* i0 = input[0];
157 assert(i0 != NULL);
158 if XNN_UNPREDICTABLE(i0 != zero) {
159 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
160 }
161 const int8_t* i1 = input[1];
162 assert(i1 != NULL);
163 if XNN_UNPREDICTABLE(i1 != zero) {
164 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
165 }
166 const int8_t* i2 = input[2];
167 assert(i2 != NULL);
168 if XNN_UNPREDICTABLE(i2 != zero) {
169 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
170 }
171 const int8_t* i3 = input[3];
172 assert(i3 != NULL);
173 if XNN_UNPREDICTABLE(i3 != zero) {
174 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
175 }
176 const int8_t* i4 = input[4];
177 assert(i4 != NULL);
178 if XNN_UNPREDICTABLE(i4 != zero) {
179 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
180 }
181 const int8_t* i5 = input[5];
182 assert(i5 != NULL);
183 if XNN_UNPREDICTABLE(i5 != zero) {
184 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
185 }
186 const int8_t* i6 = input[6];
187 assert(i6 != NULL);
188 if XNN_UNPREDICTABLE(i6 != zero) {
189 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
190 }
191 const int8_t* i7 = input[7];
192 assert(i7 != NULL);
193 if XNN_UNPREDICTABLE(i7 != zero) {
194 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
195 }
196 input += 8;
197
198 size_t c = round_up_po2(channels, 8);
199
200 for (; c >= 8; c -= 8) {
201 int32x4_t vacc0123 = vld1q_s32(b);
202 int32x4_t vacc4567 = vld1q_s32(b + 4);
203
204 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0)); i0 += 8;
205 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
206
207 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
208 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
209
210 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1)); i1 += 8;
211 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
212
213 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
214 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
215
216 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2)); i2 += 8;
217 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
218
219 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
220 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
221
222 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3)); i3 += 8;
223 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
224
225 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
226 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
227
228 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4)); i4 += 8;
229 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
230
231 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
232 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
233
234 const int16x8_t vi5x01234567 = vmovl_s8(vld1_s8(i5)); i5 += 8;
235 const int16x8_t vk5x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
236
237 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi5x01234567), vget_low_s16(vk5x01234567));
238 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi5x01234567), vget_high_s16(vk5x01234567));
239
240 const int16x8_t vi6x01234567 = vmovl_s8(vld1_s8(i6)); i6 += 8;
241 const int16x8_t vk6x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
242
243 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi6x01234567), vget_low_s16(vk6x01234567));
244 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi6x01234567), vget_high_s16(vk6x01234567));
245
246 const int16x8_t vi7x01234567 = vmovl_s8(vld1_s8(i7)); i7 += 8;
247 const int16x8_t vk7x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
248
249 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi7x01234567), vget_low_s16(vk7x01234567));
250 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi7x01234567), vget_high_s16(vk7x01234567));
251
252 vst1q_s32(b, vacc0123); b += 4;
253 vst1q_s32(b, vacc4567); b += 4;
254 }
255
256 assert(c == 0);
257 }
258
259 // Last pass to process up to 9 inputs.
260 {
261 const int32_t* b = buffer;
262
263 const int8_t* i0 = input[0];
264 assert(i0 != NULL);
265 if XNN_UNPREDICTABLE(i0 != zero) {
266 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
267 }
268 const int8_t* i1 = input[1];
269 assert(i1 != NULL);
270 if XNN_UNPREDICTABLE(i1 != zero) {
271 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
272 }
273 const int8_t* i2 = input[2];
274 assert(i2 != NULL);
275 if XNN_UNPREDICTABLE(i2 != zero) {
276 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
277 }
278 const int8_t* i3 = input[3];
279 assert(i3 != NULL);
280 if XNN_UNPREDICTABLE(i3 != zero) {
281 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
282 }
283 const int8_t* i4 = input[4];
284 assert(i4 != NULL);
285 if XNN_UNPREDICTABLE(i4 != zero) {
286 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
287 }
288 const int8_t* i5 = input[5];
289 assert(i5 != NULL);
290 if XNN_UNPREDICTABLE(i5 != zero) {
291 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
292 }
293 const int8_t* i6 = input[6];
294 assert(i6 != NULL);
295 if XNN_UNPREDICTABLE(i6 != zero) {
296 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
297 }
298 const int8_t* i7 = input[7];
299 assert(i7 != NULL);
300 if XNN_UNPREDICTABLE(i7 != zero) {
301 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
302 }
303 const int8_t* i8 = input[8];
304 assert(i8 != NULL);
305 if XNN_UNPREDICTABLE(i8 != zero) {
306 i8 = (const int8_t*) ((uintptr_t) i8 + input_offset);
307 }
308
309 size_t c = channels;
310 for (; c >= 8; c -= 8) {
311 int32x4_t vacc0123 = vld1q_s32(b); b += 4;
312 int32x4_t vacc4567 = vld1q_s32(b); b += 4;
313
314 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0)); i0 += 8;
315 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
316
317 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
318 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
319
320 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1)); i1 += 8;
321 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
322
323 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
324 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
325
326 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2)); i2 += 8;
327 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
328
329 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
330 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
331
332 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3)); i3 += 8;
333 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
334
335 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
336 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
337
338 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4)); i4 += 8;
339 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
340
341 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
342 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
343
344 const int16x8_t vi5x01234567 = vmovl_s8(vld1_s8(i5)); i5 += 8;
345 const int16x8_t vk5x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
346
347 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi5x01234567), vget_low_s16(vk5x01234567));
348 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi5x01234567), vget_high_s16(vk5x01234567));
349
350 const int16x8_t vi6x01234567 = vmovl_s8(vld1_s8(i6)); i6 += 8;
351 const int16x8_t vk6x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
352
353 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi6x01234567), vget_low_s16(vk6x01234567));
354 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi6x01234567), vget_high_s16(vk6x01234567));
355
356 const int16x8_t vi7x01234567 = vmovl_s8(vld1_s8(i7)); i7 += 8;
357 const int16x8_t vk7x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
358
359 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi7x01234567), vget_low_s16(vk7x01234567));
360 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi7x01234567), vget_high_s16(vk7x01234567));
361
362 const int16x8_t vi8x01234567 = vmovl_s8(vld1_s8(i8)); i8 += 8;
363 const int16x8_t vk8x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
364
365 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi8x01234567), vget_low_s16(vk8x01234567));
366 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi8x01234567), vget_high_s16(vk8x01234567));
367
368 float32x4_t vfpacc0123 = vcvtq_f32_s32(vacc0123);
369 float32x4_t vfpacc4567 = vcvtq_f32_s32(vacc4567);
370
371 vfpacc0123 = vmulq_f32(vfpacc0123, vscale);
372 vfpacc4567 = vmulq_f32(vfpacc4567, vscale);
373
374 vacc0123 = vreinterpretq_s32_f32(vaddq_f32(vfpacc0123, vmagic_bias));
375 vacc4567 = vreinterpretq_s32_f32(vaddq_f32(vfpacc4567, vmagic_bias));
376
377 vacc0123 = vqsubq_s32(vacc0123, vmagic_bias_less_output_zero_point);
378 vacc4567 = vqsubq_s32(vacc4567, vmagic_bias_less_output_zero_point);
379
380 #if XNN_ARCH_ARM64
381 int16x8_t vacc01234567 = vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567);
382
383
384 int8x8_t vout01234567 = vqmovn_s16(vacc01234567);
385 #else // !XNN_ARCH_ARM64
386 int16x8_t vacc01234567 = vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567));
387
388
389 int8x8_t vout01234567 = vqmovn_s16(vacc01234567);
390 #endif // !XNN_ARCH_ARM64
391
392 vout01234567 = vmax_s8(vout01234567, voutput_min);
393
394 vout01234567 = vmin_s8(vout01234567, voutput_max);
395
396 vst1_s8(output, vout01234567); output += 8;
397 }
398
399 if XNN_UNLIKELY(c != 0) {
400 {
401 int32x4_t vacc0123 = vld1q_s32(b); b += 4;
402 int32x4_t vacc4567 = vld1q_s32(b); b += 4;
403
404 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0));
405 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
406
407 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
408 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
409
410 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1));
411 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
412
413 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
414 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
415
416 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2));
417 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
418
419 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
420 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
421
422 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3));
423 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
424
425 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
426 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
427
428 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4));
429 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
430
431 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
432 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
433
434 const int16x8_t vi5x01234567 = vmovl_s8(vld1_s8(i5));
435 const int16x8_t vk5x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
436
437 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi5x01234567), vget_low_s16(vk5x01234567));
438 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi5x01234567), vget_high_s16(vk5x01234567));
439
440 const int16x8_t vi6x01234567 = vmovl_s8(vld1_s8(i6));
441 const int16x8_t vk6x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
442
443 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi6x01234567), vget_low_s16(vk6x01234567));
444 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi6x01234567), vget_high_s16(vk6x01234567));
445
446 const int16x8_t vi7x01234567 = vmovl_s8(vld1_s8(i7));
447 const int16x8_t vk7x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
448
449 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi7x01234567), vget_low_s16(vk7x01234567));
450 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi7x01234567), vget_high_s16(vk7x01234567));
451
452 const int16x8_t vi8x01234567 = vmovl_s8(vld1_s8(i8));
453 const int16x8_t vk8x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
454
455 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi8x01234567), vget_low_s16(vk8x01234567));
456 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi8x01234567), vget_high_s16(vk8x01234567));
457
458 float32x4_t vfpacc0123 = vcvtq_f32_s32(vacc0123);
459 float32x4_t vfpacc4567 = vcvtq_f32_s32(vacc4567);
460
461 vfpacc0123 = vmulq_f32(vfpacc0123, vscale);
462 vfpacc4567 = vmulq_f32(vfpacc4567, vscale);
463
464 vacc0123 = vreinterpretq_s32_f32(vaddq_f32(vfpacc0123, vmagic_bias));
465 vacc4567 = vreinterpretq_s32_f32(vaddq_f32(vfpacc4567, vmagic_bias));
466
467 vacc0123 = vqsubq_s32(vacc0123, vmagic_bias_less_output_zero_point);
468 vacc4567 = vqsubq_s32(vacc4567, vmagic_bias_less_output_zero_point);
469
470 #if XNN_ARCH_ARM64
471 int16x8_t vacc01234567 = vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567);
472 #else
473 int16x8_t vacc01234567 = vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567));
474 #endif
475
476 int8x8_t vout01234567 = vqmovn_s16(vacc01234567);
477 vout01234567 = vmax_s8(vout01234567, voutput_min);
478 vout01234567 = vmin_s8(vout01234567, voutput_max);
479
480 if (c & 4) {
481 vst1_lane_u32((void*) output, vreinterpret_u32_s8(vout01234567), 0); output += 4;
482 vout01234567 = vext_s8(vout01234567, vout01234567, 4);
483 }
484 if (c & 2) {
485 vst1_lane_u16((void*) output, vreinterpret_u16_s8(vout01234567), 0); output += 2;
486 vout01234567 = vext_s8(vout01234567, vout01234567, 2);
487 }
488 if (c & 1) {
489 vst1_lane_s8(output, vout01234567, 0); output += 1;
490 }
491 }
492 }
493 }
494
495 input = (const int8_t**) ((uintptr_t) input + input_stride);
496 output = (int8_t*) ((uintptr_t) output + output_increment);
497 } while (--output_width != 0);
498 }