Debian Codegraph @

src/qs8-dwconv/gen/qs8-dwconv-8f8m9l16c8s8r-minmax-fp32-neon-mul16.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-dwconv/multipass-neon-mul16.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12
13 #include <arm_neon.h>
14
15 #include "xnnpack/dwconv.h"
16 #include "xnnpack/math.h"
17
18
19 void xnn_qs8_dwconv_minmax_fp32_ukernel_8f8m9l16c8s8r__neon_mul16(
20 size_t channels,
21 size_t output_width,
22 const int8_t** input,
23 const void* weights,
24 int8_t* output,
25 intptr_t input_stride,
26 size_t output_increment,
27 size_t input_offset,
28 const int8_t* zero,
29 size_t kernel_size,
30 int32_t* buffer,
31 const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
32 {
33 assert(channels != 0);
34 assert(output_width != 0);
35 assert(kernel_size > 8);
36
37 const float32x4_t vscale = vld1q_dup_f32(&params->fp32_neon.scale);
38 const float32x4_t vmagic_bias = vld1q_dup_f32(&params->fp32_neon.magic_bias);
39 const int32x4_t vmagic_bias_less_output_zero_point = vld1q_dup_s32(&params->fp32_neon.magic_bias_less_output_zero_point);
40 const int8x16_t voutput_min = vld1q_dup_s8(&params->fp32_neon.output_min);
41 const int8x16_t voutput_max = vld1q_dup_s8(&params->fp32_neon.output_max);
42
43 do {
44 const void* w = weights;
45
46 // First pass to process 8 inputs.
47 {
48 int32_t* b = buffer;
49 const int8_t* i0 = input[0];
50 assert(i0 != NULL);
51 if XNN_UNPREDICTABLE(i0 != zero) {
52 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
53 }
54 const int8_t* i1 = input[1];
55 assert(i1 != NULL);
56 if XNN_UNPREDICTABLE(i1 != zero) {
57 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
58 }
59 const int8_t* i2 = input[2];
60 assert(i2 != NULL);
61 if XNN_UNPREDICTABLE(i2 != zero) {
62 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
63 }
64 const int8_t* i3 = input[3];
65 assert(i3 != NULL);
66 if XNN_UNPREDICTABLE(i3 != zero) {
67 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
68 }
69 const int8_t* i4 = input[4];
70 assert(i4 != NULL);
71 if XNN_UNPREDICTABLE(i4 != zero) {
72 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
73 }
74 const int8_t* i5 = input[5];
75 assert(i5 != NULL);
76 if XNN_UNPREDICTABLE(i5 != zero) {
77 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
78 }
79 const int8_t* i6 = input[6];
80 assert(i6 != NULL);
81 if XNN_UNPREDICTABLE(i6 != zero) {
82 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
83 }
84 const int8_t* i7 = input[7];
85 assert(i7 != NULL);
86 if XNN_UNPREDICTABLE(i7 != zero) {
87 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
88 }
89 input += 8;
90
91 size_t c = round_up_po2(channels, 8);
92
93 for (; c >= 16; c -= 16) {
94 int32x4_t vacc0123 = vld1q_s32(w); w = (const int32_t*) w + 4;
95 int32x4_t vacc4567 = vld1q_s32(w); w = (const int32_t*) w + 4;
96 int32x4_t vacc89AB = vld1q_s32(w); w = (const int32_t*) w + 4;
97 int32x4_t vaccCDEF = vld1q_s32(w); w = (const int32_t*) w + 4;
98
99 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0)); i0 += 8;
100 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
101 const int16x8_t vi0x89ABCDEF = vmovl_s8(vld1_s8(i0)); i0 += 8;
102 const int16x8_t vk0x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
103
104 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
105 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
106 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi0x89ABCDEF), vget_low_s16(vk0x89ABCDEF));
107 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi0x89ABCDEF), vget_high_s16(vk0x89ABCDEF));
108
109 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1)); i1 += 8;
110 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
111 const int16x8_t vi1x89ABCDEF = vmovl_s8(vld1_s8(i1)); i1 += 8;
112 const int16x8_t vk1x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
113
114 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
115 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
116 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi1x89ABCDEF), vget_low_s16(vk1x89ABCDEF));
117 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi1x89ABCDEF), vget_high_s16(vk1x89ABCDEF));
118
119 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2)); i2 += 8;
120 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
121 const int16x8_t vi2x89ABCDEF = vmovl_s8(vld1_s8(i2)); i2 += 8;
122 const int16x8_t vk2x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
123
124 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
125 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
126 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi2x89ABCDEF), vget_low_s16(vk2x89ABCDEF));
127 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi2x89ABCDEF), vget_high_s16(vk2x89ABCDEF));
128
129 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3)); i3 += 8;
130 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
131 const int16x8_t vi3x89ABCDEF = vmovl_s8(vld1_s8(i3)); i3 += 8;
132 const int16x8_t vk3x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
133
134 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
135 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
136 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi3x89ABCDEF), vget_low_s16(vk3x89ABCDEF));
137 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi3x89ABCDEF), vget_high_s16(vk3x89ABCDEF));
138
139 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4)); i4 += 8;
140 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
141 const int16x8_t vi4x89ABCDEF = vmovl_s8(vld1_s8(i4)); i4 += 8;
142 const int16x8_t vk4x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
143
144 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
145 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
146 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi4x89ABCDEF), vget_low_s16(vk4x89ABCDEF));
147 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi4x89ABCDEF), vget_high_s16(vk4x89ABCDEF));
148
149 const int16x8_t vi5x01234567 = vmovl_s8(vld1_s8(i5)); i5 += 8;
150 const int16x8_t vk5x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
151 const int16x8_t vi5x89ABCDEF = vmovl_s8(vld1_s8(i5)); i5 += 8;
152 const int16x8_t vk5x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
153
154 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi5x01234567), vget_low_s16(vk5x01234567));
155 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi5x01234567), vget_high_s16(vk5x01234567));
156 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi5x89ABCDEF), vget_low_s16(vk5x89ABCDEF));
157 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi5x89ABCDEF), vget_high_s16(vk5x89ABCDEF));
158
159 const int16x8_t vi6x01234567 = vmovl_s8(vld1_s8(i6)); i6 += 8;
160 const int16x8_t vk6x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
161 const int16x8_t vi6x89ABCDEF = vmovl_s8(vld1_s8(i6)); i6 += 8;
162 const int16x8_t vk6x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
163
164 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi6x01234567), vget_low_s16(vk6x01234567));
165 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi6x01234567), vget_high_s16(vk6x01234567));
166 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi6x89ABCDEF), vget_low_s16(vk6x89ABCDEF));
167 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi6x89ABCDEF), vget_high_s16(vk6x89ABCDEF));
168
169 const int16x8_t vi7x01234567 = vmovl_s8(vld1_s8(i7)); i7 += 8;
170 const int16x8_t vk7x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
171 const int16x8_t vi7x89ABCDEF = vmovl_s8(vld1_s8(i7)); i7 += 8;
172 const int16x8_t vk7x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
173
174 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi7x01234567), vget_low_s16(vk7x01234567));
175 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi7x01234567), vget_high_s16(vk7x01234567));
176 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi7x89ABCDEF), vget_low_s16(vk7x89ABCDEF));
177 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi7x89ABCDEF), vget_high_s16(vk7x89ABCDEF));
178
179 vst1q_s32(b, vacc0123); b += 4;
180 vst1q_s32(b, vacc4567); b += 4;
181 vst1q_s32(b, vacc89AB); b += 4;
182 vst1q_s32(b, vaccCDEF); b += 4;
183 }
184
185 if XNN_UNLIKELY(c != 0) {
186 do {
187 int32x4_t vacc0123 = vld1q_s32(w); w = (const int32_t*) w + 4;
188 int32x4_t vacc4567 = vld1q_s32(w); w = (const int32_t*) w + 4;
189
190 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0)); i0 += 8;
191 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
192
193 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
194 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
195
196 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1)); i1 += 8;
197 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
198
199 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
200 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
201
202 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2)); i2 += 8;
203 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
204
205 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
206 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
207
208 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3)); i3 += 8;
209 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
210
211 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
212 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
213
214 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4)); i4 += 8;
215 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
216
217 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
218 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
219
220 const int16x8_t vi5x01234567 = vmovl_s8(vld1_s8(i5)); i5 += 8;
221 const int16x8_t vk5x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
222
223 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi5x01234567), vget_low_s16(vk5x01234567));
224 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi5x01234567), vget_high_s16(vk5x01234567));
225
226 const int16x8_t vi6x01234567 = vmovl_s8(vld1_s8(i6)); i6 += 8;
227 const int16x8_t vk6x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
228
229 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi6x01234567), vget_low_s16(vk6x01234567));
230 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi6x01234567), vget_high_s16(vk6x01234567));
231
232 const int16x8_t vi7x01234567 = vmovl_s8(vld1_s8(i7)); i7 += 8;
233 const int16x8_t vk7x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
234
235 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi7x01234567), vget_low_s16(vk7x01234567));
236 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi7x01234567), vget_high_s16(vk7x01234567));
237
238 vst1q_s32(b, vacc0123); b += 4;
239 vst1q_s32(b, vacc4567); b += 4;
240 c -= 8;
241 } while (c != 0);
242 }
243 }
244
245 // Middle pass to process 8 inputs in each iteration.
246 for (size_t ks = kernel_size - 8; ks > 9; ks -= 8) {
247 int32_t* b = buffer;
248
249 const int8_t* i0 = input[0];
250 assert(i0 != NULL);
251 if XNN_UNPREDICTABLE(i0 != zero) {
252 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
253 }
254 const int8_t* i1 = input[1];
255 assert(i1 != NULL);
256 if XNN_UNPREDICTABLE(i1 != zero) {
257 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
258 }
259 const int8_t* i2 = input[2];
260 assert(i2 != NULL);
261 if XNN_UNPREDICTABLE(i2 != zero) {
262 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
263 }
264 const int8_t* i3 = input[3];
265 assert(i3 != NULL);
266 if XNN_UNPREDICTABLE(i3 != zero) {
267 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
268 }
269 const int8_t* i4 = input[4];
270 assert(i4 != NULL);
271 if XNN_UNPREDICTABLE(i4 != zero) {
272 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
273 }
274 const int8_t* i5 = input[5];
275 assert(i5 != NULL);
276 if XNN_UNPREDICTABLE(i5 != zero) {
277 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
278 }
279 const int8_t* i6 = input[6];
280 assert(i6 != NULL);
281 if XNN_UNPREDICTABLE(i6 != zero) {
282 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
283 }
284 const int8_t* i7 = input[7];
285 assert(i7 != NULL);
286 if XNN_UNPREDICTABLE(i7 != zero) {
287 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
288 }
289 input += 8;
290
291 size_t c = round_up_po2(channels, 8);
292
293 for (; c >= 16; c -= 16) {
294 int32x4_t vacc0123 = vld1q_s32(b);
295 int32x4_t vacc4567 = vld1q_s32(b + 4);
296 int32x4_t vacc89AB = vld1q_s32(b + 8);
297 int32x4_t vaccCDEF = vld1q_s32(b + 12);
298
299 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0)); i0 += 8;
300 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
301 const int16x8_t vi0x89ABCDEF = vmovl_s8(vld1_s8(i0)); i0 += 8;
302 const int16x8_t vk0x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
303
304 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
305 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
306 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi0x89ABCDEF), vget_low_s16(vk0x89ABCDEF));
307 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi0x89ABCDEF), vget_high_s16(vk0x89ABCDEF));
308
309 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1)); i1 += 8;
310 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
311 const int16x8_t vi1x89ABCDEF = vmovl_s8(vld1_s8(i1)); i1 += 8;
312 const int16x8_t vk1x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
313
314 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
315 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
316 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi1x89ABCDEF), vget_low_s16(vk1x89ABCDEF));
317 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi1x89ABCDEF), vget_high_s16(vk1x89ABCDEF));
318
319 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2)); i2 += 8;
320 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
321 const int16x8_t vi2x89ABCDEF = vmovl_s8(vld1_s8(i2)); i2 += 8;
322 const int16x8_t vk2x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
323
324 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
325 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
326 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi2x89ABCDEF), vget_low_s16(vk2x89ABCDEF));
327 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi2x89ABCDEF), vget_high_s16(vk2x89ABCDEF));
328
329 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3)); i3 += 8;
330 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
331 const int16x8_t vi3x89ABCDEF = vmovl_s8(vld1_s8(i3)); i3 += 8;
332 const int16x8_t vk3x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
333
334 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
335 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
336 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi3x89ABCDEF), vget_low_s16(vk3x89ABCDEF));
337 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi3x89ABCDEF), vget_high_s16(vk3x89ABCDEF));
338
339 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4)); i4 += 8;
340 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
341 const int16x8_t vi4x89ABCDEF = vmovl_s8(vld1_s8(i4)); i4 += 8;
342 const int16x8_t vk4x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
343
344 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
345 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
346 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi4x89ABCDEF), vget_low_s16(vk4x89ABCDEF));
347 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi4x89ABCDEF), vget_high_s16(vk4x89ABCDEF));
348
349 const int16x8_t vi5x01234567 = vmovl_s8(vld1_s8(i5)); i5 += 8;
350 const int16x8_t vk5x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
351 const int16x8_t vi5x89ABCDEF = vmovl_s8(vld1_s8(i5)); i5 += 8;
352 const int16x8_t vk5x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
353
354 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi5x01234567), vget_low_s16(vk5x01234567));
355 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi5x01234567), vget_high_s16(vk5x01234567));
356 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi5x89ABCDEF), vget_low_s16(vk5x89ABCDEF));
357 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi5x89ABCDEF), vget_high_s16(vk5x89ABCDEF));
358
359 const int16x8_t vi6x01234567 = vmovl_s8(vld1_s8(i6)); i6 += 8;
360 const int16x8_t vk6x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
361 const int16x8_t vi6x89ABCDEF = vmovl_s8(vld1_s8(i6)); i6 += 8;
362 const int16x8_t vk6x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
363
364 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi6x01234567), vget_low_s16(vk6x01234567));
365 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi6x01234567), vget_high_s16(vk6x01234567));
366 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi6x89ABCDEF), vget_low_s16(vk6x89ABCDEF));
367 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi6x89ABCDEF), vget_high_s16(vk6x89ABCDEF));
368
369 const int16x8_t vi7x01234567 = vmovl_s8(vld1_s8(i7)); i7 += 8;
370 const int16x8_t vk7x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
371 const int16x8_t vi7x89ABCDEF = vmovl_s8(vld1_s8(i7)); i7 += 8;
372 const int16x8_t vk7x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
373
374 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi7x01234567), vget_low_s16(vk7x01234567));
375 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi7x01234567), vget_high_s16(vk7x01234567));
376 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi7x89ABCDEF), vget_low_s16(vk7x89ABCDEF));
377 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi7x89ABCDEF), vget_high_s16(vk7x89ABCDEF));
378
379 vst1q_s32(b, vacc0123); b += 4;
380 vst1q_s32(b, vacc4567); b += 4;
381 vst1q_s32(b, vacc89AB); b += 4;
382 vst1q_s32(b, vaccCDEF); b += 4;
383 }
384
385 if XNN_UNLIKELY(c != 0) {
386 do {
387 int32x4_t vacc0123 = vld1q_s32(b);
388 int32x4_t vacc4567 = vld1q_s32(b + 4);
389
390 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0)); i0 += 8;
391 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
392
393 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
394 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
395
396 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1)); i1 += 8;
397 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
398
399 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
400 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
401
402 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2)); i2 += 8;
403 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
404
405 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
406 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
407
408 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3)); i3 += 8;
409 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
410
411 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
412 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
413
414 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4)); i4 += 8;
415 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
416
417 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
418 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
419
420 const int16x8_t vi5x01234567 = vmovl_s8(vld1_s8(i5)); i5 += 8;
421 const int16x8_t vk5x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
422
423 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi5x01234567), vget_low_s16(vk5x01234567));
424 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi5x01234567), vget_high_s16(vk5x01234567));
425
426 const int16x8_t vi6x01234567 = vmovl_s8(vld1_s8(i6)); i6 += 8;
427 const int16x8_t vk6x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
428
429 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi6x01234567), vget_low_s16(vk6x01234567));
430 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi6x01234567), vget_high_s16(vk6x01234567));
431
432 const int16x8_t vi7x01234567 = vmovl_s8(vld1_s8(i7)); i7 += 8;
433 const int16x8_t vk7x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
434
435 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi7x01234567), vget_low_s16(vk7x01234567));
436 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi7x01234567), vget_high_s16(vk7x01234567));
437
438 vst1q_s32(b, vacc0123); b += 4;
439 vst1q_s32(b, vacc4567); b += 4;
440 c -= 8;
441 } while (c != 0);
442 }
443 }
444
445 // Last pass to process up to 9 inputs.
446 {
447 const int32_t* b = buffer;
448
449 const int8_t* i0 = input[0];
450 assert(i0 != NULL);
451 if XNN_UNPREDICTABLE(i0 != zero) {
452 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
453 }
454 const int8_t* i1 = input[1];
455 assert(i1 != NULL);
456 if XNN_UNPREDICTABLE(i1 != zero) {
457 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
458 }
459 const int8_t* i2 = input[2];
460 assert(i2 != NULL);
461 if XNN_UNPREDICTABLE(i2 != zero) {
462 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
463 }
464 const int8_t* i3 = input[3];
465 assert(i3 != NULL);
466 if XNN_UNPREDICTABLE(i3 != zero) {
467 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
468 }
469 const int8_t* i4 = input[4];
470 assert(i4 != NULL);
471 if XNN_UNPREDICTABLE(i4 != zero) {
472 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
473 }
474 const int8_t* i5 = input[5];
475 assert(i5 != NULL);
476 if XNN_UNPREDICTABLE(i5 != zero) {
477 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
478 }
479 const int8_t* i6 = input[6];
480 assert(i6 != NULL);
481 if XNN_UNPREDICTABLE(i6 != zero) {
482 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
483 }
484 const int8_t* i7 = input[7];
485 assert(i7 != NULL);
486 if XNN_UNPREDICTABLE(i7 != zero) {
487 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
488 }
489 const int8_t* i8 = input[8];
490 assert(i8 != NULL);
491 if XNN_UNPREDICTABLE(i8 != zero) {
492 i8 = (const int8_t*) ((uintptr_t) i8 + input_offset);
493 }
494
495 size_t c = channels;
496 for (; c >= 16; c -= 16) {
497 int32x4_t vacc0123 = vld1q_s32(b); b += 4;
498 int32x4_t vacc4567 = vld1q_s32(b); b += 4;
499 int32x4_t vacc89AB = vld1q_s32(b); b += 4;
500 int32x4_t vaccCDEF = vld1q_s32(b); b += 4;
501
502 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0)); i0 += 8;
503 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
504 const int16x8_t vi0x89ABCDEF = vmovl_s8(vld1_s8(i0)); i0 += 8;
505 const int16x8_t vk0x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
506
507 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
508 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
509 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi0x89ABCDEF), vget_low_s16(vk0x89ABCDEF));
510 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi0x89ABCDEF), vget_high_s16(vk0x89ABCDEF));
511
512 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1)); i1 += 8;
513 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
514 const int16x8_t vi1x89ABCDEF = vmovl_s8(vld1_s8(i1)); i1 += 8;
515 const int16x8_t vk1x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
516
517 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
518 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
519 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi1x89ABCDEF), vget_low_s16(vk1x89ABCDEF));
520 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi1x89ABCDEF), vget_high_s16(vk1x89ABCDEF));
521
522 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2)); i2 += 8;
523 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
524 const int16x8_t vi2x89ABCDEF = vmovl_s8(vld1_s8(i2)); i2 += 8;
525 const int16x8_t vk2x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
526
527 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
528 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
529 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi2x89ABCDEF), vget_low_s16(vk2x89ABCDEF));
530 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi2x89ABCDEF), vget_high_s16(vk2x89ABCDEF));
531
532 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3)); i3 += 8;
533 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
534 const int16x8_t vi3x89ABCDEF = vmovl_s8(vld1_s8(i3)); i3 += 8;
535 const int16x8_t vk3x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
536
537 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
538 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
539 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi3x89ABCDEF), vget_low_s16(vk3x89ABCDEF));
540 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi3x89ABCDEF), vget_high_s16(vk3x89ABCDEF));
541
542 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4)); i4 += 8;
543 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
544 const int16x8_t vi4x89ABCDEF = vmovl_s8(vld1_s8(i4)); i4 += 8;
545 const int16x8_t vk4x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
546
547 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
548 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
549 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi4x89ABCDEF), vget_low_s16(vk4x89ABCDEF));
550 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi4x89ABCDEF), vget_high_s16(vk4x89ABCDEF));
551
552 const int16x8_t vi5x01234567 = vmovl_s8(vld1_s8(i5)); i5 += 8;
553 const int16x8_t vk5x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
554 const int16x8_t vi5x89ABCDEF = vmovl_s8(vld1_s8(i5)); i5 += 8;
555 const int16x8_t vk5x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
556
557 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi5x01234567), vget_low_s16(vk5x01234567));
558 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi5x01234567), vget_high_s16(vk5x01234567));
559 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi5x89ABCDEF), vget_low_s16(vk5x89ABCDEF));
560 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi5x89ABCDEF), vget_high_s16(vk5x89ABCDEF));
561
562 const int16x8_t vi6x01234567 = vmovl_s8(vld1_s8(i6)); i6 += 8;
563 const int16x8_t vk6x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
564 const int16x8_t vi6x89ABCDEF = vmovl_s8(vld1_s8(i6)); i6 += 8;
565 const int16x8_t vk6x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
566
567 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi6x01234567), vget_low_s16(vk6x01234567));
568 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi6x01234567), vget_high_s16(vk6x01234567));
569 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi6x89ABCDEF), vget_low_s16(vk6x89ABCDEF));
570 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi6x89ABCDEF), vget_high_s16(vk6x89ABCDEF));
571
572 const int16x8_t vi7x01234567 = vmovl_s8(vld1_s8(i7)); i7 += 8;
573 const int16x8_t vk7x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
574 const int16x8_t vi7x89ABCDEF = vmovl_s8(vld1_s8(i7)); i7 += 8;
575 const int16x8_t vk7x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
576
577 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi7x01234567), vget_low_s16(vk7x01234567));
578 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi7x01234567), vget_high_s16(vk7x01234567));
579 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi7x89ABCDEF), vget_low_s16(vk7x89ABCDEF));
580 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi7x89ABCDEF), vget_high_s16(vk7x89ABCDEF));
581
582 const int16x8_t vi8x01234567 = vmovl_s8(vld1_s8(i8)); i8 += 8;
583 const int16x8_t vk8x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
584 const int16x8_t vi8x89ABCDEF = vmovl_s8(vld1_s8(i8)); i8 += 8;
585 const int16x8_t vk8x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
586
587 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi8x01234567), vget_low_s16(vk8x01234567));
588 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi8x01234567), vget_high_s16(vk8x01234567));
589 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi8x89ABCDEF), vget_low_s16(vk8x89ABCDEF));
590 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi8x89ABCDEF), vget_high_s16(vk8x89ABCDEF));
591
592 float32x4_t vfpacc0123 = vcvtq_f32_s32(vacc0123);
593 float32x4_t vfpacc4567 = vcvtq_f32_s32(vacc4567);
594 float32x4_t vfpacc89AB = vcvtq_f32_s32(vacc89AB);
595 float32x4_t vfpaccCDEF = vcvtq_f32_s32(vaccCDEF);
596
597 vfpacc0123 = vmulq_f32(vfpacc0123, vscale);
598 vfpacc4567 = vmulq_f32(vfpacc4567, vscale);
599 vfpacc89AB = vmulq_f32(vfpacc89AB, vscale);
600 vfpaccCDEF = vmulq_f32(vfpaccCDEF, vscale);
601
602 vacc0123 = vreinterpretq_s32_f32(vaddq_f32(vfpacc0123, vmagic_bias));
603 vacc4567 = vreinterpretq_s32_f32(vaddq_f32(vfpacc4567, vmagic_bias));
604 vacc89AB = vreinterpretq_s32_f32(vaddq_f32(vfpacc89AB, vmagic_bias));
605 vaccCDEF = vreinterpretq_s32_f32(vaddq_f32(vfpaccCDEF, vmagic_bias));
606
607 vacc0123 = vqsubq_s32(vacc0123, vmagic_bias_less_output_zero_point);
608 vacc4567 = vqsubq_s32(vacc4567, vmagic_bias_less_output_zero_point);
609 vacc89AB = vqsubq_s32(vacc89AB, vmagic_bias_less_output_zero_point);
610 vaccCDEF = vqsubq_s32(vaccCDEF, vmagic_bias_less_output_zero_point);
611
612 #if XNN_ARCH_ARM64
613 int16x8_t vacc01234567 = vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567);
614 int16x8_t vacc89ABCDEF = vqmovn_high_s32(vqmovn_s32(vacc89AB), vaccCDEF);
615
616
617 int8x16_t vout0123456789ABCDEF = vqmovn_high_s16(vqmovn_s16(vacc01234567), vacc89ABCDEF);
618 #else // !XNN_ARCH_ARM64
619 int16x8_t vacc01234567 = vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567));
620 int16x8_t vacc89ABCDEF = vcombine_s16(vqmovn_s32(vacc89AB), vqmovn_s32(vaccCDEF));
621
622
623 int8x16_t vout0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc01234567), vqmovn_s16(vacc89ABCDEF));
624 #endif // !XNN_ARCH_ARM64
625
626 vout0123456789ABCDEF = vmaxq_s8(vout0123456789ABCDEF, voutput_min);
627
628 vout0123456789ABCDEF = vminq_s8(vout0123456789ABCDEF, voutput_max);
629
630 vst1q_s8(output, vout0123456789ABCDEF); output += 16;
631 }
632
633 if XNN_UNLIKELY(c != 0) {
634 do {
635 int32x4_t vacc0123 = vld1q_s32(b); b += 4;
636 int32x4_t vacc4567 = vld1q_s32(b); b += 4;
637
638 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0)); i0 += 8;
639 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
640
641 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
642 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
643
644 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1)); i1 += 8;
645 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
646
647 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
648 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
649
650 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2)); i2 += 8;
651 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
652
653 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
654 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
655
656 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3)); i3 += 8;
657 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
658
659 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
660 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
661
662 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4)); i4 += 8;
663 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
664
665 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
666 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
667
668 const int16x8_t vi5x01234567 = vmovl_s8(vld1_s8(i5)); i5 += 8;
669 const int16x8_t vk5x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
670
671 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi5x01234567), vget_low_s16(vk5x01234567));
672 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi5x01234567), vget_high_s16(vk5x01234567));
673
674 const int16x8_t vi6x01234567 = vmovl_s8(vld1_s8(i6)); i6 += 8;
675 const int16x8_t vk6x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
676
677 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi6x01234567), vget_low_s16(vk6x01234567));
678 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi6x01234567), vget_high_s16(vk6x01234567));
679
680 const int16x8_t vi7x01234567 = vmovl_s8(vld1_s8(i7)); i7 += 8;
681 const int16x8_t vk7x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
682
683 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi7x01234567), vget_low_s16(vk7x01234567));
684 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi7x01234567), vget_high_s16(vk7x01234567));
685
686 const int16x8_t vi8x01234567 = vmovl_s8(vld1_s8(i8)); i8 += 8;
687 const int16x8_t vk8x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
688
689 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi8x01234567), vget_low_s16(vk8x01234567));
690 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi8x01234567), vget_high_s16(vk8x01234567));
691
692 float32x4_t vfpacc0123 = vcvtq_f32_s32(vacc0123);
693 float32x4_t vfpacc4567 = vcvtq_f32_s32(vacc4567);
694
695 vfpacc0123 = vmulq_f32(vfpacc0123, vscale);
696 vfpacc4567 = vmulq_f32(vfpacc4567, vscale);
697
698 vacc0123 = vreinterpretq_s32_f32(vaddq_f32(vfpacc0123, vmagic_bias));
699 vacc4567 = vreinterpretq_s32_f32(vaddq_f32(vfpacc4567, vmagic_bias));
700
701 vacc0123 = vqsubq_s32(vacc0123, vmagic_bias_less_output_zero_point);
702 vacc4567 = vqsubq_s32(vacc4567, vmagic_bias_less_output_zero_point);
703
704 #if XNN_ARCH_ARM64
705 int16x8_t vacc01234567 = vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567);
706 #else
707 int16x8_t vacc01234567 = vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567));
708 #endif
709
710 int8x8_t vout01234567 = vqmovn_s16(vacc01234567);
711 vout01234567 = vmax_s8(vout01234567, vget_low_s8(voutput_min));
712 vout01234567 = vmin_s8(vout01234567, vget_low_s8(voutput_max));
713
714 if XNN_LIKELY(c >= 8) {
715 vst1_s8(output, vout01234567); output += 8;
716 c -= 8;
717 } else {
718 if (c & 4) {
719 vst1_lane_u32((void*) output, vreinterpret_u32_s8(vout01234567), 0); output += 4;
720 vout01234567 = vext_s8(vout01234567, vout01234567, 4);
721 }
722 if (c & 2) {
723 vst1_lane_u16((void*) output, vreinterpret_u16_s8(vout01234567), 0); output += 2;
724 vout01234567 = vext_s8(vout01234567, vout01234567, 2);
725 }
726 if (c & 1) {
727 vst1_lane_s8(output, vout01234567, 0); output += 1;
728 }
729 c = 0;
730 }
731 } while (c != 0);
732 }
733 }
734
735 input = (const int8_t**) ((uintptr_t) input + input_stride);
736 output = (int8_t*) ((uintptr_t) output + output_increment);
737 } while (--output_width != 0);
738 }