Debian Codegraph @

src/qs8-dwconv/gen/qs8-dwconv-5f5m5l16c8s8r-minmax-fp32-neonv8-mul16.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-dwconv/multipass-neon-mul16.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12
13 #include <arm_neon.h>
14
15 #include "xnnpack/dwconv.h"
16 #include "xnnpack/intrinsics-polyfill.h"
17 #include "xnnpack/math.h"
18
19
20 void xnn_qs8_dwconv_minmax_fp32_ukernel_5f5m5l16c8s8r__neonv8_mul16(
21 size_t channels,
22 size_t output_width,
23 const int8_t** input,
24 const void* weights,
25 int8_t* output,
26 intptr_t input_stride,
27 size_t output_increment,
28 size_t input_offset,
29 const int8_t* zero,
30 size_t kernel_size,
31 int32_t* buffer,
32 const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
33 {
34 assert(channels != 0);
35 assert(output_width != 0);
36 assert(kernel_size > 5);
37
38 const float32x4_t vscale = vld1q_dup_f32(&params->fp32_neonv8.scale);
39 const int16x8_t voutput_zero_point = vld1q_dup_s16(&params->fp32_neonv8.output_zero_point);
40 const int8x16_t voutput_min = vld1q_dup_s8(&params->fp32_neonv8.output_min);
41 const int8x16_t voutput_max = vld1q_dup_s8(&params->fp32_neonv8.output_max);
42
43 do {
44 const void* w = weights;
45
46 // First pass to process 5 inputs.
47 {
48 int32_t* b = buffer;
49 const int8_t* i0 = input[0];
50 assert(i0 != NULL);
51 if XNN_UNPREDICTABLE(i0 != zero) {
52 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
53 }
54 const int8_t* i1 = input[1];
55 assert(i1 != NULL);
56 if XNN_UNPREDICTABLE(i1 != zero) {
57 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
58 }
59 const int8_t* i2 = input[2];
60 assert(i2 != NULL);
61 if XNN_UNPREDICTABLE(i2 != zero) {
62 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
63 }
64 const int8_t* i3 = input[3];
65 assert(i3 != NULL);
66 if XNN_UNPREDICTABLE(i3 != zero) {
67 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
68 }
69 const int8_t* i4 = input[4];
70 assert(i4 != NULL);
71 if XNN_UNPREDICTABLE(i4 != zero) {
72 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
73 }
74 input += 5;
75
76 size_t c = round_up_po2(channels, 8);
77
78 for (; c >= 16; c -= 16) {
79 int32x4_t vacc0123 = vld1q_s32(w); w = (const int32_t*) w + 4;
80 int32x4_t vacc4567 = vld1q_s32(w); w = (const int32_t*) w + 4;
81 int32x4_t vacc89AB = vld1q_s32(w); w = (const int32_t*) w + 4;
82 int32x4_t vaccCDEF = vld1q_s32(w); w = (const int32_t*) w + 4;
83
84 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0)); i0 += 8;
85 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
86 const int16x8_t vi0x89ABCDEF = vmovl_s8(vld1_s8(i0)); i0 += 8;
87 const int16x8_t vk0x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
88
89 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
90 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
91 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi0x89ABCDEF), vget_low_s16(vk0x89ABCDEF));
92 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi0x89ABCDEF), vget_high_s16(vk0x89ABCDEF));
93
94 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1)); i1 += 8;
95 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
96 const int16x8_t vi1x89ABCDEF = vmovl_s8(vld1_s8(i1)); i1 += 8;
97 const int16x8_t vk1x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
98
99 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
100 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
101 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi1x89ABCDEF), vget_low_s16(vk1x89ABCDEF));
102 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi1x89ABCDEF), vget_high_s16(vk1x89ABCDEF));
103
104 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2)); i2 += 8;
105 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
106 const int16x8_t vi2x89ABCDEF = vmovl_s8(vld1_s8(i2)); i2 += 8;
107 const int16x8_t vk2x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
108
109 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
110 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
111 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi2x89ABCDEF), vget_low_s16(vk2x89ABCDEF));
112 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi2x89ABCDEF), vget_high_s16(vk2x89ABCDEF));
113
114 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3)); i3 += 8;
115 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
116 const int16x8_t vi3x89ABCDEF = vmovl_s8(vld1_s8(i3)); i3 += 8;
117 const int16x8_t vk3x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
118
119 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
120 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
121 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi3x89ABCDEF), vget_low_s16(vk3x89ABCDEF));
122 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi3x89ABCDEF), vget_high_s16(vk3x89ABCDEF));
123
124 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4)); i4 += 8;
125 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
126 const int16x8_t vi4x89ABCDEF = vmovl_s8(vld1_s8(i4)); i4 += 8;
127 const int16x8_t vk4x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
128
129 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
130 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
131 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi4x89ABCDEF), vget_low_s16(vk4x89ABCDEF));
132 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi4x89ABCDEF), vget_high_s16(vk4x89ABCDEF));
133
134 vst1q_s32(b, vacc0123); b += 4;
135 vst1q_s32(b, vacc4567); b += 4;
136 vst1q_s32(b, vacc89AB); b += 4;
137 vst1q_s32(b, vaccCDEF); b += 4;
138 }
139
140 if XNN_UNLIKELY(c != 0) {
141 do {
142 int32x4_t vacc0123 = vld1q_s32(w); w = (const int32_t*) w + 4;
143 int32x4_t vacc4567 = vld1q_s32(w); w = (const int32_t*) w + 4;
144
145 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0)); i0 += 8;
146 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
147
148 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
149 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
150
151 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1)); i1 += 8;
152 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
153
154 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
155 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
156
157 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2)); i2 += 8;
158 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
159
160 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
161 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
162
163 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3)); i3 += 8;
164 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
165
166 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
167 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
168
169 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4)); i4 += 8;
170 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
171
172 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
173 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
174
175 vst1q_s32(b, vacc0123); b += 4;
176 vst1q_s32(b, vacc4567); b += 4;
177 c -= 8;
178 } while (c != 0);
179 }
180 }
181
182 // Middle pass to process 5 inputs in each iteration.
183 for (size_t ks = kernel_size - 5; ks > 5; ks -= 5) {
184 int32_t* b = buffer;
185
186 const int8_t* i0 = input[0];
187 assert(i0 != NULL);
188 if XNN_UNPREDICTABLE(i0 != zero) {
189 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
190 }
191 const int8_t* i1 = input[1];
192 assert(i1 != NULL);
193 if XNN_UNPREDICTABLE(i1 != zero) {
194 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
195 }
196 const int8_t* i2 = input[2];
197 assert(i2 != NULL);
198 if XNN_UNPREDICTABLE(i2 != zero) {
199 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
200 }
201 const int8_t* i3 = input[3];
202 assert(i3 != NULL);
203 if XNN_UNPREDICTABLE(i3 != zero) {
204 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
205 }
206 const int8_t* i4 = input[4];
207 assert(i4 != NULL);
208 if XNN_UNPREDICTABLE(i4 != zero) {
209 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
210 }
211 input += 5;
212
213 size_t c = round_up_po2(channels, 8);
214
215 for (; c >= 16; c -= 16) {
216 int32x4_t vacc0123 = vld1q_s32(b);
217 int32x4_t vacc4567 = vld1q_s32(b + 4);
218 int32x4_t vacc89AB = vld1q_s32(b + 8);
219 int32x4_t vaccCDEF = vld1q_s32(b + 12);
220
221 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0)); i0 += 8;
222 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
223 const int16x8_t vi0x89ABCDEF = vmovl_s8(vld1_s8(i0)); i0 += 8;
224 const int16x8_t vk0x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
225
226 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
227 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
228 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi0x89ABCDEF), vget_low_s16(vk0x89ABCDEF));
229 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi0x89ABCDEF), vget_high_s16(vk0x89ABCDEF));
230
231 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1)); i1 += 8;
232 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
233 const int16x8_t vi1x89ABCDEF = vmovl_s8(vld1_s8(i1)); i1 += 8;
234 const int16x8_t vk1x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
235
236 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
237 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
238 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi1x89ABCDEF), vget_low_s16(vk1x89ABCDEF));
239 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi1x89ABCDEF), vget_high_s16(vk1x89ABCDEF));
240
241 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2)); i2 += 8;
242 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
243 const int16x8_t vi2x89ABCDEF = vmovl_s8(vld1_s8(i2)); i2 += 8;
244 const int16x8_t vk2x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
245
246 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
247 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
248 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi2x89ABCDEF), vget_low_s16(vk2x89ABCDEF));
249 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi2x89ABCDEF), vget_high_s16(vk2x89ABCDEF));
250
251 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3)); i3 += 8;
252 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
253 const int16x8_t vi3x89ABCDEF = vmovl_s8(vld1_s8(i3)); i3 += 8;
254 const int16x8_t vk3x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
255
256 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
257 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
258 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi3x89ABCDEF), vget_low_s16(vk3x89ABCDEF));
259 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi3x89ABCDEF), vget_high_s16(vk3x89ABCDEF));
260
261 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4)); i4 += 8;
262 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
263 const int16x8_t vi4x89ABCDEF = vmovl_s8(vld1_s8(i4)); i4 += 8;
264 const int16x8_t vk4x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
265
266 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
267 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
268 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi4x89ABCDEF), vget_low_s16(vk4x89ABCDEF));
269 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi4x89ABCDEF), vget_high_s16(vk4x89ABCDEF));
270
271 vst1q_s32(b, vacc0123); b += 4;
272 vst1q_s32(b, vacc4567); b += 4;
273 vst1q_s32(b, vacc89AB); b += 4;
274 vst1q_s32(b, vaccCDEF); b += 4;
275 }
276
277 if XNN_UNLIKELY(c != 0) {
278 do {
279 int32x4_t vacc0123 = vld1q_s32(b);
280 int32x4_t vacc4567 = vld1q_s32(b + 4);
281
282 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0)); i0 += 8;
283 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
284
285 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
286 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
287
288 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1)); i1 += 8;
289 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
290
291 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
292 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
293
294 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2)); i2 += 8;
295 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
296
297 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
298 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
299
300 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3)); i3 += 8;
301 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
302
303 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
304 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
305
306 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4)); i4 += 8;
307 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
308
309 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
310 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
311
312 vst1q_s32(b, vacc0123); b += 4;
313 vst1q_s32(b, vacc4567); b += 4;
314 c -= 8;
315 } while (c != 0);
316 }
317 }
318
319 // Last pass to process up to 5 inputs.
320 {
321 const int32_t* b = buffer;
322
323 const int8_t* i0 = input[0];
324 assert(i0 != NULL);
325 if XNN_UNPREDICTABLE(i0 != zero) {
326 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
327 }
328 const int8_t* i1 = input[1];
329 assert(i1 != NULL);
330 if XNN_UNPREDICTABLE(i1 != zero) {
331 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
332 }
333 const int8_t* i2 = input[2];
334 assert(i2 != NULL);
335 if XNN_UNPREDICTABLE(i2 != zero) {
336 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
337 }
338 const int8_t* i3 = input[3];
339 assert(i3 != NULL);
340 if XNN_UNPREDICTABLE(i3 != zero) {
341 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
342 }
343 const int8_t* i4 = input[4];
344 assert(i4 != NULL);
345 if XNN_UNPREDICTABLE(i4 != zero) {
346 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
347 }
348
349 size_t c = channels;
350 for (; c >= 16; c -= 16) {
351 int32x4_t vacc0123 = vld1q_s32(b); b += 4;
352 int32x4_t vacc4567 = vld1q_s32(b); b += 4;
353 int32x4_t vacc89AB = vld1q_s32(b); b += 4;
354 int32x4_t vaccCDEF = vld1q_s32(b); b += 4;
355
356 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0)); i0 += 8;
357 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
358 const int16x8_t vi0x89ABCDEF = vmovl_s8(vld1_s8(i0)); i0 += 8;
359 const int16x8_t vk0x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
360
361 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
362 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
363 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi0x89ABCDEF), vget_low_s16(vk0x89ABCDEF));
364 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi0x89ABCDEF), vget_high_s16(vk0x89ABCDEF));
365
366 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1)); i1 += 8;
367 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
368 const int16x8_t vi1x89ABCDEF = vmovl_s8(vld1_s8(i1)); i1 += 8;
369 const int16x8_t vk1x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
370
371 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
372 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
373 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi1x89ABCDEF), vget_low_s16(vk1x89ABCDEF));
374 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi1x89ABCDEF), vget_high_s16(vk1x89ABCDEF));
375
376 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2)); i2 += 8;
377 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
378 const int16x8_t vi2x89ABCDEF = vmovl_s8(vld1_s8(i2)); i2 += 8;
379 const int16x8_t vk2x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
380
381 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
382 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
383 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi2x89ABCDEF), vget_low_s16(vk2x89ABCDEF));
384 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi2x89ABCDEF), vget_high_s16(vk2x89ABCDEF));
385
386 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3)); i3 += 8;
387 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
388 const int16x8_t vi3x89ABCDEF = vmovl_s8(vld1_s8(i3)); i3 += 8;
389 const int16x8_t vk3x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
390
391 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
392 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
393 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi3x89ABCDEF), vget_low_s16(vk3x89ABCDEF));
394 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi3x89ABCDEF), vget_high_s16(vk3x89ABCDEF));
395
396 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4)); i4 += 8;
397 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
398 const int16x8_t vi4x89ABCDEF = vmovl_s8(vld1_s8(i4)); i4 += 8;
399 const int16x8_t vk4x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
400
401 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
402 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
403 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi4x89ABCDEF), vget_low_s16(vk4x89ABCDEF));
404 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi4x89ABCDEF), vget_high_s16(vk4x89ABCDEF));
405
406 float32x4_t vfpacc0123 = vcvtq_f32_s32(vacc0123);
407 float32x4_t vfpacc4567 = vcvtq_f32_s32(vacc4567);
408 float32x4_t vfpacc89AB = vcvtq_f32_s32(vacc89AB);
409 float32x4_t vfpaccCDEF = vcvtq_f32_s32(vaccCDEF);
410
411 vfpacc0123 = vmulq_f32(vfpacc0123, vscale);
412 vfpacc4567 = vmulq_f32(vfpacc4567, vscale);
413 vfpacc89AB = vmulq_f32(vfpacc89AB, vscale);
414 vfpaccCDEF = vmulq_f32(vfpaccCDEF, vscale);
415
416 vacc0123 = vcvtnq_s32_f32(vfpacc0123);
417 vacc4567 = vcvtnq_s32_f32(vfpacc4567);
418 vacc89AB = vcvtnq_s32_f32(vfpacc89AB);
419 vaccCDEF = vcvtnq_s32_f32(vfpaccCDEF);
420
421 #if XNN_ARCH_ARM64
422 int16x8_t vacc01234567 = vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567);
423 int16x8_t vacc89ABCDEF = vqmovn_high_s32(vqmovn_s32(vacc89AB), vaccCDEF);
424
425 vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
426 vacc89ABCDEF = vqaddq_s16(vacc89ABCDEF, voutput_zero_point);
427
428 int8x16_t vout0123456789ABCDEF = vqmovn_high_s16(vqmovn_s16(vacc01234567), vacc89ABCDEF);
429 #else // !XNN_ARCH_ARM64
430 int16x8_t vacc01234567 = vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567));
431 int16x8_t vacc89ABCDEF = vcombine_s16(vqmovn_s32(vacc89AB), vqmovn_s32(vaccCDEF));
432
433 vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
434 vacc89ABCDEF = vqaddq_s16(vacc89ABCDEF, voutput_zero_point);
435
436 int8x16_t vout0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc01234567), vqmovn_s16(vacc89ABCDEF));
437 #endif // !XNN_ARCH_ARM64
438
439 vout0123456789ABCDEF = vmaxq_s8(vout0123456789ABCDEF, voutput_min);
440
441 vout0123456789ABCDEF = vminq_s8(vout0123456789ABCDEF, voutput_max);
442
443 vst1q_s8(output, vout0123456789ABCDEF); output += 16;
444 }
445
446 if XNN_UNLIKELY(c != 0) {
447 do {
448 int32x4_t vacc0123 = vld1q_s32(b); b += 4;
449 int32x4_t vacc4567 = vld1q_s32(b); b += 4;
450
451 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0)); i0 += 8;
452 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
453
454 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
455 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
456
457 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1)); i1 += 8;
458 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
459
460 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
461 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
462
463 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2)); i2 += 8;
464 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
465
466 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
467 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
468
469 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3)); i3 += 8;
470 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
471
472 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
473 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
474
475 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4)); i4 += 8;
476 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
477
478 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
479 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
480
481 float32x4_t vfpacc0123 = vcvtq_f32_s32(vacc0123);
482 float32x4_t vfpacc4567 = vcvtq_f32_s32(vacc4567);
483
484 vfpacc0123 = vmulq_f32(vfpacc0123, vscale);
485 vfpacc4567 = vmulq_f32(vfpacc4567, vscale);
486
487 vacc0123 = vcvtnq_s32_f32(vfpacc0123);
488 vacc4567 = vcvtnq_s32_f32(vfpacc4567);
489
490 #if XNN_ARCH_ARM64
491 int16x8_t vacc01234567 = vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567);
492 #else
493 int16x8_t vacc01234567 = vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567));
494 #endif
495 vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
496
497 int8x8_t vout01234567 = vqmovn_s16(vacc01234567);
498 vout01234567 = vmax_s8(vout01234567, vget_low_s8(voutput_min));
499 vout01234567 = vmin_s8(vout01234567, vget_low_s8(voutput_max));
500
501 if XNN_LIKELY(c >= 8) {
502 vst1_s8(output, vout01234567); output += 8;
503 c -= 8;
504 } else {
505 if (c & 4) {
506 vst1_lane_u32((void*) output, vreinterpret_u32_s8(vout01234567), 0); output += 4;
507 vout01234567 = vext_s8(vout01234567, vout01234567, 4);
508 }
509 if (c & 2) {
510 vst1_lane_u16((void*) output, vreinterpret_u16_s8(vout01234567), 0); output += 2;
511 vout01234567 = vext_s8(vout01234567, vout01234567, 2);
512 }
513 if (c & 1) {
514 vst1_lane_s8(output, vout01234567, 0); output += 1;
515 }
516 c = 0;
517 }
518 } while (c != 0);
519 }
520 }
521
522 input = (const int8_t**) ((uintptr_t) input + input_stride);
523 output = (int8_t*) ((uintptr_t) output + output_increment);
524 } while (--output_width != 0);
525 }