Debian Codegraph @

src/qs8-dwconv/gen/qs8-dwconv-5f5m5l32c8s8r-minmax-rndnu-neon-mul16.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-dwconv/multipass-neon-mul16.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12
13 #include <arm_neon.h>
14
15 #include "xnnpack/dwconv.h"
16 #include "xnnpack/math.h"
17
18
19 void xnn_qs8_dwconv_minmax_rndnu_ukernel_5f5m5l32c8s8r__neon_mul16(
20 size_t channels,
21 size_t output_width,
22 const int8_t** input,
23 const void* weights,
24 int8_t* output,
25 intptr_t input_stride,
26 size_t output_increment,
27 size_t input_offset,
28 const int8_t* zero,
29 size_t kernel_size,
30 int32_t* buffer,
31 const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
32 {
33 assert(channels != 0);
34 assert(output_width != 0);
35 assert(kernel_size > 5);
36
37 const int32x4_t vright_pre_shift = vld1q_dup_s32(&params->rndnu_neon.right_pre_shift);
38 const int32x4_t vmultiplier = vld1q_dup_s32(&params->rndnu_neon.multiplier);
39 const int32x4_t vright_post_shift = vld1q_dup_s32(&params->rndnu_neon.right_post_shift);
40 const int16x8_t voutput_zero_point = vld1q_dup_s16(&params->rndnu_neon.output_zero_point);
41 const int8x16_t voutput_min = vld1q_dup_s8(&params->rndnu_neon.output_min);
42 const int8x16_t voutput_max = vld1q_dup_s8(&params->rndnu_neon.output_max);
43
44 do {
45 const void* w = weights;
46
47 // First pass to process 5 inputs.
48 {
49 int32_t* b = buffer;
50 const int8_t* i0 = input[0];
51 assert(i0 != NULL);
52 if XNN_UNPREDICTABLE(i0 != zero) {
53 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
54 }
55 const int8_t* i1 = input[1];
56 assert(i1 != NULL);
57 if XNN_UNPREDICTABLE(i1 != zero) {
58 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
59 }
60 const int8_t* i2 = input[2];
61 assert(i2 != NULL);
62 if XNN_UNPREDICTABLE(i2 != zero) {
63 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
64 }
65 const int8_t* i3 = input[3];
66 assert(i3 != NULL);
67 if XNN_UNPREDICTABLE(i3 != zero) {
68 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
69 }
70 const int8_t* i4 = input[4];
71 assert(i4 != NULL);
72 if XNN_UNPREDICTABLE(i4 != zero) {
73 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
74 }
75 input += 5;
76
77 size_t c = round_up_po2(channels, 8);
78
79 for (; c >= 32; c -= 32) {
80 int32x4_t vacc0123 = vld1q_s32(w); w = (const int32_t*) w + 4;
81 int32x4_t vacc4567 = vld1q_s32(w); w = (const int32_t*) w + 4;
82 int32x4_t vacc89AB = vld1q_s32(w); w = (const int32_t*) w + 4;
83 int32x4_t vaccCDEF = vld1q_s32(w); w = (const int32_t*) w + 4;
84 int32x4_t vaccGHIJ = vld1q_s32(w); w = (const int32_t*) w + 4;
85 int32x4_t vaccKLMN = vld1q_s32(w); w = (const int32_t*) w + 4;
86 int32x4_t vaccOPQR = vld1q_s32(w); w = (const int32_t*) w + 4;
87 int32x4_t vaccSTUV = vld1q_s32(w); w = (const int32_t*) w + 4;
88
89 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0)); i0 += 8;
90 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
91 const int16x8_t vi0x89ABCDEF = vmovl_s8(vld1_s8(i0)); i0 += 8;
92 const int16x8_t vk0x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
93 const int16x8_t vi0xGHIJKLMN = vmovl_s8(vld1_s8(i0)); i0 += 8;
94 const int16x8_t vk0xGHIJKLMN = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
95 const int16x8_t vi0xOPQRSTUV = vmovl_s8(vld1_s8(i0)); i0 += 8;
96 const int16x8_t vk0xOPQRSTUV = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
97
98 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
99 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
100 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi0x89ABCDEF), vget_low_s16(vk0x89ABCDEF));
101 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi0x89ABCDEF), vget_high_s16(vk0x89ABCDEF));
102 vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi0xGHIJKLMN), vget_low_s16(vk0xGHIJKLMN));
103 vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi0xGHIJKLMN), vget_high_s16(vk0xGHIJKLMN));
104 vaccOPQR = vmlal_s16(vaccOPQR, vget_low_s16(vi0xOPQRSTUV), vget_low_s16(vk0xOPQRSTUV));
105 vaccSTUV = vmlal_s16(vaccSTUV, vget_high_s16(vi0xOPQRSTUV), vget_high_s16(vk0xOPQRSTUV));
106
107 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1)); i1 += 8;
108 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
109 const int16x8_t vi1x89ABCDEF = vmovl_s8(vld1_s8(i1)); i1 += 8;
110 const int16x8_t vk1x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
111 const int16x8_t vi1xGHIJKLMN = vmovl_s8(vld1_s8(i1)); i1 += 8;
112 const int16x8_t vk1xGHIJKLMN = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
113 const int16x8_t vi1xOPQRSTUV = vmovl_s8(vld1_s8(i1)); i1 += 8;
114 const int16x8_t vk1xOPQRSTUV = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
115
116 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
117 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
118 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi1x89ABCDEF), vget_low_s16(vk1x89ABCDEF));
119 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi1x89ABCDEF), vget_high_s16(vk1x89ABCDEF));
120 vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi1xGHIJKLMN), vget_low_s16(vk1xGHIJKLMN));
121 vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi1xGHIJKLMN), vget_high_s16(vk1xGHIJKLMN));
122 vaccOPQR = vmlal_s16(vaccOPQR, vget_low_s16(vi1xOPQRSTUV), vget_low_s16(vk1xOPQRSTUV));
123 vaccSTUV = vmlal_s16(vaccSTUV, vget_high_s16(vi1xOPQRSTUV), vget_high_s16(vk1xOPQRSTUV));
124
125 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2)); i2 += 8;
126 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
127 const int16x8_t vi2x89ABCDEF = vmovl_s8(vld1_s8(i2)); i2 += 8;
128 const int16x8_t vk2x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
129 const int16x8_t vi2xGHIJKLMN = vmovl_s8(vld1_s8(i2)); i2 += 8;
130 const int16x8_t vk2xGHIJKLMN = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
131 const int16x8_t vi2xOPQRSTUV = vmovl_s8(vld1_s8(i2)); i2 += 8;
132 const int16x8_t vk2xOPQRSTUV = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
133
134 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
135 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
136 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi2x89ABCDEF), vget_low_s16(vk2x89ABCDEF));
137 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi2x89ABCDEF), vget_high_s16(vk2x89ABCDEF));
138 vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi2xGHIJKLMN), vget_low_s16(vk2xGHIJKLMN));
139 vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi2xGHIJKLMN), vget_high_s16(vk2xGHIJKLMN));
140 vaccOPQR = vmlal_s16(vaccOPQR, vget_low_s16(vi2xOPQRSTUV), vget_low_s16(vk2xOPQRSTUV));
141 vaccSTUV = vmlal_s16(vaccSTUV, vget_high_s16(vi2xOPQRSTUV), vget_high_s16(vk2xOPQRSTUV));
142
143 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3)); i3 += 8;
144 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
145 const int16x8_t vi3x89ABCDEF = vmovl_s8(vld1_s8(i3)); i3 += 8;
146 const int16x8_t vk3x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
147 const int16x8_t vi3xGHIJKLMN = vmovl_s8(vld1_s8(i3)); i3 += 8;
148 const int16x8_t vk3xGHIJKLMN = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
149 const int16x8_t vi3xOPQRSTUV = vmovl_s8(vld1_s8(i3)); i3 += 8;
150 const int16x8_t vk3xOPQRSTUV = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
151
152 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
153 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
154 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi3x89ABCDEF), vget_low_s16(vk3x89ABCDEF));
155 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi3x89ABCDEF), vget_high_s16(vk3x89ABCDEF));
156 vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi3xGHIJKLMN), vget_low_s16(vk3xGHIJKLMN));
157 vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi3xGHIJKLMN), vget_high_s16(vk3xGHIJKLMN));
158 vaccOPQR = vmlal_s16(vaccOPQR, vget_low_s16(vi3xOPQRSTUV), vget_low_s16(vk3xOPQRSTUV));
159 vaccSTUV = vmlal_s16(vaccSTUV, vget_high_s16(vi3xOPQRSTUV), vget_high_s16(vk3xOPQRSTUV));
160
161 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4)); i4 += 8;
162 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
163 const int16x8_t vi4x89ABCDEF = vmovl_s8(vld1_s8(i4)); i4 += 8;
164 const int16x8_t vk4x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
165 const int16x8_t vi4xGHIJKLMN = vmovl_s8(vld1_s8(i4)); i4 += 8;
166 const int16x8_t vk4xGHIJKLMN = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
167 const int16x8_t vi4xOPQRSTUV = vmovl_s8(vld1_s8(i4)); i4 += 8;
168 const int16x8_t vk4xOPQRSTUV = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
169
170 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
171 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
172 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi4x89ABCDEF), vget_low_s16(vk4x89ABCDEF));
173 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi4x89ABCDEF), vget_high_s16(vk4x89ABCDEF));
174 vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi4xGHIJKLMN), vget_low_s16(vk4xGHIJKLMN));
175 vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi4xGHIJKLMN), vget_high_s16(vk4xGHIJKLMN));
176 vaccOPQR = vmlal_s16(vaccOPQR, vget_low_s16(vi4xOPQRSTUV), vget_low_s16(vk4xOPQRSTUV));
177 vaccSTUV = vmlal_s16(vaccSTUV, vget_high_s16(vi4xOPQRSTUV), vget_high_s16(vk4xOPQRSTUV));
178
179 vst1q_s32(b, vacc0123); b += 4;
180 vst1q_s32(b, vacc4567); b += 4;
181 vst1q_s32(b, vacc89AB); b += 4;
182 vst1q_s32(b, vaccCDEF); b += 4;
183 vst1q_s32(b, vaccGHIJ); b += 4;
184 vst1q_s32(b, vaccKLMN); b += 4;
185 vst1q_s32(b, vaccOPQR); b += 4;
186 vst1q_s32(b, vaccSTUV); b += 4;
187 }
188
189 if XNN_UNLIKELY(c != 0) {
190 do {
191 int32x4_t vacc0123 = vld1q_s32(w); w = (const int32_t*) w + 4;
192 int32x4_t vacc4567 = vld1q_s32(w); w = (const int32_t*) w + 4;
193
194 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0)); i0 += 8;
195 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
196
197 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
198 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
199
200 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1)); i1 += 8;
201 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
202
203 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
204 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
205
206 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2)); i2 += 8;
207 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
208
209 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
210 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
211
212 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3)); i3 += 8;
213 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
214
215 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
216 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
217
218 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4)); i4 += 8;
219 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
220
221 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
222 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
223
224 vst1q_s32(b, vacc0123); b += 4;
225 vst1q_s32(b, vacc4567); b += 4;
226 c -= 8;
227 } while (c != 0);
228 }
229 }
230
231 // Middle pass to process 5 inputs in each iteration.
232 for (size_t ks = kernel_size - 5; ks > 5; ks -= 5) {
233 int32_t* b = buffer;
234
235 const int8_t* i0 = input[0];
236 assert(i0 != NULL);
237 if XNN_UNPREDICTABLE(i0 != zero) {
238 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
239 }
240 const int8_t* i1 = input[1];
241 assert(i1 != NULL);
242 if XNN_UNPREDICTABLE(i1 != zero) {
243 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
244 }
245 const int8_t* i2 = input[2];
246 assert(i2 != NULL);
247 if XNN_UNPREDICTABLE(i2 != zero) {
248 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
249 }
250 const int8_t* i3 = input[3];
251 assert(i3 != NULL);
252 if XNN_UNPREDICTABLE(i3 != zero) {
253 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
254 }
255 const int8_t* i4 = input[4];
256 assert(i4 != NULL);
257 if XNN_UNPREDICTABLE(i4 != zero) {
258 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
259 }
260 input += 5;
261
262 size_t c = round_up_po2(channels, 8);
263
264 for (; c >= 32; c -= 32) {
265 int32x4_t vacc0123 = vld1q_s32(b);
266 int32x4_t vacc4567 = vld1q_s32(b + 4);
267 int32x4_t vacc89AB = vld1q_s32(b + 8);
268 int32x4_t vaccCDEF = vld1q_s32(b + 12);
269 int32x4_t vaccGHIJ = vld1q_s32(b + 16);
270 int32x4_t vaccKLMN = vld1q_s32(b + 20);
271 int32x4_t vaccOPQR = vld1q_s32(b + 24);
272 int32x4_t vaccSTUV = vld1q_s32(b + 28);
273
274 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0)); i0 += 8;
275 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
276 const int16x8_t vi0x89ABCDEF = vmovl_s8(vld1_s8(i0)); i0 += 8;
277 const int16x8_t vk0x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
278 const int16x8_t vi0xGHIJKLMN = vmovl_s8(vld1_s8(i0)); i0 += 8;
279 const int16x8_t vk0xGHIJKLMN = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
280 const int16x8_t vi0xOPQRSTUV = vmovl_s8(vld1_s8(i0)); i0 += 8;
281 const int16x8_t vk0xOPQRSTUV = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
282
283 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
284 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
285 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi0x89ABCDEF), vget_low_s16(vk0x89ABCDEF));
286 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi0x89ABCDEF), vget_high_s16(vk0x89ABCDEF));
287 vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi0xGHIJKLMN), vget_low_s16(vk0xGHIJKLMN));
288 vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi0xGHIJKLMN), vget_high_s16(vk0xGHIJKLMN));
289 vaccOPQR = vmlal_s16(vaccOPQR, vget_low_s16(vi0xOPQRSTUV), vget_low_s16(vk0xOPQRSTUV));
290 vaccSTUV = vmlal_s16(vaccSTUV, vget_high_s16(vi0xOPQRSTUV), vget_high_s16(vk0xOPQRSTUV));
291
292 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1)); i1 += 8;
293 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
294 const int16x8_t vi1x89ABCDEF = vmovl_s8(vld1_s8(i1)); i1 += 8;
295 const int16x8_t vk1x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
296 const int16x8_t vi1xGHIJKLMN = vmovl_s8(vld1_s8(i1)); i1 += 8;
297 const int16x8_t vk1xGHIJKLMN = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
298 const int16x8_t vi1xOPQRSTUV = vmovl_s8(vld1_s8(i1)); i1 += 8;
299 const int16x8_t vk1xOPQRSTUV = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
300
301 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
302 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
303 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi1x89ABCDEF), vget_low_s16(vk1x89ABCDEF));
304 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi1x89ABCDEF), vget_high_s16(vk1x89ABCDEF));
305 vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi1xGHIJKLMN), vget_low_s16(vk1xGHIJKLMN));
306 vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi1xGHIJKLMN), vget_high_s16(vk1xGHIJKLMN));
307 vaccOPQR = vmlal_s16(vaccOPQR, vget_low_s16(vi1xOPQRSTUV), vget_low_s16(vk1xOPQRSTUV));
308 vaccSTUV = vmlal_s16(vaccSTUV, vget_high_s16(vi1xOPQRSTUV), vget_high_s16(vk1xOPQRSTUV));
309
310 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2)); i2 += 8;
311 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
312 const int16x8_t vi2x89ABCDEF = vmovl_s8(vld1_s8(i2)); i2 += 8;
313 const int16x8_t vk2x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
314 const int16x8_t vi2xGHIJKLMN = vmovl_s8(vld1_s8(i2)); i2 += 8;
315 const int16x8_t vk2xGHIJKLMN = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
316 const int16x8_t vi2xOPQRSTUV = vmovl_s8(vld1_s8(i2)); i2 += 8;
317 const int16x8_t vk2xOPQRSTUV = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
318
319 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
320 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
321 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi2x89ABCDEF), vget_low_s16(vk2x89ABCDEF));
322 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi2x89ABCDEF), vget_high_s16(vk2x89ABCDEF));
323 vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi2xGHIJKLMN), vget_low_s16(vk2xGHIJKLMN));
324 vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi2xGHIJKLMN), vget_high_s16(vk2xGHIJKLMN));
325 vaccOPQR = vmlal_s16(vaccOPQR, vget_low_s16(vi2xOPQRSTUV), vget_low_s16(vk2xOPQRSTUV));
326 vaccSTUV = vmlal_s16(vaccSTUV, vget_high_s16(vi2xOPQRSTUV), vget_high_s16(vk2xOPQRSTUV));
327
328 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3)); i3 += 8;
329 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
330 const int16x8_t vi3x89ABCDEF = vmovl_s8(vld1_s8(i3)); i3 += 8;
331 const int16x8_t vk3x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
332 const int16x8_t vi3xGHIJKLMN = vmovl_s8(vld1_s8(i3)); i3 += 8;
333 const int16x8_t vk3xGHIJKLMN = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
334 const int16x8_t vi3xOPQRSTUV = vmovl_s8(vld1_s8(i3)); i3 += 8;
335 const int16x8_t vk3xOPQRSTUV = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
336
337 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
338 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
339 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi3x89ABCDEF), vget_low_s16(vk3x89ABCDEF));
340 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi3x89ABCDEF), vget_high_s16(vk3x89ABCDEF));
341 vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi3xGHIJKLMN), vget_low_s16(vk3xGHIJKLMN));
342 vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi3xGHIJKLMN), vget_high_s16(vk3xGHIJKLMN));
343 vaccOPQR = vmlal_s16(vaccOPQR, vget_low_s16(vi3xOPQRSTUV), vget_low_s16(vk3xOPQRSTUV));
344 vaccSTUV = vmlal_s16(vaccSTUV, vget_high_s16(vi3xOPQRSTUV), vget_high_s16(vk3xOPQRSTUV));
345
346 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4)); i4 += 8;
347 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
348 const int16x8_t vi4x89ABCDEF = vmovl_s8(vld1_s8(i4)); i4 += 8;
349 const int16x8_t vk4x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
350 const int16x8_t vi4xGHIJKLMN = vmovl_s8(vld1_s8(i4)); i4 += 8;
351 const int16x8_t vk4xGHIJKLMN = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
352 const int16x8_t vi4xOPQRSTUV = vmovl_s8(vld1_s8(i4)); i4 += 8;
353 const int16x8_t vk4xOPQRSTUV = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
354
355 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
356 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
357 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi4x89ABCDEF), vget_low_s16(vk4x89ABCDEF));
358 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi4x89ABCDEF), vget_high_s16(vk4x89ABCDEF));
359 vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi4xGHIJKLMN), vget_low_s16(vk4xGHIJKLMN));
360 vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi4xGHIJKLMN), vget_high_s16(vk4xGHIJKLMN));
361 vaccOPQR = vmlal_s16(vaccOPQR, vget_low_s16(vi4xOPQRSTUV), vget_low_s16(vk4xOPQRSTUV));
362 vaccSTUV = vmlal_s16(vaccSTUV, vget_high_s16(vi4xOPQRSTUV), vget_high_s16(vk4xOPQRSTUV));
363
364 vst1q_s32(b, vacc0123); b += 4;
365 vst1q_s32(b, vacc4567); b += 4;
366 vst1q_s32(b, vacc89AB); b += 4;
367 vst1q_s32(b, vaccCDEF); b += 4;
368 vst1q_s32(b, vaccGHIJ); b += 4;
369 vst1q_s32(b, vaccKLMN); b += 4;
370 vst1q_s32(b, vaccOPQR); b += 4;
371 vst1q_s32(b, vaccSTUV); b += 4;
372 }
373
374 if XNN_UNLIKELY(c != 0) {
375 do {
376 int32x4_t vacc0123 = vld1q_s32(b);
377 int32x4_t vacc4567 = vld1q_s32(b + 4);
378
379 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0)); i0 += 8;
380 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
381
382 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
383 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
384
385 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1)); i1 += 8;
386 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
387
388 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
389 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
390
391 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2)); i2 += 8;
392 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
393
394 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
395 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
396
397 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3)); i3 += 8;
398 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
399
400 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
401 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
402
403 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4)); i4 += 8;
404 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
405
406 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
407 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
408
409 vst1q_s32(b, vacc0123); b += 4;
410 vst1q_s32(b, vacc4567); b += 4;
411 c -= 8;
412 } while (c != 0);
413 }
414 }
415
416 // Last pass to process up to 5 inputs.
417 {
418 const int32_t* b = buffer;
419
420 const int8_t* i0 = input[0];
421 assert(i0 != NULL);
422 if XNN_UNPREDICTABLE(i0 != zero) {
423 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
424 }
425 const int8_t* i1 = input[1];
426 assert(i1 != NULL);
427 if XNN_UNPREDICTABLE(i1 != zero) {
428 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
429 }
430 const int8_t* i2 = input[2];
431 assert(i2 != NULL);
432 if XNN_UNPREDICTABLE(i2 != zero) {
433 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
434 }
435 const int8_t* i3 = input[3];
436 assert(i3 != NULL);
437 if XNN_UNPREDICTABLE(i3 != zero) {
438 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
439 }
440 const int8_t* i4 = input[4];
441 assert(i4 != NULL);
442 if XNN_UNPREDICTABLE(i4 != zero) {
443 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
444 }
445
446 size_t c = channels;
447 for (; c >= 32; c -= 32) {
448 int32x4_t vacc0123 = vld1q_s32(b); b += 4;
449 int32x4_t vacc4567 = vld1q_s32(b); b += 4;
450 int32x4_t vacc89AB = vld1q_s32(b); b += 4;
451 int32x4_t vaccCDEF = vld1q_s32(b); b += 4;
452 int32x4_t vaccGHIJ = vld1q_s32(b); b += 4;
453 int32x4_t vaccKLMN = vld1q_s32(b); b += 4;
454 int32x4_t vaccOPQR = vld1q_s32(b); b += 4;
455 int32x4_t vaccSTUV = vld1q_s32(b); b += 4;
456
457 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0)); i0 += 8;
458 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
459 const int16x8_t vi0x89ABCDEF = vmovl_s8(vld1_s8(i0)); i0 += 8;
460 const int16x8_t vk0x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
461 const int16x8_t vi0xGHIJKLMN = vmovl_s8(vld1_s8(i0)); i0 += 8;
462 const int16x8_t vk0xGHIJKLMN = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
463 const int16x8_t vi0xOPQRSTUV = vmovl_s8(vld1_s8(i0)); i0 += 8;
464 const int16x8_t vk0xOPQRSTUV = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
465
466 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
467 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
468 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi0x89ABCDEF), vget_low_s16(vk0x89ABCDEF));
469 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi0x89ABCDEF), vget_high_s16(vk0x89ABCDEF));
470 vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi0xGHIJKLMN), vget_low_s16(vk0xGHIJKLMN));
471 vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi0xGHIJKLMN), vget_high_s16(vk0xGHIJKLMN));
472 vaccOPQR = vmlal_s16(vaccOPQR, vget_low_s16(vi0xOPQRSTUV), vget_low_s16(vk0xOPQRSTUV));
473 vaccSTUV = vmlal_s16(vaccSTUV, vget_high_s16(vi0xOPQRSTUV), vget_high_s16(vk0xOPQRSTUV));
474
475 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1)); i1 += 8;
476 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
477 const int16x8_t vi1x89ABCDEF = vmovl_s8(vld1_s8(i1)); i1 += 8;
478 const int16x8_t vk1x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
479 const int16x8_t vi1xGHIJKLMN = vmovl_s8(vld1_s8(i1)); i1 += 8;
480 const int16x8_t vk1xGHIJKLMN = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
481 const int16x8_t vi1xOPQRSTUV = vmovl_s8(vld1_s8(i1)); i1 += 8;
482 const int16x8_t vk1xOPQRSTUV = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
483
484 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
485 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
486 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi1x89ABCDEF), vget_low_s16(vk1x89ABCDEF));
487 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi1x89ABCDEF), vget_high_s16(vk1x89ABCDEF));
488 vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi1xGHIJKLMN), vget_low_s16(vk1xGHIJKLMN));
489 vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi1xGHIJKLMN), vget_high_s16(vk1xGHIJKLMN));
490 vaccOPQR = vmlal_s16(vaccOPQR, vget_low_s16(vi1xOPQRSTUV), vget_low_s16(vk1xOPQRSTUV));
491 vaccSTUV = vmlal_s16(vaccSTUV, vget_high_s16(vi1xOPQRSTUV), vget_high_s16(vk1xOPQRSTUV));
492
493 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2)); i2 += 8;
494 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
495 const int16x8_t vi2x89ABCDEF = vmovl_s8(vld1_s8(i2)); i2 += 8;
496 const int16x8_t vk2x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
497 const int16x8_t vi2xGHIJKLMN = vmovl_s8(vld1_s8(i2)); i2 += 8;
498 const int16x8_t vk2xGHIJKLMN = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
499 const int16x8_t vi2xOPQRSTUV = vmovl_s8(vld1_s8(i2)); i2 += 8;
500 const int16x8_t vk2xOPQRSTUV = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
501
502 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
503 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
504 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi2x89ABCDEF), vget_low_s16(vk2x89ABCDEF));
505 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi2x89ABCDEF), vget_high_s16(vk2x89ABCDEF));
506 vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi2xGHIJKLMN), vget_low_s16(vk2xGHIJKLMN));
507 vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi2xGHIJKLMN), vget_high_s16(vk2xGHIJKLMN));
508 vaccOPQR = vmlal_s16(vaccOPQR, vget_low_s16(vi2xOPQRSTUV), vget_low_s16(vk2xOPQRSTUV));
509 vaccSTUV = vmlal_s16(vaccSTUV, vget_high_s16(vi2xOPQRSTUV), vget_high_s16(vk2xOPQRSTUV));
510
511 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3)); i3 += 8;
512 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
513 const int16x8_t vi3x89ABCDEF = vmovl_s8(vld1_s8(i3)); i3 += 8;
514 const int16x8_t vk3x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
515 const int16x8_t vi3xGHIJKLMN = vmovl_s8(vld1_s8(i3)); i3 += 8;
516 const int16x8_t vk3xGHIJKLMN = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
517 const int16x8_t vi3xOPQRSTUV = vmovl_s8(vld1_s8(i3)); i3 += 8;
518 const int16x8_t vk3xOPQRSTUV = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
519
520 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
521 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
522 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi3x89ABCDEF), vget_low_s16(vk3x89ABCDEF));
523 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi3x89ABCDEF), vget_high_s16(vk3x89ABCDEF));
524 vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi3xGHIJKLMN), vget_low_s16(vk3xGHIJKLMN));
525 vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi3xGHIJKLMN), vget_high_s16(vk3xGHIJKLMN));
526 vaccOPQR = vmlal_s16(vaccOPQR, vget_low_s16(vi3xOPQRSTUV), vget_low_s16(vk3xOPQRSTUV));
527 vaccSTUV = vmlal_s16(vaccSTUV, vget_high_s16(vi3xOPQRSTUV), vget_high_s16(vk3xOPQRSTUV));
528
529 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4)); i4 += 8;
530 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
531 const int16x8_t vi4x89ABCDEF = vmovl_s8(vld1_s8(i4)); i4 += 8;
532 const int16x8_t vk4x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
533 const int16x8_t vi4xGHIJKLMN = vmovl_s8(vld1_s8(i4)); i4 += 8;
534 const int16x8_t vk4xGHIJKLMN = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
535 const int16x8_t vi4xOPQRSTUV = vmovl_s8(vld1_s8(i4)); i4 += 8;
536 const int16x8_t vk4xOPQRSTUV = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
537
538 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
539 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
540 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi4x89ABCDEF), vget_low_s16(vk4x89ABCDEF));
541 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi4x89ABCDEF), vget_high_s16(vk4x89ABCDEF));
542 vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi4xGHIJKLMN), vget_low_s16(vk4xGHIJKLMN));
543 vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi4xGHIJKLMN), vget_high_s16(vk4xGHIJKLMN));
544 vaccOPQR = vmlal_s16(vaccOPQR, vget_low_s16(vi4xOPQRSTUV), vget_low_s16(vk4xOPQRSTUV));
545 vaccSTUV = vmlal_s16(vaccSTUV, vget_high_s16(vi4xOPQRSTUV), vget_high_s16(vk4xOPQRSTUV));
546
547 vacc0123 = vqshlq_s32(vacc0123, vright_pre_shift);
548 vacc4567 = vqshlq_s32(vacc4567, vright_pre_shift);
549 vacc89AB = vqshlq_s32(vacc89AB, vright_pre_shift);
550 vaccCDEF = vqshlq_s32(vaccCDEF, vright_pre_shift);
551 vaccGHIJ = vqshlq_s32(vaccGHIJ, vright_pre_shift);
552 vaccKLMN = vqshlq_s32(vaccKLMN, vright_pre_shift);
553 vaccOPQR = vqshlq_s32(vaccOPQR, vright_pre_shift);
554 vaccSTUV = vqshlq_s32(vaccSTUV, vright_pre_shift);
555
556 vacc0123 = vqdmulhq_s32(vacc0123, vmultiplier);
557 vacc4567 = vqdmulhq_s32(vacc4567, vmultiplier);
558 vacc89AB = vqdmulhq_s32(vacc89AB, vmultiplier);
559 vaccCDEF = vqdmulhq_s32(vaccCDEF, vmultiplier);
560 vaccGHIJ = vqdmulhq_s32(vaccGHIJ, vmultiplier);
561 vaccKLMN = vqdmulhq_s32(vaccKLMN, vmultiplier);
562 vaccOPQR = vqdmulhq_s32(vaccOPQR, vmultiplier);
563 vaccSTUV = vqdmulhq_s32(vaccSTUV, vmultiplier);
564
565 vacc0123 = vrshlq_s32(vacc0123, vright_post_shift);
566 vacc4567 = vrshlq_s32(vacc4567, vright_post_shift);
567 vacc89AB = vrshlq_s32(vacc89AB, vright_post_shift);
568 vaccCDEF = vrshlq_s32(vaccCDEF, vright_post_shift);
569 vaccGHIJ = vrshlq_s32(vaccGHIJ, vright_post_shift);
570 vaccKLMN = vrshlq_s32(vaccKLMN, vright_post_shift);
571 vaccOPQR = vrshlq_s32(vaccOPQR, vright_post_shift);
572 vaccSTUV = vrshlq_s32(vaccSTUV, vright_post_shift);
573
574 #if XNN_ARCH_ARM64
575 int16x8_t vacc01234567 = vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567);
576 int16x8_t vacc89ABCDEF = vqmovn_high_s32(vqmovn_s32(vacc89AB), vaccCDEF);
577 int16x8_t vaccGHIJKLMN = vqmovn_high_s32(vqmovn_s32(vaccGHIJ), vaccKLMN);
578 int16x8_t vaccOPQRSTUV = vqmovn_high_s32(vqmovn_s32(vaccOPQR), vaccSTUV);
579
580 vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
581 vacc89ABCDEF = vqaddq_s16(vacc89ABCDEF, voutput_zero_point);
582 vaccGHIJKLMN = vqaddq_s16(vaccGHIJKLMN, voutput_zero_point);
583 vaccOPQRSTUV = vqaddq_s16(vaccOPQRSTUV, voutput_zero_point);
584
585 int8x16_t vout0123456789ABCDEF = vqmovn_high_s16(vqmovn_s16(vacc01234567), vacc89ABCDEF);
586 int8x16_t voutGHIJKLMNOPQRSTUV = vqmovn_high_s16(vqmovn_s16(vaccGHIJKLMN), vaccOPQRSTUV);
587 #else // !XNN_ARCH_ARM64
588 int16x8_t vacc01234567 = vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567));
589 int16x8_t vacc89ABCDEF = vcombine_s16(vqmovn_s32(vacc89AB), vqmovn_s32(vaccCDEF));
590 int16x8_t vaccGHIJKLMN = vcombine_s16(vqmovn_s32(vaccGHIJ), vqmovn_s32(vaccKLMN));
591 int16x8_t vaccOPQRSTUV = vcombine_s16(vqmovn_s32(vaccOPQR), vqmovn_s32(vaccSTUV));
592
593 vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
594 vacc89ABCDEF = vqaddq_s16(vacc89ABCDEF, voutput_zero_point);
595 vaccGHIJKLMN = vqaddq_s16(vaccGHIJKLMN, voutput_zero_point);
596 vaccOPQRSTUV = vqaddq_s16(vaccOPQRSTUV, voutput_zero_point);
597
598 int8x16_t vout0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc01234567), vqmovn_s16(vacc89ABCDEF));
599 int8x16_t voutGHIJKLMNOPQRSTUV = vcombine_s8(vqmovn_s16(vaccGHIJKLMN), vqmovn_s16(vaccOPQRSTUV));
600 #endif // !XNN_ARCH_ARM64
601
602 vout0123456789ABCDEF = vmaxq_s8(vout0123456789ABCDEF, voutput_min);
603 voutGHIJKLMNOPQRSTUV = vmaxq_s8(voutGHIJKLMNOPQRSTUV, voutput_min);
604
605 vout0123456789ABCDEF = vminq_s8(vout0123456789ABCDEF, voutput_max);
606 voutGHIJKLMNOPQRSTUV = vminq_s8(voutGHIJKLMNOPQRSTUV, voutput_max);
607
608 vst1q_s8(output, vout0123456789ABCDEF); output += 16;
609 vst1q_s8(output, voutGHIJKLMNOPQRSTUV); output += 16;
610 }
611
612 if XNN_UNLIKELY(c != 0) {
613 do {
614 int32x4_t vacc0123 = vld1q_s32(b); b += 4;
615 int32x4_t vacc4567 = vld1q_s32(b); b += 4;
616
617 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0)); i0 += 8;
618 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
619
620 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
621 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
622
623 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1)); i1 += 8;
624 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
625
626 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
627 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
628
629 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2)); i2 += 8;
630 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
631
632 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
633 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
634
635 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3)); i3 += 8;
636 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
637
638 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
639 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
640
641 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4)); i4 += 8;
642 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
643
644 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
645 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
646
647 vacc0123 = vqshlq_s32(vacc0123, vright_pre_shift);
648 vacc4567 = vqshlq_s32(vacc4567, vright_pre_shift);
649
650 vacc0123 = vqdmulhq_s32(vacc0123, vmultiplier);
651 vacc4567 = vqdmulhq_s32(vacc4567, vmultiplier);
652
653 vacc0123 = vrshlq_s32(vacc0123, vright_post_shift);
654 vacc4567 = vrshlq_s32(vacc4567, vright_post_shift);
655
656 #if XNN_ARCH_ARM64
657 int16x8_t vacc01234567 = vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567);
658 #else
659 int16x8_t vacc01234567 = vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567));
660 #endif
661 vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
662
663 int8x8_t vout01234567 = vqmovn_s16(vacc01234567);
664 vout01234567 = vmax_s8(vout01234567, vget_low_s8(voutput_min));
665 vout01234567 = vmin_s8(vout01234567, vget_low_s8(voutput_max));
666
667 if XNN_LIKELY(c >= 8) {
668 vst1_s8(output, vout01234567); output += 8;
669 c -= 8;
670 } else {
671 if (c & 4) {
672 vst1_lane_u32((void*) output, vreinterpret_u32_s8(vout01234567), 0); output += 4;
673 vout01234567 = vext_s8(vout01234567, vout01234567, 4);
674 }
675 if (c & 2) {
676 vst1_lane_u16((void*) output, vreinterpret_u16_s8(vout01234567), 0); output += 2;
677 vout01234567 = vext_s8(vout01234567, vout01234567, 2);
678 }
679 if (c & 1) {
680 vst1_lane_s8(output, vout01234567, 0); output += 1;
681 }
682 c = 0;
683 }
684 } while (c != 0);
685 }
686 }
687
688 input = (const int8_t**) ((uintptr_t) input + input_stride);
689 output = (int8_t*) ((uintptr_t) output + output_increment);
690 } while (--output_width != 0);
691 }