Debian Codegraph @

src/qs8-dwconv/gen/qs8-dwconv-5f5m5l32c8s8r-minmax-fp32-neonv8-mul16.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-dwconv/multipass-neon-mul16.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12
13 #include <arm_neon.h>
14
15 #include "xnnpack/dwconv.h"
16 #include "xnnpack/intrinsics-polyfill.h"
17 #include "xnnpack/math.h"
18
19
20 void xnn_qs8_dwconv_minmax_fp32_ukernel_5f5m5l32c8s8r__neonv8_mul16(
21 size_t channels,
22 size_t output_width,
23 const int8_t** input,
24 const void* weights,
25 int8_t* output,
26 intptr_t input_stride,
27 size_t output_increment,
28 size_t input_offset,
29 const int8_t* zero,
30 size_t kernel_size,
31 int32_t* buffer,
32 const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
33 {
34 assert(channels != 0);
35 assert(output_width != 0);
36 assert(kernel_size > 5);
37
38 const float32x4_t vscale = vld1q_dup_f32(&params->fp32_neonv8.scale);
39 const int16x8_t voutput_zero_point = vld1q_dup_s16(&params->fp32_neonv8.output_zero_point);
40 const int8x16_t voutput_min = vld1q_dup_s8(&params->fp32_neonv8.output_min);
41 const int8x16_t voutput_max = vld1q_dup_s8(&params->fp32_neonv8.output_max);
42
43 do {
44 const void* w = weights;
45
46 // First pass to process 5 inputs.
47 {
48 int32_t* b = buffer;
49 const int8_t* i0 = input[0];
50 assert(i0 != NULL);
51 if XNN_UNPREDICTABLE(i0 != zero) {
52 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
53 }
54 const int8_t* i1 = input[1];
55 assert(i1 != NULL);
56 if XNN_UNPREDICTABLE(i1 != zero) {
57 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
58 }
59 const int8_t* i2 = input[2];
60 assert(i2 != NULL);
61 if XNN_UNPREDICTABLE(i2 != zero) {
62 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
63 }
64 const int8_t* i3 = input[3];
65 assert(i3 != NULL);
66 if XNN_UNPREDICTABLE(i3 != zero) {
67 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
68 }
69 const int8_t* i4 = input[4];
70 assert(i4 != NULL);
71 if XNN_UNPREDICTABLE(i4 != zero) {
72 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
73 }
74 input += 5;
75
76 size_t c = round_up_po2(channels, 8);
77
78 for (; c >= 32; c -= 32) {
79 int32x4_t vacc0123 = vld1q_s32(w); w = (const int32_t*) w + 4;
80 int32x4_t vacc4567 = vld1q_s32(w); w = (const int32_t*) w + 4;
81 int32x4_t vacc89AB = vld1q_s32(w); w = (const int32_t*) w + 4;
82 int32x4_t vaccCDEF = vld1q_s32(w); w = (const int32_t*) w + 4;
83 int32x4_t vaccGHIJ = vld1q_s32(w); w = (const int32_t*) w + 4;
84 int32x4_t vaccKLMN = vld1q_s32(w); w = (const int32_t*) w + 4;
85 int32x4_t vaccOPQR = vld1q_s32(w); w = (const int32_t*) w + 4;
86 int32x4_t vaccSTUV = vld1q_s32(w); w = (const int32_t*) w + 4;
87
88 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0)); i0 += 8;
89 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
90 const int16x8_t vi0x89ABCDEF = vmovl_s8(vld1_s8(i0)); i0 += 8;
91 const int16x8_t vk0x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
92 const int16x8_t vi0xGHIJKLMN = vmovl_s8(vld1_s8(i0)); i0 += 8;
93 const int16x8_t vk0xGHIJKLMN = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
94 const int16x8_t vi0xOPQRSTUV = vmovl_s8(vld1_s8(i0)); i0 += 8;
95 const int16x8_t vk0xOPQRSTUV = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
96
97 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
98 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
99 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi0x89ABCDEF), vget_low_s16(vk0x89ABCDEF));
100 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi0x89ABCDEF), vget_high_s16(vk0x89ABCDEF));
101 vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi0xGHIJKLMN), vget_low_s16(vk0xGHIJKLMN));
102 vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi0xGHIJKLMN), vget_high_s16(vk0xGHIJKLMN));
103 vaccOPQR = vmlal_s16(vaccOPQR, vget_low_s16(vi0xOPQRSTUV), vget_low_s16(vk0xOPQRSTUV));
104 vaccSTUV = vmlal_s16(vaccSTUV, vget_high_s16(vi0xOPQRSTUV), vget_high_s16(vk0xOPQRSTUV));
105
106 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1)); i1 += 8;
107 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
108 const int16x8_t vi1x89ABCDEF = vmovl_s8(vld1_s8(i1)); i1 += 8;
109 const int16x8_t vk1x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
110 const int16x8_t vi1xGHIJKLMN = vmovl_s8(vld1_s8(i1)); i1 += 8;
111 const int16x8_t vk1xGHIJKLMN = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
112 const int16x8_t vi1xOPQRSTUV = vmovl_s8(vld1_s8(i1)); i1 += 8;
113 const int16x8_t vk1xOPQRSTUV = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
114
115 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
116 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
117 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi1x89ABCDEF), vget_low_s16(vk1x89ABCDEF));
118 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi1x89ABCDEF), vget_high_s16(vk1x89ABCDEF));
119 vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi1xGHIJKLMN), vget_low_s16(vk1xGHIJKLMN));
120 vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi1xGHIJKLMN), vget_high_s16(vk1xGHIJKLMN));
121 vaccOPQR = vmlal_s16(vaccOPQR, vget_low_s16(vi1xOPQRSTUV), vget_low_s16(vk1xOPQRSTUV));
122 vaccSTUV = vmlal_s16(vaccSTUV, vget_high_s16(vi1xOPQRSTUV), vget_high_s16(vk1xOPQRSTUV));
123
124 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2)); i2 += 8;
125 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
126 const int16x8_t vi2x89ABCDEF = vmovl_s8(vld1_s8(i2)); i2 += 8;
127 const int16x8_t vk2x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
128 const int16x8_t vi2xGHIJKLMN = vmovl_s8(vld1_s8(i2)); i2 += 8;
129 const int16x8_t vk2xGHIJKLMN = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
130 const int16x8_t vi2xOPQRSTUV = vmovl_s8(vld1_s8(i2)); i2 += 8;
131 const int16x8_t vk2xOPQRSTUV = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
132
133 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
134 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
135 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi2x89ABCDEF), vget_low_s16(vk2x89ABCDEF));
136 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi2x89ABCDEF), vget_high_s16(vk2x89ABCDEF));
137 vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi2xGHIJKLMN), vget_low_s16(vk2xGHIJKLMN));
138 vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi2xGHIJKLMN), vget_high_s16(vk2xGHIJKLMN));
139 vaccOPQR = vmlal_s16(vaccOPQR, vget_low_s16(vi2xOPQRSTUV), vget_low_s16(vk2xOPQRSTUV));
140 vaccSTUV = vmlal_s16(vaccSTUV, vget_high_s16(vi2xOPQRSTUV), vget_high_s16(vk2xOPQRSTUV));
141
142 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3)); i3 += 8;
143 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
144 const int16x8_t vi3x89ABCDEF = vmovl_s8(vld1_s8(i3)); i3 += 8;
145 const int16x8_t vk3x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
146 const int16x8_t vi3xGHIJKLMN = vmovl_s8(vld1_s8(i3)); i3 += 8;
147 const int16x8_t vk3xGHIJKLMN = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
148 const int16x8_t vi3xOPQRSTUV = vmovl_s8(vld1_s8(i3)); i3 += 8;
149 const int16x8_t vk3xOPQRSTUV = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
150
151 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
152 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
153 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi3x89ABCDEF), vget_low_s16(vk3x89ABCDEF));
154 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi3x89ABCDEF), vget_high_s16(vk3x89ABCDEF));
155 vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi3xGHIJKLMN), vget_low_s16(vk3xGHIJKLMN));
156 vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi3xGHIJKLMN), vget_high_s16(vk3xGHIJKLMN));
157 vaccOPQR = vmlal_s16(vaccOPQR, vget_low_s16(vi3xOPQRSTUV), vget_low_s16(vk3xOPQRSTUV));
158 vaccSTUV = vmlal_s16(vaccSTUV, vget_high_s16(vi3xOPQRSTUV), vget_high_s16(vk3xOPQRSTUV));
159
160 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4)); i4 += 8;
161 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
162 const int16x8_t vi4x89ABCDEF = vmovl_s8(vld1_s8(i4)); i4 += 8;
163 const int16x8_t vk4x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
164 const int16x8_t vi4xGHIJKLMN = vmovl_s8(vld1_s8(i4)); i4 += 8;
165 const int16x8_t vk4xGHIJKLMN = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
166 const int16x8_t vi4xOPQRSTUV = vmovl_s8(vld1_s8(i4)); i4 += 8;
167 const int16x8_t vk4xOPQRSTUV = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
168
169 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
170 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
171 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi4x89ABCDEF), vget_low_s16(vk4x89ABCDEF));
172 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi4x89ABCDEF), vget_high_s16(vk4x89ABCDEF));
173 vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi4xGHIJKLMN), vget_low_s16(vk4xGHIJKLMN));
174 vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi4xGHIJKLMN), vget_high_s16(vk4xGHIJKLMN));
175 vaccOPQR = vmlal_s16(vaccOPQR, vget_low_s16(vi4xOPQRSTUV), vget_low_s16(vk4xOPQRSTUV));
176 vaccSTUV = vmlal_s16(vaccSTUV, vget_high_s16(vi4xOPQRSTUV), vget_high_s16(vk4xOPQRSTUV));
177
178 vst1q_s32(b, vacc0123); b += 4;
179 vst1q_s32(b, vacc4567); b += 4;
180 vst1q_s32(b, vacc89AB); b += 4;
181 vst1q_s32(b, vaccCDEF); b += 4;
182 vst1q_s32(b, vaccGHIJ); b += 4;
183 vst1q_s32(b, vaccKLMN); b += 4;
184 vst1q_s32(b, vaccOPQR); b += 4;
185 vst1q_s32(b, vaccSTUV); b += 4;
186 }
187
188 if XNN_UNLIKELY(c != 0) {
189 do {
190 int32x4_t vacc0123 = vld1q_s32(w); w = (const int32_t*) w + 4;
191 int32x4_t vacc4567 = vld1q_s32(w); w = (const int32_t*) w + 4;
192
193 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0)); i0 += 8;
194 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
195
196 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
197 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
198
199 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1)); i1 += 8;
200 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
201
202 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
203 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
204
205 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2)); i2 += 8;
206 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
207
208 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
209 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
210
211 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3)); i3 += 8;
212 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
213
214 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
215 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
216
217 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4)); i4 += 8;
218 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
219
220 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
221 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
222
223 vst1q_s32(b, vacc0123); b += 4;
224 vst1q_s32(b, vacc4567); b += 4;
225 c -= 8;
226 } while (c != 0);
227 }
228 }
229
230 // Middle pass to process 5 inputs in each iteration.
231 for (size_t ks = kernel_size - 5; ks > 5; ks -= 5) {
232 int32_t* b = buffer;
233
234 const int8_t* i0 = input[0];
235 assert(i0 != NULL);
236 if XNN_UNPREDICTABLE(i0 != zero) {
237 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
238 }
239 const int8_t* i1 = input[1];
240 assert(i1 != NULL);
241 if XNN_UNPREDICTABLE(i1 != zero) {
242 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
243 }
244 const int8_t* i2 = input[2];
245 assert(i2 != NULL);
246 if XNN_UNPREDICTABLE(i2 != zero) {
247 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
248 }
249 const int8_t* i3 = input[3];
250 assert(i3 != NULL);
251 if XNN_UNPREDICTABLE(i3 != zero) {
252 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
253 }
254 const int8_t* i4 = input[4];
255 assert(i4 != NULL);
256 if XNN_UNPREDICTABLE(i4 != zero) {
257 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
258 }
259 input += 5;
260
261 size_t c = round_up_po2(channels, 8);
262
263 for (; c >= 32; c -= 32) {
264 int32x4_t vacc0123 = vld1q_s32(b);
265 int32x4_t vacc4567 = vld1q_s32(b + 4);
266 int32x4_t vacc89AB = vld1q_s32(b + 8);
267 int32x4_t vaccCDEF = vld1q_s32(b + 12);
268 int32x4_t vaccGHIJ = vld1q_s32(b + 16);
269 int32x4_t vaccKLMN = vld1q_s32(b + 20);
270 int32x4_t vaccOPQR = vld1q_s32(b + 24);
271 int32x4_t vaccSTUV = vld1q_s32(b + 28);
272
273 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0)); i0 += 8;
274 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
275 const int16x8_t vi0x89ABCDEF = vmovl_s8(vld1_s8(i0)); i0 += 8;
276 const int16x8_t vk0x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
277 const int16x8_t vi0xGHIJKLMN = vmovl_s8(vld1_s8(i0)); i0 += 8;
278 const int16x8_t vk0xGHIJKLMN = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
279 const int16x8_t vi0xOPQRSTUV = vmovl_s8(vld1_s8(i0)); i0 += 8;
280 const int16x8_t vk0xOPQRSTUV = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
281
282 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
283 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
284 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi0x89ABCDEF), vget_low_s16(vk0x89ABCDEF));
285 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi0x89ABCDEF), vget_high_s16(vk0x89ABCDEF));
286 vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi0xGHIJKLMN), vget_low_s16(vk0xGHIJKLMN));
287 vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi0xGHIJKLMN), vget_high_s16(vk0xGHIJKLMN));
288 vaccOPQR = vmlal_s16(vaccOPQR, vget_low_s16(vi0xOPQRSTUV), vget_low_s16(vk0xOPQRSTUV));
289 vaccSTUV = vmlal_s16(vaccSTUV, vget_high_s16(vi0xOPQRSTUV), vget_high_s16(vk0xOPQRSTUV));
290
291 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1)); i1 += 8;
292 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
293 const int16x8_t vi1x89ABCDEF = vmovl_s8(vld1_s8(i1)); i1 += 8;
294 const int16x8_t vk1x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
295 const int16x8_t vi1xGHIJKLMN = vmovl_s8(vld1_s8(i1)); i1 += 8;
296 const int16x8_t vk1xGHIJKLMN = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
297 const int16x8_t vi1xOPQRSTUV = vmovl_s8(vld1_s8(i1)); i1 += 8;
298 const int16x8_t vk1xOPQRSTUV = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
299
300 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
301 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
302 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi1x89ABCDEF), vget_low_s16(vk1x89ABCDEF));
303 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi1x89ABCDEF), vget_high_s16(vk1x89ABCDEF));
304 vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi1xGHIJKLMN), vget_low_s16(vk1xGHIJKLMN));
305 vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi1xGHIJKLMN), vget_high_s16(vk1xGHIJKLMN));
306 vaccOPQR = vmlal_s16(vaccOPQR, vget_low_s16(vi1xOPQRSTUV), vget_low_s16(vk1xOPQRSTUV));
307 vaccSTUV = vmlal_s16(vaccSTUV, vget_high_s16(vi1xOPQRSTUV), vget_high_s16(vk1xOPQRSTUV));
308
309 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2)); i2 += 8;
310 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
311 const int16x8_t vi2x89ABCDEF = vmovl_s8(vld1_s8(i2)); i2 += 8;
312 const int16x8_t vk2x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
313 const int16x8_t vi2xGHIJKLMN = vmovl_s8(vld1_s8(i2)); i2 += 8;
314 const int16x8_t vk2xGHIJKLMN = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
315 const int16x8_t vi2xOPQRSTUV = vmovl_s8(vld1_s8(i2)); i2 += 8;
316 const int16x8_t vk2xOPQRSTUV = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
317
318 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
319 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
320 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi2x89ABCDEF), vget_low_s16(vk2x89ABCDEF));
321 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi2x89ABCDEF), vget_high_s16(vk2x89ABCDEF));
322 vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi2xGHIJKLMN), vget_low_s16(vk2xGHIJKLMN));
323 vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi2xGHIJKLMN), vget_high_s16(vk2xGHIJKLMN));
324 vaccOPQR = vmlal_s16(vaccOPQR, vget_low_s16(vi2xOPQRSTUV), vget_low_s16(vk2xOPQRSTUV));
325 vaccSTUV = vmlal_s16(vaccSTUV, vget_high_s16(vi2xOPQRSTUV), vget_high_s16(vk2xOPQRSTUV));
326
327 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3)); i3 += 8;
328 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
329 const int16x8_t vi3x89ABCDEF = vmovl_s8(vld1_s8(i3)); i3 += 8;
330 const int16x8_t vk3x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
331 const int16x8_t vi3xGHIJKLMN = vmovl_s8(vld1_s8(i3)); i3 += 8;
332 const int16x8_t vk3xGHIJKLMN = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
333 const int16x8_t vi3xOPQRSTUV = vmovl_s8(vld1_s8(i3)); i3 += 8;
334 const int16x8_t vk3xOPQRSTUV = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
335
336 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
337 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
338 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi3x89ABCDEF), vget_low_s16(vk3x89ABCDEF));
339 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi3x89ABCDEF), vget_high_s16(vk3x89ABCDEF));
340 vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi3xGHIJKLMN), vget_low_s16(vk3xGHIJKLMN));
341 vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi3xGHIJKLMN), vget_high_s16(vk3xGHIJKLMN));
342 vaccOPQR = vmlal_s16(vaccOPQR, vget_low_s16(vi3xOPQRSTUV), vget_low_s16(vk3xOPQRSTUV));
343 vaccSTUV = vmlal_s16(vaccSTUV, vget_high_s16(vi3xOPQRSTUV), vget_high_s16(vk3xOPQRSTUV));
344
345 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4)); i4 += 8;
346 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
347 const int16x8_t vi4x89ABCDEF = vmovl_s8(vld1_s8(i4)); i4 += 8;
348 const int16x8_t vk4x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
349 const int16x8_t vi4xGHIJKLMN = vmovl_s8(vld1_s8(i4)); i4 += 8;
350 const int16x8_t vk4xGHIJKLMN = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
351 const int16x8_t vi4xOPQRSTUV = vmovl_s8(vld1_s8(i4)); i4 += 8;
352 const int16x8_t vk4xOPQRSTUV = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
353
354 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
355 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
356 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi4x89ABCDEF), vget_low_s16(vk4x89ABCDEF));
357 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi4x89ABCDEF), vget_high_s16(vk4x89ABCDEF));
358 vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi4xGHIJKLMN), vget_low_s16(vk4xGHIJKLMN));
359 vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi4xGHIJKLMN), vget_high_s16(vk4xGHIJKLMN));
360 vaccOPQR = vmlal_s16(vaccOPQR, vget_low_s16(vi4xOPQRSTUV), vget_low_s16(vk4xOPQRSTUV));
361 vaccSTUV = vmlal_s16(vaccSTUV, vget_high_s16(vi4xOPQRSTUV), vget_high_s16(vk4xOPQRSTUV));
362
363 vst1q_s32(b, vacc0123); b += 4;
364 vst1q_s32(b, vacc4567); b += 4;
365 vst1q_s32(b, vacc89AB); b += 4;
366 vst1q_s32(b, vaccCDEF); b += 4;
367 vst1q_s32(b, vaccGHIJ); b += 4;
368 vst1q_s32(b, vaccKLMN); b += 4;
369 vst1q_s32(b, vaccOPQR); b += 4;
370 vst1q_s32(b, vaccSTUV); b += 4;
371 }
372
373 if XNN_UNLIKELY(c != 0) {
374 do {
375 int32x4_t vacc0123 = vld1q_s32(b);
376 int32x4_t vacc4567 = vld1q_s32(b + 4);
377
378 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0)); i0 += 8;
379 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
380
381 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
382 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
383
384 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1)); i1 += 8;
385 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
386
387 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
388 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
389
390 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2)); i2 += 8;
391 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
392
393 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
394 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
395
396 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3)); i3 += 8;
397 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
398
399 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
400 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
401
402 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4)); i4 += 8;
403 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
404
405 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
406 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
407
408 vst1q_s32(b, vacc0123); b += 4;
409 vst1q_s32(b, vacc4567); b += 4;
410 c -= 8;
411 } while (c != 0);
412 }
413 }
414
415 // Last pass to process up to 5 inputs.
416 {
417 const int32_t* b = buffer;
418
419 const int8_t* i0 = input[0];
420 assert(i0 != NULL);
421 if XNN_UNPREDICTABLE(i0 != zero) {
422 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
423 }
424 const int8_t* i1 = input[1];
425 assert(i1 != NULL);
426 if XNN_UNPREDICTABLE(i1 != zero) {
427 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
428 }
429 const int8_t* i2 = input[2];
430 assert(i2 != NULL);
431 if XNN_UNPREDICTABLE(i2 != zero) {
432 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
433 }
434 const int8_t* i3 = input[3];
435 assert(i3 != NULL);
436 if XNN_UNPREDICTABLE(i3 != zero) {
437 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
438 }
439 const int8_t* i4 = input[4];
440 assert(i4 != NULL);
441 if XNN_UNPREDICTABLE(i4 != zero) {
442 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
443 }
444
445 size_t c = channels;
446 for (; c >= 32; c -= 32) {
447 int32x4_t vacc0123 = vld1q_s32(b); b += 4;
448 int32x4_t vacc4567 = vld1q_s32(b); b += 4;
449 int32x4_t vacc89AB = vld1q_s32(b); b += 4;
450 int32x4_t vaccCDEF = vld1q_s32(b); b += 4;
451 int32x4_t vaccGHIJ = vld1q_s32(b); b += 4;
452 int32x4_t vaccKLMN = vld1q_s32(b); b += 4;
453 int32x4_t vaccOPQR = vld1q_s32(b); b += 4;
454 int32x4_t vaccSTUV = vld1q_s32(b); b += 4;
455
456 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0)); i0 += 8;
457 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
458 const int16x8_t vi0x89ABCDEF = vmovl_s8(vld1_s8(i0)); i0 += 8;
459 const int16x8_t vk0x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
460 const int16x8_t vi0xGHIJKLMN = vmovl_s8(vld1_s8(i0)); i0 += 8;
461 const int16x8_t vk0xGHIJKLMN = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
462 const int16x8_t vi0xOPQRSTUV = vmovl_s8(vld1_s8(i0)); i0 += 8;
463 const int16x8_t vk0xOPQRSTUV = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
464
465 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
466 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
467 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi0x89ABCDEF), vget_low_s16(vk0x89ABCDEF));
468 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi0x89ABCDEF), vget_high_s16(vk0x89ABCDEF));
469 vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi0xGHIJKLMN), vget_low_s16(vk0xGHIJKLMN));
470 vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi0xGHIJKLMN), vget_high_s16(vk0xGHIJKLMN));
471 vaccOPQR = vmlal_s16(vaccOPQR, vget_low_s16(vi0xOPQRSTUV), vget_low_s16(vk0xOPQRSTUV));
472 vaccSTUV = vmlal_s16(vaccSTUV, vget_high_s16(vi0xOPQRSTUV), vget_high_s16(vk0xOPQRSTUV));
473
474 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1)); i1 += 8;
475 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
476 const int16x8_t vi1x89ABCDEF = vmovl_s8(vld1_s8(i1)); i1 += 8;
477 const int16x8_t vk1x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
478 const int16x8_t vi1xGHIJKLMN = vmovl_s8(vld1_s8(i1)); i1 += 8;
479 const int16x8_t vk1xGHIJKLMN = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
480 const int16x8_t vi1xOPQRSTUV = vmovl_s8(vld1_s8(i1)); i1 += 8;
481 const int16x8_t vk1xOPQRSTUV = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
482
483 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
484 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
485 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi1x89ABCDEF), vget_low_s16(vk1x89ABCDEF));
486 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi1x89ABCDEF), vget_high_s16(vk1x89ABCDEF));
487 vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi1xGHIJKLMN), vget_low_s16(vk1xGHIJKLMN));
488 vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi1xGHIJKLMN), vget_high_s16(vk1xGHIJKLMN));
489 vaccOPQR = vmlal_s16(vaccOPQR, vget_low_s16(vi1xOPQRSTUV), vget_low_s16(vk1xOPQRSTUV));
490 vaccSTUV = vmlal_s16(vaccSTUV, vget_high_s16(vi1xOPQRSTUV), vget_high_s16(vk1xOPQRSTUV));
491
492 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2)); i2 += 8;
493 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
494 const int16x8_t vi2x89ABCDEF = vmovl_s8(vld1_s8(i2)); i2 += 8;
495 const int16x8_t vk2x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
496 const int16x8_t vi2xGHIJKLMN = vmovl_s8(vld1_s8(i2)); i2 += 8;
497 const int16x8_t vk2xGHIJKLMN = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
498 const int16x8_t vi2xOPQRSTUV = vmovl_s8(vld1_s8(i2)); i2 += 8;
499 const int16x8_t vk2xOPQRSTUV = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
500
501 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
502 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
503 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi2x89ABCDEF), vget_low_s16(vk2x89ABCDEF));
504 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi2x89ABCDEF), vget_high_s16(vk2x89ABCDEF));
505 vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi2xGHIJKLMN), vget_low_s16(vk2xGHIJKLMN));
506 vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi2xGHIJKLMN), vget_high_s16(vk2xGHIJKLMN));
507 vaccOPQR = vmlal_s16(vaccOPQR, vget_low_s16(vi2xOPQRSTUV), vget_low_s16(vk2xOPQRSTUV));
508 vaccSTUV = vmlal_s16(vaccSTUV, vget_high_s16(vi2xOPQRSTUV), vget_high_s16(vk2xOPQRSTUV));
509
510 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3)); i3 += 8;
511 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
512 const int16x8_t vi3x89ABCDEF = vmovl_s8(vld1_s8(i3)); i3 += 8;
513 const int16x8_t vk3x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
514 const int16x8_t vi3xGHIJKLMN = vmovl_s8(vld1_s8(i3)); i3 += 8;
515 const int16x8_t vk3xGHIJKLMN = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
516 const int16x8_t vi3xOPQRSTUV = vmovl_s8(vld1_s8(i3)); i3 += 8;
517 const int16x8_t vk3xOPQRSTUV = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
518
519 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
520 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
521 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi3x89ABCDEF), vget_low_s16(vk3x89ABCDEF));
522 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi3x89ABCDEF), vget_high_s16(vk3x89ABCDEF));
523 vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi3xGHIJKLMN), vget_low_s16(vk3xGHIJKLMN));
524 vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi3xGHIJKLMN), vget_high_s16(vk3xGHIJKLMN));
525 vaccOPQR = vmlal_s16(vaccOPQR, vget_low_s16(vi3xOPQRSTUV), vget_low_s16(vk3xOPQRSTUV));
526 vaccSTUV = vmlal_s16(vaccSTUV, vget_high_s16(vi3xOPQRSTUV), vget_high_s16(vk3xOPQRSTUV));
527
528 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4)); i4 += 8;
529 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
530 const int16x8_t vi4x89ABCDEF = vmovl_s8(vld1_s8(i4)); i4 += 8;
531 const int16x8_t vk4x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
532 const int16x8_t vi4xGHIJKLMN = vmovl_s8(vld1_s8(i4)); i4 += 8;
533 const int16x8_t vk4xGHIJKLMN = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
534 const int16x8_t vi4xOPQRSTUV = vmovl_s8(vld1_s8(i4)); i4 += 8;
535 const int16x8_t vk4xOPQRSTUV = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
536
537 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
538 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
539 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi4x89ABCDEF), vget_low_s16(vk4x89ABCDEF));
540 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi4x89ABCDEF), vget_high_s16(vk4x89ABCDEF));
541 vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi4xGHIJKLMN), vget_low_s16(vk4xGHIJKLMN));
542 vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi4xGHIJKLMN), vget_high_s16(vk4xGHIJKLMN));
543 vaccOPQR = vmlal_s16(vaccOPQR, vget_low_s16(vi4xOPQRSTUV), vget_low_s16(vk4xOPQRSTUV));
544 vaccSTUV = vmlal_s16(vaccSTUV, vget_high_s16(vi4xOPQRSTUV), vget_high_s16(vk4xOPQRSTUV));
545
546 float32x4_t vfpacc0123 = vcvtq_f32_s32(vacc0123);
547 float32x4_t vfpacc4567 = vcvtq_f32_s32(vacc4567);
548 float32x4_t vfpacc89AB = vcvtq_f32_s32(vacc89AB);
549 float32x4_t vfpaccCDEF = vcvtq_f32_s32(vaccCDEF);
550 float32x4_t vfpaccGHIJ = vcvtq_f32_s32(vaccGHIJ);
551 float32x4_t vfpaccKLMN = vcvtq_f32_s32(vaccKLMN);
552 float32x4_t vfpaccOPQR = vcvtq_f32_s32(vaccOPQR);
553 float32x4_t vfpaccSTUV = vcvtq_f32_s32(vaccSTUV);
554
555 vfpacc0123 = vmulq_f32(vfpacc0123, vscale);
556 vfpacc4567 = vmulq_f32(vfpacc4567, vscale);
557 vfpacc89AB = vmulq_f32(vfpacc89AB, vscale);
558 vfpaccCDEF = vmulq_f32(vfpaccCDEF, vscale);
559 vfpaccGHIJ = vmulq_f32(vfpaccGHIJ, vscale);
560 vfpaccKLMN = vmulq_f32(vfpaccKLMN, vscale);
561 vfpaccOPQR = vmulq_f32(vfpaccOPQR, vscale);
562 vfpaccSTUV = vmulq_f32(vfpaccSTUV, vscale);
563
564 vacc0123 = vcvtnq_s32_f32(vfpacc0123);
565 vacc4567 = vcvtnq_s32_f32(vfpacc4567);
566 vacc89AB = vcvtnq_s32_f32(vfpacc89AB);
567 vaccCDEF = vcvtnq_s32_f32(vfpaccCDEF);
568 vaccGHIJ = vcvtnq_s32_f32(vfpaccGHIJ);
569 vaccKLMN = vcvtnq_s32_f32(vfpaccKLMN);
570 vaccOPQR = vcvtnq_s32_f32(vfpaccOPQR);
571 vaccSTUV = vcvtnq_s32_f32(vfpaccSTUV);
572
573 #if XNN_ARCH_ARM64
574 int16x8_t vacc01234567 = vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567);
575 int16x8_t vacc89ABCDEF = vqmovn_high_s32(vqmovn_s32(vacc89AB), vaccCDEF);
576 int16x8_t vaccGHIJKLMN = vqmovn_high_s32(vqmovn_s32(vaccGHIJ), vaccKLMN);
577 int16x8_t vaccOPQRSTUV = vqmovn_high_s32(vqmovn_s32(vaccOPQR), vaccSTUV);
578
579 vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
580 vacc89ABCDEF = vqaddq_s16(vacc89ABCDEF, voutput_zero_point);
581 vaccGHIJKLMN = vqaddq_s16(vaccGHIJKLMN, voutput_zero_point);
582 vaccOPQRSTUV = vqaddq_s16(vaccOPQRSTUV, voutput_zero_point);
583
584 int8x16_t vout0123456789ABCDEF = vqmovn_high_s16(vqmovn_s16(vacc01234567), vacc89ABCDEF);
585 int8x16_t voutGHIJKLMNOPQRSTUV = vqmovn_high_s16(vqmovn_s16(vaccGHIJKLMN), vaccOPQRSTUV);
586 #else // !XNN_ARCH_ARM64
587 int16x8_t vacc01234567 = vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567));
588 int16x8_t vacc89ABCDEF = vcombine_s16(vqmovn_s32(vacc89AB), vqmovn_s32(vaccCDEF));
589 int16x8_t vaccGHIJKLMN = vcombine_s16(vqmovn_s32(vaccGHIJ), vqmovn_s32(vaccKLMN));
590 int16x8_t vaccOPQRSTUV = vcombine_s16(vqmovn_s32(vaccOPQR), vqmovn_s32(vaccSTUV));
591
592 vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
593 vacc89ABCDEF = vqaddq_s16(vacc89ABCDEF, voutput_zero_point);
594 vaccGHIJKLMN = vqaddq_s16(vaccGHIJKLMN, voutput_zero_point);
595 vaccOPQRSTUV = vqaddq_s16(vaccOPQRSTUV, voutput_zero_point);
596
597 int8x16_t vout0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc01234567), vqmovn_s16(vacc89ABCDEF));
598 int8x16_t voutGHIJKLMNOPQRSTUV = vcombine_s8(vqmovn_s16(vaccGHIJKLMN), vqmovn_s16(vaccOPQRSTUV));
599 #endif // !XNN_ARCH_ARM64
600
601 vout0123456789ABCDEF = vmaxq_s8(vout0123456789ABCDEF, voutput_min);
602 voutGHIJKLMNOPQRSTUV = vmaxq_s8(voutGHIJKLMNOPQRSTUV, voutput_min);
603
604 vout0123456789ABCDEF = vminq_s8(vout0123456789ABCDEF, voutput_max);
605 voutGHIJKLMNOPQRSTUV = vminq_s8(voutGHIJKLMNOPQRSTUV, voutput_max);
606
607 vst1q_s8(output, vout0123456789ABCDEF); output += 16;
608 vst1q_s8(output, voutGHIJKLMNOPQRSTUV); output += 16;
609 }
610
611 if XNN_UNLIKELY(c != 0) {
612 do {
613 int32x4_t vacc0123 = vld1q_s32(b); b += 4;
614 int32x4_t vacc4567 = vld1q_s32(b); b += 4;
615
616 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0)); i0 += 8;
617 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
618
619 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
620 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
621
622 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1)); i1 += 8;
623 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
624
625 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
626 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
627
628 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2)); i2 += 8;
629 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
630
631 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
632 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
633
634 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3)); i3 += 8;
635 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
636
637 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
638 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
639
640 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4)); i4 += 8;
641 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
642
643 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
644 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
645
646 float32x4_t vfpacc0123 = vcvtq_f32_s32(vacc0123);
647 float32x4_t vfpacc4567 = vcvtq_f32_s32(vacc4567);
648
649 vfpacc0123 = vmulq_f32(vfpacc0123, vscale);
650 vfpacc4567 = vmulq_f32(vfpacc4567, vscale);
651
652 vacc0123 = vcvtnq_s32_f32(vfpacc0123);
653 vacc4567 = vcvtnq_s32_f32(vfpacc4567);
654
655 #if XNN_ARCH_ARM64
656 int16x8_t vacc01234567 = vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567);
657 #else
658 int16x8_t vacc01234567 = vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567));
659 #endif
660 vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
661
662 int8x8_t vout01234567 = vqmovn_s16(vacc01234567);
663 vout01234567 = vmax_s8(vout01234567, vget_low_s8(voutput_min));
664 vout01234567 = vmin_s8(vout01234567, vget_low_s8(voutput_max));
665
666 if XNN_LIKELY(c >= 8) {
667 vst1_s8(output, vout01234567); output += 8;
668 c -= 8;
669 } else {
670 if (c & 4) {
671 vst1_lane_u32((void*) output, vreinterpret_u32_s8(vout01234567), 0); output += 4;
672 vout01234567 = vext_s8(vout01234567, vout01234567, 4);
673 }
674 if (c & 2) {
675 vst1_lane_u16((void*) output, vreinterpret_u16_s8(vout01234567), 0); output += 2;
676 vout01234567 = vext_s8(vout01234567, vout01234567, 2);
677 }
678 if (c & 1) {
679 vst1_lane_s8(output, vout01234567, 0); output += 1;
680 }
681 c = 0;
682 }
683 } while (c != 0);
684 }
685 }
686
687 input = (const int8_t**) ((uintptr_t) input + input_stride);
688 output = (int8_t*) ((uintptr_t) output + output_increment);
689 } while (--output_width != 0);
690 }