Debian Codegraph @

src/qs8-dwconv/gen/qs8-dwconv-6f6m7l16c8s8r-minmax-fp32-neonv8-mul16.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-dwconv/multipass-neon-mul16.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12
13 #include <arm_neon.h>
14
15 #include "xnnpack/dwconv.h"
16 #include "xnnpack/intrinsics-polyfill.h"
17 #include "xnnpack/math.h"
18
19
20 void xnn_qs8_dwconv_minmax_fp32_ukernel_6f6m7l16c8s8r__neonv8_mul16(
21 size_t channels,
22 size_t output_width,
23 const int8_t** input,
24 const void* weights,
25 int8_t* output,
26 intptr_t input_stride,
27 size_t output_increment,
28 size_t input_offset,
29 const int8_t* zero,
30 size_t kernel_size,
31 int32_t* buffer,
32 const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
33 {
34 assert(channels != 0);
35 assert(output_width != 0);
36 assert(kernel_size > 6);
37
38 const float32x4_t vscale = vld1q_dup_f32(&params->fp32_neonv8.scale);
39 const int16x8_t voutput_zero_point = vld1q_dup_s16(&params->fp32_neonv8.output_zero_point);
40 const int8x16_t voutput_min = vld1q_dup_s8(&params->fp32_neonv8.output_min);
41 const int8x16_t voutput_max = vld1q_dup_s8(&params->fp32_neonv8.output_max);
42
43 do {
44 const void* w = weights;
45
46 // First pass to process 6 inputs.
47 {
48 int32_t* b = buffer;
49 const int8_t* i0 = input[0];
50 assert(i0 != NULL);
51 if XNN_UNPREDICTABLE(i0 != zero) {
52 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
53 }
54 const int8_t* i1 = input[1];
55 assert(i1 != NULL);
56 if XNN_UNPREDICTABLE(i1 != zero) {
57 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
58 }
59 const int8_t* i2 = input[2];
60 assert(i2 != NULL);
61 if XNN_UNPREDICTABLE(i2 != zero) {
62 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
63 }
64 const int8_t* i3 = input[3];
65 assert(i3 != NULL);
66 if XNN_UNPREDICTABLE(i3 != zero) {
67 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
68 }
69 const int8_t* i4 = input[4];
70 assert(i4 != NULL);
71 if XNN_UNPREDICTABLE(i4 != zero) {
72 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
73 }
74 const int8_t* i5 = input[5];
75 assert(i5 != NULL);
76 if XNN_UNPREDICTABLE(i5 != zero) {
77 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
78 }
79 input += 6;
80
81 size_t c = round_up_po2(channels, 8);
82
83 for (; c >= 16; c -= 16) {
84 int32x4_t vacc0123 = vld1q_s32(w); w = (const int32_t*) w + 4;
85 int32x4_t vacc4567 = vld1q_s32(w); w = (const int32_t*) w + 4;
86 int32x4_t vacc89AB = vld1q_s32(w); w = (const int32_t*) w + 4;
87 int32x4_t vaccCDEF = vld1q_s32(w); w = (const int32_t*) w + 4;
88
89 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0)); i0 += 8;
90 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
91 const int16x8_t vi0x89ABCDEF = vmovl_s8(vld1_s8(i0)); i0 += 8;
92 const int16x8_t vk0x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
93
94 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
95 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
96 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi0x89ABCDEF), vget_low_s16(vk0x89ABCDEF));
97 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi0x89ABCDEF), vget_high_s16(vk0x89ABCDEF));
98
99 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1)); i1 += 8;
100 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
101 const int16x8_t vi1x89ABCDEF = vmovl_s8(vld1_s8(i1)); i1 += 8;
102 const int16x8_t vk1x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
103
104 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
105 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
106 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi1x89ABCDEF), vget_low_s16(vk1x89ABCDEF));
107 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi1x89ABCDEF), vget_high_s16(vk1x89ABCDEF));
108
109 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2)); i2 += 8;
110 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
111 const int16x8_t vi2x89ABCDEF = vmovl_s8(vld1_s8(i2)); i2 += 8;
112 const int16x8_t vk2x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
113
114 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
115 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
116 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi2x89ABCDEF), vget_low_s16(vk2x89ABCDEF));
117 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi2x89ABCDEF), vget_high_s16(vk2x89ABCDEF));
118
119 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3)); i3 += 8;
120 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
121 const int16x8_t vi3x89ABCDEF = vmovl_s8(vld1_s8(i3)); i3 += 8;
122 const int16x8_t vk3x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
123
124 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
125 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
126 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi3x89ABCDEF), vget_low_s16(vk3x89ABCDEF));
127 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi3x89ABCDEF), vget_high_s16(vk3x89ABCDEF));
128
129 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4)); i4 += 8;
130 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
131 const int16x8_t vi4x89ABCDEF = vmovl_s8(vld1_s8(i4)); i4 += 8;
132 const int16x8_t vk4x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
133
134 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
135 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
136 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi4x89ABCDEF), vget_low_s16(vk4x89ABCDEF));
137 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi4x89ABCDEF), vget_high_s16(vk4x89ABCDEF));
138
139 const int16x8_t vi5x01234567 = vmovl_s8(vld1_s8(i5)); i5 += 8;
140 const int16x8_t vk5x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
141 const int16x8_t vi5x89ABCDEF = vmovl_s8(vld1_s8(i5)); i5 += 8;
142 const int16x8_t vk5x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
143
144 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi5x01234567), vget_low_s16(vk5x01234567));
145 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi5x01234567), vget_high_s16(vk5x01234567));
146 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi5x89ABCDEF), vget_low_s16(vk5x89ABCDEF));
147 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi5x89ABCDEF), vget_high_s16(vk5x89ABCDEF));
148
149 vst1q_s32(b, vacc0123); b += 4;
150 vst1q_s32(b, vacc4567); b += 4;
151 vst1q_s32(b, vacc89AB); b += 4;
152 vst1q_s32(b, vaccCDEF); b += 4;
153 }
154
155 if XNN_UNLIKELY(c != 0) {
156 do {
157 int32x4_t vacc0123 = vld1q_s32(w); w = (const int32_t*) w + 4;
158 int32x4_t vacc4567 = vld1q_s32(w); w = (const int32_t*) w + 4;
159
160 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0)); i0 += 8;
161 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
162
163 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
164 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
165
166 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1)); i1 += 8;
167 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
168
169 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
170 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
171
172 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2)); i2 += 8;
173 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
174
175 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
176 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
177
178 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3)); i3 += 8;
179 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
180
181 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
182 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
183
184 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4)); i4 += 8;
185 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
186
187 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
188 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
189
190 const int16x8_t vi5x01234567 = vmovl_s8(vld1_s8(i5)); i5 += 8;
191 const int16x8_t vk5x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
192
193 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi5x01234567), vget_low_s16(vk5x01234567));
194 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi5x01234567), vget_high_s16(vk5x01234567));
195
196 vst1q_s32(b, vacc0123); b += 4;
197 vst1q_s32(b, vacc4567); b += 4;
198 c -= 8;
199 } while (c != 0);
200 }
201 }
202
203 // Middle pass to process 6 inputs in each iteration.
204 for (size_t ks = kernel_size - 6; ks > 7; ks -= 6) {
205 int32_t* b = buffer;
206
207 const int8_t* i0 = input[0];
208 assert(i0 != NULL);
209 if XNN_UNPREDICTABLE(i0 != zero) {
210 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
211 }
212 const int8_t* i1 = input[1];
213 assert(i1 != NULL);
214 if XNN_UNPREDICTABLE(i1 != zero) {
215 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
216 }
217 const int8_t* i2 = input[2];
218 assert(i2 != NULL);
219 if XNN_UNPREDICTABLE(i2 != zero) {
220 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
221 }
222 const int8_t* i3 = input[3];
223 assert(i3 != NULL);
224 if XNN_UNPREDICTABLE(i3 != zero) {
225 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
226 }
227 const int8_t* i4 = input[4];
228 assert(i4 != NULL);
229 if XNN_UNPREDICTABLE(i4 != zero) {
230 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
231 }
232 const int8_t* i5 = input[5];
233 assert(i5 != NULL);
234 if XNN_UNPREDICTABLE(i5 != zero) {
235 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
236 }
237 input += 6;
238
239 size_t c = round_up_po2(channels, 8);
240
241 for (; c >= 16; c -= 16) {
242 int32x4_t vacc0123 = vld1q_s32(b);
243 int32x4_t vacc4567 = vld1q_s32(b + 4);
244 int32x4_t vacc89AB = vld1q_s32(b + 8);
245 int32x4_t vaccCDEF = vld1q_s32(b + 12);
246
247 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0)); i0 += 8;
248 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
249 const int16x8_t vi0x89ABCDEF = vmovl_s8(vld1_s8(i0)); i0 += 8;
250 const int16x8_t vk0x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
251
252 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
253 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
254 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi0x89ABCDEF), vget_low_s16(vk0x89ABCDEF));
255 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi0x89ABCDEF), vget_high_s16(vk0x89ABCDEF));
256
257 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1)); i1 += 8;
258 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
259 const int16x8_t vi1x89ABCDEF = vmovl_s8(vld1_s8(i1)); i1 += 8;
260 const int16x8_t vk1x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
261
262 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
263 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
264 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi1x89ABCDEF), vget_low_s16(vk1x89ABCDEF));
265 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi1x89ABCDEF), vget_high_s16(vk1x89ABCDEF));
266
267 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2)); i2 += 8;
268 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
269 const int16x8_t vi2x89ABCDEF = vmovl_s8(vld1_s8(i2)); i2 += 8;
270 const int16x8_t vk2x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
271
272 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
273 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
274 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi2x89ABCDEF), vget_low_s16(vk2x89ABCDEF));
275 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi2x89ABCDEF), vget_high_s16(vk2x89ABCDEF));
276
277 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3)); i3 += 8;
278 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
279 const int16x8_t vi3x89ABCDEF = vmovl_s8(vld1_s8(i3)); i3 += 8;
280 const int16x8_t vk3x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
281
282 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
283 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
284 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi3x89ABCDEF), vget_low_s16(vk3x89ABCDEF));
285 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi3x89ABCDEF), vget_high_s16(vk3x89ABCDEF));
286
287 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4)); i4 += 8;
288 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
289 const int16x8_t vi4x89ABCDEF = vmovl_s8(vld1_s8(i4)); i4 += 8;
290 const int16x8_t vk4x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
291
292 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
293 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
294 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi4x89ABCDEF), vget_low_s16(vk4x89ABCDEF));
295 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi4x89ABCDEF), vget_high_s16(vk4x89ABCDEF));
296
297 const int16x8_t vi5x01234567 = vmovl_s8(vld1_s8(i5)); i5 += 8;
298 const int16x8_t vk5x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
299 const int16x8_t vi5x89ABCDEF = vmovl_s8(vld1_s8(i5)); i5 += 8;
300 const int16x8_t vk5x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
301
302 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi5x01234567), vget_low_s16(vk5x01234567));
303 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi5x01234567), vget_high_s16(vk5x01234567));
304 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi5x89ABCDEF), vget_low_s16(vk5x89ABCDEF));
305 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi5x89ABCDEF), vget_high_s16(vk5x89ABCDEF));
306
307 vst1q_s32(b, vacc0123); b += 4;
308 vst1q_s32(b, vacc4567); b += 4;
309 vst1q_s32(b, vacc89AB); b += 4;
310 vst1q_s32(b, vaccCDEF); b += 4;
311 }
312
313 if XNN_UNLIKELY(c != 0) {
314 do {
315 int32x4_t vacc0123 = vld1q_s32(b);
316 int32x4_t vacc4567 = vld1q_s32(b + 4);
317
318 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0)); i0 += 8;
319 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
320
321 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
322 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
323
324 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1)); i1 += 8;
325 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
326
327 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
328 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
329
330 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2)); i2 += 8;
331 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
332
333 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
334 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
335
336 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3)); i3 += 8;
337 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
338
339 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
340 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
341
342 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4)); i4 += 8;
343 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
344
345 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
346 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
347
348 const int16x8_t vi5x01234567 = vmovl_s8(vld1_s8(i5)); i5 += 8;
349 const int16x8_t vk5x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
350
351 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi5x01234567), vget_low_s16(vk5x01234567));
352 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi5x01234567), vget_high_s16(vk5x01234567));
353
354 vst1q_s32(b, vacc0123); b += 4;
355 vst1q_s32(b, vacc4567); b += 4;
356 c -= 8;
357 } while (c != 0);
358 }
359 }
360
361 // Last pass to process up to 7 inputs.
362 {
363 const int32_t* b = buffer;
364
365 const int8_t* i0 = input[0];
366 assert(i0 != NULL);
367 if XNN_UNPREDICTABLE(i0 != zero) {
368 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
369 }
370 const int8_t* i1 = input[1];
371 assert(i1 != NULL);
372 if XNN_UNPREDICTABLE(i1 != zero) {
373 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
374 }
375 const int8_t* i2 = input[2];
376 assert(i2 != NULL);
377 if XNN_UNPREDICTABLE(i2 != zero) {
378 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
379 }
380 const int8_t* i3 = input[3];
381 assert(i3 != NULL);
382 if XNN_UNPREDICTABLE(i3 != zero) {
383 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
384 }
385 const int8_t* i4 = input[4];
386 assert(i4 != NULL);
387 if XNN_UNPREDICTABLE(i4 != zero) {
388 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
389 }
390 const int8_t* i5 = input[5];
391 assert(i5 != NULL);
392 if XNN_UNPREDICTABLE(i5 != zero) {
393 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
394 }
395 const int8_t* i6 = input[6];
396 assert(i6 != NULL);
397 if XNN_UNPREDICTABLE(i6 != zero) {
398 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
399 }
400
401 size_t c = channels;
402 for (; c >= 16; c -= 16) {
403 int32x4_t vacc0123 = vld1q_s32(b); b += 4;
404 int32x4_t vacc4567 = vld1q_s32(b); b += 4;
405 int32x4_t vacc89AB = vld1q_s32(b); b += 4;
406 int32x4_t vaccCDEF = vld1q_s32(b); b += 4;
407
408 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0)); i0 += 8;
409 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
410 const int16x8_t vi0x89ABCDEF = vmovl_s8(vld1_s8(i0)); i0 += 8;
411 const int16x8_t vk0x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
412
413 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
414 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
415 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi0x89ABCDEF), vget_low_s16(vk0x89ABCDEF));
416 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi0x89ABCDEF), vget_high_s16(vk0x89ABCDEF));
417
418 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1)); i1 += 8;
419 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
420 const int16x8_t vi1x89ABCDEF = vmovl_s8(vld1_s8(i1)); i1 += 8;
421 const int16x8_t vk1x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
422
423 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
424 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
425 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi1x89ABCDEF), vget_low_s16(vk1x89ABCDEF));
426 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi1x89ABCDEF), vget_high_s16(vk1x89ABCDEF));
427
428 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2)); i2 += 8;
429 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
430 const int16x8_t vi2x89ABCDEF = vmovl_s8(vld1_s8(i2)); i2 += 8;
431 const int16x8_t vk2x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
432
433 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
434 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
435 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi2x89ABCDEF), vget_low_s16(vk2x89ABCDEF));
436 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi2x89ABCDEF), vget_high_s16(vk2x89ABCDEF));
437
438 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3)); i3 += 8;
439 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
440 const int16x8_t vi3x89ABCDEF = vmovl_s8(vld1_s8(i3)); i3 += 8;
441 const int16x8_t vk3x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
442
443 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
444 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
445 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi3x89ABCDEF), vget_low_s16(vk3x89ABCDEF));
446 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi3x89ABCDEF), vget_high_s16(vk3x89ABCDEF));
447
448 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4)); i4 += 8;
449 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
450 const int16x8_t vi4x89ABCDEF = vmovl_s8(vld1_s8(i4)); i4 += 8;
451 const int16x8_t vk4x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
452
453 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
454 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
455 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi4x89ABCDEF), vget_low_s16(vk4x89ABCDEF));
456 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi4x89ABCDEF), vget_high_s16(vk4x89ABCDEF));
457
458 const int16x8_t vi5x01234567 = vmovl_s8(vld1_s8(i5)); i5 += 8;
459 const int16x8_t vk5x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
460 const int16x8_t vi5x89ABCDEF = vmovl_s8(vld1_s8(i5)); i5 += 8;
461 const int16x8_t vk5x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
462
463 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi5x01234567), vget_low_s16(vk5x01234567));
464 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi5x01234567), vget_high_s16(vk5x01234567));
465 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi5x89ABCDEF), vget_low_s16(vk5x89ABCDEF));
466 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi5x89ABCDEF), vget_high_s16(vk5x89ABCDEF));
467
468 const int16x8_t vi6x01234567 = vmovl_s8(vld1_s8(i6)); i6 += 8;
469 const int16x8_t vk6x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
470 const int16x8_t vi6x89ABCDEF = vmovl_s8(vld1_s8(i6)); i6 += 8;
471 const int16x8_t vk6x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
472
473 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi6x01234567), vget_low_s16(vk6x01234567));
474 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi6x01234567), vget_high_s16(vk6x01234567));
475 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi6x89ABCDEF), vget_low_s16(vk6x89ABCDEF));
476 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi6x89ABCDEF), vget_high_s16(vk6x89ABCDEF));
477
478 float32x4_t vfpacc0123 = vcvtq_f32_s32(vacc0123);
479 float32x4_t vfpacc4567 = vcvtq_f32_s32(vacc4567);
480 float32x4_t vfpacc89AB = vcvtq_f32_s32(vacc89AB);
481 float32x4_t vfpaccCDEF = vcvtq_f32_s32(vaccCDEF);
482
483 vfpacc0123 = vmulq_f32(vfpacc0123, vscale);
484 vfpacc4567 = vmulq_f32(vfpacc4567, vscale);
485 vfpacc89AB = vmulq_f32(vfpacc89AB, vscale);
486 vfpaccCDEF = vmulq_f32(vfpaccCDEF, vscale);
487
488 vacc0123 = vcvtnq_s32_f32(vfpacc0123);
489 vacc4567 = vcvtnq_s32_f32(vfpacc4567);
490 vacc89AB = vcvtnq_s32_f32(vfpacc89AB);
491 vaccCDEF = vcvtnq_s32_f32(vfpaccCDEF);
492
493 #if XNN_ARCH_ARM64
494 int16x8_t vacc01234567 = vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567);
495 int16x8_t vacc89ABCDEF = vqmovn_high_s32(vqmovn_s32(vacc89AB), vaccCDEF);
496
497 vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
498 vacc89ABCDEF = vqaddq_s16(vacc89ABCDEF, voutput_zero_point);
499
500 int8x16_t vout0123456789ABCDEF = vqmovn_high_s16(vqmovn_s16(vacc01234567), vacc89ABCDEF);
501 #else // !XNN_ARCH_ARM64
502 int16x8_t vacc01234567 = vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567));
503 int16x8_t vacc89ABCDEF = vcombine_s16(vqmovn_s32(vacc89AB), vqmovn_s32(vaccCDEF));
504
505 vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
506 vacc89ABCDEF = vqaddq_s16(vacc89ABCDEF, voutput_zero_point);
507
508 int8x16_t vout0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc01234567), vqmovn_s16(vacc89ABCDEF));
509 #endif // !XNN_ARCH_ARM64
510
511 vout0123456789ABCDEF = vmaxq_s8(vout0123456789ABCDEF, voutput_min);
512
513 vout0123456789ABCDEF = vminq_s8(vout0123456789ABCDEF, voutput_max);
514
515 vst1q_s8(output, vout0123456789ABCDEF); output += 16;
516 }
517
518 if XNN_UNLIKELY(c != 0) {
519 do {
520 int32x4_t vacc0123 = vld1q_s32(b); b += 4;
521 int32x4_t vacc4567 = vld1q_s32(b); b += 4;
522
523 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0)); i0 += 8;
524 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
525
526 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
527 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
528
529 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1)); i1 += 8;
530 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
531
532 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
533 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
534
535 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2)); i2 += 8;
536 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
537
538 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
539 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
540
541 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3)); i3 += 8;
542 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
543
544 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
545 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
546
547 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4)); i4 += 8;
548 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
549
550 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
551 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
552
553 const int16x8_t vi5x01234567 = vmovl_s8(vld1_s8(i5)); i5 += 8;
554 const int16x8_t vk5x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
555
556 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi5x01234567), vget_low_s16(vk5x01234567));
557 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi5x01234567), vget_high_s16(vk5x01234567));
558
559 const int16x8_t vi6x01234567 = vmovl_s8(vld1_s8(i6)); i6 += 8;
560 const int16x8_t vk6x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
561
562 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi6x01234567), vget_low_s16(vk6x01234567));
563 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi6x01234567), vget_high_s16(vk6x01234567));
564
565 float32x4_t vfpacc0123 = vcvtq_f32_s32(vacc0123);
566 float32x4_t vfpacc4567 = vcvtq_f32_s32(vacc4567);
567
568 vfpacc0123 = vmulq_f32(vfpacc0123, vscale);
569 vfpacc4567 = vmulq_f32(vfpacc4567, vscale);
570
571 vacc0123 = vcvtnq_s32_f32(vfpacc0123);
572 vacc4567 = vcvtnq_s32_f32(vfpacc4567);
573
574 #if XNN_ARCH_ARM64
575 int16x8_t vacc01234567 = vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567);
576 #else
577 int16x8_t vacc01234567 = vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567));
578 #endif
579 vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
580
581 int8x8_t vout01234567 = vqmovn_s16(vacc01234567);
582 vout01234567 = vmax_s8(vout01234567, vget_low_s8(voutput_min));
583 vout01234567 = vmin_s8(vout01234567, vget_low_s8(voutput_max));
584
585 if XNN_LIKELY(c >= 8) {
586 vst1_s8(output, vout01234567); output += 8;
587 c -= 8;
588 } else {
589 if (c & 4) {
590 vst1_lane_u32((void*) output, vreinterpret_u32_s8(vout01234567), 0); output += 4;
591 vout01234567 = vext_s8(vout01234567, vout01234567, 4);
592 }
593 if (c & 2) {
594 vst1_lane_u16((void*) output, vreinterpret_u16_s8(vout01234567), 0); output += 2;
595 vout01234567 = vext_s8(vout01234567, vout01234567, 2);
596 }
597 if (c & 1) {
598 vst1_lane_s8(output, vout01234567, 0); output += 1;
599 }
600 c = 0;
601 }
602 } while (c != 0);
603 }
604 }
605
606 input = (const int8_t**) ((uintptr_t) input + input_stride);
607 output = (int8_t*) ((uintptr_t) output + output_increment);
608 } while (--output_width != 0);
609 }