Debian Codegraph @

src/qs8-dwconv/gen/qs8-dwconv-6f6m7l16c8s8r-minmax-rndnu-neon-mla8-ld128.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-dwconv/multipass-neon-mul8.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12
13 #include <arm_neon.h>
14
15 #include "xnnpack/dwconv.h"
16 #include "xnnpack/math.h"
17
18
19 void xnn_qs8_dwconv_minmax_rndnu_ukernel_6f6m7l16c8s8r__neon_mla8_ld128(
20 size_t channels,
21 size_t output_width,
22 const int8_t** input,
23 const void* weights,
24 int8_t* output,
25 intptr_t input_stride,
26 size_t output_increment,
27 size_t input_offset,
28 const int8_t* zero,
29 size_t kernel_size,
30 int32_t* buffer,
31 const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
32 {
33 assert(channels != 0);
34 assert(output_width != 0);
35 assert(kernel_size > 6);
36
37 const int32x4_t vright_pre_shift = vld1q_dup_s32(&params->rndnu_neon.right_pre_shift);
38 const int32x4_t vmultiplier = vld1q_dup_s32(&params->rndnu_neon.multiplier);
39 const int32x4_t vright_post_shift = vld1q_dup_s32(&params->rndnu_neon.right_post_shift);
40 const int16x8_t voutput_zero_point = vld1q_dup_s16(&params->rndnu_neon.output_zero_point);
41 const int8x16_t voutput_min = vld1q_dup_s8(&params->rndnu_neon.output_min);
42 const int8x16_t voutput_max = vld1q_dup_s8(&params->rndnu_neon.output_max);
43 do {
44 const void* w = weights;
45
46 // First pass to process 6 inputs.
47 {
48 int32_t* b = buffer;
49 const int8_t* i0 = input[0];
50 assert(i0 != NULL);
51 if XNN_UNPREDICTABLE(i0 != zero) {
52 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
53 }
54 const int8_t* i1 = input[1];
55 assert(i1 != NULL);
56 if XNN_UNPREDICTABLE(i1 != zero) {
57 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
58 }
59 const int8_t* i2 = input[2];
60 assert(i2 != NULL);
61 if XNN_UNPREDICTABLE(i2 != zero) {
62 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
63 }
64 const int8_t* i3 = input[3];
65 assert(i3 != NULL);
66 if XNN_UNPREDICTABLE(i3 != zero) {
67 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
68 }
69 const int8_t* i4 = input[4];
70 assert(i4 != NULL);
71 if XNN_UNPREDICTABLE(i4 != zero) {
72 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
73 }
74 const int8_t* i5 = input[5];
75 assert(i5 != NULL);
76 if XNN_UNPREDICTABLE(i5 != zero) {
77 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
78 }
79 input += 6;
80
81 size_t c = round_up_po2(channels, 8);
82
83 for (; c >= 16; c -= 16) {
84 int32x4_t vacc0123 = vld1q_s32(w); w = (const int32_t*) w + 4;
85 int32x4_t vacc4567 = vld1q_s32(w); w = (const int32_t*) w + 4;
86 int32x4_t vacc89AB = vld1q_s32(w); w = (const int32_t*) w + 4;
87 int32x4_t vaccCDEF = vld1q_s32(w); w = (const int32_t*) w + 4;
88
89 const int8x16_t vi0x0123456789ABCDEF = vld1q_s8(i0); i0 += 16;
90 const int8x16_t vk0x0123456789ABCDEF = vld1q_s8(w); w = (const int8_t*) w + 16;
91
92 int16x8_t vprod01234567 = vmull_s8(vget_low_s8(vi0x0123456789ABCDEF), vget_low_s8(vk0x0123456789ABCDEF));
93 int16x8_t vprod89ABCDEF = vmull_s8(vget_high_s8(vi0x0123456789ABCDEF), vget_high_s8(vk0x0123456789ABCDEF));
94
95
96 const int8x16_t vi1x0123456789ABCDEF = vld1q_s8(i1); i1 += 16;
97 const int8x16_t vk1x0123456789ABCDEF = vld1q_s8(w); w = (const int8_t*) w + 16;
98
99 vprod01234567 = vmlal_s8(vprod01234567, vget_low_s8(vi1x0123456789ABCDEF), vget_low_s8(vk1x0123456789ABCDEF));
100 vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vget_high_s8(vi1x0123456789ABCDEF), vget_high_s8(vk1x0123456789ABCDEF));
101
102 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
103 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
104 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
105 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
106
107 const int8x16_t vi2x0123456789ABCDEF = vld1q_s8(i2); i2 += 16;
108 const int8x16_t vk2x0123456789ABCDEF = vld1q_s8(w); w = (const int8_t*) w + 16;
109
110 vprod01234567 = vmull_s8(vget_low_s8(vi2x0123456789ABCDEF), vget_low_s8(vk2x0123456789ABCDEF));
111 vprod89ABCDEF = vmull_s8(vget_high_s8(vi2x0123456789ABCDEF), vget_high_s8(vk2x0123456789ABCDEF));
112
113
114 const int8x16_t vi3x0123456789ABCDEF = vld1q_s8(i3); i3 += 16;
115 const int8x16_t vk3x0123456789ABCDEF = vld1q_s8(w); w = (const int8_t*) w + 16;
116
117 vprod01234567 = vmlal_s8(vprod01234567, vget_low_s8(vi3x0123456789ABCDEF), vget_low_s8(vk3x0123456789ABCDEF));
118 vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vget_high_s8(vi3x0123456789ABCDEF), vget_high_s8(vk3x0123456789ABCDEF));
119
120 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
121 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
122 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
123 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
124
125 const int8x16_t vi4x0123456789ABCDEF = vld1q_s8(i4); i4 += 16;
126 const int8x16_t vk4x0123456789ABCDEF = vld1q_s8(w); w = (const int8_t*) w + 16;
127
128 vprod01234567 = vmull_s8(vget_low_s8(vi4x0123456789ABCDEF), vget_low_s8(vk4x0123456789ABCDEF));
129 vprod89ABCDEF = vmull_s8(vget_high_s8(vi4x0123456789ABCDEF), vget_high_s8(vk4x0123456789ABCDEF));
130
131
132 const int8x16_t vi5x0123456789ABCDEF = vld1q_s8(i5); i5 += 16;
133 const int8x16_t vk5x0123456789ABCDEF = vld1q_s8(w); w = (const int8_t*) w + 16;
134
135 vprod01234567 = vmlal_s8(vprod01234567, vget_low_s8(vi5x0123456789ABCDEF), vget_low_s8(vk5x0123456789ABCDEF));
136 vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vget_high_s8(vi5x0123456789ABCDEF), vget_high_s8(vk5x0123456789ABCDEF));
137
138 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
139 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
140 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
141 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
142
143 vst1q_s32(b, vacc0123); b += 4;
144 vst1q_s32(b, vacc4567); b += 4;
145 vst1q_s32(b, vacc89AB); b += 4;
146 vst1q_s32(b, vaccCDEF); b += 4;
147 }
148
149 if XNN_UNLIKELY(c != 0) {
150 do {
151 int32x4_t vacc0123 = vld1q_s32(w); w = (const int32_t*) w + 4;
152 int32x4_t vacc4567 = vld1q_s32(w); w = (const int32_t*) w + 4;
153
154 const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
155 const int8x8_t vk0x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
156
157 int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
158
159
160 const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
161 const int8x8_t vk1x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
162
163 vprod01234567 = vmlal_s8(vprod01234567, vi1x01234567, vk1x01234567);
164
165 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
166 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
167
168 const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
169 const int8x8_t vk2x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
170
171 vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
172
173
174 const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
175 const int8x8_t vk3x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
176
177 vprod01234567 = vmlal_s8(vprod01234567, vi3x01234567, vk3x01234567);
178
179 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
180 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
181
182 const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
183 const int8x8_t vk4x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
184
185 vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
186
187
188 const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
189 const int8x8_t vk5x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
190
191 vprod01234567 = vmlal_s8(vprod01234567, vi5x01234567, vk5x01234567);
192
193 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
194 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
195
196 vst1q_s32(b, vacc0123); b += 4;
197 vst1q_s32(b, vacc4567); b += 4;
198 c -= 8;
199 } while (c != 0);
200 }
201 }
202
203 // Middle pass to process 6 inputs in each iteration.
204 for (size_t ks = kernel_size - 6; ks > 7; ks -= 6) {
205 int32_t* b = buffer;
206
207 const int8_t* i0 = input[0];
208 assert(i0 != NULL);
209 if XNN_UNPREDICTABLE(i0 != zero) {
210 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
211 }
212 const int8_t* i1 = input[1];
213 assert(i1 != NULL);
214 if XNN_UNPREDICTABLE(i1 != zero) {
215 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
216 }
217 const int8_t* i2 = input[2];
218 assert(i2 != NULL);
219 if XNN_UNPREDICTABLE(i2 != zero) {
220 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
221 }
222 const int8_t* i3 = input[3];
223 assert(i3 != NULL);
224 if XNN_UNPREDICTABLE(i3 != zero) {
225 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
226 }
227 const int8_t* i4 = input[4];
228 assert(i4 != NULL);
229 if XNN_UNPREDICTABLE(i4 != zero) {
230 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
231 }
232 const int8_t* i5 = input[5];
233 assert(i5 != NULL);
234 if XNN_UNPREDICTABLE(i5 != zero) {
235 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
236 }
237 input += 6;
238
239 size_t c = round_up_po2(channels, 8);
240
241 for (; c >= 16; c -= 16) {
242 int32x4_t vacc0123 = vld1q_s32(b);
243 int32x4_t vacc4567 = vld1q_s32(b + 4);
244 int32x4_t vacc89AB = vld1q_s32(b + 8);
245 int32x4_t vaccCDEF = vld1q_s32(b + 12);
246
247 const int8x16_t vi0x0123456789ABCDEF = vld1q_s8(i0); i0 += 16;
248 const int8x16_t vk0x0123456789ABCDEF = vld1q_s8(w); w = (const int8_t*) w + 16;
249
250 int16x8_t vprod01234567 = vmull_s8(vget_low_s8(vi0x0123456789ABCDEF), vget_low_s8(vk0x0123456789ABCDEF));
251 int16x8_t vprod89ABCDEF = vmull_s8(vget_high_s8(vi0x0123456789ABCDEF), vget_high_s8(vk0x0123456789ABCDEF));
252
253
254 const int8x16_t vi1x0123456789ABCDEF = vld1q_s8(i1); i1 += 16;
255 const int8x16_t vk1x0123456789ABCDEF = vld1q_s8(w); w = (const int8_t*) w + 16;
256
257 vprod01234567 = vmlal_s8(vprod01234567, vget_low_s8(vi1x0123456789ABCDEF), vget_low_s8(vk1x0123456789ABCDEF));
258 vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vget_high_s8(vi1x0123456789ABCDEF), vget_high_s8(vk1x0123456789ABCDEF));
259
260 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
261 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
262 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
263 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
264
265 const int8x16_t vi2x0123456789ABCDEF = vld1q_s8(i2); i2 += 16;
266 const int8x16_t vk2x0123456789ABCDEF = vld1q_s8(w); w = (const int8_t*) w + 16;
267
268 vprod01234567 = vmull_s8(vget_low_s8(vi2x0123456789ABCDEF), vget_low_s8(vk2x0123456789ABCDEF));
269 vprod89ABCDEF = vmull_s8(vget_high_s8(vi2x0123456789ABCDEF), vget_high_s8(vk2x0123456789ABCDEF));
270
271
272 const int8x16_t vi3x0123456789ABCDEF = vld1q_s8(i3); i3 += 16;
273 const int8x16_t vk3x0123456789ABCDEF = vld1q_s8(w); w = (const int8_t*) w + 16;
274
275 vprod01234567 = vmlal_s8(vprod01234567, vget_low_s8(vi3x0123456789ABCDEF), vget_low_s8(vk3x0123456789ABCDEF));
276 vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vget_high_s8(vi3x0123456789ABCDEF), vget_high_s8(vk3x0123456789ABCDEF));
277
278 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
279 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
280 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
281 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
282
283 const int8x16_t vi4x0123456789ABCDEF = vld1q_s8(i4); i4 += 16;
284 const int8x16_t vk4x0123456789ABCDEF = vld1q_s8(w); w = (const int8_t*) w + 16;
285
286 vprod01234567 = vmull_s8(vget_low_s8(vi4x0123456789ABCDEF), vget_low_s8(vk4x0123456789ABCDEF));
287 vprod89ABCDEF = vmull_s8(vget_high_s8(vi4x0123456789ABCDEF), vget_high_s8(vk4x0123456789ABCDEF));
288
289
290 const int8x16_t vi5x0123456789ABCDEF = vld1q_s8(i5); i5 += 16;
291 const int8x16_t vk5x0123456789ABCDEF = vld1q_s8(w); w = (const int8_t*) w + 16;
292
293 vprod01234567 = vmlal_s8(vprod01234567, vget_low_s8(vi5x0123456789ABCDEF), vget_low_s8(vk5x0123456789ABCDEF));
294 vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vget_high_s8(vi5x0123456789ABCDEF), vget_high_s8(vk5x0123456789ABCDEF));
295
296 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
297 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
298 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
299 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
300
301 vst1q_s32(b, vacc0123); b += 4;
302 vst1q_s32(b, vacc4567); b += 4;
303 vst1q_s32(b, vacc89AB); b += 4;
304 vst1q_s32(b, vaccCDEF); b += 4;
305 }
306
307 if XNN_UNLIKELY(c != 0) {
308 do {
309 int32x4_t vacc0123 = vld1q_s32(b);
310 int32x4_t vacc4567 = vld1q_s32(b + 4);
311
312 const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
313 const int8x8_t vk0x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
314
315 int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
316
317
318 const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
319 const int8x8_t vk1x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
320
321 vprod01234567 = vmlal_s8(vprod01234567, vi1x01234567, vk1x01234567);
322
323 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
324 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
325
326 const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
327 const int8x8_t vk2x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
328
329 vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
330
331
332 const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
333 const int8x8_t vk3x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
334
335 vprod01234567 = vmlal_s8(vprod01234567, vi3x01234567, vk3x01234567);
336
337 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
338 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
339
340 const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
341 const int8x8_t vk4x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
342
343 vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
344
345
346 const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
347 const int8x8_t vk5x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
348
349 vprod01234567 = vmlal_s8(vprod01234567, vi5x01234567, vk5x01234567);
350
351 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
352 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
353
354 vst1q_s32(b, vacc0123); b += 4;
355 vst1q_s32(b, vacc4567); b += 4;
356 c -= 8;
357 } while (c != 0);
358 }
359 }
360
361 // Last pass to process up to 7 inputs.
362 {
363 const int32_t* b = buffer;
364 const int8_t* i0 = input[0];
365 assert(i0 != NULL);
366 if XNN_UNPREDICTABLE(i0 != zero) {
367 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
368 }
369 const int8_t* i1 = input[1];
370 assert(i1 != NULL);
371 if XNN_UNPREDICTABLE(i1 != zero) {
372 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
373 }
374 const int8_t* i2 = input[2];
375 assert(i2 != NULL);
376 if XNN_UNPREDICTABLE(i2 != zero) {
377 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
378 }
379 const int8_t* i3 = input[3];
380 assert(i3 != NULL);
381 if XNN_UNPREDICTABLE(i3 != zero) {
382 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
383 }
384 const int8_t* i4 = input[4];
385 assert(i4 != NULL);
386 if XNN_UNPREDICTABLE(i4 != zero) {
387 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
388 }
389 const int8_t* i5 = input[5];
390 assert(i5 != NULL);
391 if XNN_UNPREDICTABLE(i5 != zero) {
392 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
393 }
394 const int8_t* i6 = input[6];
395 assert(i6 != NULL);
396 if XNN_UNPREDICTABLE(i6 != zero) {
397 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
398 }
399
400 size_t c = channels;
401 for (; c >= 16; c -= 16) {
402 int32x4_t vacc0123 = vld1q_s32(b); b += 4;
403 int32x4_t vacc4567 = vld1q_s32(b); b += 4;
404 int32x4_t vacc89AB = vld1q_s32(b); b += 4;
405 int32x4_t vaccCDEF = vld1q_s32(b); b += 4;
406
407 const int8x16_t vi0x0123456789ABCDEF = vld1q_s8(i0); i0 += 16;
408 const int8x16_t vk0x0123456789ABCDEF = vld1q_s8(w); w = (const int8_t*) w + 16;
409
410 int16x8_t vprod01234567 = vmull_s8(vget_low_s8(vi0x0123456789ABCDEF), vget_low_s8(vk0x0123456789ABCDEF));
411 int16x8_t vprod89ABCDEF = vmull_s8(vget_high_s8(vi0x0123456789ABCDEF), vget_high_s8(vk0x0123456789ABCDEF));
412
413
414 const int8x16_t vi1x0123456789ABCDEF = vld1q_s8(i1); i1 += 16;
415 const int8x16_t vk1x0123456789ABCDEF = vld1q_s8(w); w = (const int8_t*) w + 16;
416
417 vprod01234567 = vmlal_s8(vprod01234567, vget_low_s8(vi1x0123456789ABCDEF), vget_low_s8(vk1x0123456789ABCDEF));
418 vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vget_high_s8(vi1x0123456789ABCDEF), vget_high_s8(vk1x0123456789ABCDEF));
419
420 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
421 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
422 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
423 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
424
425 const int8x16_t vi2x0123456789ABCDEF = vld1q_s8(i2); i2 += 16;
426 const int8x16_t vk2x0123456789ABCDEF = vld1q_s8(w); w = (const int8_t*) w + 16;
427
428 vprod01234567 = vmull_s8(vget_low_s8(vi2x0123456789ABCDEF), vget_low_s8(vk2x0123456789ABCDEF));
429 vprod89ABCDEF = vmull_s8(vget_high_s8(vi2x0123456789ABCDEF), vget_high_s8(vk2x0123456789ABCDEF));
430
431
432 const int8x16_t vi3x0123456789ABCDEF = vld1q_s8(i3); i3 += 16;
433 const int8x16_t vk3x0123456789ABCDEF = vld1q_s8(w); w = (const int8_t*) w + 16;
434
435 vprod01234567 = vmlal_s8(vprod01234567, vget_low_s8(vi3x0123456789ABCDEF), vget_low_s8(vk3x0123456789ABCDEF));
436 vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vget_high_s8(vi3x0123456789ABCDEF), vget_high_s8(vk3x0123456789ABCDEF));
437
438 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
439 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
440 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
441 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
442
443 const int8x16_t vi4x0123456789ABCDEF = vld1q_s8(i4); i4 += 16;
444 const int8x16_t vk4x0123456789ABCDEF = vld1q_s8(w); w = (const int8_t*) w + 16;
445
446 vprod01234567 = vmull_s8(vget_low_s8(vi4x0123456789ABCDEF), vget_low_s8(vk4x0123456789ABCDEF));
447 vprod89ABCDEF = vmull_s8(vget_high_s8(vi4x0123456789ABCDEF), vget_high_s8(vk4x0123456789ABCDEF));
448
449
450 const int8x16_t vi5x0123456789ABCDEF = vld1q_s8(i5); i5 += 16;
451 const int8x16_t vk5x0123456789ABCDEF = vld1q_s8(w); w = (const int8_t*) w + 16;
452
453 vprod01234567 = vmlal_s8(vprod01234567, vget_low_s8(vi5x0123456789ABCDEF), vget_low_s8(vk5x0123456789ABCDEF));
454 vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vget_high_s8(vi5x0123456789ABCDEF), vget_high_s8(vk5x0123456789ABCDEF));
455
456 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
457 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
458 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
459 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
460
461 const int8x16_t vi6x0123456789ABCDEF = vld1q_s8(i6); i6 += 16;
462 const int8x16_t vk6x0123456789ABCDEF = vld1q_s8(w); w = (const int8_t*) w + 16;
463
464 vprod01234567 = vmull_s8(vget_low_s8(vi6x0123456789ABCDEF), vget_low_s8(vk6x0123456789ABCDEF));
465 vprod89ABCDEF = vmull_s8(vget_high_s8(vi6x0123456789ABCDEF), vget_high_s8(vk6x0123456789ABCDEF));
466
467 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
468 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
469 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
470 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
471
472 vacc0123 = vqshlq_s32(vacc0123, vright_pre_shift);
473 vacc4567 = vqshlq_s32(vacc4567, vright_pre_shift);
474 vacc89AB = vqshlq_s32(vacc89AB, vright_pre_shift);
475 vaccCDEF = vqshlq_s32(vaccCDEF, vright_pre_shift);
476
477 vacc0123 = vqdmulhq_s32(vacc0123, vmultiplier);
478 vacc4567 = vqdmulhq_s32(vacc4567, vmultiplier);
479 vacc89AB = vqdmulhq_s32(vacc89AB, vmultiplier);
480 vaccCDEF = vqdmulhq_s32(vaccCDEF, vmultiplier);
481
482 vacc0123 = vrshlq_s32(vacc0123, vright_post_shift);
483 vacc4567 = vrshlq_s32(vacc4567, vright_post_shift);
484 vacc89AB = vrshlq_s32(vacc89AB, vright_post_shift);
485 vaccCDEF = vrshlq_s32(vaccCDEF, vright_post_shift);
486
487 #if XNN_ARCH_ARM64
488 int16x8_t vacc01234567 = vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567);
489 int16x8_t vacc89ABCDEF = vqmovn_high_s32(vqmovn_s32(vacc89AB), vaccCDEF);
490
491 vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
492 vacc89ABCDEF = vqaddq_s16(vacc89ABCDEF, voutput_zero_point);
493
494 int8x16_t vout0123456789ABCDEF = vqmovn_high_s16(vqmovn_s16(vacc01234567), vacc89ABCDEF);
495 #else // !XNN_ARCH_ARM64
496 int16x8_t vacc01234567 = vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567));
497 int16x8_t vacc89ABCDEF = vcombine_s16(vqmovn_s32(vacc89AB), vqmovn_s32(vaccCDEF));
498
499 vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
500 vacc89ABCDEF = vqaddq_s16(vacc89ABCDEF, voutput_zero_point);
501
502 int8x16_t vout0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc01234567), vqmovn_s16(vacc89ABCDEF));
503 #endif // !XNN_ARCH_ARM64
504
505 vout0123456789ABCDEF = vmaxq_s8(vout0123456789ABCDEF, voutput_min);
506
507 vout0123456789ABCDEF = vminq_s8(vout0123456789ABCDEF, voutput_max);
508
509 vst1q_s8(output, vout0123456789ABCDEF); output += 16;
510 }
511
512
513 if XNN_UNLIKELY(c != 0) {
514 do {
515 int32x4_t vacc0123 = vld1q_s32(b); b += 4;
516 int32x4_t vacc4567 = vld1q_s32(b); b += 4;
517
518 const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
519 const int8x8_t vk0x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
520
521 int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
522
523 const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
524 const int8x8_t vk1x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
525
526 vprod01234567 = vmlal_s8(vprod01234567, vi1x01234567, vk1x01234567);
527
528 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
529 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
530 const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
531 const int8x8_t vk2x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
532
533 vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
534
535 const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
536 const int8x8_t vk3x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
537
538 vprod01234567 = vmlal_s8(vprod01234567, vi3x01234567, vk3x01234567);
539
540 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
541 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
542 const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
543 const int8x8_t vk4x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
544
545 vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
546
547 const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
548 const int8x8_t vk5x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
549
550 vprod01234567 = vmlal_s8(vprod01234567, vi5x01234567, vk5x01234567);
551
552 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
553 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
554 const int8x8_t vi6x01234567 = vld1_s8(i6); i6 += 8;
555 const int8x8_t vk6x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
556
557 vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
558
559 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
560 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
561
562 vacc0123 = vqshlq_s32(vacc0123, vright_pre_shift);
563 vacc4567 = vqshlq_s32(vacc4567, vright_pre_shift);
564
565 vacc0123 = vqdmulhq_s32(vacc0123, vmultiplier);
566 vacc4567 = vqdmulhq_s32(vacc4567, vmultiplier);
567
568 vacc0123 = vrshlq_s32(vacc0123, vright_post_shift);
569 vacc4567 = vrshlq_s32(vacc4567, vright_post_shift);
570
571 #if XNN_ARCH_ARM64
572 int16x8_t vacc01234567 = vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567);
573 #else
574 int16x8_t vacc01234567 = vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567));
575 #endif
576 vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
577
578 int8x8_t vout01234567 = vqmovn_s16(vacc01234567);
579 vout01234567 = vmax_s8(vout01234567, vget_low_s8(voutput_min));
580 vout01234567 = vmin_s8(vout01234567, vget_low_s8(voutput_max));
581
582 if XNN_LIKELY(c >= 8) {
583 vst1_s8(output, vout01234567); output += 8;
584 c -= 8;
585 } else {
586 if (c & 4) {
587 vst1_lane_u32((void*) output, vreinterpret_u32_s8(vout01234567), 0); output += 4;
588 vout01234567 = vext_s8(vout01234567, vout01234567, 4);
589 }
590 if (c & 2) {
591 vst1_lane_u16((void*) output, vreinterpret_u16_s8(vout01234567), 0); output += 2;
592 vout01234567 = vext_s8(vout01234567, vout01234567, 2);
593 }
594 if (c & 1) {
595 vst1_lane_s8(output, vout01234567, 0); output += 1;
596 }
597 c = 0;
598 }
599
600 } while (c != 0);
601 }
602 }
603
604 input = (const int8_t**) ((uintptr_t) input + input_stride);
605 output = (int8_t*) ((uintptr_t) output + output_increment);
606 } while (--output_width != 0);
607 }