Debian Codegraph @

src/qs8-dwconv/gen/qs8-dwconv-8f8m9l16c8s8r-minmax-rndnu-neon-mla8-ld64.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-dwconv/multipass-neon-mul8.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12
13 #include <arm_neon.h>
14
15 #include "xnnpack/dwconv.h"
16 #include "xnnpack/math.h"
17
18
19 void xnn_qs8_dwconv_minmax_rndnu_ukernel_8f8m9l16c8s8r__neon_mla8_ld64(
20 size_t channels,
21 size_t output_width,
22 const int8_t** input,
23 const void* weights,
24 int8_t* output,
25 intptr_t input_stride,
26 size_t output_increment,
27 size_t input_offset,
28 const int8_t* zero,
29 size_t kernel_size,
30 int32_t* buffer,
31 const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
32 {
33 assert(channels != 0);
34 assert(output_width != 0);
35 assert(kernel_size > 8);
36
37 const int32x4_t vright_pre_shift = vld1q_dup_s32(&params->rndnu_neon.right_pre_shift);
38 const int32x4_t vmultiplier = vld1q_dup_s32(&params->rndnu_neon.multiplier);
39 const int32x4_t vright_post_shift = vld1q_dup_s32(&params->rndnu_neon.right_post_shift);
40 const int16x8_t voutput_zero_point = vld1q_dup_s16(&params->rndnu_neon.output_zero_point);
41 const int8x16_t voutput_min = vld1q_dup_s8(&params->rndnu_neon.output_min);
42 const int8x16_t voutput_max = vld1q_dup_s8(&params->rndnu_neon.output_max);
43 do {
44 const void* w = weights;
45
46 // First pass to process 8 inputs.
47 {
48 int32_t* b = buffer;
49 const int8_t* i0 = input[0];
50 assert(i0 != NULL);
51 if XNN_UNPREDICTABLE(i0 != zero) {
52 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
53 }
54 const int8_t* i1 = input[1];
55 assert(i1 != NULL);
56 if XNN_UNPREDICTABLE(i1 != zero) {
57 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
58 }
59 const int8_t* i2 = input[2];
60 assert(i2 != NULL);
61 if XNN_UNPREDICTABLE(i2 != zero) {
62 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
63 }
64 const int8_t* i3 = input[3];
65 assert(i3 != NULL);
66 if XNN_UNPREDICTABLE(i3 != zero) {
67 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
68 }
69 const int8_t* i4 = input[4];
70 assert(i4 != NULL);
71 if XNN_UNPREDICTABLE(i4 != zero) {
72 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
73 }
74 const int8_t* i5 = input[5];
75 assert(i5 != NULL);
76 if XNN_UNPREDICTABLE(i5 != zero) {
77 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
78 }
79 const int8_t* i6 = input[6];
80 assert(i6 != NULL);
81 if XNN_UNPREDICTABLE(i6 != zero) {
82 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
83 }
84 const int8_t* i7 = input[7];
85 assert(i7 != NULL);
86 if XNN_UNPREDICTABLE(i7 != zero) {
87 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
88 }
89 input += 8;
90
91 size_t c = round_up_po2(channels, 8);
92
93 for (; c >= 16; c -= 16) {
94 int32x4_t vacc0123 = vld1q_s32(w); w = (const int32_t*) w + 4;
95 int32x4_t vacc4567 = vld1q_s32(w); w = (const int32_t*) w + 4;
96 int32x4_t vacc89AB = vld1q_s32(w); w = (const int32_t*) w + 4;
97 int32x4_t vaccCDEF = vld1q_s32(w); w = (const int32_t*) w + 4;
98
99 const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
100 const int8x8_t vk0x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
101 const int8x8_t vi0x89ABCDEF = vld1_s8(i0); i0 += 8;
102 const int8x8_t vk0x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
103
104 int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
105 int16x8_t vprod89ABCDEF = vmull_s8(vi0x89ABCDEF, vk0x89ABCDEF);
106
107
108 const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
109 const int8x8_t vk1x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
110 const int8x8_t vi1x89ABCDEF = vld1_s8(i1); i1 += 8;
111 const int8x8_t vk1x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
112
113 vprod01234567 = vmlal_s8(vprod01234567, vi1x01234567, vk1x01234567);
114 vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi1x89ABCDEF, vk1x89ABCDEF);
115
116 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
117 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
118 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
119 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
120
121 const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
122 const int8x8_t vk2x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
123 const int8x8_t vi2x89ABCDEF = vld1_s8(i2); i2 += 8;
124 const int8x8_t vk2x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
125
126 vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
127 vprod89ABCDEF = vmull_s8(vi2x89ABCDEF, vk2x89ABCDEF);
128
129
130 const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
131 const int8x8_t vk3x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
132 const int8x8_t vi3x89ABCDEF = vld1_s8(i3); i3 += 8;
133 const int8x8_t vk3x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
134
135 vprod01234567 = vmlal_s8(vprod01234567, vi3x01234567, vk3x01234567);
136 vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi3x89ABCDEF, vk3x89ABCDEF);
137
138 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
139 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
140 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
141 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
142
143 const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
144 const int8x8_t vk4x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
145 const int8x8_t vi4x89ABCDEF = vld1_s8(i4); i4 += 8;
146 const int8x8_t vk4x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
147
148 vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
149 vprod89ABCDEF = vmull_s8(vi4x89ABCDEF, vk4x89ABCDEF);
150
151
152 const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
153 const int8x8_t vk5x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
154 const int8x8_t vi5x89ABCDEF = vld1_s8(i5); i5 += 8;
155 const int8x8_t vk5x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
156
157 vprod01234567 = vmlal_s8(vprod01234567, vi5x01234567, vk5x01234567);
158 vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi5x89ABCDEF, vk5x89ABCDEF);
159
160 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
161 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
162 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
163 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
164
165 const int8x8_t vi6x01234567 = vld1_s8(i6); i6 += 8;
166 const int8x8_t vk6x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
167 const int8x8_t vi6x89ABCDEF = vld1_s8(i6); i6 += 8;
168 const int8x8_t vk6x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
169
170 vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
171 vprod89ABCDEF = vmull_s8(vi6x89ABCDEF, vk6x89ABCDEF);
172
173
174 const int8x8_t vi7x01234567 = vld1_s8(i7); i7 += 8;
175 const int8x8_t vk7x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
176 const int8x8_t vi7x89ABCDEF = vld1_s8(i7); i7 += 8;
177 const int8x8_t vk7x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
178
179 vprod01234567 = vmlal_s8(vprod01234567, vi7x01234567, vk7x01234567);
180 vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi7x89ABCDEF, vk7x89ABCDEF);
181
182 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
183 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
184 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
185 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
186
187 vst1q_s32(b, vacc0123); b += 4;
188 vst1q_s32(b, vacc4567); b += 4;
189 vst1q_s32(b, vacc89AB); b += 4;
190 vst1q_s32(b, vaccCDEF); b += 4;
191 }
192
193 if XNN_UNLIKELY(c != 0) {
194 do {
195 int32x4_t vacc0123 = vld1q_s32(w); w = (const int32_t*) w + 4;
196 int32x4_t vacc4567 = vld1q_s32(w); w = (const int32_t*) w + 4;
197
198 const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
199 const int8x8_t vk0x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
200
201 int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
202
203
204 const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
205 const int8x8_t vk1x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
206
207 vprod01234567 = vmlal_s8(vprod01234567, vi1x01234567, vk1x01234567);
208
209 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
210 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
211
212 const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
213 const int8x8_t vk2x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
214
215 vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
216
217
218 const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
219 const int8x8_t vk3x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
220
221 vprod01234567 = vmlal_s8(vprod01234567, vi3x01234567, vk3x01234567);
222
223 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
224 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
225
226 const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
227 const int8x8_t vk4x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
228
229 vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
230
231
232 const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
233 const int8x8_t vk5x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
234
235 vprod01234567 = vmlal_s8(vprod01234567, vi5x01234567, vk5x01234567);
236
237 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
238 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
239
240 const int8x8_t vi6x01234567 = vld1_s8(i6); i6 += 8;
241 const int8x8_t vk6x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
242
243 vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
244
245
246 const int8x8_t vi7x01234567 = vld1_s8(i7); i7 += 8;
247 const int8x8_t vk7x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
248
249 vprod01234567 = vmlal_s8(vprod01234567, vi7x01234567, vk7x01234567);
250
251 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
252 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
253
254 vst1q_s32(b, vacc0123); b += 4;
255 vst1q_s32(b, vacc4567); b += 4;
256 c -= 8;
257 } while (c != 0);
258 }
259 }
260
261 // Middle pass to process 8 inputs in each iteration.
262 for (size_t ks = kernel_size - 8; ks > 9; ks -= 8) {
263 int32_t* b = buffer;
264
265 const int8_t* i0 = input[0];
266 assert(i0 != NULL);
267 if XNN_UNPREDICTABLE(i0 != zero) {
268 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
269 }
270 const int8_t* i1 = input[1];
271 assert(i1 != NULL);
272 if XNN_UNPREDICTABLE(i1 != zero) {
273 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
274 }
275 const int8_t* i2 = input[2];
276 assert(i2 != NULL);
277 if XNN_UNPREDICTABLE(i2 != zero) {
278 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
279 }
280 const int8_t* i3 = input[3];
281 assert(i3 != NULL);
282 if XNN_UNPREDICTABLE(i3 != zero) {
283 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
284 }
285 const int8_t* i4 = input[4];
286 assert(i4 != NULL);
287 if XNN_UNPREDICTABLE(i4 != zero) {
288 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
289 }
290 const int8_t* i5 = input[5];
291 assert(i5 != NULL);
292 if XNN_UNPREDICTABLE(i5 != zero) {
293 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
294 }
295 const int8_t* i6 = input[6];
296 assert(i6 != NULL);
297 if XNN_UNPREDICTABLE(i6 != zero) {
298 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
299 }
300 const int8_t* i7 = input[7];
301 assert(i7 != NULL);
302 if XNN_UNPREDICTABLE(i7 != zero) {
303 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
304 }
305 input += 8;
306
307 size_t c = round_up_po2(channels, 8);
308
309 for (; c >= 16; c -= 16) {
310 int32x4_t vacc0123 = vld1q_s32(b);
311 int32x4_t vacc4567 = vld1q_s32(b + 4);
312 int32x4_t vacc89AB = vld1q_s32(b + 8);
313 int32x4_t vaccCDEF = vld1q_s32(b + 12);
314
315 const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
316 const int8x8_t vk0x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
317 const int8x8_t vi0x89ABCDEF = vld1_s8(i0); i0 += 8;
318 const int8x8_t vk0x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
319
320 int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
321 int16x8_t vprod89ABCDEF = vmull_s8(vi0x89ABCDEF, vk0x89ABCDEF);
322
323
324 const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
325 const int8x8_t vk1x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
326 const int8x8_t vi1x89ABCDEF = vld1_s8(i1); i1 += 8;
327 const int8x8_t vk1x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
328
329 vprod01234567 = vmlal_s8(vprod01234567, vi1x01234567, vk1x01234567);
330 vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi1x89ABCDEF, vk1x89ABCDEF);
331
332 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
333 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
334 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
335 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
336
337 const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
338 const int8x8_t vk2x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
339 const int8x8_t vi2x89ABCDEF = vld1_s8(i2); i2 += 8;
340 const int8x8_t vk2x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
341
342 vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
343 vprod89ABCDEF = vmull_s8(vi2x89ABCDEF, vk2x89ABCDEF);
344
345
346 const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
347 const int8x8_t vk3x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
348 const int8x8_t vi3x89ABCDEF = vld1_s8(i3); i3 += 8;
349 const int8x8_t vk3x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
350
351 vprod01234567 = vmlal_s8(vprod01234567, vi3x01234567, vk3x01234567);
352 vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi3x89ABCDEF, vk3x89ABCDEF);
353
354 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
355 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
356 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
357 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
358
359 const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
360 const int8x8_t vk4x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
361 const int8x8_t vi4x89ABCDEF = vld1_s8(i4); i4 += 8;
362 const int8x8_t vk4x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
363
364 vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
365 vprod89ABCDEF = vmull_s8(vi4x89ABCDEF, vk4x89ABCDEF);
366
367
368 const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
369 const int8x8_t vk5x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
370 const int8x8_t vi5x89ABCDEF = vld1_s8(i5); i5 += 8;
371 const int8x8_t vk5x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
372
373 vprod01234567 = vmlal_s8(vprod01234567, vi5x01234567, vk5x01234567);
374 vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi5x89ABCDEF, vk5x89ABCDEF);
375
376 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
377 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
378 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
379 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
380
381 const int8x8_t vi6x01234567 = vld1_s8(i6); i6 += 8;
382 const int8x8_t vk6x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
383 const int8x8_t vi6x89ABCDEF = vld1_s8(i6); i6 += 8;
384 const int8x8_t vk6x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
385
386 vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
387 vprod89ABCDEF = vmull_s8(vi6x89ABCDEF, vk6x89ABCDEF);
388
389
390 const int8x8_t vi7x01234567 = vld1_s8(i7); i7 += 8;
391 const int8x8_t vk7x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
392 const int8x8_t vi7x89ABCDEF = vld1_s8(i7); i7 += 8;
393 const int8x8_t vk7x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
394
395 vprod01234567 = vmlal_s8(vprod01234567, vi7x01234567, vk7x01234567);
396 vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi7x89ABCDEF, vk7x89ABCDEF);
397
398 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
399 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
400 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
401 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
402
403 vst1q_s32(b, vacc0123); b += 4;
404 vst1q_s32(b, vacc4567); b += 4;
405 vst1q_s32(b, vacc89AB); b += 4;
406 vst1q_s32(b, vaccCDEF); b += 4;
407 }
408
409 if XNN_UNLIKELY(c != 0) {
410 do {
411 int32x4_t vacc0123 = vld1q_s32(b);
412 int32x4_t vacc4567 = vld1q_s32(b + 4);
413
414 const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
415 const int8x8_t vk0x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
416
417 int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
418
419
420 const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
421 const int8x8_t vk1x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
422
423 vprod01234567 = vmlal_s8(vprod01234567, vi1x01234567, vk1x01234567);
424
425 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
426 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
427
428 const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
429 const int8x8_t vk2x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
430
431 vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
432
433
434 const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
435 const int8x8_t vk3x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
436
437 vprod01234567 = vmlal_s8(vprod01234567, vi3x01234567, vk3x01234567);
438
439 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
440 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
441
442 const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
443 const int8x8_t vk4x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
444
445 vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
446
447
448 const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
449 const int8x8_t vk5x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
450
451 vprod01234567 = vmlal_s8(vprod01234567, vi5x01234567, vk5x01234567);
452
453 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
454 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
455
456 const int8x8_t vi6x01234567 = vld1_s8(i6); i6 += 8;
457 const int8x8_t vk6x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
458
459 vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
460
461
462 const int8x8_t vi7x01234567 = vld1_s8(i7); i7 += 8;
463 const int8x8_t vk7x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
464
465 vprod01234567 = vmlal_s8(vprod01234567, vi7x01234567, vk7x01234567);
466
467 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
468 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
469
470 vst1q_s32(b, vacc0123); b += 4;
471 vst1q_s32(b, vacc4567); b += 4;
472 c -= 8;
473 } while (c != 0);
474 }
475 }
476
477 // Last pass to process up to 9 inputs.
478 {
479 const int32_t* b = buffer;
480 const int8_t* i0 = input[0];
481 assert(i0 != NULL);
482 if XNN_UNPREDICTABLE(i0 != zero) {
483 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
484 }
485 const int8_t* i1 = input[1];
486 assert(i1 != NULL);
487 if XNN_UNPREDICTABLE(i1 != zero) {
488 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
489 }
490 const int8_t* i2 = input[2];
491 assert(i2 != NULL);
492 if XNN_UNPREDICTABLE(i2 != zero) {
493 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
494 }
495 const int8_t* i3 = input[3];
496 assert(i3 != NULL);
497 if XNN_UNPREDICTABLE(i3 != zero) {
498 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
499 }
500 const int8_t* i4 = input[4];
501 assert(i4 != NULL);
502 if XNN_UNPREDICTABLE(i4 != zero) {
503 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
504 }
505 const int8_t* i5 = input[5];
506 assert(i5 != NULL);
507 if XNN_UNPREDICTABLE(i5 != zero) {
508 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
509 }
510 const int8_t* i6 = input[6];
511 assert(i6 != NULL);
512 if XNN_UNPREDICTABLE(i6 != zero) {
513 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
514 }
515 const int8_t* i7 = input[7];
516 assert(i7 != NULL);
517 if XNN_UNPREDICTABLE(i7 != zero) {
518 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
519 }
520 const int8_t* i8 = input[8];
521 assert(i8 != NULL);
522 if XNN_UNPREDICTABLE(i8 != zero) {
523 i8 = (const int8_t*) ((uintptr_t) i8 + input_offset);
524 }
525
526 size_t c = channels;
527 for (; c >= 16; c -= 16) {
528 int32x4_t vacc0123 = vld1q_s32(b); b += 4;
529 int32x4_t vacc4567 = vld1q_s32(b); b += 4;
530 int32x4_t vacc89AB = vld1q_s32(b); b += 4;
531 int32x4_t vaccCDEF = vld1q_s32(b); b += 4;
532
533 const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
534 const int8x8_t vk0x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
535 const int8x8_t vi0x89ABCDEF = vld1_s8(i0); i0 += 8;
536 const int8x8_t vk0x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
537
538 int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
539 int16x8_t vprod89ABCDEF = vmull_s8(vi0x89ABCDEF, vk0x89ABCDEF);
540
541
542 const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
543 const int8x8_t vk1x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
544 const int8x8_t vi1x89ABCDEF = vld1_s8(i1); i1 += 8;
545 const int8x8_t vk1x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
546
547 vprod01234567 = vmlal_s8(vprod01234567, vi1x01234567, vk1x01234567);
548 vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi1x89ABCDEF, vk1x89ABCDEF);
549
550 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
551 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
552 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
553 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
554
555 const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
556 const int8x8_t vk2x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
557 const int8x8_t vi2x89ABCDEF = vld1_s8(i2); i2 += 8;
558 const int8x8_t vk2x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
559
560 vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
561 vprod89ABCDEF = vmull_s8(vi2x89ABCDEF, vk2x89ABCDEF);
562
563
564 const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
565 const int8x8_t vk3x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
566 const int8x8_t vi3x89ABCDEF = vld1_s8(i3); i3 += 8;
567 const int8x8_t vk3x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
568
569 vprod01234567 = vmlal_s8(vprod01234567, vi3x01234567, vk3x01234567);
570 vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi3x89ABCDEF, vk3x89ABCDEF);
571
572 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
573 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
574 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
575 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
576
577 const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
578 const int8x8_t vk4x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
579 const int8x8_t vi4x89ABCDEF = vld1_s8(i4); i4 += 8;
580 const int8x8_t vk4x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
581
582 vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
583 vprod89ABCDEF = vmull_s8(vi4x89ABCDEF, vk4x89ABCDEF);
584
585
586 const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
587 const int8x8_t vk5x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
588 const int8x8_t vi5x89ABCDEF = vld1_s8(i5); i5 += 8;
589 const int8x8_t vk5x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
590
591 vprod01234567 = vmlal_s8(vprod01234567, vi5x01234567, vk5x01234567);
592 vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi5x89ABCDEF, vk5x89ABCDEF);
593
594 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
595 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
596 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
597 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
598
599 const int8x8_t vi6x01234567 = vld1_s8(i6); i6 += 8;
600 const int8x8_t vk6x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
601 const int8x8_t vi6x89ABCDEF = vld1_s8(i6); i6 += 8;
602 const int8x8_t vk6x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
603
604 vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
605 vprod89ABCDEF = vmull_s8(vi6x89ABCDEF, vk6x89ABCDEF);
606
607
608 const int8x8_t vi7x01234567 = vld1_s8(i7); i7 += 8;
609 const int8x8_t vk7x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
610 const int8x8_t vi7x89ABCDEF = vld1_s8(i7); i7 += 8;
611 const int8x8_t vk7x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
612
613 vprod01234567 = vmlal_s8(vprod01234567, vi7x01234567, vk7x01234567);
614 vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi7x89ABCDEF, vk7x89ABCDEF);
615
616 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
617 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
618 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
619 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
620
621 const int8x8_t vi8x01234567 = vld1_s8(i8); i8 += 8;
622 const int8x8_t vk8x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
623 const int8x8_t vi8x89ABCDEF = vld1_s8(i8); i8 += 8;
624 const int8x8_t vk8x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
625
626 vprod01234567 = vmull_s8(vi8x01234567, vk8x01234567);
627 vprod89ABCDEF = vmull_s8(vi8x89ABCDEF, vk8x89ABCDEF);
628
629 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
630 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
631 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
632 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
633
634 vacc0123 = vqshlq_s32(vacc0123, vright_pre_shift);
635 vacc4567 = vqshlq_s32(vacc4567, vright_pre_shift);
636 vacc89AB = vqshlq_s32(vacc89AB, vright_pre_shift);
637 vaccCDEF = vqshlq_s32(vaccCDEF, vright_pre_shift);
638
639 vacc0123 = vqdmulhq_s32(vacc0123, vmultiplier);
640 vacc4567 = vqdmulhq_s32(vacc4567, vmultiplier);
641 vacc89AB = vqdmulhq_s32(vacc89AB, vmultiplier);
642 vaccCDEF = vqdmulhq_s32(vaccCDEF, vmultiplier);
643
644 vacc0123 = vrshlq_s32(vacc0123, vright_post_shift);
645 vacc4567 = vrshlq_s32(vacc4567, vright_post_shift);
646 vacc89AB = vrshlq_s32(vacc89AB, vright_post_shift);
647 vaccCDEF = vrshlq_s32(vaccCDEF, vright_post_shift);
648
649 #if XNN_ARCH_ARM64
650 int16x8_t vacc01234567 = vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567);
651 int16x8_t vacc89ABCDEF = vqmovn_high_s32(vqmovn_s32(vacc89AB), vaccCDEF);
652
653 vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
654 vacc89ABCDEF = vqaddq_s16(vacc89ABCDEF, voutput_zero_point);
655
656 int8x16_t vout0123456789ABCDEF = vqmovn_high_s16(vqmovn_s16(vacc01234567), vacc89ABCDEF);
657 #else // !XNN_ARCH_ARM64
658 int16x8_t vacc01234567 = vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567));
659 int16x8_t vacc89ABCDEF = vcombine_s16(vqmovn_s32(vacc89AB), vqmovn_s32(vaccCDEF));
660
661 vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
662 vacc89ABCDEF = vqaddq_s16(vacc89ABCDEF, voutput_zero_point);
663
664 int8x16_t vout0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc01234567), vqmovn_s16(vacc89ABCDEF));
665 #endif // !XNN_ARCH_ARM64
666
667 vout0123456789ABCDEF = vmaxq_s8(vout0123456789ABCDEF, voutput_min);
668
669 vout0123456789ABCDEF = vminq_s8(vout0123456789ABCDEF, voutput_max);
670
671 vst1q_s8(output, vout0123456789ABCDEF); output += 16;
672 }
673
674
675 if XNN_UNLIKELY(c != 0) {
676 do {
677 int32x4_t vacc0123 = vld1q_s32(b); b += 4;
678 int32x4_t vacc4567 = vld1q_s32(b); b += 4;
679
680 const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
681 const int8x8_t vk0x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
682
683 int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
684
685 const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
686 const int8x8_t vk1x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
687
688 vprod01234567 = vmlal_s8(vprod01234567, vi1x01234567, vk1x01234567);
689
690 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
691 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
692 const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
693 const int8x8_t vk2x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
694
695 vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
696
697 const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
698 const int8x8_t vk3x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
699
700 vprod01234567 = vmlal_s8(vprod01234567, vi3x01234567, vk3x01234567);
701
702 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
703 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
704 const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
705 const int8x8_t vk4x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
706
707 vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
708
709 const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
710 const int8x8_t vk5x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
711
712 vprod01234567 = vmlal_s8(vprod01234567, vi5x01234567, vk5x01234567);
713
714 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
715 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
716 const int8x8_t vi6x01234567 = vld1_s8(i6); i6 += 8;
717 const int8x8_t vk6x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
718
719 vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
720
721 const int8x8_t vi7x01234567 = vld1_s8(i7); i7 += 8;
722 const int8x8_t vk7x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
723
724 vprod01234567 = vmlal_s8(vprod01234567, vi7x01234567, vk7x01234567);
725
726 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
727 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
728 const int8x8_t vi8x01234567 = vld1_s8(i8); i8 += 8;
729 const int8x8_t vk8x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
730
731 vprod01234567 = vmull_s8(vi8x01234567, vk8x01234567);
732
733 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
734 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
735
736 vacc0123 = vqshlq_s32(vacc0123, vright_pre_shift);
737 vacc4567 = vqshlq_s32(vacc4567, vright_pre_shift);
738
739 vacc0123 = vqdmulhq_s32(vacc0123, vmultiplier);
740 vacc4567 = vqdmulhq_s32(vacc4567, vmultiplier);
741
742 vacc0123 = vrshlq_s32(vacc0123, vright_post_shift);
743 vacc4567 = vrshlq_s32(vacc4567, vright_post_shift);
744
745 #if XNN_ARCH_ARM64
746 int16x8_t vacc01234567 = vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567);
747 #else
748 int16x8_t vacc01234567 = vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567));
749 #endif
750 vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
751
752 int8x8_t vout01234567 = vqmovn_s16(vacc01234567);
753 vout01234567 = vmax_s8(vout01234567, vget_low_s8(voutput_min));
754 vout01234567 = vmin_s8(vout01234567, vget_low_s8(voutput_max));
755
756 if XNN_LIKELY(c >= 8) {
757 vst1_s8(output, vout01234567); output += 8;
758 c -= 8;
759 } else {
760 if (c & 4) {
761 vst1_lane_u32((void*) output, vreinterpret_u32_s8(vout01234567), 0); output += 4;
762 vout01234567 = vext_s8(vout01234567, vout01234567, 4);
763 }
764 if (c & 2) {
765 vst1_lane_u16((void*) output, vreinterpret_u16_s8(vout01234567), 0); output += 2;
766 vout01234567 = vext_s8(vout01234567, vout01234567, 2);
767 }
768 if (c & 1) {
769 vst1_lane_s8(output, vout01234567, 0); output += 1;
770 }
771 c = 0;
772 }
773
774 } while (c != 0);
775 }
776 }
777
778 input = (const int8_t**) ((uintptr_t) input + input_stride);
779 output = (int8_t*) ((uintptr_t) output + output_increment);
780 } while (--output_width != 0);
781 }