Debian Codegraph @

src/qs8-dwconv/gen/qs8-dwconv-6f6m7l16c8s8r-minmax-rndnu-neon-mul8-ld64.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-dwconv/multipass-neon-mul8.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12
13 #include <arm_neon.h>
14
15 #include "xnnpack/dwconv.h"
16 #include "xnnpack/math.h"
17
18
19 void xnn_qs8_dwconv_minmax_rndnu_ukernel_6f6m7l16c8s8r__neon_mul8_ld64(
20 size_t channels,
21 size_t output_width,
22 const int8_t** input,
23 const void* weights,
24 int8_t* output,
25 intptr_t input_stride,
26 size_t output_increment,
27 size_t input_offset,
28 const int8_t* zero,
29 size_t kernel_size,
30 int32_t* buffer,
31 const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
32 {
33 assert(channels != 0);
34 assert(output_width != 0);
35 assert(kernel_size > 6);
36
37 const int32x4_t vright_pre_shift = vld1q_dup_s32(&params->rndnu_neon.right_pre_shift);
38 const int32x4_t vmultiplier = vld1q_dup_s32(&params->rndnu_neon.multiplier);
39 const int32x4_t vright_post_shift = vld1q_dup_s32(&params->rndnu_neon.right_post_shift);
40 const int16x8_t voutput_zero_point = vld1q_dup_s16(&params->rndnu_neon.output_zero_point);
41 const int8x16_t voutput_min = vld1q_dup_s8(&params->rndnu_neon.output_min);
42 const int8x16_t voutput_max = vld1q_dup_s8(&params->rndnu_neon.output_max);
43 do {
44 const void* w = weights;
45
46 // First pass to process 6 inputs.
47 {
48 int32_t* b = buffer;
49 const int8_t* i0 = input[0];
50 assert(i0 != NULL);
51 if XNN_UNPREDICTABLE(i0 != zero) {
52 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
53 }
54 const int8_t* i1 = input[1];
55 assert(i1 != NULL);
56 if XNN_UNPREDICTABLE(i1 != zero) {
57 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
58 }
59 const int8_t* i2 = input[2];
60 assert(i2 != NULL);
61 if XNN_UNPREDICTABLE(i2 != zero) {
62 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
63 }
64 const int8_t* i3 = input[3];
65 assert(i3 != NULL);
66 if XNN_UNPREDICTABLE(i3 != zero) {
67 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
68 }
69 const int8_t* i4 = input[4];
70 assert(i4 != NULL);
71 if XNN_UNPREDICTABLE(i4 != zero) {
72 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
73 }
74 const int8_t* i5 = input[5];
75 assert(i5 != NULL);
76 if XNN_UNPREDICTABLE(i5 != zero) {
77 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
78 }
79 input += 6;
80
81 size_t c = round_up_po2(channels, 8);
82
83 for (; c >= 16; c -= 16) {
84 int32x4_t vacc0123 = vld1q_s32(w); w = (const int32_t*) w + 4;
85 int32x4_t vacc4567 = vld1q_s32(w); w = (const int32_t*) w + 4;
86 int32x4_t vacc89AB = vld1q_s32(w); w = (const int32_t*) w + 4;
87 int32x4_t vaccCDEF = vld1q_s32(w); w = (const int32_t*) w + 4;
88
89 const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
90 const int8x8_t vk0x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
91 const int8x8_t vi0x89ABCDEF = vld1_s8(i0); i0 += 8;
92 const int8x8_t vk0x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
93
94 int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
95 int16x8_t vprod89ABCDEF = vmull_s8(vi0x89ABCDEF, vk0x89ABCDEF);
96
97 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
98 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
99 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
100 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
101
102 const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
103 const int8x8_t vk1x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
104 const int8x8_t vi1x89ABCDEF = vld1_s8(i1); i1 += 8;
105 const int8x8_t vk1x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
106
107 vprod01234567 = vmull_s8(vi1x01234567, vk1x01234567);
108 vprod89ABCDEF = vmull_s8(vi1x89ABCDEF, vk1x89ABCDEF);
109
110 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
111 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
112 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
113 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
114
115 const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
116 const int8x8_t vk2x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
117 const int8x8_t vi2x89ABCDEF = vld1_s8(i2); i2 += 8;
118 const int8x8_t vk2x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
119
120 vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
121 vprod89ABCDEF = vmull_s8(vi2x89ABCDEF, vk2x89ABCDEF);
122
123 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
124 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
125 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
126 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
127
128 const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
129 const int8x8_t vk3x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
130 const int8x8_t vi3x89ABCDEF = vld1_s8(i3); i3 += 8;
131 const int8x8_t vk3x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
132
133 vprod01234567 = vmull_s8(vi3x01234567, vk3x01234567);
134 vprod89ABCDEF = vmull_s8(vi3x89ABCDEF, vk3x89ABCDEF);
135
136 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
137 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
138 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
139 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
140
141 const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
142 const int8x8_t vk4x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
143 const int8x8_t vi4x89ABCDEF = vld1_s8(i4); i4 += 8;
144 const int8x8_t vk4x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
145
146 vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
147 vprod89ABCDEF = vmull_s8(vi4x89ABCDEF, vk4x89ABCDEF);
148
149 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
150 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
151 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
152 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
153
154 const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
155 const int8x8_t vk5x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
156 const int8x8_t vi5x89ABCDEF = vld1_s8(i5); i5 += 8;
157 const int8x8_t vk5x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
158
159 vprod01234567 = vmull_s8(vi5x01234567, vk5x01234567);
160 vprod89ABCDEF = vmull_s8(vi5x89ABCDEF, vk5x89ABCDEF);
161
162 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
163 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
164 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
165 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
166
167 vst1q_s32(b, vacc0123); b += 4;
168 vst1q_s32(b, vacc4567); b += 4;
169 vst1q_s32(b, vacc89AB); b += 4;
170 vst1q_s32(b, vaccCDEF); b += 4;
171 }
172
173 if XNN_UNLIKELY(c != 0) {
174 do {
175 int32x4_t vacc0123 = vld1q_s32(w); w = (const int32_t*) w + 4;
176 int32x4_t vacc4567 = vld1q_s32(w); w = (const int32_t*) w + 4;
177
178 const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
179 const int8x8_t vk0x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
180
181 int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
182
183 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
184 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
185
186 const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
187 const int8x8_t vk1x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
188
189 vprod01234567 = vmull_s8(vi1x01234567, vk1x01234567);
190
191 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
192 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
193
194 const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
195 const int8x8_t vk2x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
196
197 vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
198
199 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
200 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
201
202 const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
203 const int8x8_t vk3x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
204
205 vprod01234567 = vmull_s8(vi3x01234567, vk3x01234567);
206
207 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
208 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
209
210 const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
211 const int8x8_t vk4x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
212
213 vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
214
215 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
216 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
217
218 const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
219 const int8x8_t vk5x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
220
221 vprod01234567 = vmull_s8(vi5x01234567, vk5x01234567);
222
223 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
224 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
225
226 vst1q_s32(b, vacc0123); b += 4;
227 vst1q_s32(b, vacc4567); b += 4;
228 c -= 8;
229 } while (c != 0);
230 }
231 }
232
233 // Middle pass to process 6 inputs in each iteration.
234 for (size_t ks = kernel_size - 6; ks > 7; ks -= 6) {
235 int32_t* b = buffer;
236
237 const int8_t* i0 = input[0];
238 assert(i0 != NULL);
239 if XNN_UNPREDICTABLE(i0 != zero) {
240 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
241 }
242 const int8_t* i1 = input[1];
243 assert(i1 != NULL);
244 if XNN_UNPREDICTABLE(i1 != zero) {
245 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
246 }
247 const int8_t* i2 = input[2];
248 assert(i2 != NULL);
249 if XNN_UNPREDICTABLE(i2 != zero) {
250 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
251 }
252 const int8_t* i3 = input[3];
253 assert(i3 != NULL);
254 if XNN_UNPREDICTABLE(i3 != zero) {
255 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
256 }
257 const int8_t* i4 = input[4];
258 assert(i4 != NULL);
259 if XNN_UNPREDICTABLE(i4 != zero) {
260 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
261 }
262 const int8_t* i5 = input[5];
263 assert(i5 != NULL);
264 if XNN_UNPREDICTABLE(i5 != zero) {
265 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
266 }
267 input += 6;
268
269 size_t c = round_up_po2(channels, 8);
270
271 for (; c >= 16; c -= 16) {
272 int32x4_t vacc0123 = vld1q_s32(b);
273 int32x4_t vacc4567 = vld1q_s32(b + 4);
274 int32x4_t vacc89AB = vld1q_s32(b + 8);
275 int32x4_t vaccCDEF = vld1q_s32(b + 12);
276
277 const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
278 const int8x8_t vk0x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
279 const int8x8_t vi0x89ABCDEF = vld1_s8(i0); i0 += 8;
280 const int8x8_t vk0x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
281
282 int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
283 int16x8_t vprod89ABCDEF = vmull_s8(vi0x89ABCDEF, vk0x89ABCDEF);
284
285 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
286 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
287 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
288 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
289
290 const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
291 const int8x8_t vk1x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
292 const int8x8_t vi1x89ABCDEF = vld1_s8(i1); i1 += 8;
293 const int8x8_t vk1x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
294
295 vprod01234567 = vmull_s8(vi1x01234567, vk1x01234567);
296 vprod89ABCDEF = vmull_s8(vi1x89ABCDEF, vk1x89ABCDEF);
297
298 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
299 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
300 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
301 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
302
303 const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
304 const int8x8_t vk2x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
305 const int8x8_t vi2x89ABCDEF = vld1_s8(i2); i2 += 8;
306 const int8x8_t vk2x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
307
308 vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
309 vprod89ABCDEF = vmull_s8(vi2x89ABCDEF, vk2x89ABCDEF);
310
311 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
312 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
313 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
314 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
315
316 const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
317 const int8x8_t vk3x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
318 const int8x8_t vi3x89ABCDEF = vld1_s8(i3); i3 += 8;
319 const int8x8_t vk3x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
320
321 vprod01234567 = vmull_s8(vi3x01234567, vk3x01234567);
322 vprod89ABCDEF = vmull_s8(vi3x89ABCDEF, vk3x89ABCDEF);
323
324 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
325 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
326 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
327 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
328
329 const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
330 const int8x8_t vk4x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
331 const int8x8_t vi4x89ABCDEF = vld1_s8(i4); i4 += 8;
332 const int8x8_t vk4x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
333
334 vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
335 vprod89ABCDEF = vmull_s8(vi4x89ABCDEF, vk4x89ABCDEF);
336
337 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
338 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
339 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
340 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
341
342 const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
343 const int8x8_t vk5x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
344 const int8x8_t vi5x89ABCDEF = vld1_s8(i5); i5 += 8;
345 const int8x8_t vk5x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
346
347 vprod01234567 = vmull_s8(vi5x01234567, vk5x01234567);
348 vprod89ABCDEF = vmull_s8(vi5x89ABCDEF, vk5x89ABCDEF);
349
350 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
351 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
352 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
353 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
354
355 vst1q_s32(b, vacc0123); b += 4;
356 vst1q_s32(b, vacc4567); b += 4;
357 vst1q_s32(b, vacc89AB); b += 4;
358 vst1q_s32(b, vaccCDEF); b += 4;
359 }
360
361 if XNN_UNLIKELY(c != 0) {
362 do {
363 int32x4_t vacc0123 = vld1q_s32(b);
364 int32x4_t vacc4567 = vld1q_s32(b + 4);
365
366 const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
367 const int8x8_t vk0x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
368
369 int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
370
371 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
372 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
373
374 const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
375 const int8x8_t vk1x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
376
377 vprod01234567 = vmull_s8(vi1x01234567, vk1x01234567);
378
379 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
380 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
381
382 const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
383 const int8x8_t vk2x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
384
385 vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
386
387 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
388 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
389
390 const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
391 const int8x8_t vk3x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
392
393 vprod01234567 = vmull_s8(vi3x01234567, vk3x01234567);
394
395 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
396 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
397
398 const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
399 const int8x8_t vk4x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
400
401 vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
402
403 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
404 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
405
406 const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
407 const int8x8_t vk5x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
408
409 vprod01234567 = vmull_s8(vi5x01234567, vk5x01234567);
410
411 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
412 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
413
414 vst1q_s32(b, vacc0123); b += 4;
415 vst1q_s32(b, vacc4567); b += 4;
416 c -= 8;
417 } while (c != 0);
418 }
419 }
420
421 // Last pass to process up to 7 inputs.
422 {
423 const int32_t* b = buffer;
424 const int8_t* i0 = input[0];
425 assert(i0 != NULL);
426 if XNN_UNPREDICTABLE(i0 != zero) {
427 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
428 }
429 const int8_t* i1 = input[1];
430 assert(i1 != NULL);
431 if XNN_UNPREDICTABLE(i1 != zero) {
432 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
433 }
434 const int8_t* i2 = input[2];
435 assert(i2 != NULL);
436 if XNN_UNPREDICTABLE(i2 != zero) {
437 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
438 }
439 const int8_t* i3 = input[3];
440 assert(i3 != NULL);
441 if XNN_UNPREDICTABLE(i3 != zero) {
442 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
443 }
444 const int8_t* i4 = input[4];
445 assert(i4 != NULL);
446 if XNN_UNPREDICTABLE(i4 != zero) {
447 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
448 }
449 const int8_t* i5 = input[5];
450 assert(i5 != NULL);
451 if XNN_UNPREDICTABLE(i5 != zero) {
452 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
453 }
454 const int8_t* i6 = input[6];
455 assert(i6 != NULL);
456 if XNN_UNPREDICTABLE(i6 != zero) {
457 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
458 }
459
460 size_t c = channels;
461 for (; c >= 16; c -= 16) {
462 int32x4_t vacc0123 = vld1q_s32(b); b += 4;
463 int32x4_t vacc4567 = vld1q_s32(b); b += 4;
464 int32x4_t vacc89AB = vld1q_s32(b); b += 4;
465 int32x4_t vaccCDEF = vld1q_s32(b); b += 4;
466
467 const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
468 const int8x8_t vk0x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
469 const int8x8_t vi0x89ABCDEF = vld1_s8(i0); i0 += 8;
470 const int8x8_t vk0x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
471
472 int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
473 int16x8_t vprod89ABCDEF = vmull_s8(vi0x89ABCDEF, vk0x89ABCDEF);
474
475 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
476 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
477 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
478 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
479
480 const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
481 const int8x8_t vk1x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
482 const int8x8_t vi1x89ABCDEF = vld1_s8(i1); i1 += 8;
483 const int8x8_t vk1x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
484
485 vprod01234567 = vmull_s8(vi1x01234567, vk1x01234567);
486 vprod89ABCDEF = vmull_s8(vi1x89ABCDEF, vk1x89ABCDEF);
487
488 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
489 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
490 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
491 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
492
493 const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
494 const int8x8_t vk2x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
495 const int8x8_t vi2x89ABCDEF = vld1_s8(i2); i2 += 8;
496 const int8x8_t vk2x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
497
498 vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
499 vprod89ABCDEF = vmull_s8(vi2x89ABCDEF, vk2x89ABCDEF);
500
501 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
502 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
503 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
504 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
505
506 const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
507 const int8x8_t vk3x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
508 const int8x8_t vi3x89ABCDEF = vld1_s8(i3); i3 += 8;
509 const int8x8_t vk3x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
510
511 vprod01234567 = vmull_s8(vi3x01234567, vk3x01234567);
512 vprod89ABCDEF = vmull_s8(vi3x89ABCDEF, vk3x89ABCDEF);
513
514 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
515 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
516 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
517 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
518
519 const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
520 const int8x8_t vk4x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
521 const int8x8_t vi4x89ABCDEF = vld1_s8(i4); i4 += 8;
522 const int8x8_t vk4x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
523
524 vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
525 vprod89ABCDEF = vmull_s8(vi4x89ABCDEF, vk4x89ABCDEF);
526
527 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
528 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
529 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
530 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
531
532 const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
533 const int8x8_t vk5x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
534 const int8x8_t vi5x89ABCDEF = vld1_s8(i5); i5 += 8;
535 const int8x8_t vk5x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
536
537 vprod01234567 = vmull_s8(vi5x01234567, vk5x01234567);
538 vprod89ABCDEF = vmull_s8(vi5x89ABCDEF, vk5x89ABCDEF);
539
540 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
541 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
542 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
543 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
544
545 const int8x8_t vi6x01234567 = vld1_s8(i6); i6 += 8;
546 const int8x8_t vk6x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
547 const int8x8_t vi6x89ABCDEF = vld1_s8(i6); i6 += 8;
548 const int8x8_t vk6x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
549
550 vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
551 vprod89ABCDEF = vmull_s8(vi6x89ABCDEF, vk6x89ABCDEF);
552
553 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
554 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
555 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
556 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
557
558 vacc0123 = vqshlq_s32(vacc0123, vright_pre_shift);
559 vacc4567 = vqshlq_s32(vacc4567, vright_pre_shift);
560 vacc89AB = vqshlq_s32(vacc89AB, vright_pre_shift);
561 vaccCDEF = vqshlq_s32(vaccCDEF, vright_pre_shift);
562
563 vacc0123 = vqdmulhq_s32(vacc0123, vmultiplier);
564 vacc4567 = vqdmulhq_s32(vacc4567, vmultiplier);
565 vacc89AB = vqdmulhq_s32(vacc89AB, vmultiplier);
566 vaccCDEF = vqdmulhq_s32(vaccCDEF, vmultiplier);
567
568 vacc0123 = vrshlq_s32(vacc0123, vright_post_shift);
569 vacc4567 = vrshlq_s32(vacc4567, vright_post_shift);
570 vacc89AB = vrshlq_s32(vacc89AB, vright_post_shift);
571 vaccCDEF = vrshlq_s32(vaccCDEF, vright_post_shift);
572
573 #if XNN_ARCH_ARM64
574 int16x8_t vacc01234567 = vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567);
575 int16x8_t vacc89ABCDEF = vqmovn_high_s32(vqmovn_s32(vacc89AB), vaccCDEF);
576
577 vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
578 vacc89ABCDEF = vqaddq_s16(vacc89ABCDEF, voutput_zero_point);
579
580 int8x16_t vout0123456789ABCDEF = vqmovn_high_s16(vqmovn_s16(vacc01234567), vacc89ABCDEF);
581 #else // !XNN_ARCH_ARM64
582 int16x8_t vacc01234567 = vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567));
583 int16x8_t vacc89ABCDEF = vcombine_s16(vqmovn_s32(vacc89AB), vqmovn_s32(vaccCDEF));
584
585 vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
586 vacc89ABCDEF = vqaddq_s16(vacc89ABCDEF, voutput_zero_point);
587
588 int8x16_t vout0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc01234567), vqmovn_s16(vacc89ABCDEF));
589 #endif // !XNN_ARCH_ARM64
590
591 vout0123456789ABCDEF = vmaxq_s8(vout0123456789ABCDEF, voutput_min);
592
593 vout0123456789ABCDEF = vminq_s8(vout0123456789ABCDEF, voutput_max);
594
595 vst1q_s8(output, vout0123456789ABCDEF); output += 16;
596 }
597
598
599 if XNN_UNLIKELY(c != 0) {
600 do {
601 int32x4_t vacc0123 = vld1q_s32(b); b += 4;
602 int32x4_t vacc4567 = vld1q_s32(b); b += 4;
603
604 const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
605 const int8x8_t vk0x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
606
607 int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
608
609 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
610 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
611 const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
612 const int8x8_t vk1x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
613
614 vprod01234567 = vmull_s8(vi1x01234567, vk1x01234567);
615
616 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
617 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
618 const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
619 const int8x8_t vk2x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
620
621 vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
622
623 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
624 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
625 const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
626 const int8x8_t vk3x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
627
628 vprod01234567 = vmull_s8(vi3x01234567, vk3x01234567);
629
630 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
631 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
632 const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
633 const int8x8_t vk4x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
634
635 vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
636
637 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
638 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
639 const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
640 const int8x8_t vk5x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
641
642 vprod01234567 = vmull_s8(vi5x01234567, vk5x01234567);
643
644 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
645 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
646 const int8x8_t vi6x01234567 = vld1_s8(i6); i6 += 8;
647 const int8x8_t vk6x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
648
649 vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
650
651 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
652 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
653
654 vacc0123 = vqshlq_s32(vacc0123, vright_pre_shift);
655 vacc4567 = vqshlq_s32(vacc4567, vright_pre_shift);
656
657 vacc0123 = vqdmulhq_s32(vacc0123, vmultiplier);
658 vacc4567 = vqdmulhq_s32(vacc4567, vmultiplier);
659
660 vacc0123 = vrshlq_s32(vacc0123, vright_post_shift);
661 vacc4567 = vrshlq_s32(vacc4567, vright_post_shift);
662
663 #if XNN_ARCH_ARM64
664 int16x8_t vacc01234567 = vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567);
665 #else
666 int16x8_t vacc01234567 = vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567));
667 #endif
668 vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
669
670 int8x8_t vout01234567 = vqmovn_s16(vacc01234567);
671 vout01234567 = vmax_s8(vout01234567, vget_low_s8(voutput_min));
672 vout01234567 = vmin_s8(vout01234567, vget_low_s8(voutput_max));
673
674 if XNN_LIKELY(c >= 8) {
675 vst1_s8(output, vout01234567); output += 8;
676 c -= 8;
677 } else {
678 if (c & 4) {
679 vst1_lane_u32((void*) output, vreinterpret_u32_s8(vout01234567), 0); output += 4;
680 vout01234567 = vext_s8(vout01234567, vout01234567, 4);
681 }
682 if (c & 2) {
683 vst1_lane_u16((void*) output, vreinterpret_u16_s8(vout01234567), 0); output += 2;
684 vout01234567 = vext_s8(vout01234567, vout01234567, 2);
685 }
686 if (c & 1) {
687 vst1_lane_s8(output, vout01234567, 0); output += 1;
688 }
689 c = 0;
690 }
691
692 } while (c != 0);
693 }
694 }
695
696 input = (const int8_t**) ((uintptr_t) input + input_stride);
697 output = (int8_t*) ((uintptr_t) output + output_increment);
698 } while (--output_width != 0);
699 }