Debian Codegraph @

src/qs8-dwconv/gen/qs8-dwconv-5f5m5l16c8s8r-minmax-rndnu-neon-mul8-ld64.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-dwconv/multipass-neon-mul8.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12
13 #include <arm_neon.h>
14
15 #include "xnnpack/dwconv.h"
16 #include "xnnpack/math.h"
17
18
19 void xnn_qs8_dwconv_minmax_rndnu_ukernel_5f5m5l16c8s8r__neon_mul8_ld64(
20 size_t channels,
21 size_t output_width,
22 const int8_t** input,
23 const void* weights,
24 int8_t* output,
25 intptr_t input_stride,
26 size_t output_increment,
27 size_t input_offset,
28 const int8_t* zero,
29 size_t kernel_size,
30 int32_t* buffer,
31 const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
32 {
33 assert(channels != 0);
34 assert(output_width != 0);
35 assert(kernel_size > 5);
36
37 const int32x4_t vright_pre_shift = vld1q_dup_s32(&params->rndnu_neon.right_pre_shift);
38 const int32x4_t vmultiplier = vld1q_dup_s32(&params->rndnu_neon.multiplier);
39 const int32x4_t vright_post_shift = vld1q_dup_s32(&params->rndnu_neon.right_post_shift);
40 const int16x8_t voutput_zero_point = vld1q_dup_s16(&params->rndnu_neon.output_zero_point);
41 const int8x16_t voutput_min = vld1q_dup_s8(&params->rndnu_neon.output_min);
42 const int8x16_t voutput_max = vld1q_dup_s8(&params->rndnu_neon.output_max);
43 do {
44 const void* w = weights;
45
46 // First pass to process 5 inputs.
47 {
48 int32_t* b = buffer;
49 const int8_t* i0 = input[0];
50 assert(i0 != NULL);
51 if XNN_UNPREDICTABLE(i0 != zero) {
52 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
53 }
54 const int8_t* i1 = input[1];
55 assert(i1 != NULL);
56 if XNN_UNPREDICTABLE(i1 != zero) {
57 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
58 }
59 const int8_t* i2 = input[2];
60 assert(i2 != NULL);
61 if XNN_UNPREDICTABLE(i2 != zero) {
62 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
63 }
64 const int8_t* i3 = input[3];
65 assert(i3 != NULL);
66 if XNN_UNPREDICTABLE(i3 != zero) {
67 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
68 }
69 const int8_t* i4 = input[4];
70 assert(i4 != NULL);
71 if XNN_UNPREDICTABLE(i4 != zero) {
72 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
73 }
74 input += 5;
75
76 size_t c = round_up_po2(channels, 8);
77
78 for (; c >= 16; c -= 16) {
79 int32x4_t vacc0123 = vld1q_s32(w); w = (const int32_t*) w + 4;
80 int32x4_t vacc4567 = vld1q_s32(w); w = (const int32_t*) w + 4;
81 int32x4_t vacc89AB = vld1q_s32(w); w = (const int32_t*) w + 4;
82 int32x4_t vaccCDEF = vld1q_s32(w); w = (const int32_t*) w + 4;
83
84 const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
85 const int8x8_t vk0x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
86 const int8x8_t vi0x89ABCDEF = vld1_s8(i0); i0 += 8;
87 const int8x8_t vk0x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
88
89 int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
90 int16x8_t vprod89ABCDEF = vmull_s8(vi0x89ABCDEF, vk0x89ABCDEF);
91
92 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
93 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
94 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
95 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
96
97 const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
98 const int8x8_t vk1x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
99 const int8x8_t vi1x89ABCDEF = vld1_s8(i1); i1 += 8;
100 const int8x8_t vk1x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
101
102 vprod01234567 = vmull_s8(vi1x01234567, vk1x01234567);
103 vprod89ABCDEF = vmull_s8(vi1x89ABCDEF, vk1x89ABCDEF);
104
105 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
106 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
107 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
108 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
109
110 const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
111 const int8x8_t vk2x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
112 const int8x8_t vi2x89ABCDEF = vld1_s8(i2); i2 += 8;
113 const int8x8_t vk2x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
114
115 vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
116 vprod89ABCDEF = vmull_s8(vi2x89ABCDEF, vk2x89ABCDEF);
117
118 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
119 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
120 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
121 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
122
123 const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
124 const int8x8_t vk3x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
125 const int8x8_t vi3x89ABCDEF = vld1_s8(i3); i3 += 8;
126 const int8x8_t vk3x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
127
128 vprod01234567 = vmull_s8(vi3x01234567, vk3x01234567);
129 vprod89ABCDEF = vmull_s8(vi3x89ABCDEF, vk3x89ABCDEF);
130
131 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
132 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
133 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
134 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
135
136 const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
137 const int8x8_t vk4x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
138 const int8x8_t vi4x89ABCDEF = vld1_s8(i4); i4 += 8;
139 const int8x8_t vk4x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
140
141 vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
142 vprod89ABCDEF = vmull_s8(vi4x89ABCDEF, vk4x89ABCDEF);
143
144 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
145 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
146 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
147 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
148
149 vst1q_s32(b, vacc0123); b += 4;
150 vst1q_s32(b, vacc4567); b += 4;
151 vst1q_s32(b, vacc89AB); b += 4;
152 vst1q_s32(b, vaccCDEF); b += 4;
153 }
154
155 if XNN_UNLIKELY(c != 0) {
156 do {
157 int32x4_t vacc0123 = vld1q_s32(w); w = (const int32_t*) w + 4;
158 int32x4_t vacc4567 = vld1q_s32(w); w = (const int32_t*) w + 4;
159
160 const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
161 const int8x8_t vk0x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
162
163 int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
164
165 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
166 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
167
168 const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
169 const int8x8_t vk1x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
170
171 vprod01234567 = vmull_s8(vi1x01234567, vk1x01234567);
172
173 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
174 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
175
176 const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
177 const int8x8_t vk2x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
178
179 vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
180
181 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
182 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
183
184 const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
185 const int8x8_t vk3x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
186
187 vprod01234567 = vmull_s8(vi3x01234567, vk3x01234567);
188
189 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
190 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
191
192 const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
193 const int8x8_t vk4x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
194
195 vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
196
197 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
198 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
199
200 vst1q_s32(b, vacc0123); b += 4;
201 vst1q_s32(b, vacc4567); b += 4;
202 c -= 8;
203 } while (c != 0);
204 }
205 }
206
207 // Middle pass to process 5 inputs in each iteration.
208 for (size_t ks = kernel_size - 5; ks > 5; ks -= 5) {
209 int32_t* b = buffer;
210
211 const int8_t* i0 = input[0];
212 assert(i0 != NULL);
213 if XNN_UNPREDICTABLE(i0 != zero) {
214 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
215 }
216 const int8_t* i1 = input[1];
217 assert(i1 != NULL);
218 if XNN_UNPREDICTABLE(i1 != zero) {
219 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
220 }
221 const int8_t* i2 = input[2];
222 assert(i2 != NULL);
223 if XNN_UNPREDICTABLE(i2 != zero) {
224 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
225 }
226 const int8_t* i3 = input[3];
227 assert(i3 != NULL);
228 if XNN_UNPREDICTABLE(i3 != zero) {
229 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
230 }
231 const int8_t* i4 = input[4];
232 assert(i4 != NULL);
233 if XNN_UNPREDICTABLE(i4 != zero) {
234 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
235 }
236 input += 5;
237
238 size_t c = round_up_po2(channels, 8);
239
240 for (; c >= 16; c -= 16) {
241 int32x4_t vacc0123 = vld1q_s32(b);
242 int32x4_t vacc4567 = vld1q_s32(b + 4);
243 int32x4_t vacc89AB = vld1q_s32(b + 8);
244 int32x4_t vaccCDEF = vld1q_s32(b + 12);
245
246 const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
247 const int8x8_t vk0x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
248 const int8x8_t vi0x89ABCDEF = vld1_s8(i0); i0 += 8;
249 const int8x8_t vk0x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
250
251 int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
252 int16x8_t vprod89ABCDEF = vmull_s8(vi0x89ABCDEF, vk0x89ABCDEF);
253
254 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
255 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
256 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
257 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
258
259 const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
260 const int8x8_t vk1x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
261 const int8x8_t vi1x89ABCDEF = vld1_s8(i1); i1 += 8;
262 const int8x8_t vk1x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
263
264 vprod01234567 = vmull_s8(vi1x01234567, vk1x01234567);
265 vprod89ABCDEF = vmull_s8(vi1x89ABCDEF, vk1x89ABCDEF);
266
267 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
268 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
269 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
270 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
271
272 const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
273 const int8x8_t vk2x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
274 const int8x8_t vi2x89ABCDEF = vld1_s8(i2); i2 += 8;
275 const int8x8_t vk2x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
276
277 vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
278 vprod89ABCDEF = vmull_s8(vi2x89ABCDEF, vk2x89ABCDEF);
279
280 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
281 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
282 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
283 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
284
285 const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
286 const int8x8_t vk3x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
287 const int8x8_t vi3x89ABCDEF = vld1_s8(i3); i3 += 8;
288 const int8x8_t vk3x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
289
290 vprod01234567 = vmull_s8(vi3x01234567, vk3x01234567);
291 vprod89ABCDEF = vmull_s8(vi3x89ABCDEF, vk3x89ABCDEF);
292
293 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
294 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
295 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
296 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
297
298 const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
299 const int8x8_t vk4x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
300 const int8x8_t vi4x89ABCDEF = vld1_s8(i4); i4 += 8;
301 const int8x8_t vk4x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
302
303 vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
304 vprod89ABCDEF = vmull_s8(vi4x89ABCDEF, vk4x89ABCDEF);
305
306 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
307 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
308 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
309 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
310
311 vst1q_s32(b, vacc0123); b += 4;
312 vst1q_s32(b, vacc4567); b += 4;
313 vst1q_s32(b, vacc89AB); b += 4;
314 vst1q_s32(b, vaccCDEF); b += 4;
315 }
316
317 if XNN_UNLIKELY(c != 0) {
318 do {
319 int32x4_t vacc0123 = vld1q_s32(b);
320 int32x4_t vacc4567 = vld1q_s32(b + 4);
321
322 const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
323 const int8x8_t vk0x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
324
325 int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
326
327 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
328 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
329
330 const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
331 const int8x8_t vk1x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
332
333 vprod01234567 = vmull_s8(vi1x01234567, vk1x01234567);
334
335 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
336 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
337
338 const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
339 const int8x8_t vk2x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
340
341 vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
342
343 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
344 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
345
346 const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
347 const int8x8_t vk3x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
348
349 vprod01234567 = vmull_s8(vi3x01234567, vk3x01234567);
350
351 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
352 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
353
354 const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
355 const int8x8_t vk4x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
356
357 vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
358
359 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
360 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
361
362 vst1q_s32(b, vacc0123); b += 4;
363 vst1q_s32(b, vacc4567); b += 4;
364 c -= 8;
365 } while (c != 0);
366 }
367 }
368
369 // Last pass to process up to 5 inputs.
370 {
371 const int32_t* b = buffer;
372 const int8_t* i0 = input[0];
373 assert(i0 != NULL);
374 if XNN_UNPREDICTABLE(i0 != zero) {
375 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
376 }
377 const int8_t* i1 = input[1];
378 assert(i1 != NULL);
379 if XNN_UNPREDICTABLE(i1 != zero) {
380 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
381 }
382 const int8_t* i2 = input[2];
383 assert(i2 != NULL);
384 if XNN_UNPREDICTABLE(i2 != zero) {
385 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
386 }
387 const int8_t* i3 = input[3];
388 assert(i3 != NULL);
389 if XNN_UNPREDICTABLE(i3 != zero) {
390 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
391 }
392 const int8_t* i4 = input[4];
393 assert(i4 != NULL);
394 if XNN_UNPREDICTABLE(i4 != zero) {
395 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
396 }
397
398 size_t c = channels;
399 for (; c >= 16; c -= 16) {
400 int32x4_t vacc0123 = vld1q_s32(b); b += 4;
401 int32x4_t vacc4567 = vld1q_s32(b); b += 4;
402 int32x4_t vacc89AB = vld1q_s32(b); b += 4;
403 int32x4_t vaccCDEF = vld1q_s32(b); b += 4;
404
405 const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
406 const int8x8_t vk0x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
407 const int8x8_t vi0x89ABCDEF = vld1_s8(i0); i0 += 8;
408 const int8x8_t vk0x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
409
410 int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
411 int16x8_t vprod89ABCDEF = vmull_s8(vi0x89ABCDEF, vk0x89ABCDEF);
412
413 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
414 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
415 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
416 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
417
418 const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
419 const int8x8_t vk1x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
420 const int8x8_t vi1x89ABCDEF = vld1_s8(i1); i1 += 8;
421 const int8x8_t vk1x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
422
423 vprod01234567 = vmull_s8(vi1x01234567, vk1x01234567);
424 vprod89ABCDEF = vmull_s8(vi1x89ABCDEF, vk1x89ABCDEF);
425
426 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
427 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
428 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
429 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
430
431 const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
432 const int8x8_t vk2x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
433 const int8x8_t vi2x89ABCDEF = vld1_s8(i2); i2 += 8;
434 const int8x8_t vk2x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
435
436 vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
437 vprod89ABCDEF = vmull_s8(vi2x89ABCDEF, vk2x89ABCDEF);
438
439 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
440 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
441 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
442 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
443
444 const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
445 const int8x8_t vk3x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
446 const int8x8_t vi3x89ABCDEF = vld1_s8(i3); i3 += 8;
447 const int8x8_t vk3x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
448
449 vprod01234567 = vmull_s8(vi3x01234567, vk3x01234567);
450 vprod89ABCDEF = vmull_s8(vi3x89ABCDEF, vk3x89ABCDEF);
451
452 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
453 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
454 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
455 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
456
457 const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
458 const int8x8_t vk4x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
459 const int8x8_t vi4x89ABCDEF = vld1_s8(i4); i4 += 8;
460 const int8x8_t vk4x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
461
462 vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
463 vprod89ABCDEF = vmull_s8(vi4x89ABCDEF, vk4x89ABCDEF);
464
465 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
466 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
467 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
468 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
469
470 vacc0123 = vqshlq_s32(vacc0123, vright_pre_shift);
471 vacc4567 = vqshlq_s32(vacc4567, vright_pre_shift);
472 vacc89AB = vqshlq_s32(vacc89AB, vright_pre_shift);
473 vaccCDEF = vqshlq_s32(vaccCDEF, vright_pre_shift);
474
475 vacc0123 = vqdmulhq_s32(vacc0123, vmultiplier);
476 vacc4567 = vqdmulhq_s32(vacc4567, vmultiplier);
477 vacc89AB = vqdmulhq_s32(vacc89AB, vmultiplier);
478 vaccCDEF = vqdmulhq_s32(vaccCDEF, vmultiplier);
479
480 vacc0123 = vrshlq_s32(vacc0123, vright_post_shift);
481 vacc4567 = vrshlq_s32(vacc4567, vright_post_shift);
482 vacc89AB = vrshlq_s32(vacc89AB, vright_post_shift);
483 vaccCDEF = vrshlq_s32(vaccCDEF, vright_post_shift);
484
485 #if XNN_ARCH_ARM64
486 int16x8_t vacc01234567 = vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567);
487 int16x8_t vacc89ABCDEF = vqmovn_high_s32(vqmovn_s32(vacc89AB), vaccCDEF);
488
489 vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
490 vacc89ABCDEF = vqaddq_s16(vacc89ABCDEF, voutput_zero_point);
491
492 int8x16_t vout0123456789ABCDEF = vqmovn_high_s16(vqmovn_s16(vacc01234567), vacc89ABCDEF);
493 #else // !XNN_ARCH_ARM64
494 int16x8_t vacc01234567 = vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567));
495 int16x8_t vacc89ABCDEF = vcombine_s16(vqmovn_s32(vacc89AB), vqmovn_s32(vaccCDEF));
496
497 vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
498 vacc89ABCDEF = vqaddq_s16(vacc89ABCDEF, voutput_zero_point);
499
500 int8x16_t vout0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc01234567), vqmovn_s16(vacc89ABCDEF));
501 #endif // !XNN_ARCH_ARM64
502
503 vout0123456789ABCDEF = vmaxq_s8(vout0123456789ABCDEF, voutput_min);
504
505 vout0123456789ABCDEF = vminq_s8(vout0123456789ABCDEF, voutput_max);
506
507 vst1q_s8(output, vout0123456789ABCDEF); output += 16;
508 }
509
510
511 if XNN_UNLIKELY(c != 0) {
512 do {
513 int32x4_t vacc0123 = vld1q_s32(b); b += 4;
514 int32x4_t vacc4567 = vld1q_s32(b); b += 4;
515
516 const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
517 const int8x8_t vk0x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
518
519 int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
520
521 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
522 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
523 const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
524 const int8x8_t vk1x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
525
526 vprod01234567 = vmull_s8(vi1x01234567, vk1x01234567);
527
528 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
529 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
530 const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
531 const int8x8_t vk2x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
532
533 vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
534
535 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
536 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
537 const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
538 const int8x8_t vk3x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
539
540 vprod01234567 = vmull_s8(vi3x01234567, vk3x01234567);
541
542 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
543 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
544 const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
545 const int8x8_t vk4x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
546
547 vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
548
549 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
550 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
551
552 vacc0123 = vqshlq_s32(vacc0123, vright_pre_shift);
553 vacc4567 = vqshlq_s32(vacc4567, vright_pre_shift);
554
555 vacc0123 = vqdmulhq_s32(vacc0123, vmultiplier);
556 vacc4567 = vqdmulhq_s32(vacc4567, vmultiplier);
557
558 vacc0123 = vrshlq_s32(vacc0123, vright_post_shift);
559 vacc4567 = vrshlq_s32(vacc4567, vright_post_shift);
560
561 #if XNN_ARCH_ARM64
562 int16x8_t vacc01234567 = vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567);
563 #else
564 int16x8_t vacc01234567 = vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567));
565 #endif
566 vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
567
568 int8x8_t vout01234567 = vqmovn_s16(vacc01234567);
569 vout01234567 = vmax_s8(vout01234567, vget_low_s8(voutput_min));
570 vout01234567 = vmin_s8(vout01234567, vget_low_s8(voutput_max));
571
572 if XNN_LIKELY(c >= 8) {
573 vst1_s8(output, vout01234567); output += 8;
574 c -= 8;
575 } else {
576 if (c & 4) {
577 vst1_lane_u32((void*) output, vreinterpret_u32_s8(vout01234567), 0); output += 4;
578 vout01234567 = vext_s8(vout01234567, vout01234567, 4);
579 }
580 if (c & 2) {
581 vst1_lane_u16((void*) output, vreinterpret_u16_s8(vout01234567), 0); output += 2;
582 vout01234567 = vext_s8(vout01234567, vout01234567, 2);
583 }
584 if (c & 1) {
585 vst1_lane_s8(output, vout01234567, 0); output += 1;
586 }
587 c = 0;
588 }
589
590 } while (c != 0);
591 }
592 }
593
594 input = (const int8_t**) ((uintptr_t) input + input_stride);
595 output = (int8_t*) ((uintptr_t) output + output_increment);
596 } while (--output_width != 0);
597 }