Debian Codegraph @

src/qs8-dwconv/gen/qs8-dwconv-6f6m7l16c8s8r-minmax-rndnu-neon-mul8-ld128.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-dwconv/multipass-neon-mul8.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12
13 #include <arm_neon.h>
14
15 #include "xnnpack/dwconv.h"
16 #include "xnnpack/math.h"
17
18
19 void xnn_qs8_dwconv_minmax_rndnu_ukernel_6f6m7l16c8s8r__neon_mul8_ld128(
20 size_t channels,
21 size_t output_width,
22 const int8_t** input,
23 const void* weights,
24 int8_t* output,
25 intptr_t input_stride,
26 size_t output_increment,
27 size_t input_offset,
28 const int8_t* zero,
29 size_t kernel_size,
30 int32_t* buffer,
31 const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
32 {
33 assert(channels != 0);
34 assert(output_width != 0);
35 assert(kernel_size > 6);
36
37 const int32x4_t vright_pre_shift = vld1q_dup_s32(&params->rndnu_neon.right_pre_shift);
38 const int32x4_t vmultiplier = vld1q_dup_s32(&params->rndnu_neon.multiplier);
39 const int32x4_t vright_post_shift = vld1q_dup_s32(&params->rndnu_neon.right_post_shift);
40 const int16x8_t voutput_zero_point = vld1q_dup_s16(&params->rndnu_neon.output_zero_point);
41 const int8x16_t voutput_min = vld1q_dup_s8(&params->rndnu_neon.output_min);
42 const int8x16_t voutput_max = vld1q_dup_s8(&params->rndnu_neon.output_max);
43 do {
44 const void* w = weights;
45
46 // First pass to process 6 inputs.
47 {
48 int32_t* b = buffer;
49 const int8_t* i0 = input[0];
50 assert(i0 != NULL);
51 if XNN_UNPREDICTABLE(i0 != zero) {
52 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
53 }
54 const int8_t* i1 = input[1];
55 assert(i1 != NULL);
56 if XNN_UNPREDICTABLE(i1 != zero) {
57 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
58 }
59 const int8_t* i2 = input[2];
60 assert(i2 != NULL);
61 if XNN_UNPREDICTABLE(i2 != zero) {
62 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
63 }
64 const int8_t* i3 = input[3];
65 assert(i3 != NULL);
66 if XNN_UNPREDICTABLE(i3 != zero) {
67 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
68 }
69 const int8_t* i4 = input[4];
70 assert(i4 != NULL);
71 if XNN_UNPREDICTABLE(i4 != zero) {
72 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
73 }
74 const int8_t* i5 = input[5];
75 assert(i5 != NULL);
76 if XNN_UNPREDICTABLE(i5 != zero) {
77 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
78 }
79 input += 6;
80
81 size_t c = round_up_po2(channels, 8);
82
83 for (; c >= 16; c -= 16) {
84 int32x4_t vacc0123 = vld1q_s32(w); w = (const int32_t*) w + 4;
85 int32x4_t vacc4567 = vld1q_s32(w); w = (const int32_t*) w + 4;
86 int32x4_t vacc89AB = vld1q_s32(w); w = (const int32_t*) w + 4;
87 int32x4_t vaccCDEF = vld1q_s32(w); w = (const int32_t*) w + 4;
88
89 const int8x16_t vi0x0123456789ABCDEF = vld1q_s8(i0); i0 += 16;
90 const int8x16_t vk0x0123456789ABCDEF = vld1q_s8(w); w = (const int8_t*) w + 16;
91
92 int16x8_t vprod01234567 = vmull_s8(vget_low_s8(vi0x0123456789ABCDEF), vget_low_s8(vk0x0123456789ABCDEF));
93 int16x8_t vprod89ABCDEF = vmull_s8(vget_high_s8(vi0x0123456789ABCDEF), vget_high_s8(vk0x0123456789ABCDEF));
94
95 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
96 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
97 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
98 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
99
100 const int8x16_t vi1x0123456789ABCDEF = vld1q_s8(i1); i1 += 16;
101 const int8x16_t vk1x0123456789ABCDEF = vld1q_s8(w); w = (const int8_t*) w + 16;
102
103 vprod01234567 = vmull_s8(vget_low_s8(vi1x0123456789ABCDEF), vget_low_s8(vk1x0123456789ABCDEF));
104 vprod89ABCDEF = vmull_s8(vget_high_s8(vi1x0123456789ABCDEF), vget_high_s8(vk1x0123456789ABCDEF));
105
106 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
107 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
108 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
109 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
110
111 const int8x16_t vi2x0123456789ABCDEF = vld1q_s8(i2); i2 += 16;
112 const int8x16_t vk2x0123456789ABCDEF = vld1q_s8(w); w = (const int8_t*) w + 16;
113
114 vprod01234567 = vmull_s8(vget_low_s8(vi2x0123456789ABCDEF), vget_low_s8(vk2x0123456789ABCDEF));
115 vprod89ABCDEF = vmull_s8(vget_high_s8(vi2x0123456789ABCDEF), vget_high_s8(vk2x0123456789ABCDEF));
116
117 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
118 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
119 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
120 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
121
122 const int8x16_t vi3x0123456789ABCDEF = vld1q_s8(i3); i3 += 16;
123 const int8x16_t vk3x0123456789ABCDEF = vld1q_s8(w); w = (const int8_t*) w + 16;
124
125 vprod01234567 = vmull_s8(vget_low_s8(vi3x0123456789ABCDEF), vget_low_s8(vk3x0123456789ABCDEF));
126 vprod89ABCDEF = vmull_s8(vget_high_s8(vi3x0123456789ABCDEF), vget_high_s8(vk3x0123456789ABCDEF));
127
128 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
129 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
130 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
131 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
132
133 const int8x16_t vi4x0123456789ABCDEF = vld1q_s8(i4); i4 += 16;
134 const int8x16_t vk4x0123456789ABCDEF = vld1q_s8(w); w = (const int8_t*) w + 16;
135
136 vprod01234567 = vmull_s8(vget_low_s8(vi4x0123456789ABCDEF), vget_low_s8(vk4x0123456789ABCDEF));
137 vprod89ABCDEF = vmull_s8(vget_high_s8(vi4x0123456789ABCDEF), vget_high_s8(vk4x0123456789ABCDEF));
138
139 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
140 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
141 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
142 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
143
144 const int8x16_t vi5x0123456789ABCDEF = vld1q_s8(i5); i5 += 16;
145 const int8x16_t vk5x0123456789ABCDEF = vld1q_s8(w); w = (const int8_t*) w + 16;
146
147 vprod01234567 = vmull_s8(vget_low_s8(vi5x0123456789ABCDEF), vget_low_s8(vk5x0123456789ABCDEF));
148 vprod89ABCDEF = vmull_s8(vget_high_s8(vi5x0123456789ABCDEF), vget_high_s8(vk5x0123456789ABCDEF));
149
150 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
151 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
152 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
153 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
154
155 vst1q_s32(b, vacc0123); b += 4;
156 vst1q_s32(b, vacc4567); b += 4;
157 vst1q_s32(b, vacc89AB); b += 4;
158 vst1q_s32(b, vaccCDEF); b += 4;
159 }
160
161 if XNN_UNLIKELY(c != 0) {
162 do {
163 int32x4_t vacc0123 = vld1q_s32(w); w = (const int32_t*) w + 4;
164 int32x4_t vacc4567 = vld1q_s32(w); w = (const int32_t*) w + 4;
165
166 const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
167 const int8x8_t vk0x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
168
169 int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
170
171 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
172 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
173
174 const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
175 const int8x8_t vk1x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
176
177 vprod01234567 = vmull_s8(vi1x01234567, vk1x01234567);
178
179 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
180 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
181
182 const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
183 const int8x8_t vk2x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
184
185 vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
186
187 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
188 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
189
190 const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
191 const int8x8_t vk3x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
192
193 vprod01234567 = vmull_s8(vi3x01234567, vk3x01234567);
194
195 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
196 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
197
198 const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
199 const int8x8_t vk4x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
200
201 vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
202
203 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
204 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
205
206 const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
207 const int8x8_t vk5x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
208
209 vprod01234567 = vmull_s8(vi5x01234567, vk5x01234567);
210
211 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
212 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
213
214 vst1q_s32(b, vacc0123); b += 4;
215 vst1q_s32(b, vacc4567); b += 4;
216 c -= 8;
217 } while (c != 0);
218 }
219 }
220
221 // Middle pass to process 6 inputs in each iteration.
222 for (size_t ks = kernel_size - 6; ks > 7; ks -= 6) {
223 int32_t* b = buffer;
224
225 const int8_t* i0 = input[0];
226 assert(i0 != NULL);
227 if XNN_UNPREDICTABLE(i0 != zero) {
228 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
229 }
230 const int8_t* i1 = input[1];
231 assert(i1 != NULL);
232 if XNN_UNPREDICTABLE(i1 != zero) {
233 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
234 }
235 const int8_t* i2 = input[2];
236 assert(i2 != NULL);
237 if XNN_UNPREDICTABLE(i2 != zero) {
238 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
239 }
240 const int8_t* i3 = input[3];
241 assert(i3 != NULL);
242 if XNN_UNPREDICTABLE(i3 != zero) {
243 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
244 }
245 const int8_t* i4 = input[4];
246 assert(i4 != NULL);
247 if XNN_UNPREDICTABLE(i4 != zero) {
248 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
249 }
250 const int8_t* i5 = input[5];
251 assert(i5 != NULL);
252 if XNN_UNPREDICTABLE(i5 != zero) {
253 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
254 }
255 input += 6;
256
257 size_t c = round_up_po2(channels, 8);
258
259 for (; c >= 16; c -= 16) {
260 int32x4_t vacc0123 = vld1q_s32(b);
261 int32x4_t vacc4567 = vld1q_s32(b + 4);
262 int32x4_t vacc89AB = vld1q_s32(b + 8);
263 int32x4_t vaccCDEF = vld1q_s32(b + 12);
264
265 const int8x16_t vi0x0123456789ABCDEF = vld1q_s8(i0); i0 += 16;
266 const int8x16_t vk0x0123456789ABCDEF = vld1q_s8(w); w = (const int8_t*) w + 16;
267
268 int16x8_t vprod01234567 = vmull_s8(vget_low_s8(vi0x0123456789ABCDEF), vget_low_s8(vk0x0123456789ABCDEF));
269 int16x8_t vprod89ABCDEF = vmull_s8(vget_high_s8(vi0x0123456789ABCDEF), vget_high_s8(vk0x0123456789ABCDEF));
270
271 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
272 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
273 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
274 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
275
276 const int8x16_t vi1x0123456789ABCDEF = vld1q_s8(i1); i1 += 16;
277 const int8x16_t vk1x0123456789ABCDEF = vld1q_s8(w); w = (const int8_t*) w + 16;
278
279 vprod01234567 = vmull_s8(vget_low_s8(vi1x0123456789ABCDEF), vget_low_s8(vk1x0123456789ABCDEF));
280 vprod89ABCDEF = vmull_s8(vget_high_s8(vi1x0123456789ABCDEF), vget_high_s8(vk1x0123456789ABCDEF));
281
282 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
283 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
284 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
285 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
286
287 const int8x16_t vi2x0123456789ABCDEF = vld1q_s8(i2); i2 += 16;
288 const int8x16_t vk2x0123456789ABCDEF = vld1q_s8(w); w = (const int8_t*) w + 16;
289
290 vprod01234567 = vmull_s8(vget_low_s8(vi2x0123456789ABCDEF), vget_low_s8(vk2x0123456789ABCDEF));
291 vprod89ABCDEF = vmull_s8(vget_high_s8(vi2x0123456789ABCDEF), vget_high_s8(vk2x0123456789ABCDEF));
292
293 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
294 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
295 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
296 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
297
298 const int8x16_t vi3x0123456789ABCDEF = vld1q_s8(i3); i3 += 16;
299 const int8x16_t vk3x0123456789ABCDEF = vld1q_s8(w); w = (const int8_t*) w + 16;
300
301 vprod01234567 = vmull_s8(vget_low_s8(vi3x0123456789ABCDEF), vget_low_s8(vk3x0123456789ABCDEF));
302 vprod89ABCDEF = vmull_s8(vget_high_s8(vi3x0123456789ABCDEF), vget_high_s8(vk3x0123456789ABCDEF));
303
304 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
305 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
306 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
307 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
308
309 const int8x16_t vi4x0123456789ABCDEF = vld1q_s8(i4); i4 += 16;
310 const int8x16_t vk4x0123456789ABCDEF = vld1q_s8(w); w = (const int8_t*) w + 16;
311
312 vprod01234567 = vmull_s8(vget_low_s8(vi4x0123456789ABCDEF), vget_low_s8(vk4x0123456789ABCDEF));
313 vprod89ABCDEF = vmull_s8(vget_high_s8(vi4x0123456789ABCDEF), vget_high_s8(vk4x0123456789ABCDEF));
314
315 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
316 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
317 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
318 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
319
320 const int8x16_t vi5x0123456789ABCDEF = vld1q_s8(i5); i5 += 16;
321 const int8x16_t vk5x0123456789ABCDEF = vld1q_s8(w); w = (const int8_t*) w + 16;
322
323 vprod01234567 = vmull_s8(vget_low_s8(vi5x0123456789ABCDEF), vget_low_s8(vk5x0123456789ABCDEF));
324 vprod89ABCDEF = vmull_s8(vget_high_s8(vi5x0123456789ABCDEF), vget_high_s8(vk5x0123456789ABCDEF));
325
326 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
327 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
328 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
329 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
330
331 vst1q_s32(b, vacc0123); b += 4;
332 vst1q_s32(b, vacc4567); b += 4;
333 vst1q_s32(b, vacc89AB); b += 4;
334 vst1q_s32(b, vaccCDEF); b += 4;
335 }
336
337 if XNN_UNLIKELY(c != 0) {
338 do {
339 int32x4_t vacc0123 = vld1q_s32(b);
340 int32x4_t vacc4567 = vld1q_s32(b + 4);
341
342 const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
343 const int8x8_t vk0x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
344
345 int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
346
347 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
348 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
349
350 const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
351 const int8x8_t vk1x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
352
353 vprod01234567 = vmull_s8(vi1x01234567, vk1x01234567);
354
355 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
356 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
357
358 const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
359 const int8x8_t vk2x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
360
361 vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
362
363 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
364 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
365
366 const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
367 const int8x8_t vk3x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
368
369 vprod01234567 = vmull_s8(vi3x01234567, vk3x01234567);
370
371 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
372 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
373
374 const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
375 const int8x8_t vk4x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
376
377 vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
378
379 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
380 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
381
382 const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
383 const int8x8_t vk5x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
384
385 vprod01234567 = vmull_s8(vi5x01234567, vk5x01234567);
386
387 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
388 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
389
390 vst1q_s32(b, vacc0123); b += 4;
391 vst1q_s32(b, vacc4567); b += 4;
392 c -= 8;
393 } while (c != 0);
394 }
395 }
396
397 // Last pass to process up to 7 inputs.
398 {
399 const int32_t* b = buffer;
400 const int8_t* i0 = input[0];
401 assert(i0 != NULL);
402 if XNN_UNPREDICTABLE(i0 != zero) {
403 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
404 }
405 const int8_t* i1 = input[1];
406 assert(i1 != NULL);
407 if XNN_UNPREDICTABLE(i1 != zero) {
408 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
409 }
410 const int8_t* i2 = input[2];
411 assert(i2 != NULL);
412 if XNN_UNPREDICTABLE(i2 != zero) {
413 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
414 }
415 const int8_t* i3 = input[3];
416 assert(i3 != NULL);
417 if XNN_UNPREDICTABLE(i3 != zero) {
418 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
419 }
420 const int8_t* i4 = input[4];
421 assert(i4 != NULL);
422 if XNN_UNPREDICTABLE(i4 != zero) {
423 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
424 }
425 const int8_t* i5 = input[5];
426 assert(i5 != NULL);
427 if XNN_UNPREDICTABLE(i5 != zero) {
428 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
429 }
430 const int8_t* i6 = input[6];
431 assert(i6 != NULL);
432 if XNN_UNPREDICTABLE(i6 != zero) {
433 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
434 }
435
436 size_t c = channels;
437 for (; c >= 16; c -= 16) {
438 int32x4_t vacc0123 = vld1q_s32(b); b += 4;
439 int32x4_t vacc4567 = vld1q_s32(b); b += 4;
440 int32x4_t vacc89AB = vld1q_s32(b); b += 4;
441 int32x4_t vaccCDEF = vld1q_s32(b); b += 4;
442
443 const int8x16_t vi0x0123456789ABCDEF = vld1q_s8(i0); i0 += 16;
444 const int8x16_t vk0x0123456789ABCDEF = vld1q_s8(w); w = (const int8_t*) w + 16;
445
446 int16x8_t vprod01234567 = vmull_s8(vget_low_s8(vi0x0123456789ABCDEF), vget_low_s8(vk0x0123456789ABCDEF));
447 int16x8_t vprod89ABCDEF = vmull_s8(vget_high_s8(vi0x0123456789ABCDEF), vget_high_s8(vk0x0123456789ABCDEF));
448
449 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
450 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
451 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
452 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
453
454 const int8x16_t vi1x0123456789ABCDEF = vld1q_s8(i1); i1 += 16;
455 const int8x16_t vk1x0123456789ABCDEF = vld1q_s8(w); w = (const int8_t*) w + 16;
456
457 vprod01234567 = vmull_s8(vget_low_s8(vi1x0123456789ABCDEF), vget_low_s8(vk1x0123456789ABCDEF));
458 vprod89ABCDEF = vmull_s8(vget_high_s8(vi1x0123456789ABCDEF), vget_high_s8(vk1x0123456789ABCDEF));
459
460 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
461 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
462 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
463 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
464
465 const int8x16_t vi2x0123456789ABCDEF = vld1q_s8(i2); i2 += 16;
466 const int8x16_t vk2x0123456789ABCDEF = vld1q_s8(w); w = (const int8_t*) w + 16;
467
468 vprod01234567 = vmull_s8(vget_low_s8(vi2x0123456789ABCDEF), vget_low_s8(vk2x0123456789ABCDEF));
469 vprod89ABCDEF = vmull_s8(vget_high_s8(vi2x0123456789ABCDEF), vget_high_s8(vk2x0123456789ABCDEF));
470
471 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
472 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
473 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
474 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
475
476 const int8x16_t vi3x0123456789ABCDEF = vld1q_s8(i3); i3 += 16;
477 const int8x16_t vk3x0123456789ABCDEF = vld1q_s8(w); w = (const int8_t*) w + 16;
478
479 vprod01234567 = vmull_s8(vget_low_s8(vi3x0123456789ABCDEF), vget_low_s8(vk3x0123456789ABCDEF));
480 vprod89ABCDEF = vmull_s8(vget_high_s8(vi3x0123456789ABCDEF), vget_high_s8(vk3x0123456789ABCDEF));
481
482 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
483 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
484 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
485 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
486
487 const int8x16_t vi4x0123456789ABCDEF = vld1q_s8(i4); i4 += 16;
488 const int8x16_t vk4x0123456789ABCDEF = vld1q_s8(w); w = (const int8_t*) w + 16;
489
490 vprod01234567 = vmull_s8(vget_low_s8(vi4x0123456789ABCDEF), vget_low_s8(vk4x0123456789ABCDEF));
491 vprod89ABCDEF = vmull_s8(vget_high_s8(vi4x0123456789ABCDEF), vget_high_s8(vk4x0123456789ABCDEF));
492
493 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
494 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
495 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
496 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
497
498 const int8x16_t vi5x0123456789ABCDEF = vld1q_s8(i5); i5 += 16;
499 const int8x16_t vk5x0123456789ABCDEF = vld1q_s8(w); w = (const int8_t*) w + 16;
500
501 vprod01234567 = vmull_s8(vget_low_s8(vi5x0123456789ABCDEF), vget_low_s8(vk5x0123456789ABCDEF));
502 vprod89ABCDEF = vmull_s8(vget_high_s8(vi5x0123456789ABCDEF), vget_high_s8(vk5x0123456789ABCDEF));
503
504 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
505 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
506 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
507 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
508
509 const int8x16_t vi6x0123456789ABCDEF = vld1q_s8(i6); i6 += 16;
510 const int8x16_t vk6x0123456789ABCDEF = vld1q_s8(w); w = (const int8_t*) w + 16;
511
512 vprod01234567 = vmull_s8(vget_low_s8(vi6x0123456789ABCDEF), vget_low_s8(vk6x0123456789ABCDEF));
513 vprod89ABCDEF = vmull_s8(vget_high_s8(vi6x0123456789ABCDEF), vget_high_s8(vk6x0123456789ABCDEF));
514
515 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
516 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
517 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
518 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
519
520 vacc0123 = vqshlq_s32(vacc0123, vright_pre_shift);
521 vacc4567 = vqshlq_s32(vacc4567, vright_pre_shift);
522 vacc89AB = vqshlq_s32(vacc89AB, vright_pre_shift);
523 vaccCDEF = vqshlq_s32(vaccCDEF, vright_pre_shift);
524
525 vacc0123 = vqdmulhq_s32(vacc0123, vmultiplier);
526 vacc4567 = vqdmulhq_s32(vacc4567, vmultiplier);
527 vacc89AB = vqdmulhq_s32(vacc89AB, vmultiplier);
528 vaccCDEF = vqdmulhq_s32(vaccCDEF, vmultiplier);
529
530 vacc0123 = vrshlq_s32(vacc0123, vright_post_shift);
531 vacc4567 = vrshlq_s32(vacc4567, vright_post_shift);
532 vacc89AB = vrshlq_s32(vacc89AB, vright_post_shift);
533 vaccCDEF = vrshlq_s32(vaccCDEF, vright_post_shift);
534
535 #if XNN_ARCH_ARM64
536 int16x8_t vacc01234567 = vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567);
537 int16x8_t vacc89ABCDEF = vqmovn_high_s32(vqmovn_s32(vacc89AB), vaccCDEF);
538
539 vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
540 vacc89ABCDEF = vqaddq_s16(vacc89ABCDEF, voutput_zero_point);
541
542 int8x16_t vout0123456789ABCDEF = vqmovn_high_s16(vqmovn_s16(vacc01234567), vacc89ABCDEF);
543 #else // !XNN_ARCH_ARM64
544 int16x8_t vacc01234567 = vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567));
545 int16x8_t vacc89ABCDEF = vcombine_s16(vqmovn_s32(vacc89AB), vqmovn_s32(vaccCDEF));
546
547 vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
548 vacc89ABCDEF = vqaddq_s16(vacc89ABCDEF, voutput_zero_point);
549
550 int8x16_t vout0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc01234567), vqmovn_s16(vacc89ABCDEF));
551 #endif // !XNN_ARCH_ARM64
552
553 vout0123456789ABCDEF = vmaxq_s8(vout0123456789ABCDEF, voutput_min);
554
555 vout0123456789ABCDEF = vminq_s8(vout0123456789ABCDEF, voutput_max);
556
557 vst1q_s8(output, vout0123456789ABCDEF); output += 16;
558 }
559
560
561 if XNN_UNLIKELY(c != 0) {
562 do {
563 int32x4_t vacc0123 = vld1q_s32(b); b += 4;
564 int32x4_t vacc4567 = vld1q_s32(b); b += 4;
565
566 const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
567 const int8x8_t vk0x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
568
569 int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
570
571 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
572 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
573 const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
574 const int8x8_t vk1x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
575
576 vprod01234567 = vmull_s8(vi1x01234567, vk1x01234567);
577
578 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
579 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
580 const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
581 const int8x8_t vk2x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
582
583 vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
584
585 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
586 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
587 const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
588 const int8x8_t vk3x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
589
590 vprod01234567 = vmull_s8(vi3x01234567, vk3x01234567);
591
592 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
593 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
594 const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
595 const int8x8_t vk4x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
596
597 vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
598
599 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
600 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
601 const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
602 const int8x8_t vk5x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
603
604 vprod01234567 = vmull_s8(vi5x01234567, vk5x01234567);
605
606 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
607 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
608 const int8x8_t vi6x01234567 = vld1_s8(i6); i6 += 8;
609 const int8x8_t vk6x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
610
611 vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
612
613 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
614 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
615
616 vacc0123 = vqshlq_s32(vacc0123, vright_pre_shift);
617 vacc4567 = vqshlq_s32(vacc4567, vright_pre_shift);
618
619 vacc0123 = vqdmulhq_s32(vacc0123, vmultiplier);
620 vacc4567 = vqdmulhq_s32(vacc4567, vmultiplier);
621
622 vacc0123 = vrshlq_s32(vacc0123, vright_post_shift);
623 vacc4567 = vrshlq_s32(vacc4567, vright_post_shift);
624
625 #if XNN_ARCH_ARM64
626 int16x8_t vacc01234567 = vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567);
627 #else
628 int16x8_t vacc01234567 = vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567));
629 #endif
630 vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
631
632 int8x8_t vout01234567 = vqmovn_s16(vacc01234567);
633 vout01234567 = vmax_s8(vout01234567, vget_low_s8(voutput_min));
634 vout01234567 = vmin_s8(vout01234567, vget_low_s8(voutput_max));
635
636 if XNN_LIKELY(c >= 8) {
637 vst1_s8(output, vout01234567); output += 8;
638 c -= 8;
639 } else {
640 if (c & 4) {
641 vst1_lane_u32((void*) output, vreinterpret_u32_s8(vout01234567), 0); output += 4;
642 vout01234567 = vext_s8(vout01234567, vout01234567, 4);
643 }
644 if (c & 2) {
645 vst1_lane_u16((void*) output, vreinterpret_u16_s8(vout01234567), 0); output += 2;
646 vout01234567 = vext_s8(vout01234567, vout01234567, 2);
647 }
648 if (c & 1) {
649 vst1_lane_s8(output, vout01234567, 0); output += 1;
650 }
651 c = 0;
652 }
653
654 } while (c != 0);
655 }
656 }
657
658 input = (const int8_t**) ((uintptr_t) input + input_stride);
659 output = (int8_t*) ((uintptr_t) output + output_increment);
660 } while (--output_width != 0);
661 }