Debian Codegraph @

src/qs8-dwconv/gen/qs8-dwconv-6f6m7l16c8s8r-minmax-rndnu-neon-mla8-ld64.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-dwconv/multipass-neon-mul8.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12
13 #include <arm_neon.h>
14
15 #include "xnnpack/dwconv.h"
16 #include "xnnpack/math.h"
17
18
19 void xnn_qs8_dwconv_minmax_rndnu_ukernel_6f6m7l16c8s8r__neon_mla8_ld64(
20 size_t channels,
21 size_t output_width,
22 const int8_t** input,
23 const void* weights,
24 int8_t* output,
25 intptr_t input_stride,
26 size_t output_increment,
27 size_t input_offset,
28 const int8_t* zero,
29 size_t kernel_size,
30 int32_t* buffer,
31 const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
32 {
33 assert(channels != 0);
34 assert(output_width != 0);
35 assert(kernel_size > 6);
36
37 const int32x4_t vright_pre_shift = vld1q_dup_s32(&params->rndnu_neon.right_pre_shift);
38 const int32x4_t vmultiplier = vld1q_dup_s32(&params->rndnu_neon.multiplier);
39 const int32x4_t vright_post_shift = vld1q_dup_s32(&params->rndnu_neon.right_post_shift);
40 const int16x8_t voutput_zero_point = vld1q_dup_s16(&params->rndnu_neon.output_zero_point);
41 const int8x16_t voutput_min = vld1q_dup_s8(&params->rndnu_neon.output_min);
42 const int8x16_t voutput_max = vld1q_dup_s8(&params->rndnu_neon.output_max);
43 do {
44 const void* w = weights;
45
46 // First pass to process 6 inputs.
47 {
48 int32_t* b = buffer;
49 const int8_t* i0 = input[0];
50 assert(i0 != NULL);
51 if XNN_UNPREDICTABLE(i0 != zero) {
52 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
53 }
54 const int8_t* i1 = input[1];
55 assert(i1 != NULL);
56 if XNN_UNPREDICTABLE(i1 != zero) {
57 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
58 }
59 const int8_t* i2 = input[2];
60 assert(i2 != NULL);
61 if XNN_UNPREDICTABLE(i2 != zero) {
62 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
63 }
64 const int8_t* i3 = input[3];
65 assert(i3 != NULL);
66 if XNN_UNPREDICTABLE(i3 != zero) {
67 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
68 }
69 const int8_t* i4 = input[4];
70 assert(i4 != NULL);
71 if XNN_UNPREDICTABLE(i4 != zero) {
72 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
73 }
74 const int8_t* i5 = input[5];
75 assert(i5 != NULL);
76 if XNN_UNPREDICTABLE(i5 != zero) {
77 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
78 }
79 input += 6;
80
81 size_t c = round_up_po2(channels, 8);
82
83 for (; c >= 16; c -= 16) {
84 int32x4_t vacc0123 = vld1q_s32(w); w = (const int32_t*) w + 4;
85 int32x4_t vacc4567 = vld1q_s32(w); w = (const int32_t*) w + 4;
86 int32x4_t vacc89AB = vld1q_s32(w); w = (const int32_t*) w + 4;
87 int32x4_t vaccCDEF = vld1q_s32(w); w = (const int32_t*) w + 4;
88
89 const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
90 const int8x8_t vk0x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
91 const int8x8_t vi0x89ABCDEF = vld1_s8(i0); i0 += 8;
92 const int8x8_t vk0x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
93
94 int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
95 int16x8_t vprod89ABCDEF = vmull_s8(vi0x89ABCDEF, vk0x89ABCDEF);
96
97
98 const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
99 const int8x8_t vk1x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
100 const int8x8_t vi1x89ABCDEF = vld1_s8(i1); i1 += 8;
101 const int8x8_t vk1x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
102
103 vprod01234567 = vmlal_s8(vprod01234567, vi1x01234567, vk1x01234567);
104 vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi1x89ABCDEF, vk1x89ABCDEF);
105
106 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
107 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
108 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
109 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
110
111 const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
112 const int8x8_t vk2x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
113 const int8x8_t vi2x89ABCDEF = vld1_s8(i2); i2 += 8;
114 const int8x8_t vk2x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
115
116 vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
117 vprod89ABCDEF = vmull_s8(vi2x89ABCDEF, vk2x89ABCDEF);
118
119
120 const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
121 const int8x8_t vk3x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
122 const int8x8_t vi3x89ABCDEF = vld1_s8(i3); i3 += 8;
123 const int8x8_t vk3x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
124
125 vprod01234567 = vmlal_s8(vprod01234567, vi3x01234567, vk3x01234567);
126 vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi3x89ABCDEF, vk3x89ABCDEF);
127
128 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
129 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
130 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
131 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
132
133 const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
134 const int8x8_t vk4x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
135 const int8x8_t vi4x89ABCDEF = vld1_s8(i4); i4 += 8;
136 const int8x8_t vk4x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
137
138 vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
139 vprod89ABCDEF = vmull_s8(vi4x89ABCDEF, vk4x89ABCDEF);
140
141
142 const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
143 const int8x8_t vk5x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
144 const int8x8_t vi5x89ABCDEF = vld1_s8(i5); i5 += 8;
145 const int8x8_t vk5x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
146
147 vprod01234567 = vmlal_s8(vprod01234567, vi5x01234567, vk5x01234567);
148 vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi5x89ABCDEF, vk5x89ABCDEF);
149
150 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
151 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
152 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
153 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
154
155 vst1q_s32(b, vacc0123); b += 4;
156 vst1q_s32(b, vacc4567); b += 4;
157 vst1q_s32(b, vacc89AB); b += 4;
158 vst1q_s32(b, vaccCDEF); b += 4;
159 }
160
161 if XNN_UNLIKELY(c != 0) {
162 do {
163 int32x4_t vacc0123 = vld1q_s32(w); w = (const int32_t*) w + 4;
164 int32x4_t vacc4567 = vld1q_s32(w); w = (const int32_t*) w + 4;
165
166 const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
167 const int8x8_t vk0x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
168
169 int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
170
171
172 const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
173 const int8x8_t vk1x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
174
175 vprod01234567 = vmlal_s8(vprod01234567, vi1x01234567, vk1x01234567);
176
177 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
178 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
179
180 const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
181 const int8x8_t vk2x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
182
183 vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
184
185
186 const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
187 const int8x8_t vk3x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
188
189 vprod01234567 = vmlal_s8(vprod01234567, vi3x01234567, vk3x01234567);
190
191 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
192 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
193
194 const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
195 const int8x8_t vk4x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
196
197 vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
198
199
200 const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
201 const int8x8_t vk5x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
202
203 vprod01234567 = vmlal_s8(vprod01234567, vi5x01234567, vk5x01234567);
204
205 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
206 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
207
208 vst1q_s32(b, vacc0123); b += 4;
209 vst1q_s32(b, vacc4567); b += 4;
210 c -= 8;
211 } while (c != 0);
212 }
213 }
214
215 // Middle pass to process 6 inputs in each iteration.
216 for (size_t ks = kernel_size - 6; ks > 7; ks -= 6) {
217 int32_t* b = buffer;
218
219 const int8_t* i0 = input[0];
220 assert(i0 != NULL);
221 if XNN_UNPREDICTABLE(i0 != zero) {
222 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
223 }
224 const int8_t* i1 = input[1];
225 assert(i1 != NULL);
226 if XNN_UNPREDICTABLE(i1 != zero) {
227 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
228 }
229 const int8_t* i2 = input[2];
230 assert(i2 != NULL);
231 if XNN_UNPREDICTABLE(i2 != zero) {
232 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
233 }
234 const int8_t* i3 = input[3];
235 assert(i3 != NULL);
236 if XNN_UNPREDICTABLE(i3 != zero) {
237 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
238 }
239 const int8_t* i4 = input[4];
240 assert(i4 != NULL);
241 if XNN_UNPREDICTABLE(i4 != zero) {
242 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
243 }
244 const int8_t* i5 = input[5];
245 assert(i5 != NULL);
246 if XNN_UNPREDICTABLE(i5 != zero) {
247 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
248 }
249 input += 6;
250
251 size_t c = round_up_po2(channels, 8);
252
253 for (; c >= 16; c -= 16) {
254 int32x4_t vacc0123 = vld1q_s32(b);
255 int32x4_t vacc4567 = vld1q_s32(b + 4);
256 int32x4_t vacc89AB = vld1q_s32(b + 8);
257 int32x4_t vaccCDEF = vld1q_s32(b + 12);
258
259 const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
260 const int8x8_t vk0x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
261 const int8x8_t vi0x89ABCDEF = vld1_s8(i0); i0 += 8;
262 const int8x8_t vk0x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
263
264 int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
265 int16x8_t vprod89ABCDEF = vmull_s8(vi0x89ABCDEF, vk0x89ABCDEF);
266
267
268 const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
269 const int8x8_t vk1x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
270 const int8x8_t vi1x89ABCDEF = vld1_s8(i1); i1 += 8;
271 const int8x8_t vk1x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
272
273 vprod01234567 = vmlal_s8(vprod01234567, vi1x01234567, vk1x01234567);
274 vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi1x89ABCDEF, vk1x89ABCDEF);
275
276 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
277 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
278 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
279 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
280
281 const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
282 const int8x8_t vk2x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
283 const int8x8_t vi2x89ABCDEF = vld1_s8(i2); i2 += 8;
284 const int8x8_t vk2x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
285
286 vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
287 vprod89ABCDEF = vmull_s8(vi2x89ABCDEF, vk2x89ABCDEF);
288
289
290 const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
291 const int8x8_t vk3x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
292 const int8x8_t vi3x89ABCDEF = vld1_s8(i3); i3 += 8;
293 const int8x8_t vk3x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
294
295 vprod01234567 = vmlal_s8(vprod01234567, vi3x01234567, vk3x01234567);
296 vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi3x89ABCDEF, vk3x89ABCDEF);
297
298 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
299 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
300 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
301 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
302
303 const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
304 const int8x8_t vk4x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
305 const int8x8_t vi4x89ABCDEF = vld1_s8(i4); i4 += 8;
306 const int8x8_t vk4x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
307
308 vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
309 vprod89ABCDEF = vmull_s8(vi4x89ABCDEF, vk4x89ABCDEF);
310
311
312 const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
313 const int8x8_t vk5x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
314 const int8x8_t vi5x89ABCDEF = vld1_s8(i5); i5 += 8;
315 const int8x8_t vk5x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
316
317 vprod01234567 = vmlal_s8(vprod01234567, vi5x01234567, vk5x01234567);
318 vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi5x89ABCDEF, vk5x89ABCDEF);
319
320 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
321 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
322 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
323 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
324
325 vst1q_s32(b, vacc0123); b += 4;
326 vst1q_s32(b, vacc4567); b += 4;
327 vst1q_s32(b, vacc89AB); b += 4;
328 vst1q_s32(b, vaccCDEF); b += 4;
329 }
330
331 if XNN_UNLIKELY(c != 0) {
332 do {
333 int32x4_t vacc0123 = vld1q_s32(b);
334 int32x4_t vacc4567 = vld1q_s32(b + 4);
335
336 const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
337 const int8x8_t vk0x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
338
339 int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
340
341
342 const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
343 const int8x8_t vk1x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
344
345 vprod01234567 = vmlal_s8(vprod01234567, vi1x01234567, vk1x01234567);
346
347 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
348 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
349
350 const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
351 const int8x8_t vk2x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
352
353 vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
354
355
356 const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
357 const int8x8_t vk3x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
358
359 vprod01234567 = vmlal_s8(vprod01234567, vi3x01234567, vk3x01234567);
360
361 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
362 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
363
364 const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
365 const int8x8_t vk4x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
366
367 vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
368
369
370 const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
371 const int8x8_t vk5x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
372
373 vprod01234567 = vmlal_s8(vprod01234567, vi5x01234567, vk5x01234567);
374
375 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
376 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
377
378 vst1q_s32(b, vacc0123); b += 4;
379 vst1q_s32(b, vacc4567); b += 4;
380 c -= 8;
381 } while (c != 0);
382 }
383 }
384
385 // Last pass to process up to 7 inputs.
386 {
387 const int32_t* b = buffer;
388 const int8_t* i0 = input[0];
389 assert(i0 != NULL);
390 if XNN_UNPREDICTABLE(i0 != zero) {
391 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
392 }
393 const int8_t* i1 = input[1];
394 assert(i1 != NULL);
395 if XNN_UNPREDICTABLE(i1 != zero) {
396 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
397 }
398 const int8_t* i2 = input[2];
399 assert(i2 != NULL);
400 if XNN_UNPREDICTABLE(i2 != zero) {
401 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
402 }
403 const int8_t* i3 = input[3];
404 assert(i3 != NULL);
405 if XNN_UNPREDICTABLE(i3 != zero) {
406 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
407 }
408 const int8_t* i4 = input[4];
409 assert(i4 != NULL);
410 if XNN_UNPREDICTABLE(i4 != zero) {
411 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
412 }
413 const int8_t* i5 = input[5];
414 assert(i5 != NULL);
415 if XNN_UNPREDICTABLE(i5 != zero) {
416 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
417 }
418 const int8_t* i6 = input[6];
419 assert(i6 != NULL);
420 if XNN_UNPREDICTABLE(i6 != zero) {
421 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
422 }
423
424 size_t c = channels;
425 for (; c >= 16; c -= 16) {
426 int32x4_t vacc0123 = vld1q_s32(b); b += 4;
427 int32x4_t vacc4567 = vld1q_s32(b); b += 4;
428 int32x4_t vacc89AB = vld1q_s32(b); b += 4;
429 int32x4_t vaccCDEF = vld1q_s32(b); b += 4;
430
431 const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
432 const int8x8_t vk0x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
433 const int8x8_t vi0x89ABCDEF = vld1_s8(i0); i0 += 8;
434 const int8x8_t vk0x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
435
436 int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
437 int16x8_t vprod89ABCDEF = vmull_s8(vi0x89ABCDEF, vk0x89ABCDEF);
438
439
440 const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
441 const int8x8_t vk1x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
442 const int8x8_t vi1x89ABCDEF = vld1_s8(i1); i1 += 8;
443 const int8x8_t vk1x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
444
445 vprod01234567 = vmlal_s8(vprod01234567, vi1x01234567, vk1x01234567);
446 vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi1x89ABCDEF, vk1x89ABCDEF);
447
448 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
449 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
450 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
451 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
452
453 const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
454 const int8x8_t vk2x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
455 const int8x8_t vi2x89ABCDEF = vld1_s8(i2); i2 += 8;
456 const int8x8_t vk2x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
457
458 vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
459 vprod89ABCDEF = vmull_s8(vi2x89ABCDEF, vk2x89ABCDEF);
460
461
462 const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
463 const int8x8_t vk3x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
464 const int8x8_t vi3x89ABCDEF = vld1_s8(i3); i3 += 8;
465 const int8x8_t vk3x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
466
467 vprod01234567 = vmlal_s8(vprod01234567, vi3x01234567, vk3x01234567);
468 vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi3x89ABCDEF, vk3x89ABCDEF);
469
470 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
471 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
472 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
473 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
474
475 const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
476 const int8x8_t vk4x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
477 const int8x8_t vi4x89ABCDEF = vld1_s8(i4); i4 += 8;
478 const int8x8_t vk4x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
479
480 vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
481 vprod89ABCDEF = vmull_s8(vi4x89ABCDEF, vk4x89ABCDEF);
482
483
484 const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
485 const int8x8_t vk5x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
486 const int8x8_t vi5x89ABCDEF = vld1_s8(i5); i5 += 8;
487 const int8x8_t vk5x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
488
489 vprod01234567 = vmlal_s8(vprod01234567, vi5x01234567, vk5x01234567);
490 vprod89ABCDEF = vmlal_s8(vprod89ABCDEF, vi5x89ABCDEF, vk5x89ABCDEF);
491
492 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
493 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
494 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
495 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
496
497 const int8x8_t vi6x01234567 = vld1_s8(i6); i6 += 8;
498 const int8x8_t vk6x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
499 const int8x8_t vi6x89ABCDEF = vld1_s8(i6); i6 += 8;
500 const int8x8_t vk6x89ABCDEF = vld1_s8(w); w = (const int8_t*) w + 8;
501
502 vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
503 vprod89ABCDEF = vmull_s8(vi6x89ABCDEF, vk6x89ABCDEF);
504
505 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
506 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
507 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
508 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
509
510 vacc0123 = vqshlq_s32(vacc0123, vright_pre_shift);
511 vacc4567 = vqshlq_s32(vacc4567, vright_pre_shift);
512 vacc89AB = vqshlq_s32(vacc89AB, vright_pre_shift);
513 vaccCDEF = vqshlq_s32(vaccCDEF, vright_pre_shift);
514
515 vacc0123 = vqdmulhq_s32(vacc0123, vmultiplier);
516 vacc4567 = vqdmulhq_s32(vacc4567, vmultiplier);
517 vacc89AB = vqdmulhq_s32(vacc89AB, vmultiplier);
518 vaccCDEF = vqdmulhq_s32(vaccCDEF, vmultiplier);
519
520 vacc0123 = vrshlq_s32(vacc0123, vright_post_shift);
521 vacc4567 = vrshlq_s32(vacc4567, vright_post_shift);
522 vacc89AB = vrshlq_s32(vacc89AB, vright_post_shift);
523 vaccCDEF = vrshlq_s32(vaccCDEF, vright_post_shift);
524
525 #if XNN_ARCH_ARM64
526 int16x8_t vacc01234567 = vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567);
527 int16x8_t vacc89ABCDEF = vqmovn_high_s32(vqmovn_s32(vacc89AB), vaccCDEF);
528
529 vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
530 vacc89ABCDEF = vqaddq_s16(vacc89ABCDEF, voutput_zero_point);
531
532 int8x16_t vout0123456789ABCDEF = vqmovn_high_s16(vqmovn_s16(vacc01234567), vacc89ABCDEF);
533 #else // !XNN_ARCH_ARM64
534 int16x8_t vacc01234567 = vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567));
535 int16x8_t vacc89ABCDEF = vcombine_s16(vqmovn_s32(vacc89AB), vqmovn_s32(vaccCDEF));
536
537 vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
538 vacc89ABCDEF = vqaddq_s16(vacc89ABCDEF, voutput_zero_point);
539
540 int8x16_t vout0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc01234567), vqmovn_s16(vacc89ABCDEF));
541 #endif // !XNN_ARCH_ARM64
542
543 vout0123456789ABCDEF = vmaxq_s8(vout0123456789ABCDEF, voutput_min);
544
545 vout0123456789ABCDEF = vminq_s8(vout0123456789ABCDEF, voutput_max);
546
547 vst1q_s8(output, vout0123456789ABCDEF); output += 16;
548 }
549
550
551 if XNN_UNLIKELY(c != 0) {
552 do {
553 int32x4_t vacc0123 = vld1q_s32(b); b += 4;
554 int32x4_t vacc4567 = vld1q_s32(b); b += 4;
555
556 const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
557 const int8x8_t vk0x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
558
559 int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
560
561 const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
562 const int8x8_t vk1x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
563
564 vprod01234567 = vmlal_s8(vprod01234567, vi1x01234567, vk1x01234567);
565
566 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
567 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
568 const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
569 const int8x8_t vk2x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
570
571 vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
572
573 const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
574 const int8x8_t vk3x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
575
576 vprod01234567 = vmlal_s8(vprod01234567, vi3x01234567, vk3x01234567);
577
578 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
579 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
580 const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
581 const int8x8_t vk4x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
582
583 vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
584
585 const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
586 const int8x8_t vk5x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
587
588 vprod01234567 = vmlal_s8(vprod01234567, vi5x01234567, vk5x01234567);
589
590 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
591 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
592 const int8x8_t vi6x01234567 = vld1_s8(i6); i6 += 8;
593 const int8x8_t vk6x01234567 = vld1_s8(w); w = (const int8_t*) w + 8;
594
595 vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
596
597 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
598 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
599
600 vacc0123 = vqshlq_s32(vacc0123, vright_pre_shift);
601 vacc4567 = vqshlq_s32(vacc4567, vright_pre_shift);
602
603 vacc0123 = vqdmulhq_s32(vacc0123, vmultiplier);
604 vacc4567 = vqdmulhq_s32(vacc4567, vmultiplier);
605
606 vacc0123 = vrshlq_s32(vacc0123, vright_post_shift);
607 vacc4567 = vrshlq_s32(vacc4567, vright_post_shift);
608
609 #if XNN_ARCH_ARM64
610 int16x8_t vacc01234567 = vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567);
611 #else
612 int16x8_t vacc01234567 = vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567));
613 #endif
614 vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
615
616 int8x8_t vout01234567 = vqmovn_s16(vacc01234567);
617 vout01234567 = vmax_s8(vout01234567, vget_low_s8(voutput_min));
618 vout01234567 = vmin_s8(vout01234567, vget_low_s8(voutput_max));
619
620 if XNN_LIKELY(c >= 8) {
621 vst1_s8(output, vout01234567); output += 8;
622 c -= 8;
623 } else {
624 if (c & 4) {
625 vst1_lane_u32((void*) output, vreinterpret_u32_s8(vout01234567), 0); output += 4;
626 vout01234567 = vext_s8(vout01234567, vout01234567, 4);
627 }
628 if (c & 2) {
629 vst1_lane_u16((void*) output, vreinterpret_u16_s8(vout01234567), 0); output += 2;
630 vout01234567 = vext_s8(vout01234567, vout01234567, 2);
631 }
632 if (c & 1) {
633 vst1_lane_s8(output, vout01234567, 0); output += 1;
634 }
635 c = 0;
636 }
637
638 } while (c != 0);
639 }
640 }
641
642 input = (const int8_t**) ((uintptr_t) input + input_stride);
643 output = (int8_t*) ((uintptr_t) output + output_increment);
644 } while (--output_width != 0);
645 }