Debian Codegraph @

src/qs8-dwconv/gen/qs8-dwconv-6f6m7l16c8s8r-minmax-rndnu-neon-mul16.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-dwconv/multipass-neon-mul16.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12
13 #include <arm_neon.h>
14
15 #include "xnnpack/dwconv.h"
16 #include "xnnpack/math.h"
17
18
19 void xnn_qs8_dwconv_minmax_rndnu_ukernel_6f6m7l16c8s8r__neon_mul16(
20 size_t channels,
21 size_t output_width,
22 const int8_t** input,
23 const void* weights,
24 int8_t* output,
25 intptr_t input_stride,
26 size_t output_increment,
27 size_t input_offset,
28 const int8_t* zero,
29 size_t kernel_size,
30 int32_t* buffer,
31 const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
32 {
33 assert(channels != 0);
34 assert(output_width != 0);
35 assert(kernel_size > 6);
36
37 const int32x4_t vright_pre_shift = vld1q_dup_s32(&params->rndnu_neon.right_pre_shift);
38 const int32x4_t vmultiplier = vld1q_dup_s32(&params->rndnu_neon.multiplier);
39 const int32x4_t vright_post_shift = vld1q_dup_s32(&params->rndnu_neon.right_post_shift);
40 const int16x8_t voutput_zero_point = vld1q_dup_s16(&params->rndnu_neon.output_zero_point);
41 const int8x16_t voutput_min = vld1q_dup_s8(&params->rndnu_neon.output_min);
42 const int8x16_t voutput_max = vld1q_dup_s8(&params->rndnu_neon.output_max);
43
44 do {
45 const void* w = weights;
46
47 // First pass to process 6 inputs.
48 {
49 int32_t* b = buffer;
50 const int8_t* i0 = input[0];
51 assert(i0 != NULL);
52 if XNN_UNPREDICTABLE(i0 != zero) {
53 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
54 }
55 const int8_t* i1 = input[1];
56 assert(i1 != NULL);
57 if XNN_UNPREDICTABLE(i1 != zero) {
58 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
59 }
60 const int8_t* i2 = input[2];
61 assert(i2 != NULL);
62 if XNN_UNPREDICTABLE(i2 != zero) {
63 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
64 }
65 const int8_t* i3 = input[3];
66 assert(i3 != NULL);
67 if XNN_UNPREDICTABLE(i3 != zero) {
68 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
69 }
70 const int8_t* i4 = input[4];
71 assert(i4 != NULL);
72 if XNN_UNPREDICTABLE(i4 != zero) {
73 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
74 }
75 const int8_t* i5 = input[5];
76 assert(i5 != NULL);
77 if XNN_UNPREDICTABLE(i5 != zero) {
78 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
79 }
80 input += 6;
81
82 size_t c = round_up_po2(channels, 8);
83
84 for (; c >= 16; c -= 16) {
85 int32x4_t vacc0123 = vld1q_s32(w); w = (const int32_t*) w + 4;
86 int32x4_t vacc4567 = vld1q_s32(w); w = (const int32_t*) w + 4;
87 int32x4_t vacc89AB = vld1q_s32(w); w = (const int32_t*) w + 4;
88 int32x4_t vaccCDEF = vld1q_s32(w); w = (const int32_t*) w + 4;
89
90 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0)); i0 += 8;
91 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
92 const int16x8_t vi0x89ABCDEF = vmovl_s8(vld1_s8(i0)); i0 += 8;
93 const int16x8_t vk0x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
94
95 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
96 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
97 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi0x89ABCDEF), vget_low_s16(vk0x89ABCDEF));
98 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi0x89ABCDEF), vget_high_s16(vk0x89ABCDEF));
99
100 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1)); i1 += 8;
101 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
102 const int16x8_t vi1x89ABCDEF = vmovl_s8(vld1_s8(i1)); i1 += 8;
103 const int16x8_t vk1x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
104
105 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
106 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
107 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi1x89ABCDEF), vget_low_s16(vk1x89ABCDEF));
108 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi1x89ABCDEF), vget_high_s16(vk1x89ABCDEF));
109
110 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2)); i2 += 8;
111 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
112 const int16x8_t vi2x89ABCDEF = vmovl_s8(vld1_s8(i2)); i2 += 8;
113 const int16x8_t vk2x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
114
115 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
116 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
117 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi2x89ABCDEF), vget_low_s16(vk2x89ABCDEF));
118 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi2x89ABCDEF), vget_high_s16(vk2x89ABCDEF));
119
120 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3)); i3 += 8;
121 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
122 const int16x8_t vi3x89ABCDEF = vmovl_s8(vld1_s8(i3)); i3 += 8;
123 const int16x8_t vk3x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
124
125 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
126 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
127 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi3x89ABCDEF), vget_low_s16(vk3x89ABCDEF));
128 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi3x89ABCDEF), vget_high_s16(vk3x89ABCDEF));
129
130 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4)); i4 += 8;
131 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
132 const int16x8_t vi4x89ABCDEF = vmovl_s8(vld1_s8(i4)); i4 += 8;
133 const int16x8_t vk4x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
134
135 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
136 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
137 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi4x89ABCDEF), vget_low_s16(vk4x89ABCDEF));
138 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi4x89ABCDEF), vget_high_s16(vk4x89ABCDEF));
139
140 const int16x8_t vi5x01234567 = vmovl_s8(vld1_s8(i5)); i5 += 8;
141 const int16x8_t vk5x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
142 const int16x8_t vi5x89ABCDEF = vmovl_s8(vld1_s8(i5)); i5 += 8;
143 const int16x8_t vk5x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
144
145 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi5x01234567), vget_low_s16(vk5x01234567));
146 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi5x01234567), vget_high_s16(vk5x01234567));
147 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi5x89ABCDEF), vget_low_s16(vk5x89ABCDEF));
148 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi5x89ABCDEF), vget_high_s16(vk5x89ABCDEF));
149
150 vst1q_s32(b, vacc0123); b += 4;
151 vst1q_s32(b, vacc4567); b += 4;
152 vst1q_s32(b, vacc89AB); b += 4;
153 vst1q_s32(b, vaccCDEF); b += 4;
154 }
155
156 if XNN_UNLIKELY(c != 0) {
157 do {
158 int32x4_t vacc0123 = vld1q_s32(w); w = (const int32_t*) w + 4;
159 int32x4_t vacc4567 = vld1q_s32(w); w = (const int32_t*) w + 4;
160
161 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0)); i0 += 8;
162 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
163
164 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
165 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
166
167 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1)); i1 += 8;
168 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
169
170 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
171 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
172
173 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2)); i2 += 8;
174 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
175
176 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
177 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
178
179 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3)); i3 += 8;
180 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
181
182 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
183 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
184
185 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4)); i4 += 8;
186 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
187
188 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
189 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
190
191 const int16x8_t vi5x01234567 = vmovl_s8(vld1_s8(i5)); i5 += 8;
192 const int16x8_t vk5x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
193
194 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi5x01234567), vget_low_s16(vk5x01234567));
195 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi5x01234567), vget_high_s16(vk5x01234567));
196
197 vst1q_s32(b, vacc0123); b += 4;
198 vst1q_s32(b, vacc4567); b += 4;
199 c -= 8;
200 } while (c != 0);
201 }
202 }
203
204 // Middle pass to process 6 inputs in each iteration.
205 for (size_t ks = kernel_size - 6; ks > 7; ks -= 6) {
206 int32_t* b = buffer;
207
208 const int8_t* i0 = input[0];
209 assert(i0 != NULL);
210 if XNN_UNPREDICTABLE(i0 != zero) {
211 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
212 }
213 const int8_t* i1 = input[1];
214 assert(i1 != NULL);
215 if XNN_UNPREDICTABLE(i1 != zero) {
216 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
217 }
218 const int8_t* i2 = input[2];
219 assert(i2 != NULL);
220 if XNN_UNPREDICTABLE(i2 != zero) {
221 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
222 }
223 const int8_t* i3 = input[3];
224 assert(i3 != NULL);
225 if XNN_UNPREDICTABLE(i3 != zero) {
226 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
227 }
228 const int8_t* i4 = input[4];
229 assert(i4 != NULL);
230 if XNN_UNPREDICTABLE(i4 != zero) {
231 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
232 }
233 const int8_t* i5 = input[5];
234 assert(i5 != NULL);
235 if XNN_UNPREDICTABLE(i5 != zero) {
236 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
237 }
238 input += 6;
239
240 size_t c = round_up_po2(channels, 8);
241
242 for (; c >= 16; c -= 16) {
243 int32x4_t vacc0123 = vld1q_s32(b);
244 int32x4_t vacc4567 = vld1q_s32(b + 4);
245 int32x4_t vacc89AB = vld1q_s32(b + 8);
246 int32x4_t vaccCDEF = vld1q_s32(b + 12);
247
248 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0)); i0 += 8;
249 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
250 const int16x8_t vi0x89ABCDEF = vmovl_s8(vld1_s8(i0)); i0 += 8;
251 const int16x8_t vk0x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
252
253 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
254 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
255 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi0x89ABCDEF), vget_low_s16(vk0x89ABCDEF));
256 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi0x89ABCDEF), vget_high_s16(vk0x89ABCDEF));
257
258 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1)); i1 += 8;
259 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
260 const int16x8_t vi1x89ABCDEF = vmovl_s8(vld1_s8(i1)); i1 += 8;
261 const int16x8_t vk1x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
262
263 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
264 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
265 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi1x89ABCDEF), vget_low_s16(vk1x89ABCDEF));
266 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi1x89ABCDEF), vget_high_s16(vk1x89ABCDEF));
267
268 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2)); i2 += 8;
269 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
270 const int16x8_t vi2x89ABCDEF = vmovl_s8(vld1_s8(i2)); i2 += 8;
271 const int16x8_t vk2x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
272
273 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
274 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
275 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi2x89ABCDEF), vget_low_s16(vk2x89ABCDEF));
276 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi2x89ABCDEF), vget_high_s16(vk2x89ABCDEF));
277
278 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3)); i3 += 8;
279 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
280 const int16x8_t vi3x89ABCDEF = vmovl_s8(vld1_s8(i3)); i3 += 8;
281 const int16x8_t vk3x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
282
283 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
284 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
285 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi3x89ABCDEF), vget_low_s16(vk3x89ABCDEF));
286 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi3x89ABCDEF), vget_high_s16(vk3x89ABCDEF));
287
288 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4)); i4 += 8;
289 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
290 const int16x8_t vi4x89ABCDEF = vmovl_s8(vld1_s8(i4)); i4 += 8;
291 const int16x8_t vk4x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
292
293 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
294 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
295 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi4x89ABCDEF), vget_low_s16(vk4x89ABCDEF));
296 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi4x89ABCDEF), vget_high_s16(vk4x89ABCDEF));
297
298 const int16x8_t vi5x01234567 = vmovl_s8(vld1_s8(i5)); i5 += 8;
299 const int16x8_t vk5x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
300 const int16x8_t vi5x89ABCDEF = vmovl_s8(vld1_s8(i5)); i5 += 8;
301 const int16x8_t vk5x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
302
303 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi5x01234567), vget_low_s16(vk5x01234567));
304 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi5x01234567), vget_high_s16(vk5x01234567));
305 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi5x89ABCDEF), vget_low_s16(vk5x89ABCDEF));
306 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi5x89ABCDEF), vget_high_s16(vk5x89ABCDEF));
307
308 vst1q_s32(b, vacc0123); b += 4;
309 vst1q_s32(b, vacc4567); b += 4;
310 vst1q_s32(b, vacc89AB); b += 4;
311 vst1q_s32(b, vaccCDEF); b += 4;
312 }
313
314 if XNN_UNLIKELY(c != 0) {
315 do {
316 int32x4_t vacc0123 = vld1q_s32(b);
317 int32x4_t vacc4567 = vld1q_s32(b + 4);
318
319 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0)); i0 += 8;
320 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
321
322 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
323 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
324
325 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1)); i1 += 8;
326 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
327
328 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
329 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
330
331 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2)); i2 += 8;
332 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
333
334 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
335 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
336
337 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3)); i3 += 8;
338 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
339
340 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
341 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
342
343 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4)); i4 += 8;
344 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
345
346 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
347 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
348
349 const int16x8_t vi5x01234567 = vmovl_s8(vld1_s8(i5)); i5 += 8;
350 const int16x8_t vk5x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
351
352 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi5x01234567), vget_low_s16(vk5x01234567));
353 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi5x01234567), vget_high_s16(vk5x01234567));
354
355 vst1q_s32(b, vacc0123); b += 4;
356 vst1q_s32(b, vacc4567); b += 4;
357 c -= 8;
358 } while (c != 0);
359 }
360 }
361
362 // Last pass to process up to 7 inputs.
363 {
364 const int32_t* b = buffer;
365
366 const int8_t* i0 = input[0];
367 assert(i0 != NULL);
368 if XNN_UNPREDICTABLE(i0 != zero) {
369 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
370 }
371 const int8_t* i1 = input[1];
372 assert(i1 != NULL);
373 if XNN_UNPREDICTABLE(i1 != zero) {
374 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
375 }
376 const int8_t* i2 = input[2];
377 assert(i2 != NULL);
378 if XNN_UNPREDICTABLE(i2 != zero) {
379 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
380 }
381 const int8_t* i3 = input[3];
382 assert(i3 != NULL);
383 if XNN_UNPREDICTABLE(i3 != zero) {
384 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
385 }
386 const int8_t* i4 = input[4];
387 assert(i4 != NULL);
388 if XNN_UNPREDICTABLE(i4 != zero) {
389 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
390 }
391 const int8_t* i5 = input[5];
392 assert(i5 != NULL);
393 if XNN_UNPREDICTABLE(i5 != zero) {
394 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
395 }
396 const int8_t* i6 = input[6];
397 assert(i6 != NULL);
398 if XNN_UNPREDICTABLE(i6 != zero) {
399 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
400 }
401
402 size_t c = channels;
403 for (; c >= 16; c -= 16) {
404 int32x4_t vacc0123 = vld1q_s32(b); b += 4;
405 int32x4_t vacc4567 = vld1q_s32(b); b += 4;
406 int32x4_t vacc89AB = vld1q_s32(b); b += 4;
407 int32x4_t vaccCDEF = vld1q_s32(b); b += 4;
408
409 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0)); i0 += 8;
410 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
411 const int16x8_t vi0x89ABCDEF = vmovl_s8(vld1_s8(i0)); i0 += 8;
412 const int16x8_t vk0x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
413
414 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
415 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
416 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi0x89ABCDEF), vget_low_s16(vk0x89ABCDEF));
417 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi0x89ABCDEF), vget_high_s16(vk0x89ABCDEF));
418
419 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1)); i1 += 8;
420 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
421 const int16x8_t vi1x89ABCDEF = vmovl_s8(vld1_s8(i1)); i1 += 8;
422 const int16x8_t vk1x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
423
424 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
425 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
426 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi1x89ABCDEF), vget_low_s16(vk1x89ABCDEF));
427 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi1x89ABCDEF), vget_high_s16(vk1x89ABCDEF));
428
429 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2)); i2 += 8;
430 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
431 const int16x8_t vi2x89ABCDEF = vmovl_s8(vld1_s8(i2)); i2 += 8;
432 const int16x8_t vk2x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
433
434 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
435 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
436 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi2x89ABCDEF), vget_low_s16(vk2x89ABCDEF));
437 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi2x89ABCDEF), vget_high_s16(vk2x89ABCDEF));
438
439 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3)); i3 += 8;
440 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
441 const int16x8_t vi3x89ABCDEF = vmovl_s8(vld1_s8(i3)); i3 += 8;
442 const int16x8_t vk3x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
443
444 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
445 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
446 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi3x89ABCDEF), vget_low_s16(vk3x89ABCDEF));
447 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi3x89ABCDEF), vget_high_s16(vk3x89ABCDEF));
448
449 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4)); i4 += 8;
450 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
451 const int16x8_t vi4x89ABCDEF = vmovl_s8(vld1_s8(i4)); i4 += 8;
452 const int16x8_t vk4x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
453
454 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
455 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
456 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi4x89ABCDEF), vget_low_s16(vk4x89ABCDEF));
457 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi4x89ABCDEF), vget_high_s16(vk4x89ABCDEF));
458
459 const int16x8_t vi5x01234567 = vmovl_s8(vld1_s8(i5)); i5 += 8;
460 const int16x8_t vk5x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
461 const int16x8_t vi5x89ABCDEF = vmovl_s8(vld1_s8(i5)); i5 += 8;
462 const int16x8_t vk5x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
463
464 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi5x01234567), vget_low_s16(vk5x01234567));
465 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi5x01234567), vget_high_s16(vk5x01234567));
466 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi5x89ABCDEF), vget_low_s16(vk5x89ABCDEF));
467 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi5x89ABCDEF), vget_high_s16(vk5x89ABCDEF));
468
469 const int16x8_t vi6x01234567 = vmovl_s8(vld1_s8(i6)); i6 += 8;
470 const int16x8_t vk6x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
471 const int16x8_t vi6x89ABCDEF = vmovl_s8(vld1_s8(i6)); i6 += 8;
472 const int16x8_t vk6x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
473
474 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi6x01234567), vget_low_s16(vk6x01234567));
475 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi6x01234567), vget_high_s16(vk6x01234567));
476 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi6x89ABCDEF), vget_low_s16(vk6x89ABCDEF));
477 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi6x89ABCDEF), vget_high_s16(vk6x89ABCDEF));
478
479 vacc0123 = vqshlq_s32(vacc0123, vright_pre_shift);
480 vacc4567 = vqshlq_s32(vacc4567, vright_pre_shift);
481 vacc89AB = vqshlq_s32(vacc89AB, vright_pre_shift);
482 vaccCDEF = vqshlq_s32(vaccCDEF, vright_pre_shift);
483
484 vacc0123 = vqdmulhq_s32(vacc0123, vmultiplier);
485 vacc4567 = vqdmulhq_s32(vacc4567, vmultiplier);
486 vacc89AB = vqdmulhq_s32(vacc89AB, vmultiplier);
487 vaccCDEF = vqdmulhq_s32(vaccCDEF, vmultiplier);
488
489 vacc0123 = vrshlq_s32(vacc0123, vright_post_shift);
490 vacc4567 = vrshlq_s32(vacc4567, vright_post_shift);
491 vacc89AB = vrshlq_s32(vacc89AB, vright_post_shift);
492 vaccCDEF = vrshlq_s32(vaccCDEF, vright_post_shift);
493
494 #if XNN_ARCH_ARM64
495 int16x8_t vacc01234567 = vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567);
496 int16x8_t vacc89ABCDEF = vqmovn_high_s32(vqmovn_s32(vacc89AB), vaccCDEF);
497
498 vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
499 vacc89ABCDEF = vqaddq_s16(vacc89ABCDEF, voutput_zero_point);
500
501 int8x16_t vout0123456789ABCDEF = vqmovn_high_s16(vqmovn_s16(vacc01234567), vacc89ABCDEF);
502 #else // !XNN_ARCH_ARM64
503 int16x8_t vacc01234567 = vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567));
504 int16x8_t vacc89ABCDEF = vcombine_s16(vqmovn_s32(vacc89AB), vqmovn_s32(vaccCDEF));
505
506 vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
507 vacc89ABCDEF = vqaddq_s16(vacc89ABCDEF, voutput_zero_point);
508
509 int8x16_t vout0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc01234567), vqmovn_s16(vacc89ABCDEF));
510 #endif // !XNN_ARCH_ARM64
511
512 vout0123456789ABCDEF = vmaxq_s8(vout0123456789ABCDEF, voutput_min);
513
514 vout0123456789ABCDEF = vminq_s8(vout0123456789ABCDEF, voutput_max);
515
516 vst1q_s8(output, vout0123456789ABCDEF); output += 16;
517 }
518
519 if XNN_UNLIKELY(c != 0) {
520 do {
521 int32x4_t vacc0123 = vld1q_s32(b); b += 4;
522 int32x4_t vacc4567 = vld1q_s32(b); b += 4;
523
524 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0)); i0 += 8;
525 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
526
527 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
528 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
529
530 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1)); i1 += 8;
531 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
532
533 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
534 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
535
536 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2)); i2 += 8;
537 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
538
539 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
540 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
541
542 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3)); i3 += 8;
543 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
544
545 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
546 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
547
548 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4)); i4 += 8;
549 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
550
551 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
552 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
553
554 const int16x8_t vi5x01234567 = vmovl_s8(vld1_s8(i5)); i5 += 8;
555 const int16x8_t vk5x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
556
557 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi5x01234567), vget_low_s16(vk5x01234567));
558 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi5x01234567), vget_high_s16(vk5x01234567));
559
560 const int16x8_t vi6x01234567 = vmovl_s8(vld1_s8(i6)); i6 += 8;
561 const int16x8_t vk6x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
562
563 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi6x01234567), vget_low_s16(vk6x01234567));
564 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi6x01234567), vget_high_s16(vk6x01234567));
565
566 vacc0123 = vqshlq_s32(vacc0123, vright_pre_shift);
567 vacc4567 = vqshlq_s32(vacc4567, vright_pre_shift);
568
569 vacc0123 = vqdmulhq_s32(vacc0123, vmultiplier);
570 vacc4567 = vqdmulhq_s32(vacc4567, vmultiplier);
571
572 vacc0123 = vrshlq_s32(vacc0123, vright_post_shift);
573 vacc4567 = vrshlq_s32(vacc4567, vright_post_shift);
574
575 #if XNN_ARCH_ARM64
576 int16x8_t vacc01234567 = vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567);
577 #else
578 int16x8_t vacc01234567 = vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567));
579 #endif
580 vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
581
582 int8x8_t vout01234567 = vqmovn_s16(vacc01234567);
583 vout01234567 = vmax_s8(vout01234567, vget_low_s8(voutput_min));
584 vout01234567 = vmin_s8(vout01234567, vget_low_s8(voutput_max));
585
586 if XNN_LIKELY(c >= 8) {
587 vst1_s8(output, vout01234567); output += 8;
588 c -= 8;
589 } else {
590 if (c & 4) {
591 vst1_lane_u32((void*) output, vreinterpret_u32_s8(vout01234567), 0); output += 4;
592 vout01234567 = vext_s8(vout01234567, vout01234567, 4);
593 }
594 if (c & 2) {
595 vst1_lane_u16((void*) output, vreinterpret_u16_s8(vout01234567), 0); output += 2;
596 vout01234567 = vext_s8(vout01234567, vout01234567, 2);
597 }
598 if (c & 1) {
599 vst1_lane_s8(output, vout01234567, 0); output += 1;
600 }
601 c = 0;
602 }
603 } while (c != 0);
604 }
605 }
606
607 input = (const int8_t**) ((uintptr_t) input + input_stride);
608 output = (int8_t*) ((uintptr_t) output + output_increment);
609 } while (--output_width != 0);
610 }