Debian Codegraph @

src/qs8-dwconv/gen/qs8-dwconv-5f5m5l16c8s8r-minmax-fp32-neon-mul16.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-dwconv/multipass-neon-mul16.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12
13 #include <arm_neon.h>
14
15 #include "xnnpack/dwconv.h"
16 #include "xnnpack/math.h"
17
18
19 void xnn_qs8_dwconv_minmax_fp32_ukernel_5f5m5l16c8s8r__neon_mul16(
20 size_t channels,
21 size_t output_width,
22 const int8_t** input,
23 const void* weights,
24 int8_t* output,
25 intptr_t input_stride,
26 size_t output_increment,
27 size_t input_offset,
28 const int8_t* zero,
29 size_t kernel_size,
30 int32_t* buffer,
31 const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
32 {
33 assert(channels != 0);
34 assert(output_width != 0);
35 assert(kernel_size > 5);
36
37 const float32x4_t vscale = vld1q_dup_f32(&params->fp32_neon.scale);
38 const float32x4_t vmagic_bias = vld1q_dup_f32(&params->fp32_neon.magic_bias);
39 const int32x4_t vmagic_bias_less_output_zero_point = vld1q_dup_s32(&params->fp32_neon.magic_bias_less_output_zero_point);
40 const int8x16_t voutput_min = vld1q_dup_s8(&params->fp32_neon.output_min);
41 const int8x16_t voutput_max = vld1q_dup_s8(&params->fp32_neon.output_max);
42
43 do {
44 const void* w = weights;
45
46 // First pass to process 5 inputs.
47 {
48 int32_t* b = buffer;
49 const int8_t* i0 = input[0];
50 assert(i0 != NULL);
51 if XNN_UNPREDICTABLE(i0 != zero) {
52 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
53 }
54 const int8_t* i1 = input[1];
55 assert(i1 != NULL);
56 if XNN_UNPREDICTABLE(i1 != zero) {
57 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
58 }
59 const int8_t* i2 = input[2];
60 assert(i2 != NULL);
61 if XNN_UNPREDICTABLE(i2 != zero) {
62 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
63 }
64 const int8_t* i3 = input[3];
65 assert(i3 != NULL);
66 if XNN_UNPREDICTABLE(i3 != zero) {
67 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
68 }
69 const int8_t* i4 = input[4];
70 assert(i4 != NULL);
71 if XNN_UNPREDICTABLE(i4 != zero) {
72 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
73 }
74 input += 5;
75
76 size_t c = round_up_po2(channels, 8);
77
78 for (; c >= 16; c -= 16) {
79 int32x4_t vacc0123 = vld1q_s32(w); w = (const int32_t*) w + 4;
80 int32x4_t vacc4567 = vld1q_s32(w); w = (const int32_t*) w + 4;
81 int32x4_t vacc89AB = vld1q_s32(w); w = (const int32_t*) w + 4;
82 int32x4_t vaccCDEF = vld1q_s32(w); w = (const int32_t*) w + 4;
83
84 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0)); i0 += 8;
85 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
86 const int16x8_t vi0x89ABCDEF = vmovl_s8(vld1_s8(i0)); i0 += 8;
87 const int16x8_t vk0x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
88
89 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
90 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
91 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi0x89ABCDEF), vget_low_s16(vk0x89ABCDEF));
92 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi0x89ABCDEF), vget_high_s16(vk0x89ABCDEF));
93
94 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1)); i1 += 8;
95 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
96 const int16x8_t vi1x89ABCDEF = vmovl_s8(vld1_s8(i1)); i1 += 8;
97 const int16x8_t vk1x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
98
99 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
100 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
101 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi1x89ABCDEF), vget_low_s16(vk1x89ABCDEF));
102 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi1x89ABCDEF), vget_high_s16(vk1x89ABCDEF));
103
104 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2)); i2 += 8;
105 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
106 const int16x8_t vi2x89ABCDEF = vmovl_s8(vld1_s8(i2)); i2 += 8;
107 const int16x8_t vk2x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
108
109 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
110 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
111 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi2x89ABCDEF), vget_low_s16(vk2x89ABCDEF));
112 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi2x89ABCDEF), vget_high_s16(vk2x89ABCDEF));
113
114 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3)); i3 += 8;
115 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
116 const int16x8_t vi3x89ABCDEF = vmovl_s8(vld1_s8(i3)); i3 += 8;
117 const int16x8_t vk3x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
118
119 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
120 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
121 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi3x89ABCDEF), vget_low_s16(vk3x89ABCDEF));
122 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi3x89ABCDEF), vget_high_s16(vk3x89ABCDEF));
123
124 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4)); i4 += 8;
125 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
126 const int16x8_t vi4x89ABCDEF = vmovl_s8(vld1_s8(i4)); i4 += 8;
127 const int16x8_t vk4x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
128
129 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
130 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
131 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi4x89ABCDEF), vget_low_s16(vk4x89ABCDEF));
132 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi4x89ABCDEF), vget_high_s16(vk4x89ABCDEF));
133
134 vst1q_s32(b, vacc0123); b += 4;
135 vst1q_s32(b, vacc4567); b += 4;
136 vst1q_s32(b, vacc89AB); b += 4;
137 vst1q_s32(b, vaccCDEF); b += 4;
138 }
139
140 if XNN_UNLIKELY(c != 0) {
141 do {
142 int32x4_t vacc0123 = vld1q_s32(w); w = (const int32_t*) w + 4;
143 int32x4_t vacc4567 = vld1q_s32(w); w = (const int32_t*) w + 4;
144
145 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0)); i0 += 8;
146 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
147
148 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
149 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
150
151 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1)); i1 += 8;
152 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
153
154 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
155 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
156
157 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2)); i2 += 8;
158 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
159
160 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
161 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
162
163 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3)); i3 += 8;
164 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
165
166 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
167 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
168
169 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4)); i4 += 8;
170 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
171
172 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
173 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
174
175 vst1q_s32(b, vacc0123); b += 4;
176 vst1q_s32(b, vacc4567); b += 4;
177 c -= 8;
178 } while (c != 0);
179 }
180 }
181
182 // Middle pass to process 5 inputs in each iteration.
183 for (size_t ks = kernel_size - 5; ks > 5; ks -= 5) {
184 int32_t* b = buffer;
185
186 const int8_t* i0 = input[0];
187 assert(i0 != NULL);
188 if XNN_UNPREDICTABLE(i0 != zero) {
189 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
190 }
191 const int8_t* i1 = input[1];
192 assert(i1 != NULL);
193 if XNN_UNPREDICTABLE(i1 != zero) {
194 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
195 }
196 const int8_t* i2 = input[2];
197 assert(i2 != NULL);
198 if XNN_UNPREDICTABLE(i2 != zero) {
199 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
200 }
201 const int8_t* i3 = input[3];
202 assert(i3 != NULL);
203 if XNN_UNPREDICTABLE(i3 != zero) {
204 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
205 }
206 const int8_t* i4 = input[4];
207 assert(i4 != NULL);
208 if XNN_UNPREDICTABLE(i4 != zero) {
209 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
210 }
211 input += 5;
212
213 size_t c = round_up_po2(channels, 8);
214
215 for (; c >= 16; c -= 16) {
216 int32x4_t vacc0123 = vld1q_s32(b);
217 int32x4_t vacc4567 = vld1q_s32(b + 4);
218 int32x4_t vacc89AB = vld1q_s32(b + 8);
219 int32x4_t vaccCDEF = vld1q_s32(b + 12);
220
221 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0)); i0 += 8;
222 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
223 const int16x8_t vi0x89ABCDEF = vmovl_s8(vld1_s8(i0)); i0 += 8;
224 const int16x8_t vk0x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
225
226 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
227 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
228 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi0x89ABCDEF), vget_low_s16(vk0x89ABCDEF));
229 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi0x89ABCDEF), vget_high_s16(vk0x89ABCDEF));
230
231 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1)); i1 += 8;
232 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
233 const int16x8_t vi1x89ABCDEF = vmovl_s8(vld1_s8(i1)); i1 += 8;
234 const int16x8_t vk1x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
235
236 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
237 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
238 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi1x89ABCDEF), vget_low_s16(vk1x89ABCDEF));
239 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi1x89ABCDEF), vget_high_s16(vk1x89ABCDEF));
240
241 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2)); i2 += 8;
242 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
243 const int16x8_t vi2x89ABCDEF = vmovl_s8(vld1_s8(i2)); i2 += 8;
244 const int16x8_t vk2x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
245
246 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
247 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
248 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi2x89ABCDEF), vget_low_s16(vk2x89ABCDEF));
249 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi2x89ABCDEF), vget_high_s16(vk2x89ABCDEF));
250
251 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3)); i3 += 8;
252 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
253 const int16x8_t vi3x89ABCDEF = vmovl_s8(vld1_s8(i3)); i3 += 8;
254 const int16x8_t vk3x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
255
256 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
257 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
258 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi3x89ABCDEF), vget_low_s16(vk3x89ABCDEF));
259 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi3x89ABCDEF), vget_high_s16(vk3x89ABCDEF));
260
261 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4)); i4 += 8;
262 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
263 const int16x8_t vi4x89ABCDEF = vmovl_s8(vld1_s8(i4)); i4 += 8;
264 const int16x8_t vk4x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
265
266 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
267 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
268 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi4x89ABCDEF), vget_low_s16(vk4x89ABCDEF));
269 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi4x89ABCDEF), vget_high_s16(vk4x89ABCDEF));
270
271 vst1q_s32(b, vacc0123); b += 4;
272 vst1q_s32(b, vacc4567); b += 4;
273 vst1q_s32(b, vacc89AB); b += 4;
274 vst1q_s32(b, vaccCDEF); b += 4;
275 }
276
277 if XNN_UNLIKELY(c != 0) {
278 do {
279 int32x4_t vacc0123 = vld1q_s32(b);
280 int32x4_t vacc4567 = vld1q_s32(b + 4);
281
282 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0)); i0 += 8;
283 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
284
285 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
286 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
287
288 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1)); i1 += 8;
289 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
290
291 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
292 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
293
294 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2)); i2 += 8;
295 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
296
297 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
298 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
299
300 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3)); i3 += 8;
301 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
302
303 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
304 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
305
306 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4)); i4 += 8;
307 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
308
309 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
310 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
311
312 vst1q_s32(b, vacc0123); b += 4;
313 vst1q_s32(b, vacc4567); b += 4;
314 c -= 8;
315 } while (c != 0);
316 }
317 }
318
319 // Last pass to process up to 5 inputs.
320 {
321 const int32_t* b = buffer;
322
323 const int8_t* i0 = input[0];
324 assert(i0 != NULL);
325 if XNN_UNPREDICTABLE(i0 != zero) {
326 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
327 }
328 const int8_t* i1 = input[1];
329 assert(i1 != NULL);
330 if XNN_UNPREDICTABLE(i1 != zero) {
331 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
332 }
333 const int8_t* i2 = input[2];
334 assert(i2 != NULL);
335 if XNN_UNPREDICTABLE(i2 != zero) {
336 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
337 }
338 const int8_t* i3 = input[3];
339 assert(i3 != NULL);
340 if XNN_UNPREDICTABLE(i3 != zero) {
341 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
342 }
343 const int8_t* i4 = input[4];
344 assert(i4 != NULL);
345 if XNN_UNPREDICTABLE(i4 != zero) {
346 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
347 }
348
349 size_t c = channels;
350 for (; c >= 16; c -= 16) {
351 int32x4_t vacc0123 = vld1q_s32(b); b += 4;
352 int32x4_t vacc4567 = vld1q_s32(b); b += 4;
353 int32x4_t vacc89AB = vld1q_s32(b); b += 4;
354 int32x4_t vaccCDEF = vld1q_s32(b); b += 4;
355
356 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0)); i0 += 8;
357 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
358 const int16x8_t vi0x89ABCDEF = vmovl_s8(vld1_s8(i0)); i0 += 8;
359 const int16x8_t vk0x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
360
361 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
362 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
363 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi0x89ABCDEF), vget_low_s16(vk0x89ABCDEF));
364 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi0x89ABCDEF), vget_high_s16(vk0x89ABCDEF));
365
366 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1)); i1 += 8;
367 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
368 const int16x8_t vi1x89ABCDEF = vmovl_s8(vld1_s8(i1)); i1 += 8;
369 const int16x8_t vk1x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
370
371 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
372 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
373 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi1x89ABCDEF), vget_low_s16(vk1x89ABCDEF));
374 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi1x89ABCDEF), vget_high_s16(vk1x89ABCDEF));
375
376 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2)); i2 += 8;
377 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
378 const int16x8_t vi2x89ABCDEF = vmovl_s8(vld1_s8(i2)); i2 += 8;
379 const int16x8_t vk2x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
380
381 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
382 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
383 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi2x89ABCDEF), vget_low_s16(vk2x89ABCDEF));
384 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi2x89ABCDEF), vget_high_s16(vk2x89ABCDEF));
385
386 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3)); i3 += 8;
387 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
388 const int16x8_t vi3x89ABCDEF = vmovl_s8(vld1_s8(i3)); i3 += 8;
389 const int16x8_t vk3x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
390
391 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
392 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
393 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi3x89ABCDEF), vget_low_s16(vk3x89ABCDEF));
394 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi3x89ABCDEF), vget_high_s16(vk3x89ABCDEF));
395
396 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4)); i4 += 8;
397 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
398 const int16x8_t vi4x89ABCDEF = vmovl_s8(vld1_s8(i4)); i4 += 8;
399 const int16x8_t vk4x89ABCDEF = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
400
401 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
402 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
403 vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi4x89ABCDEF), vget_low_s16(vk4x89ABCDEF));
404 vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi4x89ABCDEF), vget_high_s16(vk4x89ABCDEF));
405
406 float32x4_t vfpacc0123 = vcvtq_f32_s32(vacc0123);
407 float32x4_t vfpacc4567 = vcvtq_f32_s32(vacc4567);
408 float32x4_t vfpacc89AB = vcvtq_f32_s32(vacc89AB);
409 float32x4_t vfpaccCDEF = vcvtq_f32_s32(vaccCDEF);
410
411 vfpacc0123 = vmulq_f32(vfpacc0123, vscale);
412 vfpacc4567 = vmulq_f32(vfpacc4567, vscale);
413 vfpacc89AB = vmulq_f32(vfpacc89AB, vscale);
414 vfpaccCDEF = vmulq_f32(vfpaccCDEF, vscale);
415
416 vacc0123 = vreinterpretq_s32_f32(vaddq_f32(vfpacc0123, vmagic_bias));
417 vacc4567 = vreinterpretq_s32_f32(vaddq_f32(vfpacc4567, vmagic_bias));
418 vacc89AB = vreinterpretq_s32_f32(vaddq_f32(vfpacc89AB, vmagic_bias));
419 vaccCDEF = vreinterpretq_s32_f32(vaddq_f32(vfpaccCDEF, vmagic_bias));
420
421 vacc0123 = vqsubq_s32(vacc0123, vmagic_bias_less_output_zero_point);
422 vacc4567 = vqsubq_s32(vacc4567, vmagic_bias_less_output_zero_point);
423 vacc89AB = vqsubq_s32(vacc89AB, vmagic_bias_less_output_zero_point);
424 vaccCDEF = vqsubq_s32(vaccCDEF, vmagic_bias_less_output_zero_point);
425
426 #if XNN_ARCH_ARM64
427 int16x8_t vacc01234567 = vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567);
428 int16x8_t vacc89ABCDEF = vqmovn_high_s32(vqmovn_s32(vacc89AB), vaccCDEF);
429
430
431 int8x16_t vout0123456789ABCDEF = vqmovn_high_s16(vqmovn_s16(vacc01234567), vacc89ABCDEF);
432 #else // !XNN_ARCH_ARM64
433 int16x8_t vacc01234567 = vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567));
434 int16x8_t vacc89ABCDEF = vcombine_s16(vqmovn_s32(vacc89AB), vqmovn_s32(vaccCDEF));
435
436
437 int8x16_t vout0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc01234567), vqmovn_s16(vacc89ABCDEF));
438 #endif // !XNN_ARCH_ARM64
439
440 vout0123456789ABCDEF = vmaxq_s8(vout0123456789ABCDEF, voutput_min);
441
442 vout0123456789ABCDEF = vminq_s8(vout0123456789ABCDEF, voutput_max);
443
444 vst1q_s8(output, vout0123456789ABCDEF); output += 16;
445 }
446
447 if XNN_UNLIKELY(c != 0) {
448 do {
449 int32x4_t vacc0123 = vld1q_s32(b); b += 4;
450 int32x4_t vacc4567 = vld1q_s32(b); b += 4;
451
452 const int16x8_t vi0x01234567 = vmovl_s8(vld1_s8(i0)); i0 += 8;
453 const int16x8_t vk0x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
454
455 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
456 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
457
458 const int16x8_t vi1x01234567 = vmovl_s8(vld1_s8(i1)); i1 += 8;
459 const int16x8_t vk1x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
460
461 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
462 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
463
464 const int16x8_t vi2x01234567 = vmovl_s8(vld1_s8(i2)); i2 += 8;
465 const int16x8_t vk2x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
466
467 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
468 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
469
470 const int16x8_t vi3x01234567 = vmovl_s8(vld1_s8(i3)); i3 += 8;
471 const int16x8_t vk3x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
472
473 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
474 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
475
476 const int16x8_t vi4x01234567 = vmovl_s8(vld1_s8(i4)); i4 += 8;
477 const int16x8_t vk4x01234567 = vmovl_s8(vld1_s8(w)); w = (const int8_t*) w + 8;
478
479 vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
480 vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
481
482 float32x4_t vfpacc0123 = vcvtq_f32_s32(vacc0123);
483 float32x4_t vfpacc4567 = vcvtq_f32_s32(vacc4567);
484
485 vfpacc0123 = vmulq_f32(vfpacc0123, vscale);
486 vfpacc4567 = vmulq_f32(vfpacc4567, vscale);
487
488 vacc0123 = vreinterpretq_s32_f32(vaddq_f32(vfpacc0123, vmagic_bias));
489 vacc4567 = vreinterpretq_s32_f32(vaddq_f32(vfpacc4567, vmagic_bias));
490
491 vacc0123 = vqsubq_s32(vacc0123, vmagic_bias_less_output_zero_point);
492 vacc4567 = vqsubq_s32(vacc4567, vmagic_bias_less_output_zero_point);
493
494 #if XNN_ARCH_ARM64
495 int16x8_t vacc01234567 = vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567);
496 #else
497 int16x8_t vacc01234567 = vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567));
498 #endif
499
500 int8x8_t vout01234567 = vqmovn_s16(vacc01234567);
501 vout01234567 = vmax_s8(vout01234567, vget_low_s8(voutput_min));
502 vout01234567 = vmin_s8(vout01234567, vget_low_s8(voutput_max));
503
504 if XNN_LIKELY(c >= 8) {
505 vst1_s8(output, vout01234567); output += 8;
506 c -= 8;
507 } else {
508 if (c & 4) {
509 vst1_lane_u32((void*) output, vreinterpret_u32_s8(vout01234567), 0); output += 4;
510 vout01234567 = vext_s8(vout01234567, vout01234567, 4);
511 }
512 if (c & 2) {
513 vst1_lane_u16((void*) output, vreinterpret_u16_s8(vout01234567), 0); output += 2;
514 vout01234567 = vext_s8(vout01234567, vout01234567, 2);
515 }
516 if (c & 1) {
517 vst1_lane_s8(output, vout01234567, 0); output += 1;
518 }
519 c = 0;
520 }
521 } while (c != 0);
522 }
523 }
524
525 input = (const int8_t**) ((uintptr_t) input + input_stride);
526 output = (int8_t*) ((uintptr_t) output + output_increment);
527 } while (--output_width != 0);
528 }