Debian Codegraph @

src/qs8-dwconv/gen/qs8-dwconv-6f6m7l16c8s8r-minmax-fp32-sse41-mul16-add16.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-dwconv/multipass-sse-mul16.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include <smmintrin.h>
13
14 #include "xnnpack/dwconv.h"
15 #include "xnnpack/math.h"
16 #include "xnnpack/unaligned.h"
17
18
19 void xnn_qs8_dwconv_minmax_fp32_ukernel_6f6m7l16c8s8r__sse41_mul16_add16(
20 size_t channels,
21 size_t output_width,
22 const int8_t** input,
23 const void* weights,
24 int8_t* output,
25 intptr_t input_stride,
26 size_t output_increment,
27 size_t input_offset,
28 const int8_t* zero,
29 size_t kernel_size,
30 int32_t* buffer,
31 const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
32 {
33 assert(channels != 0);
34 assert(output_width != 0);
35 assert(kernel_size > 6);
36
37
38 const __m128 vscale = _mm_set1_ps(params->fp32_scalar.scale);
39 XNN_FORCE_REALIZATION(vscale);
40
41 const __m128 voutput_max_less_zero_point = _mm_set1_ps((int32_t) params->fp32_scalar.output_max - (int32_t) params->fp32_scalar.output_zero_point);
42 const __m128i voutput_zero_point = _mm_set1_epi16(params->fp32_scalar.output_zero_point);
43 const __m128i voutput_min = _mm_set1_epi8(params->fp32_scalar.output_min);
44 XNN_FORCE_REALIZATION(voutput_max_less_zero_point);
45 XNN_FORCE_REALIZATION(voutput_zero_point);
46 XNN_FORCE_REALIZATION(voutput_min);
47
48 do {
49 const void* w = weights;
50
51 // First pass to process 6 inputs.
52 {
53 int32_t* b = buffer;
54 const int8_t* i0 = input[0];
55 assert(i0 != NULL);
56 if XNN_UNPREDICTABLE(i0 != zero) {
57 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
58 }
59 const int8_t* i1 = input[1];
60 assert(i1 != NULL);
61 if XNN_UNPREDICTABLE(i1 != zero) {
62 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
63 }
64 const int8_t* i2 = input[2];
65 assert(i2 != NULL);
66 if XNN_UNPREDICTABLE(i2 != zero) {
67 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
68 }
69 const int8_t* i3 = input[3];
70 assert(i3 != NULL);
71 if XNN_UNPREDICTABLE(i3 != zero) {
72 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
73 }
74 const int8_t* i4 = input[4];
75 assert(i4 != NULL);
76 if XNN_UNPREDICTABLE(i4 != zero) {
77 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
78 }
79 const int8_t* i5 = input[5];
80 assert(i5 != NULL);
81 if XNN_UNPREDICTABLE(i5 != zero) {
82 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
83 }
84 input += 6;
85
86 size_t c = round_up_po2(channels, 8);
87
88 for (; c >= 16; c -= 16) {
89 __m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
90 __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
91 __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
92 __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
93
94
95
96 const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
97 const __m128i vxi0x01234567 = _mm_cvtepi8_epi16(vi0x01234567);
98 const __m128i vk0x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 0 * sizeof(int8_t)));
99 const __m128i vxk0x01234567 = _mm_cvtepi8_epi16(vk0x01234567);
100 const __m128i vi0x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i0 + 8));
101 const __m128i vxi0x89ABCDEF = _mm_cvtepi8_epi16(vi0x89ABCDEF);
102 const __m128i vk0x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 8 * sizeof(int8_t)));
103 const __m128i vxk0x89ABCDEF = _mm_cvtepi8_epi16(vk0x89ABCDEF);
104 i0 += 16;
105
106
107 __m128i vprod01234567 = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
108 __m128i vprod89ABCDEF = _mm_mullo_epi16(vxi0x89ABCDEF, vxk0x89ABCDEF);
109
110
111 const __m128i vi1x01234567 = _mm_loadl_epi64((const __m128i*) i1);
112 const __m128i vxi1x01234567 = _mm_cvtepi8_epi16(vi1x01234567);
113 const __m128i vk1x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 16 * sizeof(int8_t)));
114 const __m128i vxk1x01234567 = _mm_cvtepi8_epi16(vk1x01234567);
115 const __m128i vi1x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i1 + 8));
116 const __m128i vxi1x89ABCDEF = _mm_cvtepi8_epi16(vi1x89ABCDEF);
117 const __m128i vk1x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 24 * sizeof(int8_t)));
118 const __m128i vxk1x89ABCDEF = _mm_cvtepi8_epi16(vk1x89ABCDEF);
119 i1 += 16;
120
121
122 vprod01234567 = _mm_add_epi16(vprod01234567, _mm_mullo_epi16(vxi1x01234567, vxk1x01234567));
123 vprod89ABCDEF = _mm_add_epi16(vprod89ABCDEF, _mm_mullo_epi16(vxi1x89ABCDEF, vxk1x89ABCDEF));
124
125 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
126 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
127 vacc89AB = _mm_add_epi32(vacc89AB, _mm_cvtepi16_epi32(vprod89ABCDEF));
128 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_srai_epi32(_mm_unpackhi_epi16(vprod89ABCDEF, vprod89ABCDEF), 16));
129
130 const __m128i vi2x01234567 = _mm_loadl_epi64((const __m128i*) i2);
131 const __m128i vxi2x01234567 = _mm_cvtepi8_epi16(vi2x01234567);
132 const __m128i vk2x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 32 * sizeof(int8_t)));
133 const __m128i vxk2x01234567 = _mm_cvtepi8_epi16(vk2x01234567);
134 const __m128i vi2x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i2 + 8));
135 const __m128i vxi2x89ABCDEF = _mm_cvtepi8_epi16(vi2x89ABCDEF);
136 const __m128i vk2x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 40 * sizeof(int8_t)));
137 const __m128i vxk2x89ABCDEF = _mm_cvtepi8_epi16(vk2x89ABCDEF);
138 i2 += 16;
139
140
141 vprod01234567 = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
142 vprod89ABCDEF = _mm_mullo_epi16(vxi2x89ABCDEF, vxk2x89ABCDEF);
143
144
145 const __m128i vi3x01234567 = _mm_loadl_epi64((const __m128i*) i3);
146 const __m128i vxi3x01234567 = _mm_cvtepi8_epi16(vi3x01234567);
147 const __m128i vk3x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 48 * sizeof(int8_t)));
148 const __m128i vxk3x01234567 = _mm_cvtepi8_epi16(vk3x01234567);
149 const __m128i vi3x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i3 + 8));
150 const __m128i vxi3x89ABCDEF = _mm_cvtepi8_epi16(vi3x89ABCDEF);
151 const __m128i vk3x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 56 * sizeof(int8_t)));
152 const __m128i vxk3x89ABCDEF = _mm_cvtepi8_epi16(vk3x89ABCDEF);
153 i3 += 16;
154
155
156 vprod01234567 = _mm_add_epi16(vprod01234567, _mm_mullo_epi16(vxi3x01234567, vxk3x01234567));
157 vprod89ABCDEF = _mm_add_epi16(vprod89ABCDEF, _mm_mullo_epi16(vxi3x89ABCDEF, vxk3x89ABCDEF));
158
159 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
160 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
161 vacc89AB = _mm_add_epi32(vacc89AB, _mm_cvtepi16_epi32(vprod89ABCDEF));
162 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_srai_epi32(_mm_unpackhi_epi16(vprod89ABCDEF, vprod89ABCDEF), 16));
163
164 const __m128i vi4x01234567 = _mm_loadl_epi64((const __m128i*) i4);
165 const __m128i vxi4x01234567 = _mm_cvtepi8_epi16(vi4x01234567);
166 const __m128i vk4x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 64 * sizeof(int8_t)));
167 const __m128i vxk4x01234567 = _mm_cvtepi8_epi16(vk4x01234567);
168 const __m128i vi4x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i4 + 8));
169 const __m128i vxi4x89ABCDEF = _mm_cvtepi8_epi16(vi4x89ABCDEF);
170 const __m128i vk4x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 72 * sizeof(int8_t)));
171 const __m128i vxk4x89ABCDEF = _mm_cvtepi8_epi16(vk4x89ABCDEF);
172 i4 += 16;
173
174
175 vprod01234567 = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
176 vprod89ABCDEF = _mm_mullo_epi16(vxi4x89ABCDEF, vxk4x89ABCDEF);
177
178
179 const __m128i vi5x01234567 = _mm_loadl_epi64((const __m128i*) i5);
180 const __m128i vxi5x01234567 = _mm_cvtepi8_epi16(vi5x01234567);
181 const __m128i vk5x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 80 * sizeof(int8_t)));
182 const __m128i vxk5x01234567 = _mm_cvtepi8_epi16(vk5x01234567);
183 const __m128i vi5x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i5 + 8));
184 const __m128i vxi5x89ABCDEF = _mm_cvtepi8_epi16(vi5x89ABCDEF);
185 const __m128i vk5x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 88 * sizeof(int8_t)));
186 const __m128i vxk5x89ABCDEF = _mm_cvtepi8_epi16(vk5x89ABCDEF);
187 i5 += 16;
188
189
190 vprod01234567 = _mm_add_epi16(vprod01234567, _mm_mullo_epi16(vxi5x01234567, vxk5x01234567));
191 vprod89ABCDEF = _mm_add_epi16(vprod89ABCDEF, _mm_mullo_epi16(vxi5x89ABCDEF, vxk5x89ABCDEF));
192
193 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
194 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
195 vacc89AB = _mm_add_epi32(vacc89AB, _mm_cvtepi16_epi32(vprod89ABCDEF));
196 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_srai_epi32(_mm_unpackhi_epi16(vprod89ABCDEF, vprod89ABCDEF), 16));
197
198 w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 96 * sizeof(int8_t));
199
200 _mm_storeu_si128((__m128i*) b, vacc0123);
201 _mm_storeu_si128((__m128i*) (b + 4), vacc4567);
202 _mm_storeu_si128((__m128i*) (b + 8), vacc89AB);
203 _mm_storeu_si128((__m128i*) (b + 12), vaccCDEF);
204 b += 16;
205 }
206
207 if XNN_UNLIKELY(c != 0) {
208 do {
209 __m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
210 __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
211
212
213 const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
214 const __m128i vxi0x01234567 = _mm_cvtepi8_epi16(vi0x01234567);
215 const __m128i vk0x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + sizeof(int32_t) * 8 * sizeof(int8_t)));
216 const __m128i vxk0x01234567 = _mm_cvtepi8_epi16(vk0x01234567);
217 i0 += 8;
218
219
220 __m128i vprod01234567 = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
221
222
223 const __m128i vi1x01234567 = _mm_loadl_epi64((const __m128i*) i1);
224 const __m128i vxi1x01234567 = _mm_cvtepi8_epi16(vi1x01234567);
225 const __m128i vk1x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + sizeof(int32_t) * 8 + 8 * sizeof(int8_t)));
226 const __m128i vxk1x01234567 = _mm_cvtepi8_epi16(vk1x01234567);
227 i1 += 8;
228
229
230 vprod01234567 = _mm_add_epi16(vprod01234567, _mm_mullo_epi16(vxi1x01234567, vxk1x01234567));
231
232 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
233 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
234
235 const __m128i vi2x01234567 = _mm_loadl_epi64((const __m128i*) i2);
236 const __m128i vxi2x01234567 = _mm_cvtepi8_epi16(vi2x01234567);
237 const __m128i vk2x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + sizeof(int32_t) * 8 + 16 * sizeof(int8_t)));
238 const __m128i vxk2x01234567 = _mm_cvtepi8_epi16(vk2x01234567);
239 i2 += 8;
240
241
242 vprod01234567 = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
243
244
245 const __m128i vi3x01234567 = _mm_loadl_epi64((const __m128i*) i3);
246 const __m128i vxi3x01234567 = _mm_cvtepi8_epi16(vi3x01234567);
247 const __m128i vk3x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + sizeof(int32_t) * 8 + 24 * sizeof(int8_t)));
248 const __m128i vxk3x01234567 = _mm_cvtepi8_epi16(vk3x01234567);
249 i3 += 8;
250
251
252 vprod01234567 = _mm_add_epi16(vprod01234567, _mm_mullo_epi16(vxi3x01234567, vxk3x01234567));
253
254 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
255 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
256
257 const __m128i vi4x01234567 = _mm_loadl_epi64((const __m128i*) i4);
258 const __m128i vxi4x01234567 = _mm_cvtepi8_epi16(vi4x01234567);
259 const __m128i vk4x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + sizeof(int32_t) * 8 + 32 * sizeof(int8_t)));
260 const __m128i vxk4x01234567 = _mm_cvtepi8_epi16(vk4x01234567);
261 i4 += 8;
262
263
264 vprod01234567 = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
265
266
267 const __m128i vi5x01234567 = _mm_loadl_epi64((const __m128i*) i5);
268 const __m128i vxi5x01234567 = _mm_cvtepi8_epi16(vi5x01234567);
269 const __m128i vk5x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + sizeof(int32_t) * 8 + 40 * sizeof(int8_t)));
270 const __m128i vxk5x01234567 = _mm_cvtepi8_epi16(vk5x01234567);
271 i5 += 8;
272
273
274 vprod01234567 = _mm_add_epi16(vprod01234567, _mm_mullo_epi16(vxi5x01234567, vxk5x01234567));
275
276 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
277 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
278
279 w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 48 * sizeof(int8_t));
280
281 _mm_storeu_si128((__m128i*) (b), vacc0123);
282 _mm_storeu_si128((__m128i*) (b + 4), vacc4567);
283 b += 8;
284 c -= 8;
285 } while (c != 0);
286 }
287 }
288
289 // Middle pass to process 6 inputs in each iteration.
290 for (size_t ks = kernel_size - 6; ks > 7; ks -= 6) {
291 int32_t* b = buffer;
292 const int8_t* i0 = input[0];
293 assert(i0 != NULL);
294 if XNN_UNPREDICTABLE(i0 != zero) {
295 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
296 }
297 const int8_t* i1 = input[1];
298 assert(i1 != NULL);
299 if XNN_UNPREDICTABLE(i1 != zero) {
300 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
301 }
302 const int8_t* i2 = input[2];
303 assert(i2 != NULL);
304 if XNN_UNPREDICTABLE(i2 != zero) {
305 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
306 }
307 const int8_t* i3 = input[3];
308 assert(i3 != NULL);
309 if XNN_UNPREDICTABLE(i3 != zero) {
310 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
311 }
312 const int8_t* i4 = input[4];
313 assert(i4 != NULL);
314 if XNN_UNPREDICTABLE(i4 != zero) {
315 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
316 }
317 const int8_t* i5 = input[5];
318 assert(i5 != NULL);
319 if XNN_UNPREDICTABLE(i5 != zero) {
320 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
321 }
322 input += 6;
323
324 size_t c = round_up_po2(channels, 8);
325
326 for (; c >= 16; c -= 16) {
327 __m128i vacc0123 = _mm_loadu_si128((const __m128i*) b);
328 __m128i vacc4567 = _mm_loadu_si128((const __m128i*) (b + 4));
329 __m128i vacc89AB = _mm_loadu_si128((const __m128i*) (b + 8));
330 __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) (b + 12));
331
332
333
334 const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
335 const __m128i vxi0x01234567 = _mm_cvtepi8_epi16(vi0x01234567);
336 const __m128i vk0x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 0 * sizeof(int8_t)));
337 const __m128i vxk0x01234567 = _mm_cvtepi8_epi16(vk0x01234567);
338 const __m128i vi0x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i0 + 8));
339 const __m128i vxi0x89ABCDEF = _mm_cvtepi8_epi16(vi0x89ABCDEF);
340 const __m128i vk0x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
341 const __m128i vxk0x89ABCDEF = _mm_cvtepi8_epi16(vk0x89ABCDEF);
342 i0 += 16;
343
344
345 __m128i vprod01234567 = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
346 __m128i vprod89ABCDEF = _mm_mullo_epi16(vxi0x89ABCDEF, vxk0x89ABCDEF);
347
348
349 const __m128i vi1x01234567 = _mm_loadl_epi64((const __m128i*) i1);
350 const __m128i vxi1x01234567 = _mm_cvtepi8_epi16(vi1x01234567);
351 const __m128i vk1x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
352 const __m128i vxk1x01234567 = _mm_cvtepi8_epi16(vk1x01234567);
353 const __m128i vi1x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i1 + 8));
354 const __m128i vxi1x89ABCDEF = _mm_cvtepi8_epi16(vi1x89ABCDEF);
355 const __m128i vk1x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
356 const __m128i vxk1x89ABCDEF = _mm_cvtepi8_epi16(vk1x89ABCDEF);
357 i1 += 16;
358
359
360 vprod01234567 = _mm_add_epi16(vprod01234567, _mm_mullo_epi16(vxi1x01234567, vxk1x01234567));
361 vprod89ABCDEF = _mm_add_epi16(vprod89ABCDEF, _mm_mullo_epi16(vxi1x89ABCDEF, vxk1x89ABCDEF));
362
363 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
364 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
365 vacc89AB = _mm_add_epi32(vacc89AB, _mm_cvtepi16_epi32(vprod89ABCDEF));
366 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_srai_epi32(_mm_unpackhi_epi16(vprod89ABCDEF, vprod89ABCDEF), 16));
367
368 const __m128i vi2x01234567 = _mm_loadl_epi64((const __m128i*) i2);
369 const __m128i vxi2x01234567 = _mm_cvtepi8_epi16(vi2x01234567);
370 const __m128i vk2x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t)));
371 const __m128i vxk2x01234567 = _mm_cvtepi8_epi16(vk2x01234567);
372 const __m128i vi2x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i2 + 8));
373 const __m128i vxi2x89ABCDEF = _mm_cvtepi8_epi16(vi2x89ABCDEF);
374 const __m128i vk2x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 40 * sizeof(int8_t)));
375 const __m128i vxk2x89ABCDEF = _mm_cvtepi8_epi16(vk2x89ABCDEF);
376 i2 += 16;
377
378
379 vprod01234567 = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
380 vprod89ABCDEF = _mm_mullo_epi16(vxi2x89ABCDEF, vxk2x89ABCDEF);
381
382
383 const __m128i vi3x01234567 = _mm_loadl_epi64((const __m128i*) i3);
384 const __m128i vxi3x01234567 = _mm_cvtepi8_epi16(vi3x01234567);
385 const __m128i vk3x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t)));
386 const __m128i vxk3x01234567 = _mm_cvtepi8_epi16(vk3x01234567);
387 const __m128i vi3x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i3 + 8));
388 const __m128i vxi3x89ABCDEF = _mm_cvtepi8_epi16(vi3x89ABCDEF);
389 const __m128i vk3x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 56 * sizeof(int8_t)));
390 const __m128i vxk3x89ABCDEF = _mm_cvtepi8_epi16(vk3x89ABCDEF);
391 i3 += 16;
392
393
394 vprod01234567 = _mm_add_epi16(vprod01234567, _mm_mullo_epi16(vxi3x01234567, vxk3x01234567));
395 vprod89ABCDEF = _mm_add_epi16(vprod89ABCDEF, _mm_mullo_epi16(vxi3x89ABCDEF, vxk3x89ABCDEF));
396
397 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
398 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
399 vacc89AB = _mm_add_epi32(vacc89AB, _mm_cvtepi16_epi32(vprod89ABCDEF));
400 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_srai_epi32(_mm_unpackhi_epi16(vprod89ABCDEF, vprod89ABCDEF), 16));
401
402 const __m128i vi4x01234567 = _mm_loadl_epi64((const __m128i*) i4);
403 const __m128i vxi4x01234567 = _mm_cvtepi8_epi16(vi4x01234567);
404 const __m128i vk4x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 64 * sizeof(int8_t)));
405 const __m128i vxk4x01234567 = _mm_cvtepi8_epi16(vk4x01234567);
406 const __m128i vi4x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i4 + 8));
407 const __m128i vxi4x89ABCDEF = _mm_cvtepi8_epi16(vi4x89ABCDEF);
408 const __m128i vk4x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 72 * sizeof(int8_t)));
409 const __m128i vxk4x89ABCDEF = _mm_cvtepi8_epi16(vk4x89ABCDEF);
410 i4 += 16;
411
412
413 vprod01234567 = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
414 vprod89ABCDEF = _mm_mullo_epi16(vxi4x89ABCDEF, vxk4x89ABCDEF);
415
416
417 const __m128i vi5x01234567 = _mm_loadl_epi64((const __m128i*) i5);
418 const __m128i vxi5x01234567 = _mm_cvtepi8_epi16(vi5x01234567);
419 const __m128i vk5x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 80 * sizeof(int8_t)));
420 const __m128i vxk5x01234567 = _mm_cvtepi8_epi16(vk5x01234567);
421 const __m128i vi5x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i5 + 8));
422 const __m128i vxi5x89ABCDEF = _mm_cvtepi8_epi16(vi5x89ABCDEF);
423 const __m128i vk5x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 88 * sizeof(int8_t)));
424 const __m128i vxk5x89ABCDEF = _mm_cvtepi8_epi16(vk5x89ABCDEF);
425 i5 += 16;
426
427
428 vprod01234567 = _mm_add_epi16(vprod01234567, _mm_mullo_epi16(vxi5x01234567, vxk5x01234567));
429 vprod89ABCDEF = _mm_add_epi16(vprod89ABCDEF, _mm_mullo_epi16(vxi5x89ABCDEF, vxk5x89ABCDEF));
430
431 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
432 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
433 vacc89AB = _mm_add_epi32(vacc89AB, _mm_cvtepi16_epi32(vprod89ABCDEF));
434 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_srai_epi32(_mm_unpackhi_epi16(vprod89ABCDEF, vprod89ABCDEF), 16));
435
436 w = (const void*) ((uintptr_t) w + 96 * sizeof(int8_t));
437
438 _mm_storeu_si128((__m128i*) (b), vacc0123);
439 _mm_storeu_si128((__m128i*) (b + 4), vacc4567);
440 _mm_storeu_si128((__m128i*) (b + 8), vacc89AB);
441 _mm_storeu_si128((__m128i*) (b + 12), vaccCDEF);
442 b += 16;
443 }
444
445 if XNN_UNLIKELY(c != 0) {
446 do {
447 __m128i vacc0123 = _mm_loadu_si128((const __m128i*) b);
448 __m128i vacc4567 = _mm_loadu_si128((const __m128i*) (b + 4));
449
450
451 const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
452 const __m128i vxi0x01234567 = _mm_cvtepi8_epi16(vi0x01234567);
453 const __m128i vk0x01234567 = _mm_loadl_epi64((const __m128i*) w);
454 const __m128i vxk0x01234567 = _mm_cvtepi8_epi16(vk0x01234567);
455 i0 += 8;
456
457
458 __m128i vprod01234567 = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
459
460
461 const __m128i vi1x01234567 = _mm_loadl_epi64((const __m128i*) i1);
462 const __m128i vxi1x01234567 = _mm_cvtepi8_epi16(vi1x01234567);
463 const __m128i vk1x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
464 const __m128i vxk1x01234567 = _mm_cvtepi8_epi16(vk1x01234567);
465 i1 += 8;
466
467
468 vprod01234567 = _mm_add_epi16(vprod01234567, _mm_mullo_epi16(vxi1x01234567, vxk1x01234567));
469
470 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
471 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
472
473 const __m128i vi2x01234567 = _mm_loadl_epi64((const __m128i*) i2);
474 const __m128i vxi2x01234567 = _mm_cvtepi8_epi16(vi2x01234567);
475 const __m128i vk2x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
476 const __m128i vxk2x01234567 = _mm_cvtepi8_epi16(vk2x01234567);
477 i2 += 8;
478
479
480 vprod01234567 = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
481
482
483 const __m128i vi3x01234567 = _mm_loadl_epi64((const __m128i*) i3);
484 const __m128i vxi3x01234567 = _mm_cvtepi8_epi16(vi3x01234567);
485 const __m128i vk3x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
486 const __m128i vxk3x01234567 = _mm_cvtepi8_epi16(vk3x01234567);
487 i3 += 8;
488
489
490 vprod01234567 = _mm_add_epi16(vprod01234567, _mm_mullo_epi16(vxi3x01234567, vxk3x01234567));
491
492 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
493 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
494
495 const __m128i vi4x01234567 = _mm_loadl_epi64((const __m128i*) i4);
496 const __m128i vxi4x01234567 = _mm_cvtepi8_epi16(vi4x01234567);
497 const __m128i vk4x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t)));
498 const __m128i vxk4x01234567 = _mm_cvtepi8_epi16(vk4x01234567);
499 i4 += 8;
500
501
502 vprod01234567 = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
503
504
505 const __m128i vi5x01234567 = _mm_loadl_epi64((const __m128i*) i5);
506 const __m128i vxi5x01234567 = _mm_cvtepi8_epi16(vi5x01234567);
507 const __m128i vk5x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 40 * sizeof(int8_t)));
508 const __m128i vxk5x01234567 = _mm_cvtepi8_epi16(vk5x01234567);
509 i5 += 8;
510
511
512 vprod01234567 = _mm_add_epi16(vprod01234567, _mm_mullo_epi16(vxi5x01234567, vxk5x01234567));
513
514 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
515 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
516
517 w = (const void*) ((uintptr_t) w + 48 * sizeof(int8_t));
518
519 _mm_storeu_si128((__m128i*) (b), vacc0123);
520 _mm_storeu_si128((__m128i*) (b + 4), vacc4567);
521 b += 8;
522 c -= 8;
523 } while (c != 0);
524 }
525 }
526
527 // Last pass to process up to 7 inputs.
528 {
529 const int32_t* b = buffer;
530 const int8_t* i0 = input[0];
531 assert(i0 != NULL);
532 if XNN_UNPREDICTABLE(i0 != zero) {
533 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
534 }
535 const int8_t* i1 = input[1];
536 assert(i1 != NULL);
537 if XNN_UNPREDICTABLE(i1 != zero) {
538 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
539 }
540 const int8_t* i2 = input[2];
541 assert(i2 != NULL);
542 if XNN_UNPREDICTABLE(i2 != zero) {
543 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
544 }
545 const int8_t* i3 = input[3];
546 assert(i3 != NULL);
547 if XNN_UNPREDICTABLE(i3 != zero) {
548 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
549 }
550 const int8_t* i4 = input[4];
551 assert(i4 != NULL);
552 if XNN_UNPREDICTABLE(i4 != zero) {
553 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
554 }
555 const int8_t* i5 = input[5];
556 assert(i5 != NULL);
557 if XNN_UNPREDICTABLE(i5 != zero) {
558 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
559 }
560 const int8_t* i6 = input[6];
561 assert(i6 != NULL);
562 if XNN_UNPREDICTABLE(i6 != zero) {
563 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
564 }
565
566 size_t c = channels;
567
568 for (; c >= 16; c -= 16) {
569 __m128i vacc0123 = _mm_loadu_si128((const __m128i*) b);
570 __m128i vacc4567 = _mm_loadu_si128((const __m128i*) (b + 4));
571 __m128i vacc89AB = _mm_loadu_si128((const __m128i*) (b + 8));
572 __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) (b + 12));
573 b += 16;
574
575
576 const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
577 const __m128i vxi0x01234567 = _mm_cvtepi8_epi16(vi0x01234567);
578 const __m128i vk0x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 0 * sizeof(int8_t)));
579 const __m128i vxk0x01234567 = _mm_cvtepi8_epi16(vk0x01234567);
580 const __m128i vi0x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i0 + 8));
581 const __m128i vxi0x89ABCDEF = _mm_cvtepi8_epi16(vi0x89ABCDEF);
582 const __m128i vk0x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
583 const __m128i vxk0x89ABCDEF = _mm_cvtepi8_epi16(vk0x89ABCDEF);
584 i0 += 16;
585
586
587 __m128i vprod01234567 = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
588 __m128i vprod89ABCDEF = _mm_mullo_epi16(vxi0x89ABCDEF, vxk0x89ABCDEF);
589
590
591 const __m128i vi1x01234567 = _mm_loadl_epi64((const __m128i*) i1);
592 const __m128i vxi1x01234567 = _mm_cvtepi8_epi16(vi1x01234567);
593 const __m128i vk1x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
594 const __m128i vxk1x01234567 = _mm_cvtepi8_epi16(vk1x01234567);
595 const __m128i vi1x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i1 + 8));
596 const __m128i vxi1x89ABCDEF = _mm_cvtepi8_epi16(vi1x89ABCDEF);
597 const __m128i vk1x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
598 const __m128i vxk1x89ABCDEF = _mm_cvtepi8_epi16(vk1x89ABCDEF);
599 i1 += 16;
600
601
602 vprod01234567 = _mm_add_epi16(vprod01234567, _mm_mullo_epi16(vxi1x01234567, vxk1x01234567));
603 vprod89ABCDEF = _mm_add_epi16(vprod89ABCDEF, _mm_mullo_epi16(vxi1x89ABCDEF, vxk1x89ABCDEF));
604
605 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
606 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
607 vacc89AB = _mm_add_epi32(vacc89AB, _mm_cvtepi16_epi32(vprod89ABCDEF));
608 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_srai_epi32(_mm_unpackhi_epi16(vprod89ABCDEF, vprod89ABCDEF), 16));
609
610 const __m128i vi2x01234567 = _mm_loadl_epi64((const __m128i*) i2);
611 const __m128i vxi2x01234567 = _mm_cvtepi8_epi16(vi2x01234567);
612 const __m128i vk2x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t)));
613 const __m128i vxk2x01234567 = _mm_cvtepi8_epi16(vk2x01234567);
614 const __m128i vi2x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i2 + 8));
615 const __m128i vxi2x89ABCDEF = _mm_cvtepi8_epi16(vi2x89ABCDEF);
616 const __m128i vk2x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 40 * sizeof(int8_t)));
617 const __m128i vxk2x89ABCDEF = _mm_cvtepi8_epi16(vk2x89ABCDEF);
618 i2 += 16;
619
620
621 vprod01234567 = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
622 vprod89ABCDEF = _mm_mullo_epi16(vxi2x89ABCDEF, vxk2x89ABCDEF);
623
624
625 const __m128i vi3x01234567 = _mm_loadl_epi64((const __m128i*) i3);
626 const __m128i vxi3x01234567 = _mm_cvtepi8_epi16(vi3x01234567);
627 const __m128i vk3x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t)));
628 const __m128i vxk3x01234567 = _mm_cvtepi8_epi16(vk3x01234567);
629 const __m128i vi3x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i3 + 8));
630 const __m128i vxi3x89ABCDEF = _mm_cvtepi8_epi16(vi3x89ABCDEF);
631 const __m128i vk3x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 56 * sizeof(int8_t)));
632 const __m128i vxk3x89ABCDEF = _mm_cvtepi8_epi16(vk3x89ABCDEF);
633 i3 += 16;
634
635
636 vprod01234567 = _mm_add_epi16(vprod01234567, _mm_mullo_epi16(vxi3x01234567, vxk3x01234567));
637 vprod89ABCDEF = _mm_add_epi16(vprod89ABCDEF, _mm_mullo_epi16(vxi3x89ABCDEF, vxk3x89ABCDEF));
638
639 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
640 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
641 vacc89AB = _mm_add_epi32(vacc89AB, _mm_cvtepi16_epi32(vprod89ABCDEF));
642 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_srai_epi32(_mm_unpackhi_epi16(vprod89ABCDEF, vprod89ABCDEF), 16));
643
644 const __m128i vi4x01234567 = _mm_loadl_epi64((const __m128i*) i4);
645 const __m128i vxi4x01234567 = _mm_cvtepi8_epi16(vi4x01234567);
646 const __m128i vk4x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 64 * sizeof(int8_t)));
647 const __m128i vxk4x01234567 = _mm_cvtepi8_epi16(vk4x01234567);
648 const __m128i vi4x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i4 + 8));
649 const __m128i vxi4x89ABCDEF = _mm_cvtepi8_epi16(vi4x89ABCDEF);
650 const __m128i vk4x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 72 * sizeof(int8_t)));
651 const __m128i vxk4x89ABCDEF = _mm_cvtepi8_epi16(vk4x89ABCDEF);
652 i4 += 16;
653
654
655 vprod01234567 = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
656 vprod89ABCDEF = _mm_mullo_epi16(vxi4x89ABCDEF, vxk4x89ABCDEF);
657
658
659 const __m128i vi5x01234567 = _mm_loadl_epi64((const __m128i*) i5);
660 const __m128i vxi5x01234567 = _mm_cvtepi8_epi16(vi5x01234567);
661 const __m128i vk5x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 80 * sizeof(int8_t)));
662 const __m128i vxk5x01234567 = _mm_cvtepi8_epi16(vk5x01234567);
663 const __m128i vi5x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i5 + 8));
664 const __m128i vxi5x89ABCDEF = _mm_cvtepi8_epi16(vi5x89ABCDEF);
665 const __m128i vk5x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 88 * sizeof(int8_t)));
666 const __m128i vxk5x89ABCDEF = _mm_cvtepi8_epi16(vk5x89ABCDEF);
667 i5 += 16;
668
669
670 vprod01234567 = _mm_add_epi16(vprod01234567, _mm_mullo_epi16(vxi5x01234567, vxk5x01234567));
671 vprod89ABCDEF = _mm_add_epi16(vprod89ABCDEF, _mm_mullo_epi16(vxi5x89ABCDEF, vxk5x89ABCDEF));
672
673 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
674 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
675 vacc89AB = _mm_add_epi32(vacc89AB, _mm_cvtepi16_epi32(vprod89ABCDEF));
676 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_srai_epi32(_mm_unpackhi_epi16(vprod89ABCDEF, vprod89ABCDEF), 16));
677
678 const __m128i vi6x01234567 = _mm_loadl_epi64((const __m128i*) i6);
679 const __m128i vxi6x01234567 = _mm_cvtepi8_epi16(vi6x01234567);
680 const __m128i vk6x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 96 * sizeof(int8_t)));
681 const __m128i vxk6x01234567 = _mm_cvtepi8_epi16(vk6x01234567);
682 const __m128i vi6x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i6 + 8));
683 const __m128i vxi6x89ABCDEF = _mm_cvtepi8_epi16(vi6x89ABCDEF);
684 const __m128i vk6x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 104 * sizeof(int8_t)));
685 const __m128i vxk6x89ABCDEF = _mm_cvtepi8_epi16(vk6x89ABCDEF);
686 i6 += 16;
687
688
689 vprod01234567 = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
690 vprod89ABCDEF = _mm_mullo_epi16(vxi6x89ABCDEF, vxk6x89ABCDEF);
691
692 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
693 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
694 vacc89AB = _mm_add_epi32(vacc89AB, _mm_cvtepi16_epi32(vprod89ABCDEF));
695 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_srai_epi32(_mm_unpackhi_epi16(vprod89ABCDEF, vprod89ABCDEF), 16));
696
697 w = (const void*) ((uintptr_t) w + 112 * sizeof(int8_t));
698
699 __m128 vscaled0123 = _mm_cvtepi32_ps(vacc0123);
700 __m128 vscaled4567 = _mm_cvtepi32_ps(vacc4567);
701 __m128 vscaled89AB = _mm_cvtepi32_ps(vacc89AB);
702 __m128 vscaledCDEF = _mm_cvtepi32_ps(vaccCDEF);
703
704 vscaled0123 = _mm_mul_ps(vscaled0123, vscale);
705 vscaled4567 = _mm_mul_ps(vscaled4567, vscale);
706 vscaled89AB = _mm_mul_ps(vscaled89AB, vscale);
707 vscaledCDEF = _mm_mul_ps(vscaledCDEF, vscale);
708
709 vscaled0123 = _mm_min_ps(vscaled0123, voutput_max_less_zero_point);
710 vscaled4567 = _mm_min_ps(vscaled4567, voutput_max_less_zero_point);
711 vscaled89AB = _mm_min_ps(vscaled89AB, voutput_max_less_zero_point);
712 vscaledCDEF = _mm_min_ps(vscaledCDEF, voutput_max_less_zero_point);
713
714 vacc0123 = _mm_cvtps_epi32(vscaled0123);
715 vacc4567 = _mm_cvtps_epi32(vscaled4567);
716 vacc89AB = _mm_cvtps_epi32(vscaled89AB);
717 vaccCDEF = _mm_cvtps_epi32(vscaledCDEF);
718
719 __m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
720 __m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(vacc89AB, vaccCDEF), voutput_zero_point);
721
722
723 __m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
724
725 vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
726
727 _mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
728 output += 16;
729 }
730
731 if XNN_UNLIKELY(c != 0) {
732 do {
733 __m128i vacc0123 = _mm_loadu_si128((const __m128i*) b);
734 __m128i vacc4567 = _mm_loadu_si128((const __m128i*) (b + 4));
735 b += 8;
736
737
738 const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
739 const __m128i vxi0x01234567 = _mm_cvtepi8_epi16(vi0x01234567);
740 const __m128i vk0x01234567 = _mm_loadl_epi64((const __m128i*) w);
741 const __m128i vxk0x01234567 = _mm_cvtepi8_epi16(vk0x01234567);
742 i0 += 8;
743
744
745 __m128i vprod01234567 = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
746
747
748 const __m128i vi1x01234567 = _mm_loadl_epi64((const __m128i*) i1);
749 const __m128i vxi1x01234567 = _mm_cvtepi8_epi16(vi1x01234567);
750 const __m128i vk1x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
751 const __m128i vxk1x01234567 = _mm_cvtepi8_epi16(vk1x01234567);
752 i1 += 8;
753
754
755 vprod01234567 = _mm_add_epi16(vprod01234567, _mm_mullo_epi16(vxi1x01234567, vxk1x01234567));
756
757 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
758 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
759
760 const __m128i vi2x01234567 = _mm_loadl_epi64((const __m128i*) i2);
761 const __m128i vxi2x01234567 = _mm_cvtepi8_epi16(vi2x01234567);
762 const __m128i vk2x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
763 const __m128i vxk2x01234567 = _mm_cvtepi8_epi16(vk2x01234567);
764 i2 += 8;
765
766
767 vprod01234567 = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
768
769
770 const __m128i vi3x01234567 = _mm_loadl_epi64((const __m128i*) i3);
771 const __m128i vxi3x01234567 = _mm_cvtepi8_epi16(vi3x01234567);
772 const __m128i vk3x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
773 const __m128i vxk3x01234567 = _mm_cvtepi8_epi16(vk3x01234567);
774 i3 += 8;
775
776
777 vprod01234567 = _mm_add_epi16(vprod01234567, _mm_mullo_epi16(vxi3x01234567, vxk3x01234567));
778
779 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
780 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
781
782 const __m128i vi4x01234567 = _mm_loadl_epi64((const __m128i*) i4);
783 const __m128i vxi4x01234567 = _mm_cvtepi8_epi16(vi4x01234567);
784 const __m128i vk4x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t)));
785 const __m128i vxk4x01234567 = _mm_cvtepi8_epi16(vk4x01234567);
786 i4 += 8;
787
788
789 vprod01234567 = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
790
791
792 const __m128i vi5x01234567 = _mm_loadl_epi64((const __m128i*) i5);
793 const __m128i vxi5x01234567 = _mm_cvtepi8_epi16(vi5x01234567);
794 const __m128i vk5x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 40 * sizeof(int8_t)));
795 const __m128i vxk5x01234567 = _mm_cvtepi8_epi16(vk5x01234567);
796 i5 += 8;
797
798
799 vprod01234567 = _mm_add_epi16(vprod01234567, _mm_mullo_epi16(vxi5x01234567, vxk5x01234567));
800
801 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
802 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
803
804 const __m128i vi6x01234567 = _mm_loadl_epi64((const __m128i*) i6);
805 const __m128i vxi6x01234567 = _mm_cvtepi8_epi16(vi6x01234567);
806 const __m128i vk6x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t)));
807 const __m128i vxk6x01234567 = _mm_cvtepi8_epi16(vk6x01234567);
808 i6 += 8;
809
810
811 vprod01234567 = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
812
813 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
814 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
815
816 __m128 vscaled0123 = _mm_cvtepi32_ps(vacc0123);
817 __m128 vscaled4567 = _mm_cvtepi32_ps(vacc4567);
818
819 vscaled0123 = _mm_mul_ps(vscaled0123, vscale);
820 vscaled4567 = _mm_mul_ps(vscaled4567, vscale);
821
822 w = (void*) ((uintptr_t) w + 56);
823
824 vscaled0123 = _mm_min_ps(vscaled0123, voutput_max_less_zero_point);
825 vscaled4567 = _mm_min_ps(vscaled4567, voutput_max_less_zero_point);
826
827 vacc0123 = _mm_cvtps_epi32(vscaled0123);
828 vacc4567 = _mm_cvtps_epi32(vscaled4567);
829
830 __m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
831
832
833 __m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
834
835 vout0123456701234567 = _mm_max_epi8(vout0123456701234567, voutput_min);
836
837 if XNN_LIKELY(c >= 8) {
838 _mm_storel_epi64((__m128i*) output, vout0123456701234567);
839 output += 8;
840 c -= 8;
841 } else {
842 if (c & 4) {
843 unaligned_store_u32(output, (uint32_t) _mm_cvtsi128_si32(vout0123456701234567));
844 vout0123456701234567 = _mm_srli_epi64(vout0123456701234567, 32);
845 output += 4;
846 }
847 if (c & 2) {
848 unaligned_store_u16(output, (uint16_t) _mm_extract_epi16(vout0123456701234567, 0));
849 vout0123456701234567 = _mm_srli_epi32(vout0123456701234567, 16);
850 output += 2;
851 }
852 if (c & 1) {
853 *output = (int8_t) _mm_extract_epi8(vout0123456701234567, 0);
854 output += 1;
855 }
856 c = 0;
857 }
858 } while (c != 0);
859 }
860 }
861
862 input = (const int8_t**) ((uintptr_t) input + input_stride);
863 output = (int8_t*) ((uintptr_t) output + output_increment);
864 } while (--output_width != 0);
865 }