Debian Codegraph @

src/qs8-dwconv/gen/qs8-dwconv-6f6m7l16c8s8r-minmax-fp32-sse41-mul16.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-dwconv/multipass-sse-mul16.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include <smmintrin.h>
13
14 #include "xnnpack/dwconv.h"
15 #include "xnnpack/math.h"
16 #include "xnnpack/unaligned.h"
17
18
19 void xnn_qs8_dwconv_minmax_fp32_ukernel_6f6m7l16c8s8r__sse41_mul16(
20 size_t channels,
21 size_t output_width,
22 const int8_t** input,
23 const void* weights,
24 int8_t* output,
25 intptr_t input_stride,
26 size_t output_increment,
27 size_t input_offset,
28 const int8_t* zero,
29 size_t kernel_size,
30 int32_t* buffer,
31 const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
32 {
33 assert(channels != 0);
34 assert(output_width != 0);
35 assert(kernel_size > 6);
36
37
38 const __m128 vscale = _mm_set1_ps(params->fp32_scalar.scale);
39 XNN_FORCE_REALIZATION(vscale);
40
41 const __m128 voutput_max_less_zero_point = _mm_set1_ps((int32_t) params->fp32_scalar.output_max - (int32_t) params->fp32_scalar.output_zero_point);
42 const __m128i voutput_zero_point = _mm_set1_epi16(params->fp32_scalar.output_zero_point);
43 const __m128i voutput_min = _mm_set1_epi8(params->fp32_scalar.output_min);
44 XNN_FORCE_REALIZATION(voutput_max_less_zero_point);
45 XNN_FORCE_REALIZATION(voutput_zero_point);
46 XNN_FORCE_REALIZATION(voutput_min);
47
48 do {
49 const void* w = weights;
50
51 // First pass to process 6 inputs.
52 {
53 int32_t* b = buffer;
54 const int8_t* i0 = input[0];
55 assert(i0 != NULL);
56 if XNN_UNPREDICTABLE(i0 != zero) {
57 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
58 }
59 const int8_t* i1 = input[1];
60 assert(i1 != NULL);
61 if XNN_UNPREDICTABLE(i1 != zero) {
62 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
63 }
64 const int8_t* i2 = input[2];
65 assert(i2 != NULL);
66 if XNN_UNPREDICTABLE(i2 != zero) {
67 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
68 }
69 const int8_t* i3 = input[3];
70 assert(i3 != NULL);
71 if XNN_UNPREDICTABLE(i3 != zero) {
72 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
73 }
74 const int8_t* i4 = input[4];
75 assert(i4 != NULL);
76 if XNN_UNPREDICTABLE(i4 != zero) {
77 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
78 }
79 const int8_t* i5 = input[5];
80 assert(i5 != NULL);
81 if XNN_UNPREDICTABLE(i5 != zero) {
82 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
83 }
84 input += 6;
85
86 size_t c = round_up_po2(channels, 8);
87
88 for (; c >= 16; c -= 16) {
89 __m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
90 __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
91 __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
92 __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
93
94
95
96 const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
97 const __m128i vxi0x01234567 = _mm_cvtepi8_epi16(vi0x01234567);
98 const __m128i vk0x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 0 * sizeof(int8_t)));
99 const __m128i vxk0x01234567 = _mm_cvtepi8_epi16(vk0x01234567);
100 const __m128i vi0x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i0 + 8));
101 const __m128i vxi0x89ABCDEF = _mm_cvtepi8_epi16(vi0x89ABCDEF);
102 const __m128i vk0x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 8 * sizeof(int8_t)));
103 const __m128i vxk0x89ABCDEF = _mm_cvtepi8_epi16(vk0x89ABCDEF);
104 i0 += 16;
105
106
107 __m128i vprod01234567 = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
108 __m128i vprod89ABCDEF = _mm_mullo_epi16(vxi0x89ABCDEF, vxk0x89ABCDEF);
109
110 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
111 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
112 vacc89AB = _mm_add_epi32(vacc89AB, _mm_cvtepi16_epi32(vprod89ABCDEF));
113 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_srai_epi32(_mm_unpackhi_epi16(vprod89ABCDEF, vprod89ABCDEF), 16));
114
115 const __m128i vi1x01234567 = _mm_loadl_epi64((const __m128i*) i1);
116 const __m128i vxi1x01234567 = _mm_cvtepi8_epi16(vi1x01234567);
117 const __m128i vk1x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 16 * sizeof(int8_t)));
118 const __m128i vxk1x01234567 = _mm_cvtepi8_epi16(vk1x01234567);
119 const __m128i vi1x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i1 + 8));
120 const __m128i vxi1x89ABCDEF = _mm_cvtepi8_epi16(vi1x89ABCDEF);
121 const __m128i vk1x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 24 * sizeof(int8_t)));
122 const __m128i vxk1x89ABCDEF = _mm_cvtepi8_epi16(vk1x89ABCDEF);
123 i1 += 16;
124
125
126 vprod01234567 = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
127 vprod89ABCDEF = _mm_mullo_epi16(vxi1x89ABCDEF, vxk1x89ABCDEF);
128
129 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
130 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
131 vacc89AB = _mm_add_epi32(vacc89AB, _mm_cvtepi16_epi32(vprod89ABCDEF));
132 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_srai_epi32(_mm_unpackhi_epi16(vprod89ABCDEF, vprod89ABCDEF), 16));
133
134 const __m128i vi2x01234567 = _mm_loadl_epi64((const __m128i*) i2);
135 const __m128i vxi2x01234567 = _mm_cvtepi8_epi16(vi2x01234567);
136 const __m128i vk2x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 32 * sizeof(int8_t)));
137 const __m128i vxk2x01234567 = _mm_cvtepi8_epi16(vk2x01234567);
138 const __m128i vi2x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i2 + 8));
139 const __m128i vxi2x89ABCDEF = _mm_cvtepi8_epi16(vi2x89ABCDEF);
140 const __m128i vk2x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 40 * sizeof(int8_t)));
141 const __m128i vxk2x89ABCDEF = _mm_cvtepi8_epi16(vk2x89ABCDEF);
142 i2 += 16;
143
144
145 vprod01234567 = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
146 vprod89ABCDEF = _mm_mullo_epi16(vxi2x89ABCDEF, vxk2x89ABCDEF);
147
148 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
149 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
150 vacc89AB = _mm_add_epi32(vacc89AB, _mm_cvtepi16_epi32(vprod89ABCDEF));
151 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_srai_epi32(_mm_unpackhi_epi16(vprod89ABCDEF, vprod89ABCDEF), 16));
152
153 const __m128i vi3x01234567 = _mm_loadl_epi64((const __m128i*) i3);
154 const __m128i vxi3x01234567 = _mm_cvtepi8_epi16(vi3x01234567);
155 const __m128i vk3x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 48 * sizeof(int8_t)));
156 const __m128i vxk3x01234567 = _mm_cvtepi8_epi16(vk3x01234567);
157 const __m128i vi3x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i3 + 8));
158 const __m128i vxi3x89ABCDEF = _mm_cvtepi8_epi16(vi3x89ABCDEF);
159 const __m128i vk3x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 56 * sizeof(int8_t)));
160 const __m128i vxk3x89ABCDEF = _mm_cvtepi8_epi16(vk3x89ABCDEF);
161 i3 += 16;
162
163
164 vprod01234567 = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
165 vprod89ABCDEF = _mm_mullo_epi16(vxi3x89ABCDEF, vxk3x89ABCDEF);
166
167 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
168 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
169 vacc89AB = _mm_add_epi32(vacc89AB, _mm_cvtepi16_epi32(vprod89ABCDEF));
170 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_srai_epi32(_mm_unpackhi_epi16(vprod89ABCDEF, vprod89ABCDEF), 16));
171
172 const __m128i vi4x01234567 = _mm_loadl_epi64((const __m128i*) i4);
173 const __m128i vxi4x01234567 = _mm_cvtepi8_epi16(vi4x01234567);
174 const __m128i vk4x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 64 * sizeof(int8_t)));
175 const __m128i vxk4x01234567 = _mm_cvtepi8_epi16(vk4x01234567);
176 const __m128i vi4x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i4 + 8));
177 const __m128i vxi4x89ABCDEF = _mm_cvtepi8_epi16(vi4x89ABCDEF);
178 const __m128i vk4x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 72 * sizeof(int8_t)));
179 const __m128i vxk4x89ABCDEF = _mm_cvtepi8_epi16(vk4x89ABCDEF);
180 i4 += 16;
181
182
183 vprod01234567 = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
184 vprod89ABCDEF = _mm_mullo_epi16(vxi4x89ABCDEF, vxk4x89ABCDEF);
185
186 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
187 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
188 vacc89AB = _mm_add_epi32(vacc89AB, _mm_cvtepi16_epi32(vprod89ABCDEF));
189 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_srai_epi32(_mm_unpackhi_epi16(vprod89ABCDEF, vprod89ABCDEF), 16));
190
191 const __m128i vi5x01234567 = _mm_loadl_epi64((const __m128i*) i5);
192 const __m128i vxi5x01234567 = _mm_cvtepi8_epi16(vi5x01234567);
193 const __m128i vk5x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 80 * sizeof(int8_t)));
194 const __m128i vxk5x01234567 = _mm_cvtepi8_epi16(vk5x01234567);
195 const __m128i vi5x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i5 + 8));
196 const __m128i vxi5x89ABCDEF = _mm_cvtepi8_epi16(vi5x89ABCDEF);
197 const __m128i vk5x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 88 * sizeof(int8_t)));
198 const __m128i vxk5x89ABCDEF = _mm_cvtepi8_epi16(vk5x89ABCDEF);
199 i5 += 16;
200
201
202 vprod01234567 = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
203 vprod89ABCDEF = _mm_mullo_epi16(vxi5x89ABCDEF, vxk5x89ABCDEF);
204
205 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
206 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
207 vacc89AB = _mm_add_epi32(vacc89AB, _mm_cvtepi16_epi32(vprod89ABCDEF));
208 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_srai_epi32(_mm_unpackhi_epi16(vprod89ABCDEF, vprod89ABCDEF), 16));
209
210 w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 96 * sizeof(int8_t));
211
212 _mm_storeu_si128((__m128i*) b, vacc0123);
213 _mm_storeu_si128((__m128i*) (b + 4), vacc4567);
214 _mm_storeu_si128((__m128i*) (b + 8), vacc89AB);
215 _mm_storeu_si128((__m128i*) (b + 12), vaccCDEF);
216 b += 16;
217 }
218
219 if XNN_UNLIKELY(c != 0) {
220 do {
221 __m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
222 __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
223
224
225 const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
226 const __m128i vxi0x01234567 = _mm_cvtepi8_epi16(vi0x01234567);
227 const __m128i vk0x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + sizeof(int32_t) * 8 * sizeof(int8_t)));
228 const __m128i vxk0x01234567 = _mm_cvtepi8_epi16(vk0x01234567);
229 i0 += 8;
230
231
232 __m128i vprod01234567 = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
233
234 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
235 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
236
237 const __m128i vi1x01234567 = _mm_loadl_epi64((const __m128i*) i1);
238 const __m128i vxi1x01234567 = _mm_cvtepi8_epi16(vi1x01234567);
239 const __m128i vk1x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + sizeof(int32_t) * 8 + 8 * sizeof(int8_t)));
240 const __m128i vxk1x01234567 = _mm_cvtepi8_epi16(vk1x01234567);
241 i1 += 8;
242
243
244 vprod01234567 = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
245
246 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
247 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
248
249 const __m128i vi2x01234567 = _mm_loadl_epi64((const __m128i*) i2);
250 const __m128i vxi2x01234567 = _mm_cvtepi8_epi16(vi2x01234567);
251 const __m128i vk2x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + sizeof(int32_t) * 8 + 16 * sizeof(int8_t)));
252 const __m128i vxk2x01234567 = _mm_cvtepi8_epi16(vk2x01234567);
253 i2 += 8;
254
255
256 vprod01234567 = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
257
258 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
259 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
260
261 const __m128i vi3x01234567 = _mm_loadl_epi64((const __m128i*) i3);
262 const __m128i vxi3x01234567 = _mm_cvtepi8_epi16(vi3x01234567);
263 const __m128i vk3x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + sizeof(int32_t) * 8 + 24 * sizeof(int8_t)));
264 const __m128i vxk3x01234567 = _mm_cvtepi8_epi16(vk3x01234567);
265 i3 += 8;
266
267
268 vprod01234567 = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
269
270 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
271 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
272
273 const __m128i vi4x01234567 = _mm_loadl_epi64((const __m128i*) i4);
274 const __m128i vxi4x01234567 = _mm_cvtepi8_epi16(vi4x01234567);
275 const __m128i vk4x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + sizeof(int32_t) * 8 + 32 * sizeof(int8_t)));
276 const __m128i vxk4x01234567 = _mm_cvtepi8_epi16(vk4x01234567);
277 i4 += 8;
278
279
280 vprod01234567 = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
281
282 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
283 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
284
285 const __m128i vi5x01234567 = _mm_loadl_epi64((const __m128i*) i5);
286 const __m128i vxi5x01234567 = _mm_cvtepi8_epi16(vi5x01234567);
287 const __m128i vk5x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + sizeof(int32_t) * 8 + 40 * sizeof(int8_t)));
288 const __m128i vxk5x01234567 = _mm_cvtepi8_epi16(vk5x01234567);
289 i5 += 8;
290
291
292 vprod01234567 = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
293
294 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
295 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
296
297 w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 48 * sizeof(int8_t));
298
299 _mm_storeu_si128((__m128i*) (b), vacc0123);
300 _mm_storeu_si128((__m128i*) (b + 4), vacc4567);
301 b += 8;
302 c -= 8;
303 } while (c != 0);
304 }
305 }
306
307 // Middle pass to process 6 inputs in each iteration.
308 for (size_t ks = kernel_size - 6; ks > 7; ks -= 6) {
309 int32_t* b = buffer;
310 const int8_t* i0 = input[0];
311 assert(i0 != NULL);
312 if XNN_UNPREDICTABLE(i0 != zero) {
313 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
314 }
315 const int8_t* i1 = input[1];
316 assert(i1 != NULL);
317 if XNN_UNPREDICTABLE(i1 != zero) {
318 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
319 }
320 const int8_t* i2 = input[2];
321 assert(i2 != NULL);
322 if XNN_UNPREDICTABLE(i2 != zero) {
323 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
324 }
325 const int8_t* i3 = input[3];
326 assert(i3 != NULL);
327 if XNN_UNPREDICTABLE(i3 != zero) {
328 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
329 }
330 const int8_t* i4 = input[4];
331 assert(i4 != NULL);
332 if XNN_UNPREDICTABLE(i4 != zero) {
333 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
334 }
335 const int8_t* i5 = input[5];
336 assert(i5 != NULL);
337 if XNN_UNPREDICTABLE(i5 != zero) {
338 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
339 }
340 input += 6;
341
342 size_t c = round_up_po2(channels, 8);
343
344 for (; c >= 16; c -= 16) {
345 __m128i vacc0123 = _mm_loadu_si128((const __m128i*) b);
346 __m128i vacc4567 = _mm_loadu_si128((const __m128i*) (b + 4));
347 __m128i vacc89AB = _mm_loadu_si128((const __m128i*) (b + 8));
348 __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) (b + 12));
349
350
351
352 const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
353 const __m128i vxi0x01234567 = _mm_cvtepi8_epi16(vi0x01234567);
354 const __m128i vk0x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 0 * sizeof(int8_t)));
355 const __m128i vxk0x01234567 = _mm_cvtepi8_epi16(vk0x01234567);
356 const __m128i vi0x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i0 + 8));
357 const __m128i vxi0x89ABCDEF = _mm_cvtepi8_epi16(vi0x89ABCDEF);
358 const __m128i vk0x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
359 const __m128i vxk0x89ABCDEF = _mm_cvtepi8_epi16(vk0x89ABCDEF);
360 i0 += 16;
361
362
363 __m128i vprod01234567 = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
364 __m128i vprod89ABCDEF = _mm_mullo_epi16(vxi0x89ABCDEF, vxk0x89ABCDEF);
365
366 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
367 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
368 vacc89AB = _mm_add_epi32(vacc89AB, _mm_cvtepi16_epi32(vprod89ABCDEF));
369 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_srai_epi32(_mm_unpackhi_epi16(vprod89ABCDEF, vprod89ABCDEF), 16));
370
371 const __m128i vi1x01234567 = _mm_loadl_epi64((const __m128i*) i1);
372 const __m128i vxi1x01234567 = _mm_cvtepi8_epi16(vi1x01234567);
373 const __m128i vk1x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
374 const __m128i vxk1x01234567 = _mm_cvtepi8_epi16(vk1x01234567);
375 const __m128i vi1x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i1 + 8));
376 const __m128i vxi1x89ABCDEF = _mm_cvtepi8_epi16(vi1x89ABCDEF);
377 const __m128i vk1x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
378 const __m128i vxk1x89ABCDEF = _mm_cvtepi8_epi16(vk1x89ABCDEF);
379 i1 += 16;
380
381
382 vprod01234567 = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
383 vprod89ABCDEF = _mm_mullo_epi16(vxi1x89ABCDEF, vxk1x89ABCDEF);
384
385 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
386 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
387 vacc89AB = _mm_add_epi32(vacc89AB, _mm_cvtepi16_epi32(vprod89ABCDEF));
388 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_srai_epi32(_mm_unpackhi_epi16(vprod89ABCDEF, vprod89ABCDEF), 16));
389
390 const __m128i vi2x01234567 = _mm_loadl_epi64((const __m128i*) i2);
391 const __m128i vxi2x01234567 = _mm_cvtepi8_epi16(vi2x01234567);
392 const __m128i vk2x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t)));
393 const __m128i vxk2x01234567 = _mm_cvtepi8_epi16(vk2x01234567);
394 const __m128i vi2x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i2 + 8));
395 const __m128i vxi2x89ABCDEF = _mm_cvtepi8_epi16(vi2x89ABCDEF);
396 const __m128i vk2x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 40 * sizeof(int8_t)));
397 const __m128i vxk2x89ABCDEF = _mm_cvtepi8_epi16(vk2x89ABCDEF);
398 i2 += 16;
399
400
401 vprod01234567 = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
402 vprod89ABCDEF = _mm_mullo_epi16(vxi2x89ABCDEF, vxk2x89ABCDEF);
403
404 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
405 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
406 vacc89AB = _mm_add_epi32(vacc89AB, _mm_cvtepi16_epi32(vprod89ABCDEF));
407 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_srai_epi32(_mm_unpackhi_epi16(vprod89ABCDEF, vprod89ABCDEF), 16));
408
409 const __m128i vi3x01234567 = _mm_loadl_epi64((const __m128i*) i3);
410 const __m128i vxi3x01234567 = _mm_cvtepi8_epi16(vi3x01234567);
411 const __m128i vk3x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t)));
412 const __m128i vxk3x01234567 = _mm_cvtepi8_epi16(vk3x01234567);
413 const __m128i vi3x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i3 + 8));
414 const __m128i vxi3x89ABCDEF = _mm_cvtepi8_epi16(vi3x89ABCDEF);
415 const __m128i vk3x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 56 * sizeof(int8_t)));
416 const __m128i vxk3x89ABCDEF = _mm_cvtepi8_epi16(vk3x89ABCDEF);
417 i3 += 16;
418
419
420 vprod01234567 = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
421 vprod89ABCDEF = _mm_mullo_epi16(vxi3x89ABCDEF, vxk3x89ABCDEF);
422
423 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
424 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
425 vacc89AB = _mm_add_epi32(vacc89AB, _mm_cvtepi16_epi32(vprod89ABCDEF));
426 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_srai_epi32(_mm_unpackhi_epi16(vprod89ABCDEF, vprod89ABCDEF), 16));
427
428 const __m128i vi4x01234567 = _mm_loadl_epi64((const __m128i*) i4);
429 const __m128i vxi4x01234567 = _mm_cvtepi8_epi16(vi4x01234567);
430 const __m128i vk4x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 64 * sizeof(int8_t)));
431 const __m128i vxk4x01234567 = _mm_cvtepi8_epi16(vk4x01234567);
432 const __m128i vi4x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i4 + 8));
433 const __m128i vxi4x89ABCDEF = _mm_cvtepi8_epi16(vi4x89ABCDEF);
434 const __m128i vk4x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 72 * sizeof(int8_t)));
435 const __m128i vxk4x89ABCDEF = _mm_cvtepi8_epi16(vk4x89ABCDEF);
436 i4 += 16;
437
438
439 vprod01234567 = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
440 vprod89ABCDEF = _mm_mullo_epi16(vxi4x89ABCDEF, vxk4x89ABCDEF);
441
442 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
443 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
444 vacc89AB = _mm_add_epi32(vacc89AB, _mm_cvtepi16_epi32(vprod89ABCDEF));
445 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_srai_epi32(_mm_unpackhi_epi16(vprod89ABCDEF, vprod89ABCDEF), 16));
446
447 const __m128i vi5x01234567 = _mm_loadl_epi64((const __m128i*) i5);
448 const __m128i vxi5x01234567 = _mm_cvtepi8_epi16(vi5x01234567);
449 const __m128i vk5x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 80 * sizeof(int8_t)));
450 const __m128i vxk5x01234567 = _mm_cvtepi8_epi16(vk5x01234567);
451 const __m128i vi5x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i5 + 8));
452 const __m128i vxi5x89ABCDEF = _mm_cvtepi8_epi16(vi5x89ABCDEF);
453 const __m128i vk5x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 88 * sizeof(int8_t)));
454 const __m128i vxk5x89ABCDEF = _mm_cvtepi8_epi16(vk5x89ABCDEF);
455 i5 += 16;
456
457
458 vprod01234567 = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
459 vprod89ABCDEF = _mm_mullo_epi16(vxi5x89ABCDEF, vxk5x89ABCDEF);
460
461 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
462 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
463 vacc89AB = _mm_add_epi32(vacc89AB, _mm_cvtepi16_epi32(vprod89ABCDEF));
464 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_srai_epi32(_mm_unpackhi_epi16(vprod89ABCDEF, vprod89ABCDEF), 16));
465
466 w = (const void*) ((uintptr_t) w + 96 * sizeof(int8_t));
467
468 _mm_storeu_si128((__m128i*) (b), vacc0123);
469 _mm_storeu_si128((__m128i*) (b + 4), vacc4567);
470 _mm_storeu_si128((__m128i*) (b + 8), vacc89AB);
471 _mm_storeu_si128((__m128i*) (b + 12), vaccCDEF);
472 b += 16;
473 }
474
475 if XNN_UNLIKELY(c != 0) {
476 do {
477 __m128i vacc0123 = _mm_loadu_si128((const __m128i*) b);
478 __m128i vacc4567 = _mm_loadu_si128((const __m128i*) (b + 4));
479
480
481 const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
482 const __m128i vxi0x01234567 = _mm_cvtepi8_epi16(vi0x01234567);
483 const __m128i vk0x01234567 = _mm_loadl_epi64((const __m128i*) w);
484 const __m128i vxk0x01234567 = _mm_cvtepi8_epi16(vk0x01234567);
485 i0 += 8;
486
487
488 __m128i vprod01234567 = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
489
490 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
491 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
492
493 const __m128i vi1x01234567 = _mm_loadl_epi64((const __m128i*) i1);
494 const __m128i vxi1x01234567 = _mm_cvtepi8_epi16(vi1x01234567);
495 const __m128i vk1x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
496 const __m128i vxk1x01234567 = _mm_cvtepi8_epi16(vk1x01234567);
497 i1 += 8;
498
499
500 vprod01234567 = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
501
502 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
503 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
504
505 const __m128i vi2x01234567 = _mm_loadl_epi64((const __m128i*) i2);
506 const __m128i vxi2x01234567 = _mm_cvtepi8_epi16(vi2x01234567);
507 const __m128i vk2x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
508 const __m128i vxk2x01234567 = _mm_cvtepi8_epi16(vk2x01234567);
509 i2 += 8;
510
511
512 vprod01234567 = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
513
514 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
515 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
516
517 const __m128i vi3x01234567 = _mm_loadl_epi64((const __m128i*) i3);
518 const __m128i vxi3x01234567 = _mm_cvtepi8_epi16(vi3x01234567);
519 const __m128i vk3x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
520 const __m128i vxk3x01234567 = _mm_cvtepi8_epi16(vk3x01234567);
521 i3 += 8;
522
523
524 vprod01234567 = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
525
526 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
527 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
528
529 const __m128i vi4x01234567 = _mm_loadl_epi64((const __m128i*) i4);
530 const __m128i vxi4x01234567 = _mm_cvtepi8_epi16(vi4x01234567);
531 const __m128i vk4x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t)));
532 const __m128i vxk4x01234567 = _mm_cvtepi8_epi16(vk4x01234567);
533 i4 += 8;
534
535
536 vprod01234567 = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
537
538 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
539 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
540
541 const __m128i vi5x01234567 = _mm_loadl_epi64((const __m128i*) i5);
542 const __m128i vxi5x01234567 = _mm_cvtepi8_epi16(vi5x01234567);
543 const __m128i vk5x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 40 * sizeof(int8_t)));
544 const __m128i vxk5x01234567 = _mm_cvtepi8_epi16(vk5x01234567);
545 i5 += 8;
546
547
548 vprod01234567 = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
549
550 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
551 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
552
553 w = (const void*) ((uintptr_t) w + 48 * sizeof(int8_t));
554
555 _mm_storeu_si128((__m128i*) (b), vacc0123);
556 _mm_storeu_si128((__m128i*) (b + 4), vacc4567);
557 b += 8;
558 c -= 8;
559 } while (c != 0);
560 }
561 }
562
563 // Last pass to process up to 7 inputs.
564 {
565 const int32_t* b = buffer;
566 const int8_t* i0 = input[0];
567 assert(i0 != NULL);
568 if XNN_UNPREDICTABLE(i0 != zero) {
569 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
570 }
571 const int8_t* i1 = input[1];
572 assert(i1 != NULL);
573 if XNN_UNPREDICTABLE(i1 != zero) {
574 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
575 }
576 const int8_t* i2 = input[2];
577 assert(i2 != NULL);
578 if XNN_UNPREDICTABLE(i2 != zero) {
579 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
580 }
581 const int8_t* i3 = input[3];
582 assert(i3 != NULL);
583 if XNN_UNPREDICTABLE(i3 != zero) {
584 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
585 }
586 const int8_t* i4 = input[4];
587 assert(i4 != NULL);
588 if XNN_UNPREDICTABLE(i4 != zero) {
589 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
590 }
591 const int8_t* i5 = input[5];
592 assert(i5 != NULL);
593 if XNN_UNPREDICTABLE(i5 != zero) {
594 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
595 }
596 const int8_t* i6 = input[6];
597 assert(i6 != NULL);
598 if XNN_UNPREDICTABLE(i6 != zero) {
599 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
600 }
601
602 size_t c = channels;
603
604 for (; c >= 16; c -= 16) {
605 __m128i vacc0123 = _mm_loadu_si128((const __m128i*) b);
606 __m128i vacc4567 = _mm_loadu_si128((const __m128i*) (b + 4));
607 __m128i vacc89AB = _mm_loadu_si128((const __m128i*) (b + 8));
608 __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) (b + 12));
609 b += 16;
610
611
612 const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
613 const __m128i vxi0x01234567 = _mm_cvtepi8_epi16(vi0x01234567);
614 const __m128i vk0x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 0 * sizeof(int8_t)));
615 const __m128i vxk0x01234567 = _mm_cvtepi8_epi16(vk0x01234567);
616 const __m128i vi0x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i0 + 8));
617 const __m128i vxi0x89ABCDEF = _mm_cvtepi8_epi16(vi0x89ABCDEF);
618 const __m128i vk0x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
619 const __m128i vxk0x89ABCDEF = _mm_cvtepi8_epi16(vk0x89ABCDEF);
620 i0 += 16;
621
622
623 __m128i vprod01234567 = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
624 __m128i vprod89ABCDEF = _mm_mullo_epi16(vxi0x89ABCDEF, vxk0x89ABCDEF);
625
626 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
627 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
628 vacc89AB = _mm_add_epi32(vacc89AB, _mm_cvtepi16_epi32(vprod89ABCDEF));
629 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_srai_epi32(_mm_unpackhi_epi16(vprod89ABCDEF, vprod89ABCDEF), 16));
630
631 const __m128i vi1x01234567 = _mm_loadl_epi64((const __m128i*) i1);
632 const __m128i vxi1x01234567 = _mm_cvtepi8_epi16(vi1x01234567);
633 const __m128i vk1x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
634 const __m128i vxk1x01234567 = _mm_cvtepi8_epi16(vk1x01234567);
635 const __m128i vi1x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i1 + 8));
636 const __m128i vxi1x89ABCDEF = _mm_cvtepi8_epi16(vi1x89ABCDEF);
637 const __m128i vk1x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
638 const __m128i vxk1x89ABCDEF = _mm_cvtepi8_epi16(vk1x89ABCDEF);
639 i1 += 16;
640
641
642 vprod01234567 = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
643 vprod89ABCDEF = _mm_mullo_epi16(vxi1x89ABCDEF, vxk1x89ABCDEF);
644
645 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
646 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
647 vacc89AB = _mm_add_epi32(vacc89AB, _mm_cvtepi16_epi32(vprod89ABCDEF));
648 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_srai_epi32(_mm_unpackhi_epi16(vprod89ABCDEF, vprod89ABCDEF), 16));
649
650 const __m128i vi2x01234567 = _mm_loadl_epi64((const __m128i*) i2);
651 const __m128i vxi2x01234567 = _mm_cvtepi8_epi16(vi2x01234567);
652 const __m128i vk2x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t)));
653 const __m128i vxk2x01234567 = _mm_cvtepi8_epi16(vk2x01234567);
654 const __m128i vi2x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i2 + 8));
655 const __m128i vxi2x89ABCDEF = _mm_cvtepi8_epi16(vi2x89ABCDEF);
656 const __m128i vk2x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 40 * sizeof(int8_t)));
657 const __m128i vxk2x89ABCDEF = _mm_cvtepi8_epi16(vk2x89ABCDEF);
658 i2 += 16;
659
660
661 vprod01234567 = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
662 vprod89ABCDEF = _mm_mullo_epi16(vxi2x89ABCDEF, vxk2x89ABCDEF);
663
664 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
665 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
666 vacc89AB = _mm_add_epi32(vacc89AB, _mm_cvtepi16_epi32(vprod89ABCDEF));
667 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_srai_epi32(_mm_unpackhi_epi16(vprod89ABCDEF, vprod89ABCDEF), 16));
668
669 const __m128i vi3x01234567 = _mm_loadl_epi64((const __m128i*) i3);
670 const __m128i vxi3x01234567 = _mm_cvtepi8_epi16(vi3x01234567);
671 const __m128i vk3x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t)));
672 const __m128i vxk3x01234567 = _mm_cvtepi8_epi16(vk3x01234567);
673 const __m128i vi3x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i3 + 8));
674 const __m128i vxi3x89ABCDEF = _mm_cvtepi8_epi16(vi3x89ABCDEF);
675 const __m128i vk3x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 56 * sizeof(int8_t)));
676 const __m128i vxk3x89ABCDEF = _mm_cvtepi8_epi16(vk3x89ABCDEF);
677 i3 += 16;
678
679
680 vprod01234567 = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
681 vprod89ABCDEF = _mm_mullo_epi16(vxi3x89ABCDEF, vxk3x89ABCDEF);
682
683 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
684 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
685 vacc89AB = _mm_add_epi32(vacc89AB, _mm_cvtepi16_epi32(vprod89ABCDEF));
686 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_srai_epi32(_mm_unpackhi_epi16(vprod89ABCDEF, vprod89ABCDEF), 16));
687
688 const __m128i vi4x01234567 = _mm_loadl_epi64((const __m128i*) i4);
689 const __m128i vxi4x01234567 = _mm_cvtepi8_epi16(vi4x01234567);
690 const __m128i vk4x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 64 * sizeof(int8_t)));
691 const __m128i vxk4x01234567 = _mm_cvtepi8_epi16(vk4x01234567);
692 const __m128i vi4x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i4 + 8));
693 const __m128i vxi4x89ABCDEF = _mm_cvtepi8_epi16(vi4x89ABCDEF);
694 const __m128i vk4x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 72 * sizeof(int8_t)));
695 const __m128i vxk4x89ABCDEF = _mm_cvtepi8_epi16(vk4x89ABCDEF);
696 i4 += 16;
697
698
699 vprod01234567 = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
700 vprod89ABCDEF = _mm_mullo_epi16(vxi4x89ABCDEF, vxk4x89ABCDEF);
701
702 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
703 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
704 vacc89AB = _mm_add_epi32(vacc89AB, _mm_cvtepi16_epi32(vprod89ABCDEF));
705 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_srai_epi32(_mm_unpackhi_epi16(vprod89ABCDEF, vprod89ABCDEF), 16));
706
707 const __m128i vi5x01234567 = _mm_loadl_epi64((const __m128i*) i5);
708 const __m128i vxi5x01234567 = _mm_cvtepi8_epi16(vi5x01234567);
709 const __m128i vk5x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 80 * sizeof(int8_t)));
710 const __m128i vxk5x01234567 = _mm_cvtepi8_epi16(vk5x01234567);
711 const __m128i vi5x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i5 + 8));
712 const __m128i vxi5x89ABCDEF = _mm_cvtepi8_epi16(vi5x89ABCDEF);
713 const __m128i vk5x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 88 * sizeof(int8_t)));
714 const __m128i vxk5x89ABCDEF = _mm_cvtepi8_epi16(vk5x89ABCDEF);
715 i5 += 16;
716
717
718 vprod01234567 = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
719 vprod89ABCDEF = _mm_mullo_epi16(vxi5x89ABCDEF, vxk5x89ABCDEF);
720
721 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
722 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
723 vacc89AB = _mm_add_epi32(vacc89AB, _mm_cvtepi16_epi32(vprod89ABCDEF));
724 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_srai_epi32(_mm_unpackhi_epi16(vprod89ABCDEF, vprod89ABCDEF), 16));
725
726 const __m128i vi6x01234567 = _mm_loadl_epi64((const __m128i*) i6);
727 const __m128i vxi6x01234567 = _mm_cvtepi8_epi16(vi6x01234567);
728 const __m128i vk6x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 96 * sizeof(int8_t)));
729 const __m128i vxk6x01234567 = _mm_cvtepi8_epi16(vk6x01234567);
730 const __m128i vi6x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i6 + 8));
731 const __m128i vxi6x89ABCDEF = _mm_cvtepi8_epi16(vi6x89ABCDEF);
732 const __m128i vk6x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 104 * sizeof(int8_t)));
733 const __m128i vxk6x89ABCDEF = _mm_cvtepi8_epi16(vk6x89ABCDEF);
734 i6 += 16;
735
736
737 vprod01234567 = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
738 vprod89ABCDEF = _mm_mullo_epi16(vxi6x89ABCDEF, vxk6x89ABCDEF);
739
740 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
741 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
742 vacc89AB = _mm_add_epi32(vacc89AB, _mm_cvtepi16_epi32(vprod89ABCDEF));
743 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_srai_epi32(_mm_unpackhi_epi16(vprod89ABCDEF, vprod89ABCDEF), 16));
744
745 w = (const void*) ((uintptr_t) w + 112 * sizeof(int8_t));
746
747 __m128 vscaled0123 = _mm_cvtepi32_ps(vacc0123);
748 __m128 vscaled4567 = _mm_cvtepi32_ps(vacc4567);
749 __m128 vscaled89AB = _mm_cvtepi32_ps(vacc89AB);
750 __m128 vscaledCDEF = _mm_cvtepi32_ps(vaccCDEF);
751
752 vscaled0123 = _mm_mul_ps(vscaled0123, vscale);
753 vscaled4567 = _mm_mul_ps(vscaled4567, vscale);
754 vscaled89AB = _mm_mul_ps(vscaled89AB, vscale);
755 vscaledCDEF = _mm_mul_ps(vscaledCDEF, vscale);
756
757 vscaled0123 = _mm_min_ps(vscaled0123, voutput_max_less_zero_point);
758 vscaled4567 = _mm_min_ps(vscaled4567, voutput_max_less_zero_point);
759 vscaled89AB = _mm_min_ps(vscaled89AB, voutput_max_less_zero_point);
760 vscaledCDEF = _mm_min_ps(vscaledCDEF, voutput_max_less_zero_point);
761
762 vacc0123 = _mm_cvtps_epi32(vscaled0123);
763 vacc4567 = _mm_cvtps_epi32(vscaled4567);
764 vacc89AB = _mm_cvtps_epi32(vscaled89AB);
765 vaccCDEF = _mm_cvtps_epi32(vscaledCDEF);
766
767 __m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
768 __m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(vacc89AB, vaccCDEF), voutput_zero_point);
769
770
771 __m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
772
773 vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
774
775 _mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
776 output += 16;
777 }
778
779 if XNN_UNLIKELY(c != 0) {
780 do {
781 __m128i vacc0123 = _mm_loadu_si128((const __m128i*) b);
782 __m128i vacc4567 = _mm_loadu_si128((const __m128i*) (b + 4));
783 b += 8;
784
785
786 const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
787 const __m128i vxi0x01234567 = _mm_cvtepi8_epi16(vi0x01234567);
788 const __m128i vk0x01234567 = _mm_loadl_epi64((const __m128i*) w);
789 const __m128i vxk0x01234567 = _mm_cvtepi8_epi16(vk0x01234567);
790 i0 += 8;
791
792
793 __m128i vprod01234567 = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
794
795 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
796 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
797
798 const __m128i vi1x01234567 = _mm_loadl_epi64((const __m128i*) i1);
799 const __m128i vxi1x01234567 = _mm_cvtepi8_epi16(vi1x01234567);
800 const __m128i vk1x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
801 const __m128i vxk1x01234567 = _mm_cvtepi8_epi16(vk1x01234567);
802 i1 += 8;
803
804
805 vprod01234567 = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
806
807 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
808 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
809
810 const __m128i vi2x01234567 = _mm_loadl_epi64((const __m128i*) i2);
811 const __m128i vxi2x01234567 = _mm_cvtepi8_epi16(vi2x01234567);
812 const __m128i vk2x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
813 const __m128i vxk2x01234567 = _mm_cvtepi8_epi16(vk2x01234567);
814 i2 += 8;
815
816
817 vprod01234567 = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
818
819 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
820 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
821
822 const __m128i vi3x01234567 = _mm_loadl_epi64((const __m128i*) i3);
823 const __m128i vxi3x01234567 = _mm_cvtepi8_epi16(vi3x01234567);
824 const __m128i vk3x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
825 const __m128i vxk3x01234567 = _mm_cvtepi8_epi16(vk3x01234567);
826 i3 += 8;
827
828
829 vprod01234567 = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
830
831 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
832 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
833
834 const __m128i vi4x01234567 = _mm_loadl_epi64((const __m128i*) i4);
835 const __m128i vxi4x01234567 = _mm_cvtepi8_epi16(vi4x01234567);
836 const __m128i vk4x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t)));
837 const __m128i vxk4x01234567 = _mm_cvtepi8_epi16(vk4x01234567);
838 i4 += 8;
839
840
841 vprod01234567 = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
842
843 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
844 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
845
846 const __m128i vi5x01234567 = _mm_loadl_epi64((const __m128i*) i5);
847 const __m128i vxi5x01234567 = _mm_cvtepi8_epi16(vi5x01234567);
848 const __m128i vk5x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 40 * sizeof(int8_t)));
849 const __m128i vxk5x01234567 = _mm_cvtepi8_epi16(vk5x01234567);
850 i5 += 8;
851
852
853 vprod01234567 = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
854
855 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
856 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
857
858 const __m128i vi6x01234567 = _mm_loadl_epi64((const __m128i*) i6);
859 const __m128i vxi6x01234567 = _mm_cvtepi8_epi16(vi6x01234567);
860 const __m128i vk6x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t)));
861 const __m128i vxk6x01234567 = _mm_cvtepi8_epi16(vk6x01234567);
862 i6 += 8;
863
864
865 vprod01234567 = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
866
867 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
868 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
869
870 __m128 vscaled0123 = _mm_cvtepi32_ps(vacc0123);
871 __m128 vscaled4567 = _mm_cvtepi32_ps(vacc4567);
872
873 vscaled0123 = _mm_mul_ps(vscaled0123, vscale);
874 vscaled4567 = _mm_mul_ps(vscaled4567, vscale);
875
876 w = (void*) ((uintptr_t) w + 56);
877
878 vscaled0123 = _mm_min_ps(vscaled0123, voutput_max_less_zero_point);
879 vscaled4567 = _mm_min_ps(vscaled4567, voutput_max_less_zero_point);
880
881 vacc0123 = _mm_cvtps_epi32(vscaled0123);
882 vacc4567 = _mm_cvtps_epi32(vscaled4567);
883
884 __m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
885
886
887 __m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
888
889 vout0123456701234567 = _mm_max_epi8(vout0123456701234567, voutput_min);
890
891 if XNN_LIKELY(c >= 8) {
892 _mm_storel_epi64((__m128i*) output, vout0123456701234567);
893 output += 8;
894 c -= 8;
895 } else {
896 if (c & 4) {
897 unaligned_store_u32(output, (uint32_t) _mm_cvtsi128_si32(vout0123456701234567));
898 vout0123456701234567 = _mm_srli_epi64(vout0123456701234567, 32);
899 output += 4;
900 }
901 if (c & 2) {
902 unaligned_store_u16(output, (uint16_t) _mm_extract_epi16(vout0123456701234567, 0));
903 vout0123456701234567 = _mm_srli_epi32(vout0123456701234567, 16);
904 output += 2;
905 }
906 if (c & 1) {
907 *output = (int8_t) _mm_extract_epi8(vout0123456701234567, 0);
908 output += 1;
909 }
910 c = 0;
911 }
912 } while (c != 0);
913 }
914 }
915
916 input = (const int8_t**) ((uintptr_t) input + input_stride);
917 output = (int8_t*) ((uintptr_t) output + output_increment);
918 } while (--output_width != 0);
919 }