Debian Codegraph @

src/qs8-dwconv/gen/qs8-dwconv-8f8m9l16c8s8r-minmax-fp32-sse41-mul16.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-dwconv/multipass-sse-mul16.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include <smmintrin.h>
13
14 #include "xnnpack/dwconv.h"
15 #include "xnnpack/math.h"
16 #include "xnnpack/unaligned.h"
17
18
19 void xnn_qs8_dwconv_minmax_fp32_ukernel_8f8m9l16c8s8r__sse41_mul16(
20 size_t channels,
21 size_t output_width,
22 const int8_t** input,
23 const void* weights,
24 int8_t* output,
25 intptr_t input_stride,
26 size_t output_increment,
27 size_t input_offset,
28 const int8_t* zero,
29 size_t kernel_size,
30 int32_t* buffer,
31 const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
32 {
33 assert(channels != 0);
34 assert(output_width != 0);
35 assert(kernel_size > 8);
36
37
38 const __m128 vscale = _mm_set1_ps(params->fp32_scalar.scale);
39 XNN_FORCE_REALIZATION(vscale);
40
41 const __m128 voutput_max_less_zero_point = _mm_set1_ps((int32_t) params->fp32_scalar.output_max - (int32_t) params->fp32_scalar.output_zero_point);
42 const __m128i voutput_zero_point = _mm_set1_epi16(params->fp32_scalar.output_zero_point);
43 const __m128i voutput_min = _mm_set1_epi8(params->fp32_scalar.output_min);
44 XNN_FORCE_REALIZATION(voutput_max_less_zero_point);
45 XNN_FORCE_REALIZATION(voutput_zero_point);
46 XNN_FORCE_REALIZATION(voutput_min);
47
48 do {
49 const void* w = weights;
50
51 // First pass to process 8 inputs.
52 {
53 int32_t* b = buffer;
54 const int8_t* i0 = input[0];
55 assert(i0 != NULL);
56 if XNN_UNPREDICTABLE(i0 != zero) {
57 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
58 }
59 const int8_t* i1 = input[1];
60 assert(i1 != NULL);
61 if XNN_UNPREDICTABLE(i1 != zero) {
62 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
63 }
64 const int8_t* i2 = input[2];
65 assert(i2 != NULL);
66 if XNN_UNPREDICTABLE(i2 != zero) {
67 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
68 }
69 const int8_t* i3 = input[3];
70 assert(i3 != NULL);
71 if XNN_UNPREDICTABLE(i3 != zero) {
72 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
73 }
74 const int8_t* i4 = input[4];
75 assert(i4 != NULL);
76 if XNN_UNPREDICTABLE(i4 != zero) {
77 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
78 }
79 const int8_t* i5 = input[5];
80 assert(i5 != NULL);
81 if XNN_UNPREDICTABLE(i5 != zero) {
82 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
83 }
84 const int8_t* i6 = input[6];
85 assert(i6 != NULL);
86 if XNN_UNPREDICTABLE(i6 != zero) {
87 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
88 }
89 const int8_t* i7 = input[7];
90 assert(i7 != NULL);
91 if XNN_UNPREDICTABLE(i7 != zero) {
92 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
93 }
94 input += 8;
95
96 size_t c = round_up_po2(channels, 8);
97
98 for (; c >= 16; c -= 16) {
99 __m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
100 __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
101 __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
102 __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
103
104
105
106 const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
107 const __m128i vxi0x01234567 = _mm_cvtepi8_epi16(vi0x01234567);
108 const __m128i vk0x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 0 * sizeof(int8_t)));
109 const __m128i vxk0x01234567 = _mm_cvtepi8_epi16(vk0x01234567);
110 const __m128i vi0x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i0 + 8));
111 const __m128i vxi0x89ABCDEF = _mm_cvtepi8_epi16(vi0x89ABCDEF);
112 const __m128i vk0x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 8 * sizeof(int8_t)));
113 const __m128i vxk0x89ABCDEF = _mm_cvtepi8_epi16(vk0x89ABCDEF);
114 i0 += 16;
115
116
117 __m128i vprod01234567 = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
118 __m128i vprod89ABCDEF = _mm_mullo_epi16(vxi0x89ABCDEF, vxk0x89ABCDEF);
119
120 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
121 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
122 vacc89AB = _mm_add_epi32(vacc89AB, _mm_cvtepi16_epi32(vprod89ABCDEF));
123 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_srai_epi32(_mm_unpackhi_epi16(vprod89ABCDEF, vprod89ABCDEF), 16));
124
125 const __m128i vi1x01234567 = _mm_loadl_epi64((const __m128i*) i1);
126 const __m128i vxi1x01234567 = _mm_cvtepi8_epi16(vi1x01234567);
127 const __m128i vk1x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 16 * sizeof(int8_t)));
128 const __m128i vxk1x01234567 = _mm_cvtepi8_epi16(vk1x01234567);
129 const __m128i vi1x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i1 + 8));
130 const __m128i vxi1x89ABCDEF = _mm_cvtepi8_epi16(vi1x89ABCDEF);
131 const __m128i vk1x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 24 * sizeof(int8_t)));
132 const __m128i vxk1x89ABCDEF = _mm_cvtepi8_epi16(vk1x89ABCDEF);
133 i1 += 16;
134
135
136 vprod01234567 = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
137 vprod89ABCDEF = _mm_mullo_epi16(vxi1x89ABCDEF, vxk1x89ABCDEF);
138
139 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
140 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
141 vacc89AB = _mm_add_epi32(vacc89AB, _mm_cvtepi16_epi32(vprod89ABCDEF));
142 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_srai_epi32(_mm_unpackhi_epi16(vprod89ABCDEF, vprod89ABCDEF), 16));
143
144 const __m128i vi2x01234567 = _mm_loadl_epi64((const __m128i*) i2);
145 const __m128i vxi2x01234567 = _mm_cvtepi8_epi16(vi2x01234567);
146 const __m128i vk2x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 32 * sizeof(int8_t)));
147 const __m128i vxk2x01234567 = _mm_cvtepi8_epi16(vk2x01234567);
148 const __m128i vi2x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i2 + 8));
149 const __m128i vxi2x89ABCDEF = _mm_cvtepi8_epi16(vi2x89ABCDEF);
150 const __m128i vk2x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 40 * sizeof(int8_t)));
151 const __m128i vxk2x89ABCDEF = _mm_cvtepi8_epi16(vk2x89ABCDEF);
152 i2 += 16;
153
154
155 vprod01234567 = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
156 vprod89ABCDEF = _mm_mullo_epi16(vxi2x89ABCDEF, vxk2x89ABCDEF);
157
158 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
159 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
160 vacc89AB = _mm_add_epi32(vacc89AB, _mm_cvtepi16_epi32(vprod89ABCDEF));
161 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_srai_epi32(_mm_unpackhi_epi16(vprod89ABCDEF, vprod89ABCDEF), 16));
162
163 const __m128i vi3x01234567 = _mm_loadl_epi64((const __m128i*) i3);
164 const __m128i vxi3x01234567 = _mm_cvtepi8_epi16(vi3x01234567);
165 const __m128i vk3x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 48 * sizeof(int8_t)));
166 const __m128i vxk3x01234567 = _mm_cvtepi8_epi16(vk3x01234567);
167 const __m128i vi3x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i3 + 8));
168 const __m128i vxi3x89ABCDEF = _mm_cvtepi8_epi16(vi3x89ABCDEF);
169 const __m128i vk3x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 56 * sizeof(int8_t)));
170 const __m128i vxk3x89ABCDEF = _mm_cvtepi8_epi16(vk3x89ABCDEF);
171 i3 += 16;
172
173
174 vprod01234567 = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
175 vprod89ABCDEF = _mm_mullo_epi16(vxi3x89ABCDEF, vxk3x89ABCDEF);
176
177 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
178 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
179 vacc89AB = _mm_add_epi32(vacc89AB, _mm_cvtepi16_epi32(vprod89ABCDEF));
180 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_srai_epi32(_mm_unpackhi_epi16(vprod89ABCDEF, vprod89ABCDEF), 16));
181
182 const __m128i vi4x01234567 = _mm_loadl_epi64((const __m128i*) i4);
183 const __m128i vxi4x01234567 = _mm_cvtepi8_epi16(vi4x01234567);
184 const __m128i vk4x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 64 * sizeof(int8_t)));
185 const __m128i vxk4x01234567 = _mm_cvtepi8_epi16(vk4x01234567);
186 const __m128i vi4x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i4 + 8));
187 const __m128i vxi4x89ABCDEF = _mm_cvtepi8_epi16(vi4x89ABCDEF);
188 const __m128i vk4x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 72 * sizeof(int8_t)));
189 const __m128i vxk4x89ABCDEF = _mm_cvtepi8_epi16(vk4x89ABCDEF);
190 i4 += 16;
191
192
193 vprod01234567 = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
194 vprod89ABCDEF = _mm_mullo_epi16(vxi4x89ABCDEF, vxk4x89ABCDEF);
195
196 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
197 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
198 vacc89AB = _mm_add_epi32(vacc89AB, _mm_cvtepi16_epi32(vprod89ABCDEF));
199 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_srai_epi32(_mm_unpackhi_epi16(vprod89ABCDEF, vprod89ABCDEF), 16));
200
201 const __m128i vi5x01234567 = _mm_loadl_epi64((const __m128i*) i5);
202 const __m128i vxi5x01234567 = _mm_cvtepi8_epi16(vi5x01234567);
203 const __m128i vk5x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 80 * sizeof(int8_t)));
204 const __m128i vxk5x01234567 = _mm_cvtepi8_epi16(vk5x01234567);
205 const __m128i vi5x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i5 + 8));
206 const __m128i vxi5x89ABCDEF = _mm_cvtepi8_epi16(vi5x89ABCDEF);
207 const __m128i vk5x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 88 * sizeof(int8_t)));
208 const __m128i vxk5x89ABCDEF = _mm_cvtepi8_epi16(vk5x89ABCDEF);
209 i5 += 16;
210
211
212 vprod01234567 = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
213 vprod89ABCDEF = _mm_mullo_epi16(vxi5x89ABCDEF, vxk5x89ABCDEF);
214
215 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
216 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
217 vacc89AB = _mm_add_epi32(vacc89AB, _mm_cvtepi16_epi32(vprod89ABCDEF));
218 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_srai_epi32(_mm_unpackhi_epi16(vprod89ABCDEF, vprod89ABCDEF), 16));
219
220 const __m128i vi6x01234567 = _mm_loadl_epi64((const __m128i*) i6);
221 const __m128i vxi6x01234567 = _mm_cvtepi8_epi16(vi6x01234567);
222 const __m128i vk6x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 96 * sizeof(int8_t)));
223 const __m128i vxk6x01234567 = _mm_cvtepi8_epi16(vk6x01234567);
224 const __m128i vi6x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i6 + 8));
225 const __m128i vxi6x89ABCDEF = _mm_cvtepi8_epi16(vi6x89ABCDEF);
226 const __m128i vk6x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 104 * sizeof(int8_t)));
227 const __m128i vxk6x89ABCDEF = _mm_cvtepi8_epi16(vk6x89ABCDEF);
228 i6 += 16;
229
230
231 vprod01234567 = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
232 vprod89ABCDEF = _mm_mullo_epi16(vxi6x89ABCDEF, vxk6x89ABCDEF);
233
234 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
235 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
236 vacc89AB = _mm_add_epi32(vacc89AB, _mm_cvtepi16_epi32(vprod89ABCDEF));
237 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_srai_epi32(_mm_unpackhi_epi16(vprod89ABCDEF, vprod89ABCDEF), 16));
238
239 const __m128i vi7x01234567 = _mm_loadl_epi64((const __m128i*) i7);
240 const __m128i vxi7x01234567 = _mm_cvtepi8_epi16(vi7x01234567);
241 const __m128i vk7x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 112 * sizeof(int8_t)));
242 const __m128i vxk7x01234567 = _mm_cvtepi8_epi16(vk7x01234567);
243 const __m128i vi7x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i7 + 8));
244 const __m128i vxi7x89ABCDEF = _mm_cvtepi8_epi16(vi7x89ABCDEF);
245 const __m128i vk7x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 120 * sizeof(int8_t)));
246 const __m128i vxk7x89ABCDEF = _mm_cvtepi8_epi16(vk7x89ABCDEF);
247 i7 += 16;
248
249
250 vprod01234567 = _mm_mullo_epi16(vxi7x01234567, vxk7x01234567);
251 vprod89ABCDEF = _mm_mullo_epi16(vxi7x89ABCDEF, vxk7x89ABCDEF);
252
253 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
254 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
255 vacc89AB = _mm_add_epi32(vacc89AB, _mm_cvtepi16_epi32(vprod89ABCDEF));
256 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_srai_epi32(_mm_unpackhi_epi16(vprod89ABCDEF, vprod89ABCDEF), 16));
257
258 w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 128 * sizeof(int8_t));
259
260 _mm_storeu_si128((__m128i*) b, vacc0123);
261 _mm_storeu_si128((__m128i*) (b + 4), vacc4567);
262 _mm_storeu_si128((__m128i*) (b + 8), vacc89AB);
263 _mm_storeu_si128((__m128i*) (b + 12), vaccCDEF);
264 b += 16;
265 }
266
267 if XNN_UNLIKELY(c != 0) {
268 do {
269 __m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
270 __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
271
272
273 const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
274 const __m128i vxi0x01234567 = _mm_cvtepi8_epi16(vi0x01234567);
275 const __m128i vk0x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + sizeof(int32_t) * 8 * sizeof(int8_t)));
276 const __m128i vxk0x01234567 = _mm_cvtepi8_epi16(vk0x01234567);
277 i0 += 8;
278
279
280 __m128i vprod01234567 = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
281
282 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
283 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
284
285 const __m128i vi1x01234567 = _mm_loadl_epi64((const __m128i*) i1);
286 const __m128i vxi1x01234567 = _mm_cvtepi8_epi16(vi1x01234567);
287 const __m128i vk1x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + sizeof(int32_t) * 8 + 8 * sizeof(int8_t)));
288 const __m128i vxk1x01234567 = _mm_cvtepi8_epi16(vk1x01234567);
289 i1 += 8;
290
291
292 vprod01234567 = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
293
294 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
295 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
296
297 const __m128i vi2x01234567 = _mm_loadl_epi64((const __m128i*) i2);
298 const __m128i vxi2x01234567 = _mm_cvtepi8_epi16(vi2x01234567);
299 const __m128i vk2x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + sizeof(int32_t) * 8 + 16 * sizeof(int8_t)));
300 const __m128i vxk2x01234567 = _mm_cvtepi8_epi16(vk2x01234567);
301 i2 += 8;
302
303
304 vprod01234567 = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
305
306 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
307 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
308
309 const __m128i vi3x01234567 = _mm_loadl_epi64((const __m128i*) i3);
310 const __m128i vxi3x01234567 = _mm_cvtepi8_epi16(vi3x01234567);
311 const __m128i vk3x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + sizeof(int32_t) * 8 + 24 * sizeof(int8_t)));
312 const __m128i vxk3x01234567 = _mm_cvtepi8_epi16(vk3x01234567);
313 i3 += 8;
314
315
316 vprod01234567 = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
317
318 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
319 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
320
321 const __m128i vi4x01234567 = _mm_loadl_epi64((const __m128i*) i4);
322 const __m128i vxi4x01234567 = _mm_cvtepi8_epi16(vi4x01234567);
323 const __m128i vk4x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + sizeof(int32_t) * 8 + 32 * sizeof(int8_t)));
324 const __m128i vxk4x01234567 = _mm_cvtepi8_epi16(vk4x01234567);
325 i4 += 8;
326
327
328 vprod01234567 = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
329
330 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
331 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
332
333 const __m128i vi5x01234567 = _mm_loadl_epi64((const __m128i*) i5);
334 const __m128i vxi5x01234567 = _mm_cvtepi8_epi16(vi5x01234567);
335 const __m128i vk5x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + sizeof(int32_t) * 8 + 40 * sizeof(int8_t)));
336 const __m128i vxk5x01234567 = _mm_cvtepi8_epi16(vk5x01234567);
337 i5 += 8;
338
339
340 vprod01234567 = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
341
342 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
343 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
344
345 const __m128i vi6x01234567 = _mm_loadl_epi64((const __m128i*) i6);
346 const __m128i vxi6x01234567 = _mm_cvtepi8_epi16(vi6x01234567);
347 const __m128i vk6x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + sizeof(int32_t) * 8 + 48 * sizeof(int8_t)));
348 const __m128i vxk6x01234567 = _mm_cvtepi8_epi16(vk6x01234567);
349 i6 += 8;
350
351
352 vprod01234567 = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
353
354 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
355 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
356
357 const __m128i vi7x01234567 = _mm_loadl_epi64((const __m128i*) i7);
358 const __m128i vxi7x01234567 = _mm_cvtepi8_epi16(vi7x01234567);
359 const __m128i vk7x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + sizeof(int32_t) * 8 + 56 * sizeof(int8_t)));
360 const __m128i vxk7x01234567 = _mm_cvtepi8_epi16(vk7x01234567);
361 i7 += 8;
362
363
364 vprod01234567 = _mm_mullo_epi16(vxi7x01234567, vxk7x01234567);
365
366 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
367 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
368
369 w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 64 * sizeof(int8_t));
370
371 _mm_storeu_si128((__m128i*) (b), vacc0123);
372 _mm_storeu_si128((__m128i*) (b + 4), vacc4567);
373 b += 8;
374 c -= 8;
375 } while (c != 0);
376 }
377 }
378
379 // Middle pass to process 8 inputs in each iteration.
380 for (size_t ks = kernel_size - 8; ks > 9; ks -= 8) {
381 int32_t* b = buffer;
382 const int8_t* i0 = input[0];
383 assert(i0 != NULL);
384 if XNN_UNPREDICTABLE(i0 != zero) {
385 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
386 }
387 const int8_t* i1 = input[1];
388 assert(i1 != NULL);
389 if XNN_UNPREDICTABLE(i1 != zero) {
390 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
391 }
392 const int8_t* i2 = input[2];
393 assert(i2 != NULL);
394 if XNN_UNPREDICTABLE(i2 != zero) {
395 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
396 }
397 const int8_t* i3 = input[3];
398 assert(i3 != NULL);
399 if XNN_UNPREDICTABLE(i3 != zero) {
400 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
401 }
402 const int8_t* i4 = input[4];
403 assert(i4 != NULL);
404 if XNN_UNPREDICTABLE(i4 != zero) {
405 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
406 }
407 const int8_t* i5 = input[5];
408 assert(i5 != NULL);
409 if XNN_UNPREDICTABLE(i5 != zero) {
410 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
411 }
412 const int8_t* i6 = input[6];
413 assert(i6 != NULL);
414 if XNN_UNPREDICTABLE(i6 != zero) {
415 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
416 }
417 const int8_t* i7 = input[7];
418 assert(i7 != NULL);
419 if XNN_UNPREDICTABLE(i7 != zero) {
420 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
421 }
422 input += 8;
423
424 size_t c = round_up_po2(channels, 8);
425
426 for (; c >= 16; c -= 16) {
427 __m128i vacc0123 = _mm_loadu_si128((const __m128i*) b);
428 __m128i vacc4567 = _mm_loadu_si128((const __m128i*) (b + 4));
429 __m128i vacc89AB = _mm_loadu_si128((const __m128i*) (b + 8));
430 __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) (b + 12));
431
432
433
434 const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
435 const __m128i vxi0x01234567 = _mm_cvtepi8_epi16(vi0x01234567);
436 const __m128i vk0x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 0 * sizeof(int8_t)));
437 const __m128i vxk0x01234567 = _mm_cvtepi8_epi16(vk0x01234567);
438 const __m128i vi0x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i0 + 8));
439 const __m128i vxi0x89ABCDEF = _mm_cvtepi8_epi16(vi0x89ABCDEF);
440 const __m128i vk0x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
441 const __m128i vxk0x89ABCDEF = _mm_cvtepi8_epi16(vk0x89ABCDEF);
442 i0 += 16;
443
444
445 __m128i vprod01234567 = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
446 __m128i vprod89ABCDEF = _mm_mullo_epi16(vxi0x89ABCDEF, vxk0x89ABCDEF);
447
448 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
449 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
450 vacc89AB = _mm_add_epi32(vacc89AB, _mm_cvtepi16_epi32(vprod89ABCDEF));
451 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_srai_epi32(_mm_unpackhi_epi16(vprod89ABCDEF, vprod89ABCDEF), 16));
452
453 const __m128i vi1x01234567 = _mm_loadl_epi64((const __m128i*) i1);
454 const __m128i vxi1x01234567 = _mm_cvtepi8_epi16(vi1x01234567);
455 const __m128i vk1x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
456 const __m128i vxk1x01234567 = _mm_cvtepi8_epi16(vk1x01234567);
457 const __m128i vi1x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i1 + 8));
458 const __m128i vxi1x89ABCDEF = _mm_cvtepi8_epi16(vi1x89ABCDEF);
459 const __m128i vk1x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
460 const __m128i vxk1x89ABCDEF = _mm_cvtepi8_epi16(vk1x89ABCDEF);
461 i1 += 16;
462
463
464 vprod01234567 = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
465 vprod89ABCDEF = _mm_mullo_epi16(vxi1x89ABCDEF, vxk1x89ABCDEF);
466
467 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
468 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
469 vacc89AB = _mm_add_epi32(vacc89AB, _mm_cvtepi16_epi32(vprod89ABCDEF));
470 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_srai_epi32(_mm_unpackhi_epi16(vprod89ABCDEF, vprod89ABCDEF), 16));
471
472 const __m128i vi2x01234567 = _mm_loadl_epi64((const __m128i*) i2);
473 const __m128i vxi2x01234567 = _mm_cvtepi8_epi16(vi2x01234567);
474 const __m128i vk2x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t)));
475 const __m128i vxk2x01234567 = _mm_cvtepi8_epi16(vk2x01234567);
476 const __m128i vi2x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i2 + 8));
477 const __m128i vxi2x89ABCDEF = _mm_cvtepi8_epi16(vi2x89ABCDEF);
478 const __m128i vk2x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 40 * sizeof(int8_t)));
479 const __m128i vxk2x89ABCDEF = _mm_cvtepi8_epi16(vk2x89ABCDEF);
480 i2 += 16;
481
482
483 vprod01234567 = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
484 vprod89ABCDEF = _mm_mullo_epi16(vxi2x89ABCDEF, vxk2x89ABCDEF);
485
486 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
487 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
488 vacc89AB = _mm_add_epi32(vacc89AB, _mm_cvtepi16_epi32(vprod89ABCDEF));
489 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_srai_epi32(_mm_unpackhi_epi16(vprod89ABCDEF, vprod89ABCDEF), 16));
490
491 const __m128i vi3x01234567 = _mm_loadl_epi64((const __m128i*) i3);
492 const __m128i vxi3x01234567 = _mm_cvtepi8_epi16(vi3x01234567);
493 const __m128i vk3x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t)));
494 const __m128i vxk3x01234567 = _mm_cvtepi8_epi16(vk3x01234567);
495 const __m128i vi3x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i3 + 8));
496 const __m128i vxi3x89ABCDEF = _mm_cvtepi8_epi16(vi3x89ABCDEF);
497 const __m128i vk3x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 56 * sizeof(int8_t)));
498 const __m128i vxk3x89ABCDEF = _mm_cvtepi8_epi16(vk3x89ABCDEF);
499 i3 += 16;
500
501
502 vprod01234567 = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
503 vprod89ABCDEF = _mm_mullo_epi16(vxi3x89ABCDEF, vxk3x89ABCDEF);
504
505 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
506 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
507 vacc89AB = _mm_add_epi32(vacc89AB, _mm_cvtepi16_epi32(vprod89ABCDEF));
508 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_srai_epi32(_mm_unpackhi_epi16(vprod89ABCDEF, vprod89ABCDEF), 16));
509
510 const __m128i vi4x01234567 = _mm_loadl_epi64((const __m128i*) i4);
511 const __m128i vxi4x01234567 = _mm_cvtepi8_epi16(vi4x01234567);
512 const __m128i vk4x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 64 * sizeof(int8_t)));
513 const __m128i vxk4x01234567 = _mm_cvtepi8_epi16(vk4x01234567);
514 const __m128i vi4x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i4 + 8));
515 const __m128i vxi4x89ABCDEF = _mm_cvtepi8_epi16(vi4x89ABCDEF);
516 const __m128i vk4x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 72 * sizeof(int8_t)));
517 const __m128i vxk4x89ABCDEF = _mm_cvtepi8_epi16(vk4x89ABCDEF);
518 i4 += 16;
519
520
521 vprod01234567 = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
522 vprod89ABCDEF = _mm_mullo_epi16(vxi4x89ABCDEF, vxk4x89ABCDEF);
523
524 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
525 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
526 vacc89AB = _mm_add_epi32(vacc89AB, _mm_cvtepi16_epi32(vprod89ABCDEF));
527 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_srai_epi32(_mm_unpackhi_epi16(vprod89ABCDEF, vprod89ABCDEF), 16));
528
529 const __m128i vi5x01234567 = _mm_loadl_epi64((const __m128i*) i5);
530 const __m128i vxi5x01234567 = _mm_cvtepi8_epi16(vi5x01234567);
531 const __m128i vk5x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 80 * sizeof(int8_t)));
532 const __m128i vxk5x01234567 = _mm_cvtepi8_epi16(vk5x01234567);
533 const __m128i vi5x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i5 + 8));
534 const __m128i vxi5x89ABCDEF = _mm_cvtepi8_epi16(vi5x89ABCDEF);
535 const __m128i vk5x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 88 * sizeof(int8_t)));
536 const __m128i vxk5x89ABCDEF = _mm_cvtepi8_epi16(vk5x89ABCDEF);
537 i5 += 16;
538
539
540 vprod01234567 = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
541 vprod89ABCDEF = _mm_mullo_epi16(vxi5x89ABCDEF, vxk5x89ABCDEF);
542
543 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
544 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
545 vacc89AB = _mm_add_epi32(vacc89AB, _mm_cvtepi16_epi32(vprod89ABCDEF));
546 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_srai_epi32(_mm_unpackhi_epi16(vprod89ABCDEF, vprod89ABCDEF), 16));
547
548 const __m128i vi6x01234567 = _mm_loadl_epi64((const __m128i*) i6);
549 const __m128i vxi6x01234567 = _mm_cvtepi8_epi16(vi6x01234567);
550 const __m128i vk6x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 96 * sizeof(int8_t)));
551 const __m128i vxk6x01234567 = _mm_cvtepi8_epi16(vk6x01234567);
552 const __m128i vi6x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i6 + 8));
553 const __m128i vxi6x89ABCDEF = _mm_cvtepi8_epi16(vi6x89ABCDEF);
554 const __m128i vk6x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 104 * sizeof(int8_t)));
555 const __m128i vxk6x89ABCDEF = _mm_cvtepi8_epi16(vk6x89ABCDEF);
556 i6 += 16;
557
558
559 vprod01234567 = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
560 vprod89ABCDEF = _mm_mullo_epi16(vxi6x89ABCDEF, vxk6x89ABCDEF);
561
562 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
563 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
564 vacc89AB = _mm_add_epi32(vacc89AB, _mm_cvtepi16_epi32(vprod89ABCDEF));
565 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_srai_epi32(_mm_unpackhi_epi16(vprod89ABCDEF, vprod89ABCDEF), 16));
566
567 const __m128i vi7x01234567 = _mm_loadl_epi64((const __m128i*) i7);
568 const __m128i vxi7x01234567 = _mm_cvtepi8_epi16(vi7x01234567);
569 const __m128i vk7x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 112 * sizeof(int8_t)));
570 const __m128i vxk7x01234567 = _mm_cvtepi8_epi16(vk7x01234567);
571 const __m128i vi7x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i7 + 8));
572 const __m128i vxi7x89ABCDEF = _mm_cvtepi8_epi16(vi7x89ABCDEF);
573 const __m128i vk7x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 120 * sizeof(int8_t)));
574 const __m128i vxk7x89ABCDEF = _mm_cvtepi8_epi16(vk7x89ABCDEF);
575 i7 += 16;
576
577
578 vprod01234567 = _mm_mullo_epi16(vxi7x01234567, vxk7x01234567);
579 vprod89ABCDEF = _mm_mullo_epi16(vxi7x89ABCDEF, vxk7x89ABCDEF);
580
581 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
582 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
583 vacc89AB = _mm_add_epi32(vacc89AB, _mm_cvtepi16_epi32(vprod89ABCDEF));
584 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_srai_epi32(_mm_unpackhi_epi16(vprod89ABCDEF, vprod89ABCDEF), 16));
585
586 w = (const void*) ((uintptr_t) w + 128 * sizeof(int8_t));
587
588 _mm_storeu_si128((__m128i*) (b), vacc0123);
589 _mm_storeu_si128((__m128i*) (b + 4), vacc4567);
590 _mm_storeu_si128((__m128i*) (b + 8), vacc89AB);
591 _mm_storeu_si128((__m128i*) (b + 12), vaccCDEF);
592 b += 16;
593 }
594
595 if XNN_UNLIKELY(c != 0) {
596 do {
597 __m128i vacc0123 = _mm_loadu_si128((const __m128i*) b);
598 __m128i vacc4567 = _mm_loadu_si128((const __m128i*) (b + 4));
599
600
601 const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
602 const __m128i vxi0x01234567 = _mm_cvtepi8_epi16(vi0x01234567);
603 const __m128i vk0x01234567 = _mm_loadl_epi64((const __m128i*) w);
604 const __m128i vxk0x01234567 = _mm_cvtepi8_epi16(vk0x01234567);
605 i0 += 8;
606
607
608 __m128i vprod01234567 = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
609
610 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
611 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
612
613 const __m128i vi1x01234567 = _mm_loadl_epi64((const __m128i*) i1);
614 const __m128i vxi1x01234567 = _mm_cvtepi8_epi16(vi1x01234567);
615 const __m128i vk1x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
616 const __m128i vxk1x01234567 = _mm_cvtepi8_epi16(vk1x01234567);
617 i1 += 8;
618
619
620 vprod01234567 = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
621
622 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
623 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
624
625 const __m128i vi2x01234567 = _mm_loadl_epi64((const __m128i*) i2);
626 const __m128i vxi2x01234567 = _mm_cvtepi8_epi16(vi2x01234567);
627 const __m128i vk2x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
628 const __m128i vxk2x01234567 = _mm_cvtepi8_epi16(vk2x01234567);
629 i2 += 8;
630
631
632 vprod01234567 = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
633
634 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
635 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
636
637 const __m128i vi3x01234567 = _mm_loadl_epi64((const __m128i*) i3);
638 const __m128i vxi3x01234567 = _mm_cvtepi8_epi16(vi3x01234567);
639 const __m128i vk3x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
640 const __m128i vxk3x01234567 = _mm_cvtepi8_epi16(vk3x01234567);
641 i3 += 8;
642
643
644 vprod01234567 = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
645
646 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
647 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
648
649 const __m128i vi4x01234567 = _mm_loadl_epi64((const __m128i*) i4);
650 const __m128i vxi4x01234567 = _mm_cvtepi8_epi16(vi4x01234567);
651 const __m128i vk4x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t)));
652 const __m128i vxk4x01234567 = _mm_cvtepi8_epi16(vk4x01234567);
653 i4 += 8;
654
655
656 vprod01234567 = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
657
658 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
659 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
660
661 const __m128i vi5x01234567 = _mm_loadl_epi64((const __m128i*) i5);
662 const __m128i vxi5x01234567 = _mm_cvtepi8_epi16(vi5x01234567);
663 const __m128i vk5x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 40 * sizeof(int8_t)));
664 const __m128i vxk5x01234567 = _mm_cvtepi8_epi16(vk5x01234567);
665 i5 += 8;
666
667
668 vprod01234567 = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
669
670 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
671 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
672
673 const __m128i vi6x01234567 = _mm_loadl_epi64((const __m128i*) i6);
674 const __m128i vxi6x01234567 = _mm_cvtepi8_epi16(vi6x01234567);
675 const __m128i vk6x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t)));
676 const __m128i vxk6x01234567 = _mm_cvtepi8_epi16(vk6x01234567);
677 i6 += 8;
678
679
680 vprod01234567 = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
681
682 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
683 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
684
685 const __m128i vi7x01234567 = _mm_loadl_epi64((const __m128i*) i7);
686 const __m128i vxi7x01234567 = _mm_cvtepi8_epi16(vi7x01234567);
687 const __m128i vk7x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 56 * sizeof(int8_t)));
688 const __m128i vxk7x01234567 = _mm_cvtepi8_epi16(vk7x01234567);
689 i7 += 8;
690
691
692 vprod01234567 = _mm_mullo_epi16(vxi7x01234567, vxk7x01234567);
693
694 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
695 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
696
697 w = (const void*) ((uintptr_t) w + 64 * sizeof(int8_t));
698
699 _mm_storeu_si128((__m128i*) (b), vacc0123);
700 _mm_storeu_si128((__m128i*) (b + 4), vacc4567);
701 b += 8;
702 c -= 8;
703 } while (c != 0);
704 }
705 }
706
707 // Last pass to process up to 9 inputs.
708 {
709 const int32_t* b = buffer;
710 const int8_t* i0 = input[0];
711 assert(i0 != NULL);
712 if XNN_UNPREDICTABLE(i0 != zero) {
713 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
714 }
715 const int8_t* i1 = input[1];
716 assert(i1 != NULL);
717 if XNN_UNPREDICTABLE(i1 != zero) {
718 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
719 }
720 const int8_t* i2 = input[2];
721 assert(i2 != NULL);
722 if XNN_UNPREDICTABLE(i2 != zero) {
723 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
724 }
725 const int8_t* i3 = input[3];
726 assert(i3 != NULL);
727 if XNN_UNPREDICTABLE(i3 != zero) {
728 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
729 }
730 const int8_t* i4 = input[4];
731 assert(i4 != NULL);
732 if XNN_UNPREDICTABLE(i4 != zero) {
733 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
734 }
735 const int8_t* i5 = input[5];
736 assert(i5 != NULL);
737 if XNN_UNPREDICTABLE(i5 != zero) {
738 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
739 }
740 const int8_t* i6 = input[6];
741 assert(i6 != NULL);
742 if XNN_UNPREDICTABLE(i6 != zero) {
743 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
744 }
745 const int8_t* i7 = input[7];
746 assert(i7 != NULL);
747 if XNN_UNPREDICTABLE(i7 != zero) {
748 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
749 }
750 const int8_t* i8 = input[8];
751 assert(i8 != NULL);
752 if XNN_UNPREDICTABLE(i8 != zero) {
753 i8 = (const int8_t*) ((uintptr_t) i8 + input_offset);
754 }
755
756 size_t c = channels;
757
758 for (; c >= 16; c -= 16) {
759 __m128i vacc0123 = _mm_loadu_si128((const __m128i*) b);
760 __m128i vacc4567 = _mm_loadu_si128((const __m128i*) (b + 4));
761 __m128i vacc89AB = _mm_loadu_si128((const __m128i*) (b + 8));
762 __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) (b + 12));
763 b += 16;
764
765
766 const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
767 const __m128i vxi0x01234567 = _mm_cvtepi8_epi16(vi0x01234567);
768 const __m128i vk0x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 0 * sizeof(int8_t)));
769 const __m128i vxk0x01234567 = _mm_cvtepi8_epi16(vk0x01234567);
770 const __m128i vi0x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i0 + 8));
771 const __m128i vxi0x89ABCDEF = _mm_cvtepi8_epi16(vi0x89ABCDEF);
772 const __m128i vk0x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
773 const __m128i vxk0x89ABCDEF = _mm_cvtepi8_epi16(vk0x89ABCDEF);
774 i0 += 16;
775
776
777 __m128i vprod01234567 = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
778 __m128i vprod89ABCDEF = _mm_mullo_epi16(vxi0x89ABCDEF, vxk0x89ABCDEF);
779
780 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
781 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
782 vacc89AB = _mm_add_epi32(vacc89AB, _mm_cvtepi16_epi32(vprod89ABCDEF));
783 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_srai_epi32(_mm_unpackhi_epi16(vprod89ABCDEF, vprod89ABCDEF), 16));
784
785 const __m128i vi1x01234567 = _mm_loadl_epi64((const __m128i*) i1);
786 const __m128i vxi1x01234567 = _mm_cvtepi8_epi16(vi1x01234567);
787 const __m128i vk1x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
788 const __m128i vxk1x01234567 = _mm_cvtepi8_epi16(vk1x01234567);
789 const __m128i vi1x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i1 + 8));
790 const __m128i vxi1x89ABCDEF = _mm_cvtepi8_epi16(vi1x89ABCDEF);
791 const __m128i vk1x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
792 const __m128i vxk1x89ABCDEF = _mm_cvtepi8_epi16(vk1x89ABCDEF);
793 i1 += 16;
794
795
796 vprod01234567 = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
797 vprod89ABCDEF = _mm_mullo_epi16(vxi1x89ABCDEF, vxk1x89ABCDEF);
798
799 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
800 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
801 vacc89AB = _mm_add_epi32(vacc89AB, _mm_cvtepi16_epi32(vprod89ABCDEF));
802 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_srai_epi32(_mm_unpackhi_epi16(vprod89ABCDEF, vprod89ABCDEF), 16));
803
804 const __m128i vi2x01234567 = _mm_loadl_epi64((const __m128i*) i2);
805 const __m128i vxi2x01234567 = _mm_cvtepi8_epi16(vi2x01234567);
806 const __m128i vk2x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t)));
807 const __m128i vxk2x01234567 = _mm_cvtepi8_epi16(vk2x01234567);
808 const __m128i vi2x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i2 + 8));
809 const __m128i vxi2x89ABCDEF = _mm_cvtepi8_epi16(vi2x89ABCDEF);
810 const __m128i vk2x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 40 * sizeof(int8_t)));
811 const __m128i vxk2x89ABCDEF = _mm_cvtepi8_epi16(vk2x89ABCDEF);
812 i2 += 16;
813
814
815 vprod01234567 = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
816 vprod89ABCDEF = _mm_mullo_epi16(vxi2x89ABCDEF, vxk2x89ABCDEF);
817
818 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
819 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
820 vacc89AB = _mm_add_epi32(vacc89AB, _mm_cvtepi16_epi32(vprod89ABCDEF));
821 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_srai_epi32(_mm_unpackhi_epi16(vprod89ABCDEF, vprod89ABCDEF), 16));
822
823 const __m128i vi3x01234567 = _mm_loadl_epi64((const __m128i*) i3);
824 const __m128i vxi3x01234567 = _mm_cvtepi8_epi16(vi3x01234567);
825 const __m128i vk3x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t)));
826 const __m128i vxk3x01234567 = _mm_cvtepi8_epi16(vk3x01234567);
827 const __m128i vi3x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i3 + 8));
828 const __m128i vxi3x89ABCDEF = _mm_cvtepi8_epi16(vi3x89ABCDEF);
829 const __m128i vk3x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 56 * sizeof(int8_t)));
830 const __m128i vxk3x89ABCDEF = _mm_cvtepi8_epi16(vk3x89ABCDEF);
831 i3 += 16;
832
833
834 vprod01234567 = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
835 vprod89ABCDEF = _mm_mullo_epi16(vxi3x89ABCDEF, vxk3x89ABCDEF);
836
837 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
838 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
839 vacc89AB = _mm_add_epi32(vacc89AB, _mm_cvtepi16_epi32(vprod89ABCDEF));
840 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_srai_epi32(_mm_unpackhi_epi16(vprod89ABCDEF, vprod89ABCDEF), 16));
841
842 const __m128i vi4x01234567 = _mm_loadl_epi64((const __m128i*) i4);
843 const __m128i vxi4x01234567 = _mm_cvtepi8_epi16(vi4x01234567);
844 const __m128i vk4x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 64 * sizeof(int8_t)));
845 const __m128i vxk4x01234567 = _mm_cvtepi8_epi16(vk4x01234567);
846 const __m128i vi4x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i4 + 8));
847 const __m128i vxi4x89ABCDEF = _mm_cvtepi8_epi16(vi4x89ABCDEF);
848 const __m128i vk4x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 72 * sizeof(int8_t)));
849 const __m128i vxk4x89ABCDEF = _mm_cvtepi8_epi16(vk4x89ABCDEF);
850 i4 += 16;
851
852
853 vprod01234567 = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
854 vprod89ABCDEF = _mm_mullo_epi16(vxi4x89ABCDEF, vxk4x89ABCDEF);
855
856 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
857 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
858 vacc89AB = _mm_add_epi32(vacc89AB, _mm_cvtepi16_epi32(vprod89ABCDEF));
859 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_srai_epi32(_mm_unpackhi_epi16(vprod89ABCDEF, vprod89ABCDEF), 16));
860
861 const __m128i vi5x01234567 = _mm_loadl_epi64((const __m128i*) i5);
862 const __m128i vxi5x01234567 = _mm_cvtepi8_epi16(vi5x01234567);
863 const __m128i vk5x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 80 * sizeof(int8_t)));
864 const __m128i vxk5x01234567 = _mm_cvtepi8_epi16(vk5x01234567);
865 const __m128i vi5x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i5 + 8));
866 const __m128i vxi5x89ABCDEF = _mm_cvtepi8_epi16(vi5x89ABCDEF);
867 const __m128i vk5x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 88 * sizeof(int8_t)));
868 const __m128i vxk5x89ABCDEF = _mm_cvtepi8_epi16(vk5x89ABCDEF);
869 i5 += 16;
870
871
872 vprod01234567 = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
873 vprod89ABCDEF = _mm_mullo_epi16(vxi5x89ABCDEF, vxk5x89ABCDEF);
874
875 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
876 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
877 vacc89AB = _mm_add_epi32(vacc89AB, _mm_cvtepi16_epi32(vprod89ABCDEF));
878 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_srai_epi32(_mm_unpackhi_epi16(vprod89ABCDEF, vprod89ABCDEF), 16));
879
880 const __m128i vi6x01234567 = _mm_loadl_epi64((const __m128i*) i6);
881 const __m128i vxi6x01234567 = _mm_cvtepi8_epi16(vi6x01234567);
882 const __m128i vk6x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 96 * sizeof(int8_t)));
883 const __m128i vxk6x01234567 = _mm_cvtepi8_epi16(vk6x01234567);
884 const __m128i vi6x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i6 + 8));
885 const __m128i vxi6x89ABCDEF = _mm_cvtepi8_epi16(vi6x89ABCDEF);
886 const __m128i vk6x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 104 * sizeof(int8_t)));
887 const __m128i vxk6x89ABCDEF = _mm_cvtepi8_epi16(vk6x89ABCDEF);
888 i6 += 16;
889
890
891 vprod01234567 = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
892 vprod89ABCDEF = _mm_mullo_epi16(vxi6x89ABCDEF, vxk6x89ABCDEF);
893
894 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
895 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
896 vacc89AB = _mm_add_epi32(vacc89AB, _mm_cvtepi16_epi32(vprod89ABCDEF));
897 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_srai_epi32(_mm_unpackhi_epi16(vprod89ABCDEF, vprod89ABCDEF), 16));
898
899 const __m128i vi7x01234567 = _mm_loadl_epi64((const __m128i*) i7);
900 const __m128i vxi7x01234567 = _mm_cvtepi8_epi16(vi7x01234567);
901 const __m128i vk7x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 112 * sizeof(int8_t)));
902 const __m128i vxk7x01234567 = _mm_cvtepi8_epi16(vk7x01234567);
903 const __m128i vi7x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i7 + 8));
904 const __m128i vxi7x89ABCDEF = _mm_cvtepi8_epi16(vi7x89ABCDEF);
905 const __m128i vk7x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 120 * sizeof(int8_t)));
906 const __m128i vxk7x89ABCDEF = _mm_cvtepi8_epi16(vk7x89ABCDEF);
907 i7 += 16;
908
909
910 vprod01234567 = _mm_mullo_epi16(vxi7x01234567, vxk7x01234567);
911 vprod89ABCDEF = _mm_mullo_epi16(vxi7x89ABCDEF, vxk7x89ABCDEF);
912
913 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
914 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
915 vacc89AB = _mm_add_epi32(vacc89AB, _mm_cvtepi16_epi32(vprod89ABCDEF));
916 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_srai_epi32(_mm_unpackhi_epi16(vprod89ABCDEF, vprod89ABCDEF), 16));
917
918 const __m128i vi8x01234567 = _mm_loadl_epi64((const __m128i*) i8);
919 const __m128i vxi8x01234567 = _mm_cvtepi8_epi16(vi8x01234567);
920 const __m128i vk8x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 128 * sizeof(int8_t)));
921 const __m128i vxk8x01234567 = _mm_cvtepi8_epi16(vk8x01234567);
922 const __m128i vi8x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i8 + 8));
923 const __m128i vxi8x89ABCDEF = _mm_cvtepi8_epi16(vi8x89ABCDEF);
924 const __m128i vk8x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 136 * sizeof(int8_t)));
925 const __m128i vxk8x89ABCDEF = _mm_cvtepi8_epi16(vk8x89ABCDEF);
926 i8 += 16;
927
928
929 vprod01234567 = _mm_mullo_epi16(vxi8x01234567, vxk8x01234567);
930 vprod89ABCDEF = _mm_mullo_epi16(vxi8x89ABCDEF, vxk8x89ABCDEF);
931
932 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
933 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
934 vacc89AB = _mm_add_epi32(vacc89AB, _mm_cvtepi16_epi32(vprod89ABCDEF));
935 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_srai_epi32(_mm_unpackhi_epi16(vprod89ABCDEF, vprod89ABCDEF), 16));
936
937 w = (const void*) ((uintptr_t) w + 144 * sizeof(int8_t));
938
939 __m128 vscaled0123 = _mm_cvtepi32_ps(vacc0123);
940 __m128 vscaled4567 = _mm_cvtepi32_ps(vacc4567);
941 __m128 vscaled89AB = _mm_cvtepi32_ps(vacc89AB);
942 __m128 vscaledCDEF = _mm_cvtepi32_ps(vaccCDEF);
943
944 vscaled0123 = _mm_mul_ps(vscaled0123, vscale);
945 vscaled4567 = _mm_mul_ps(vscaled4567, vscale);
946 vscaled89AB = _mm_mul_ps(vscaled89AB, vscale);
947 vscaledCDEF = _mm_mul_ps(vscaledCDEF, vscale);
948
949 vscaled0123 = _mm_min_ps(vscaled0123, voutput_max_less_zero_point);
950 vscaled4567 = _mm_min_ps(vscaled4567, voutput_max_less_zero_point);
951 vscaled89AB = _mm_min_ps(vscaled89AB, voutput_max_less_zero_point);
952 vscaledCDEF = _mm_min_ps(vscaledCDEF, voutput_max_less_zero_point);
953
954 vacc0123 = _mm_cvtps_epi32(vscaled0123);
955 vacc4567 = _mm_cvtps_epi32(vscaled4567);
956 vacc89AB = _mm_cvtps_epi32(vscaled89AB);
957 vaccCDEF = _mm_cvtps_epi32(vscaledCDEF);
958
959 __m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
960 __m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(vacc89AB, vaccCDEF), voutput_zero_point);
961
962
963 __m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
964
965 vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
966
967 _mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
968 output += 16;
969 }
970
971 if XNN_UNLIKELY(c != 0) {
972 do {
973 __m128i vacc0123 = _mm_loadu_si128((const __m128i*) b);
974 __m128i vacc4567 = _mm_loadu_si128((const __m128i*) (b + 4));
975 b += 8;
976
977
978 const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
979 const __m128i vxi0x01234567 = _mm_cvtepi8_epi16(vi0x01234567);
980 const __m128i vk0x01234567 = _mm_loadl_epi64((const __m128i*) w);
981 const __m128i vxk0x01234567 = _mm_cvtepi8_epi16(vk0x01234567);
982 i0 += 8;
983
984
985 __m128i vprod01234567 = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
986
987 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
988 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
989
990 const __m128i vi1x01234567 = _mm_loadl_epi64((const __m128i*) i1);
991 const __m128i vxi1x01234567 = _mm_cvtepi8_epi16(vi1x01234567);
992 const __m128i vk1x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
993 const __m128i vxk1x01234567 = _mm_cvtepi8_epi16(vk1x01234567);
994 i1 += 8;
995
996
997 vprod01234567 = _mm_mullo_epi16(vxi1x01234567, vxk1x01234567);
998
999 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
1000 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
1001
1002 const __m128i vi2x01234567 = _mm_loadl_epi64((const __m128i*) i2);
1003 const __m128i vxi2x01234567 = _mm_cvtepi8_epi16(vi2x01234567);
1004 const __m128i vk2x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
1005 const __m128i vxk2x01234567 = _mm_cvtepi8_epi16(vk2x01234567);
1006 i2 += 8;
1007
1008
1009 vprod01234567 = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
1010
1011 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
1012 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
1013
1014 const __m128i vi3x01234567 = _mm_loadl_epi64((const __m128i*) i3);
1015 const __m128i vxi3x01234567 = _mm_cvtepi8_epi16(vi3x01234567);
1016 const __m128i vk3x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
1017 const __m128i vxk3x01234567 = _mm_cvtepi8_epi16(vk3x01234567);
1018 i3 += 8;
1019
1020
1021 vprod01234567 = _mm_mullo_epi16(vxi3x01234567, vxk3x01234567);
1022
1023 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
1024 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
1025
1026 const __m128i vi4x01234567 = _mm_loadl_epi64((const __m128i*) i4);
1027 const __m128i vxi4x01234567 = _mm_cvtepi8_epi16(vi4x01234567);
1028 const __m128i vk4x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t)));
1029 const __m128i vxk4x01234567 = _mm_cvtepi8_epi16(vk4x01234567);
1030 i4 += 8;
1031
1032
1033 vprod01234567 = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
1034
1035 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
1036 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
1037
1038 const __m128i vi5x01234567 = _mm_loadl_epi64((const __m128i*) i5);
1039 const __m128i vxi5x01234567 = _mm_cvtepi8_epi16(vi5x01234567);
1040 const __m128i vk5x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 40 * sizeof(int8_t)));
1041 const __m128i vxk5x01234567 = _mm_cvtepi8_epi16(vk5x01234567);
1042 i5 += 8;
1043
1044
1045 vprod01234567 = _mm_mullo_epi16(vxi5x01234567, vxk5x01234567);
1046
1047 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
1048 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
1049
1050 const __m128i vi6x01234567 = _mm_loadl_epi64((const __m128i*) i6);
1051 const __m128i vxi6x01234567 = _mm_cvtepi8_epi16(vi6x01234567);
1052 const __m128i vk6x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t)));
1053 const __m128i vxk6x01234567 = _mm_cvtepi8_epi16(vk6x01234567);
1054 i6 += 8;
1055
1056
1057 vprod01234567 = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
1058
1059 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
1060 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
1061
1062 const __m128i vi7x01234567 = _mm_loadl_epi64((const __m128i*) i7);
1063 const __m128i vxi7x01234567 = _mm_cvtepi8_epi16(vi7x01234567);
1064 const __m128i vk7x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 56 * sizeof(int8_t)));
1065 const __m128i vxk7x01234567 = _mm_cvtepi8_epi16(vk7x01234567);
1066 i7 += 8;
1067
1068
1069 vprod01234567 = _mm_mullo_epi16(vxi7x01234567, vxk7x01234567);
1070
1071 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
1072 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
1073
1074 const __m128i vi8x01234567 = _mm_loadl_epi64((const __m128i*) i8);
1075 const __m128i vxi8x01234567 = _mm_cvtepi8_epi16(vi8x01234567);
1076 const __m128i vk8x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 64 * sizeof(int8_t)));
1077 const __m128i vxk8x01234567 = _mm_cvtepi8_epi16(vk8x01234567);
1078 i8 += 8;
1079
1080
1081 vprod01234567 = _mm_mullo_epi16(vxi8x01234567, vxk8x01234567);
1082
1083 vacc0123 = _mm_add_epi32(vacc0123, _mm_cvtepi16_epi32(vprod01234567));
1084 vacc4567 = _mm_add_epi32(vacc4567, _mm_srai_epi32(_mm_unpackhi_epi16(vprod01234567, vprod01234567), 16));
1085
1086 __m128 vscaled0123 = _mm_cvtepi32_ps(vacc0123);
1087 __m128 vscaled4567 = _mm_cvtepi32_ps(vacc4567);
1088
1089 vscaled0123 = _mm_mul_ps(vscaled0123, vscale);
1090 vscaled4567 = _mm_mul_ps(vscaled4567, vscale);
1091
1092 w = (void*) ((uintptr_t) w + 72);
1093
1094 vscaled0123 = _mm_min_ps(vscaled0123, voutput_max_less_zero_point);
1095 vscaled4567 = _mm_min_ps(vscaled4567, voutput_max_less_zero_point);
1096
1097 vacc0123 = _mm_cvtps_epi32(vscaled0123);
1098 vacc4567 = _mm_cvtps_epi32(vscaled4567);
1099
1100 __m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
1101
1102
1103 __m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
1104
1105 vout0123456701234567 = _mm_max_epi8(vout0123456701234567, voutput_min);
1106
1107 if XNN_LIKELY(c >= 8) {
1108 _mm_storel_epi64((__m128i*) output, vout0123456701234567);
1109 output += 8;
1110 c -= 8;
1111 } else {
1112 if (c & 4) {
1113 unaligned_store_u32(output, (uint32_t) _mm_cvtsi128_si32(vout0123456701234567));
1114 vout0123456701234567 = _mm_srli_epi64(vout0123456701234567, 32);
1115 output += 4;
1116 }
1117 if (c & 2) {
1118 unaligned_store_u16(output, (uint16_t) _mm_extract_epi16(vout0123456701234567, 0));
1119 vout0123456701234567 = _mm_srli_epi32(vout0123456701234567, 16);
1120 output += 2;
1121 }
1122 if (c & 1) {
1123 *output = (int8_t) _mm_extract_epi8(vout0123456701234567, 0);
1124 output += 1;
1125 }
1126 c = 0;
1127 }
1128 } while (c != 0);
1129 }
1130 }
1131
1132 input = (const int8_t**) ((uintptr_t) input + input_stride);
1133 output = (int8_t*) ((uintptr_t) output + output_increment);
1134 } while (--output_width != 0);
1135 }