Debian Codegraph @

src/qs8-dwconv/gen/qs8-dwconv-6f6m7l16c8s8r-minmax-fp32-sse2-mul16-add16.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-dwconv/multipass-sse-mul16.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include <emmintrin.h>
13
14 #include "xnnpack/dwconv.h"
15 #include "xnnpack/math.h"
16 #include "xnnpack/unaligned.h"
17
18
19 void xnn_qs8_dwconv_minmax_fp32_ukernel_6f6m7l16c8s8r__sse2_mul16_add16(
20 size_t channels,
21 size_t output_width,
22 const int8_t** input,
23 const void* weights,
24 int8_t* output,
25 intptr_t input_stride,
26 size_t output_increment,
27 size_t input_offset,
28 const int8_t* zero,
29 size_t kernel_size,
30 int32_t* buffer,
31 const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
32 {
33 assert(channels != 0);
34 assert(output_width != 0);
35 assert(kernel_size > 6);
36
37
38 const __m128 vscale = _mm_set1_ps(params->fp32_scalar.scale);
39 XNN_FORCE_REALIZATION(vscale);
40
41 const __m128 voutput_max_less_zero_point = _mm_set1_ps((int32_t) params->fp32_scalar.output_max - (int32_t) params->fp32_scalar.output_zero_point);
42 const __m128i voutput_zero_point = _mm_set1_epi16(params->fp32_scalar.output_zero_point);
43 const __m128i voutput_min = _mm_set1_epi16(params->fp32_scalar.output_min);
44 XNN_FORCE_REALIZATION(voutput_max_less_zero_point);
45 XNN_FORCE_REALIZATION(voutput_zero_point);
46 XNN_FORCE_REALIZATION(voutput_min);
47
48 do {
49 const void* w = weights;
50
51 // First pass to process 6 inputs.
52 {
53 int32_t* b = buffer;
54 const int8_t* i0 = input[0];
55 assert(i0 != NULL);
56 if XNN_UNPREDICTABLE(i0 != zero) {
57 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
58 }
59 const int8_t* i1 = input[1];
60 assert(i1 != NULL);
61 if XNN_UNPREDICTABLE(i1 != zero) {
62 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
63 }
64 const int8_t* i2 = input[2];
65 assert(i2 != NULL);
66 if XNN_UNPREDICTABLE(i2 != zero) {
67 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
68 }
69 const int8_t* i3 = input[3];
70 assert(i3 != NULL);
71 if XNN_UNPREDICTABLE(i3 != zero) {
72 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
73 }
74 const int8_t* i4 = input[4];
75 assert(i4 != NULL);
76 if XNN_UNPREDICTABLE(i4 != zero) {
77 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
78 }
79 const int8_t* i5 = input[5];
80 assert(i5 != NULL);
81 if XNN_UNPREDICTABLE(i5 != zero) {
82 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
83 }
84 input += 6;
85
86 size_t c = round_up_po2(channels, 8);
87
88 for (; c >= 16; c -= 16) {
89 __m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
90 __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
91 __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
92 __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
93
94
95
96 const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
97 const __m128i vk0x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 0 * sizeof(int8_t)));
98 const __m128i vi0x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i0 + 8));
99 const __m128i vk0x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 8 * sizeof(int8_t)));
100 i0 += 16;
101
102 const __m128i vxi0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x01234567, vi0x01234567), 8);
103 const __m128i vxk0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x01234567, vk0x01234567), 8);
104 const __m128i vxi0x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x89ABCDEF, vi0x89ABCDEF), 8);
105 const __m128i vxk0x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x89ABCDEF, vk0x89ABCDEF), 8);
106
107 __m128i vprod01234567 = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
108 __m128i vprod89ABCDEF = _mm_mullo_epi16(vxi0x89ABCDEF, vxk0x89ABCDEF);
109
110
111 const __m128i vi1x01234567 = _mm_loadl_epi64((const __m128i*) i1);
112 const __m128i vk1x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 16 * sizeof(int8_t)));
113 const __m128i vi1x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i1 + 8));
114 const __m128i vk1x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 24 * sizeof(int8_t)));
115 i1 += 16;
116
117 const __m128i vxi1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x01234567, vi1x01234567), 8);
118 const __m128i vxk1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x01234567, vk1x01234567), 8);
119 const __m128i vxi1x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x89ABCDEF, vi1x89ABCDEF), 8);
120 const __m128i vxk1x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x89ABCDEF, vk1x89ABCDEF), 8);
121
122 vprod01234567 = _mm_add_epi16(vprod01234567, _mm_mullo_epi16(vxi1x01234567, vxk1x01234567));
123 vprod89ABCDEF = _mm_add_epi16(vprod89ABCDEF, _mm_mullo_epi16(vxi1x89ABCDEF, vxk1x89ABCDEF));
124
125 const __m128i vsignprod1x01234567 = _mm_cmpgt_epi16(_mm_setzero_si128(), vprod01234567);
126 vacc0123 = _mm_add_epi32(vacc0123, _mm_unpacklo_epi16(vprod01234567, vsignprod1x01234567));
127 vacc4567 = _mm_add_epi32(vacc4567, _mm_unpackhi_epi16(vprod01234567, vsignprod1x01234567));
128 const __m128i vsignprod1x89ABCDEF = _mm_cmpgt_epi16(_mm_setzero_si128(), vprod89ABCDEF);
129 vacc89AB = _mm_add_epi32(vacc89AB, _mm_unpacklo_epi16(vprod89ABCDEF, vsignprod1x89ABCDEF));
130 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_unpackhi_epi16(vprod89ABCDEF, vsignprod1x89ABCDEF));
131
132 const __m128i vi2x01234567 = _mm_loadl_epi64((const __m128i*) i2);
133 const __m128i vk2x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 32 * sizeof(int8_t)));
134 const __m128i vi2x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i2 + 8));
135 const __m128i vk2x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 40 * sizeof(int8_t)));
136 i2 += 16;
137
138 const __m128i vxi2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x01234567, vi2x01234567), 8);
139 const __m128i vxk2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x01234567, vk2x01234567), 8);
140 const __m128i vxi2x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x89ABCDEF, vi2x89ABCDEF), 8);
141 const __m128i vxk2x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x89ABCDEF, vk2x89ABCDEF), 8);
142
143 vprod01234567 = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
144 vprod89ABCDEF = _mm_mullo_epi16(vxi2x89ABCDEF, vxk2x89ABCDEF);
145
146
147 const __m128i vi3x01234567 = _mm_loadl_epi64((const __m128i*) i3);
148 const __m128i vk3x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 48 * sizeof(int8_t)));
149 const __m128i vi3x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i3 + 8));
150 const __m128i vk3x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 56 * sizeof(int8_t)));
151 i3 += 16;
152
153 const __m128i vxi3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x01234567, vi3x01234567), 8);
154 const __m128i vxk3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x01234567, vk3x01234567), 8);
155 const __m128i vxi3x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x89ABCDEF, vi3x89ABCDEF), 8);
156 const __m128i vxk3x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x89ABCDEF, vk3x89ABCDEF), 8);
157
158 vprod01234567 = _mm_add_epi16(vprod01234567, _mm_mullo_epi16(vxi3x01234567, vxk3x01234567));
159 vprod89ABCDEF = _mm_add_epi16(vprod89ABCDEF, _mm_mullo_epi16(vxi3x89ABCDEF, vxk3x89ABCDEF));
160
161 const __m128i vsignprod3x01234567 = _mm_cmpgt_epi16(_mm_setzero_si128(), vprod01234567);
162 vacc0123 = _mm_add_epi32(vacc0123, _mm_unpacklo_epi16(vprod01234567, vsignprod3x01234567));
163 vacc4567 = _mm_add_epi32(vacc4567, _mm_unpackhi_epi16(vprod01234567, vsignprod3x01234567));
164 const __m128i vsignprod3x89ABCDEF = _mm_cmpgt_epi16(_mm_setzero_si128(), vprod89ABCDEF);
165 vacc89AB = _mm_add_epi32(vacc89AB, _mm_unpacklo_epi16(vprod89ABCDEF, vsignprod3x89ABCDEF));
166 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_unpackhi_epi16(vprod89ABCDEF, vsignprod3x89ABCDEF));
167
168 const __m128i vi4x01234567 = _mm_loadl_epi64((const __m128i*) i4);
169 const __m128i vk4x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 64 * sizeof(int8_t)));
170 const __m128i vi4x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i4 + 8));
171 const __m128i vk4x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 72 * sizeof(int8_t)));
172 i4 += 16;
173
174 const __m128i vxi4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x01234567, vi4x01234567), 8);
175 const __m128i vxk4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x01234567, vk4x01234567), 8);
176 const __m128i vxi4x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x89ABCDEF, vi4x89ABCDEF), 8);
177 const __m128i vxk4x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x89ABCDEF, vk4x89ABCDEF), 8);
178
179 vprod01234567 = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
180 vprod89ABCDEF = _mm_mullo_epi16(vxi4x89ABCDEF, vxk4x89ABCDEF);
181
182
183 const __m128i vi5x01234567 = _mm_loadl_epi64((const __m128i*) i5);
184 const __m128i vk5x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 80 * sizeof(int8_t)));
185 const __m128i vi5x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i5 + 8));
186 const __m128i vk5x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 88 * sizeof(int8_t)));
187 i5 += 16;
188
189 const __m128i vxi5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x01234567, vi5x01234567), 8);
190 const __m128i vxk5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x01234567, vk5x01234567), 8);
191 const __m128i vxi5x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x89ABCDEF, vi5x89ABCDEF), 8);
192 const __m128i vxk5x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x89ABCDEF, vk5x89ABCDEF), 8);
193
194 vprod01234567 = _mm_add_epi16(vprod01234567, _mm_mullo_epi16(vxi5x01234567, vxk5x01234567));
195 vprod89ABCDEF = _mm_add_epi16(vprod89ABCDEF, _mm_mullo_epi16(vxi5x89ABCDEF, vxk5x89ABCDEF));
196
197 const __m128i vsignprod5x01234567 = _mm_cmpgt_epi16(_mm_setzero_si128(), vprod01234567);
198 vacc0123 = _mm_add_epi32(vacc0123, _mm_unpacklo_epi16(vprod01234567, vsignprod5x01234567));
199 vacc4567 = _mm_add_epi32(vacc4567, _mm_unpackhi_epi16(vprod01234567, vsignprod5x01234567));
200 const __m128i vsignprod5x89ABCDEF = _mm_cmpgt_epi16(_mm_setzero_si128(), vprod89ABCDEF);
201 vacc89AB = _mm_add_epi32(vacc89AB, _mm_unpacklo_epi16(vprod89ABCDEF, vsignprod5x89ABCDEF));
202 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_unpackhi_epi16(vprod89ABCDEF, vsignprod5x89ABCDEF));
203
204 w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 96 * sizeof(int8_t));
205
206 _mm_storeu_si128((__m128i*) b, vacc0123);
207 _mm_storeu_si128((__m128i*) (b + 4), vacc4567);
208 _mm_storeu_si128((__m128i*) (b + 8), vacc89AB);
209 _mm_storeu_si128((__m128i*) (b + 12), vaccCDEF);
210 b += 16;
211 }
212
213 if XNN_UNLIKELY(c != 0) {
214 do {
215 __m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
216 __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
217
218
219 const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
220 const __m128i vk0x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + sizeof(int32_t) * 8 * sizeof(int8_t)));
221 i0 += 8;
222
223 const __m128i vxi0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x01234567, vi0x01234567), 8);
224 const __m128i vxk0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x01234567, vk0x01234567), 8);
225
226 __m128i vprod01234567 = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
227
228
229 const __m128i vi1x01234567 = _mm_loadl_epi64((const __m128i*) i1);
230 const __m128i vk1x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + sizeof(int32_t) * 8 + 8 * sizeof(int8_t)));
231 i1 += 8;
232
233 const __m128i vxi1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x01234567, vi1x01234567), 8);
234 const __m128i vxk1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x01234567, vk1x01234567), 8);
235
236 vprod01234567 = _mm_add_epi16(vprod01234567, _mm_mullo_epi16(vxi1x01234567, vxk1x01234567));
237
238 const __m128i vsignprod1x01234567 = _mm_cmpgt_epi16(_mm_setzero_si128(), vprod01234567);
239 vacc0123 = _mm_add_epi32(vacc0123, _mm_unpacklo_epi16(vprod01234567, vsignprod1x01234567));
240 vacc4567 = _mm_add_epi32(vacc4567, _mm_unpackhi_epi16(vprod01234567, vsignprod1x01234567));
241
242 const __m128i vi2x01234567 = _mm_loadl_epi64((const __m128i*) i2);
243 const __m128i vk2x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + sizeof(int32_t) * 8 + 16 * sizeof(int8_t)));
244 i2 += 8;
245
246 const __m128i vxi2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x01234567, vi2x01234567), 8);
247 const __m128i vxk2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x01234567, vk2x01234567), 8);
248
249 vprod01234567 = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
250
251
252 const __m128i vi3x01234567 = _mm_loadl_epi64((const __m128i*) i3);
253 const __m128i vk3x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + sizeof(int32_t) * 8 + 24 * sizeof(int8_t)));
254 i3 += 8;
255
256 const __m128i vxi3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x01234567, vi3x01234567), 8);
257 const __m128i vxk3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x01234567, vk3x01234567), 8);
258
259 vprod01234567 = _mm_add_epi16(vprod01234567, _mm_mullo_epi16(vxi3x01234567, vxk3x01234567));
260
261 const __m128i vsignprod3x01234567 = _mm_cmpgt_epi16(_mm_setzero_si128(), vprod01234567);
262 vacc0123 = _mm_add_epi32(vacc0123, _mm_unpacklo_epi16(vprod01234567, vsignprod3x01234567));
263 vacc4567 = _mm_add_epi32(vacc4567, _mm_unpackhi_epi16(vprod01234567, vsignprod3x01234567));
264
265 const __m128i vi4x01234567 = _mm_loadl_epi64((const __m128i*) i4);
266 const __m128i vk4x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + sizeof(int32_t) * 8 + 32 * sizeof(int8_t)));
267 i4 += 8;
268
269 const __m128i vxi4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x01234567, vi4x01234567), 8);
270 const __m128i vxk4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x01234567, vk4x01234567), 8);
271
272 vprod01234567 = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
273
274
275 const __m128i vi5x01234567 = _mm_loadl_epi64((const __m128i*) i5);
276 const __m128i vk5x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + sizeof(int32_t) * 8 + 40 * sizeof(int8_t)));
277 i5 += 8;
278
279 const __m128i vxi5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x01234567, vi5x01234567), 8);
280 const __m128i vxk5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x01234567, vk5x01234567), 8);
281
282 vprod01234567 = _mm_add_epi16(vprod01234567, _mm_mullo_epi16(vxi5x01234567, vxk5x01234567));
283
284 const __m128i vsignprod5x01234567 = _mm_cmpgt_epi16(_mm_setzero_si128(), vprod01234567);
285 vacc0123 = _mm_add_epi32(vacc0123, _mm_unpacklo_epi16(vprod01234567, vsignprod5x01234567));
286 vacc4567 = _mm_add_epi32(vacc4567, _mm_unpackhi_epi16(vprod01234567, vsignprod5x01234567));
287
288 w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 48 * sizeof(int8_t));
289
290 _mm_storeu_si128((__m128i*) (b), vacc0123);
291 _mm_storeu_si128((__m128i*) (b + 4), vacc4567);
292 b += 8;
293 c -= 8;
294 } while (c != 0);
295 }
296 }
297
298 // Middle pass to process 6 inputs in each iteration.
299 for (size_t ks = kernel_size - 6; ks > 7; ks -= 6) {
300 int32_t* b = buffer;
301 const int8_t* i0 = input[0];
302 assert(i0 != NULL);
303 if XNN_UNPREDICTABLE(i0 != zero) {
304 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
305 }
306 const int8_t* i1 = input[1];
307 assert(i1 != NULL);
308 if XNN_UNPREDICTABLE(i1 != zero) {
309 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
310 }
311 const int8_t* i2 = input[2];
312 assert(i2 != NULL);
313 if XNN_UNPREDICTABLE(i2 != zero) {
314 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
315 }
316 const int8_t* i3 = input[3];
317 assert(i3 != NULL);
318 if XNN_UNPREDICTABLE(i3 != zero) {
319 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
320 }
321 const int8_t* i4 = input[4];
322 assert(i4 != NULL);
323 if XNN_UNPREDICTABLE(i4 != zero) {
324 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
325 }
326 const int8_t* i5 = input[5];
327 assert(i5 != NULL);
328 if XNN_UNPREDICTABLE(i5 != zero) {
329 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
330 }
331 input += 6;
332
333 size_t c = round_up_po2(channels, 8);
334
335 for (; c >= 16; c -= 16) {
336 __m128i vacc0123 = _mm_loadu_si128((const __m128i*) b);
337 __m128i vacc4567 = _mm_loadu_si128((const __m128i*) (b + 4));
338 __m128i vacc89AB = _mm_loadu_si128((const __m128i*) (b + 8));
339 __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) (b + 12));
340
341
342
343 const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
344 const __m128i vk0x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 0 * sizeof(int8_t)));
345 const __m128i vi0x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i0 + 8));
346 const __m128i vk0x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
347 i0 += 16;
348
349 const __m128i vxi0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x01234567, vi0x01234567), 8);
350 const __m128i vxk0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x01234567, vk0x01234567), 8);
351 const __m128i vxi0x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x89ABCDEF, vi0x89ABCDEF), 8);
352 const __m128i vxk0x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x89ABCDEF, vk0x89ABCDEF), 8);
353
354 __m128i vprod01234567 = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
355 __m128i vprod89ABCDEF = _mm_mullo_epi16(vxi0x89ABCDEF, vxk0x89ABCDEF);
356
357
358 const __m128i vi1x01234567 = _mm_loadl_epi64((const __m128i*) i1);
359 const __m128i vk1x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
360 const __m128i vi1x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i1 + 8));
361 const __m128i vk1x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
362 i1 += 16;
363
364 const __m128i vxi1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x01234567, vi1x01234567), 8);
365 const __m128i vxk1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x01234567, vk1x01234567), 8);
366 const __m128i vxi1x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x89ABCDEF, vi1x89ABCDEF), 8);
367 const __m128i vxk1x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x89ABCDEF, vk1x89ABCDEF), 8);
368
369 vprod01234567 = _mm_add_epi16(vprod01234567, _mm_mullo_epi16(vxi1x01234567, vxk1x01234567));
370 vprod89ABCDEF = _mm_add_epi16(vprod89ABCDEF, _mm_mullo_epi16(vxi1x89ABCDEF, vxk1x89ABCDEF));
371
372 const __m128i vsignprod1x01234567 = _mm_cmpgt_epi16(_mm_setzero_si128(), vprod01234567);
373 vacc0123 = _mm_add_epi32(vacc0123, _mm_unpacklo_epi16(vprod01234567, vsignprod1x01234567));
374 vacc4567 = _mm_add_epi32(vacc4567, _mm_unpackhi_epi16(vprod01234567, vsignprod1x01234567));
375 const __m128i vsignprod1x89ABCDEF = _mm_cmpgt_epi16(_mm_setzero_si128(), vprod89ABCDEF);
376 vacc89AB = _mm_add_epi32(vacc89AB, _mm_unpacklo_epi16(vprod89ABCDEF, vsignprod1x89ABCDEF));
377 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_unpackhi_epi16(vprod89ABCDEF, vsignprod1x89ABCDEF));
378
379 const __m128i vi2x01234567 = _mm_loadl_epi64((const __m128i*) i2);
380 const __m128i vk2x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t)));
381 const __m128i vi2x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i2 + 8));
382 const __m128i vk2x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 40 * sizeof(int8_t)));
383 i2 += 16;
384
385 const __m128i vxi2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x01234567, vi2x01234567), 8);
386 const __m128i vxk2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x01234567, vk2x01234567), 8);
387 const __m128i vxi2x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x89ABCDEF, vi2x89ABCDEF), 8);
388 const __m128i vxk2x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x89ABCDEF, vk2x89ABCDEF), 8);
389
390 vprod01234567 = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
391 vprod89ABCDEF = _mm_mullo_epi16(vxi2x89ABCDEF, vxk2x89ABCDEF);
392
393
394 const __m128i vi3x01234567 = _mm_loadl_epi64((const __m128i*) i3);
395 const __m128i vk3x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t)));
396 const __m128i vi3x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i3 + 8));
397 const __m128i vk3x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 56 * sizeof(int8_t)));
398 i3 += 16;
399
400 const __m128i vxi3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x01234567, vi3x01234567), 8);
401 const __m128i vxk3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x01234567, vk3x01234567), 8);
402 const __m128i vxi3x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x89ABCDEF, vi3x89ABCDEF), 8);
403 const __m128i vxk3x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x89ABCDEF, vk3x89ABCDEF), 8);
404
405 vprod01234567 = _mm_add_epi16(vprod01234567, _mm_mullo_epi16(vxi3x01234567, vxk3x01234567));
406 vprod89ABCDEF = _mm_add_epi16(vprod89ABCDEF, _mm_mullo_epi16(vxi3x89ABCDEF, vxk3x89ABCDEF));
407
408 const __m128i vsignprod3x01234567 = _mm_cmpgt_epi16(_mm_setzero_si128(), vprod01234567);
409 vacc0123 = _mm_add_epi32(vacc0123, _mm_unpacklo_epi16(vprod01234567, vsignprod3x01234567));
410 vacc4567 = _mm_add_epi32(vacc4567, _mm_unpackhi_epi16(vprod01234567, vsignprod3x01234567));
411 const __m128i vsignprod3x89ABCDEF = _mm_cmpgt_epi16(_mm_setzero_si128(), vprod89ABCDEF);
412 vacc89AB = _mm_add_epi32(vacc89AB, _mm_unpacklo_epi16(vprod89ABCDEF, vsignprod3x89ABCDEF));
413 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_unpackhi_epi16(vprod89ABCDEF, vsignprod3x89ABCDEF));
414
415 const __m128i vi4x01234567 = _mm_loadl_epi64((const __m128i*) i4);
416 const __m128i vk4x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 64 * sizeof(int8_t)));
417 const __m128i vi4x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i4 + 8));
418 const __m128i vk4x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 72 * sizeof(int8_t)));
419 i4 += 16;
420
421 const __m128i vxi4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x01234567, vi4x01234567), 8);
422 const __m128i vxk4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x01234567, vk4x01234567), 8);
423 const __m128i vxi4x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x89ABCDEF, vi4x89ABCDEF), 8);
424 const __m128i vxk4x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x89ABCDEF, vk4x89ABCDEF), 8);
425
426 vprod01234567 = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
427 vprod89ABCDEF = _mm_mullo_epi16(vxi4x89ABCDEF, vxk4x89ABCDEF);
428
429
430 const __m128i vi5x01234567 = _mm_loadl_epi64((const __m128i*) i5);
431 const __m128i vk5x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 80 * sizeof(int8_t)));
432 const __m128i vi5x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i5 + 8));
433 const __m128i vk5x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 88 * sizeof(int8_t)));
434 i5 += 16;
435
436 const __m128i vxi5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x01234567, vi5x01234567), 8);
437 const __m128i vxk5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x01234567, vk5x01234567), 8);
438 const __m128i vxi5x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x89ABCDEF, vi5x89ABCDEF), 8);
439 const __m128i vxk5x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x89ABCDEF, vk5x89ABCDEF), 8);
440
441 vprod01234567 = _mm_add_epi16(vprod01234567, _mm_mullo_epi16(vxi5x01234567, vxk5x01234567));
442 vprod89ABCDEF = _mm_add_epi16(vprod89ABCDEF, _mm_mullo_epi16(vxi5x89ABCDEF, vxk5x89ABCDEF));
443
444 const __m128i vsignprod5x01234567 = _mm_cmpgt_epi16(_mm_setzero_si128(), vprod01234567);
445 vacc0123 = _mm_add_epi32(vacc0123, _mm_unpacklo_epi16(vprod01234567, vsignprod5x01234567));
446 vacc4567 = _mm_add_epi32(vacc4567, _mm_unpackhi_epi16(vprod01234567, vsignprod5x01234567));
447 const __m128i vsignprod5x89ABCDEF = _mm_cmpgt_epi16(_mm_setzero_si128(), vprod89ABCDEF);
448 vacc89AB = _mm_add_epi32(vacc89AB, _mm_unpacklo_epi16(vprod89ABCDEF, vsignprod5x89ABCDEF));
449 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_unpackhi_epi16(vprod89ABCDEF, vsignprod5x89ABCDEF));
450
451 w = (const void*) ((uintptr_t) w + 96 * sizeof(int8_t));
452
453 _mm_storeu_si128((__m128i*) (b), vacc0123);
454 _mm_storeu_si128((__m128i*) (b + 4), vacc4567);
455 _mm_storeu_si128((__m128i*) (b + 8), vacc89AB);
456 _mm_storeu_si128((__m128i*) (b + 12), vaccCDEF);
457 b += 16;
458 }
459
460 if XNN_UNLIKELY(c != 0) {
461 do {
462 __m128i vacc0123 = _mm_loadu_si128((const __m128i*) b);
463 __m128i vacc4567 = _mm_loadu_si128((const __m128i*) (b + 4));
464
465
466 const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
467 const __m128i vk0x01234567 = _mm_loadl_epi64((const __m128i*) w);
468 i0 += 8;
469
470 const __m128i vxi0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x01234567, vi0x01234567), 8);
471 const __m128i vxk0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x01234567, vk0x01234567), 8);
472
473 __m128i vprod01234567 = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
474
475
476 const __m128i vi1x01234567 = _mm_loadl_epi64((const __m128i*) i1);
477 const __m128i vk1x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
478 i1 += 8;
479
480 const __m128i vxi1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x01234567, vi1x01234567), 8);
481 const __m128i vxk1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x01234567, vk1x01234567), 8);
482
483 vprod01234567 = _mm_add_epi16(vprod01234567, _mm_mullo_epi16(vxi1x01234567, vxk1x01234567));
484
485 const __m128i vsignprod1x01234567 = _mm_cmpgt_epi16(_mm_setzero_si128(), vprod01234567);
486 vacc0123 = _mm_add_epi32(vacc0123, _mm_unpacklo_epi16(vprod01234567, vsignprod1x01234567));
487 vacc4567 = _mm_add_epi32(vacc4567, _mm_unpackhi_epi16(vprod01234567, vsignprod1x01234567));
488
489 const __m128i vi2x01234567 = _mm_loadl_epi64((const __m128i*) i2);
490 const __m128i vk2x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
491 i2 += 8;
492
493 const __m128i vxi2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x01234567, vi2x01234567), 8);
494 const __m128i vxk2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x01234567, vk2x01234567), 8);
495
496 vprod01234567 = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
497
498
499 const __m128i vi3x01234567 = _mm_loadl_epi64((const __m128i*) i3);
500 const __m128i vk3x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
501 i3 += 8;
502
503 const __m128i vxi3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x01234567, vi3x01234567), 8);
504 const __m128i vxk3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x01234567, vk3x01234567), 8);
505
506 vprod01234567 = _mm_add_epi16(vprod01234567, _mm_mullo_epi16(vxi3x01234567, vxk3x01234567));
507
508 const __m128i vsignprod3x01234567 = _mm_cmpgt_epi16(_mm_setzero_si128(), vprod01234567);
509 vacc0123 = _mm_add_epi32(vacc0123, _mm_unpacklo_epi16(vprod01234567, vsignprod3x01234567));
510 vacc4567 = _mm_add_epi32(vacc4567, _mm_unpackhi_epi16(vprod01234567, vsignprod3x01234567));
511
512 const __m128i vi4x01234567 = _mm_loadl_epi64((const __m128i*) i4);
513 const __m128i vk4x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t)));
514 i4 += 8;
515
516 const __m128i vxi4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x01234567, vi4x01234567), 8);
517 const __m128i vxk4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x01234567, vk4x01234567), 8);
518
519 vprod01234567 = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
520
521
522 const __m128i vi5x01234567 = _mm_loadl_epi64((const __m128i*) i5);
523 const __m128i vk5x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 40 * sizeof(int8_t)));
524 i5 += 8;
525
526 const __m128i vxi5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x01234567, vi5x01234567), 8);
527 const __m128i vxk5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x01234567, vk5x01234567), 8);
528
529 vprod01234567 = _mm_add_epi16(vprod01234567, _mm_mullo_epi16(vxi5x01234567, vxk5x01234567));
530
531 const __m128i vsignprod5x01234567 = _mm_cmpgt_epi16(_mm_setzero_si128(), vprod01234567);
532 vacc0123 = _mm_add_epi32(vacc0123, _mm_unpacklo_epi16(vprod01234567, vsignprod5x01234567));
533 vacc4567 = _mm_add_epi32(vacc4567, _mm_unpackhi_epi16(vprod01234567, vsignprod5x01234567));
534
535 w = (const void*) ((uintptr_t) w + 48 * sizeof(int8_t));
536
537 _mm_storeu_si128((__m128i*) (b), vacc0123);
538 _mm_storeu_si128((__m128i*) (b + 4), vacc4567);
539 b += 8;
540 c -= 8;
541 } while (c != 0);
542 }
543 }
544
545 // Last pass to process up to 7 inputs.
546 {
547 const int32_t* b = buffer;
548 const int8_t* i0 = input[0];
549 assert(i0 != NULL);
550 if XNN_UNPREDICTABLE(i0 != zero) {
551 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
552 }
553 const int8_t* i1 = input[1];
554 assert(i1 != NULL);
555 if XNN_UNPREDICTABLE(i1 != zero) {
556 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
557 }
558 const int8_t* i2 = input[2];
559 assert(i2 != NULL);
560 if XNN_UNPREDICTABLE(i2 != zero) {
561 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
562 }
563 const int8_t* i3 = input[3];
564 assert(i3 != NULL);
565 if XNN_UNPREDICTABLE(i3 != zero) {
566 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
567 }
568 const int8_t* i4 = input[4];
569 assert(i4 != NULL);
570 if XNN_UNPREDICTABLE(i4 != zero) {
571 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
572 }
573 const int8_t* i5 = input[5];
574 assert(i5 != NULL);
575 if XNN_UNPREDICTABLE(i5 != zero) {
576 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
577 }
578 const int8_t* i6 = input[6];
579 assert(i6 != NULL);
580 if XNN_UNPREDICTABLE(i6 != zero) {
581 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
582 }
583
584 size_t c = channels;
585
586 for (; c >= 16; c -= 16) {
587 __m128i vacc0123 = _mm_loadu_si128((const __m128i*) b);
588 __m128i vacc4567 = _mm_loadu_si128((const __m128i*) (b + 4));
589 __m128i vacc89AB = _mm_loadu_si128((const __m128i*) (b + 8));
590 __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) (b + 12));
591 b += 16;
592
593
594 const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
595 const __m128i vk0x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 0 * sizeof(int8_t)));
596 const __m128i vi0x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i0 + 8));
597 const __m128i vk0x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
598 i0 += 16;
599
600 const __m128i vxi0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x01234567, vi0x01234567), 8);
601 const __m128i vxk0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x01234567, vk0x01234567), 8);
602 const __m128i vxi0x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x89ABCDEF, vi0x89ABCDEF), 8);
603 const __m128i vxk0x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x89ABCDEF, vk0x89ABCDEF), 8);
604
605 __m128i vprod01234567 = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
606 __m128i vprod89ABCDEF = _mm_mullo_epi16(vxi0x89ABCDEF, vxk0x89ABCDEF);
607
608
609 const __m128i vi1x01234567 = _mm_loadl_epi64((const __m128i*) i1);
610 const __m128i vk1x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
611 const __m128i vi1x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i1 + 8));
612 const __m128i vk1x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
613 i1 += 16;
614
615 const __m128i vxi1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x01234567, vi1x01234567), 8);
616 const __m128i vxk1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x01234567, vk1x01234567), 8);
617 const __m128i vxi1x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x89ABCDEF, vi1x89ABCDEF), 8);
618 const __m128i vxk1x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x89ABCDEF, vk1x89ABCDEF), 8);
619
620 vprod01234567 = _mm_add_epi16(vprod01234567, _mm_mullo_epi16(vxi1x01234567, vxk1x01234567));
621 vprod89ABCDEF = _mm_add_epi16(vprod89ABCDEF, _mm_mullo_epi16(vxi1x89ABCDEF, vxk1x89ABCDEF));
622
623 const __m128i vsignprod1x01234567 = _mm_cmpgt_epi16(_mm_setzero_si128(), vprod01234567);
624 vacc0123 = _mm_add_epi32(vacc0123, _mm_unpacklo_epi16(vprod01234567, vsignprod1x01234567));
625 vacc4567 = _mm_add_epi32(vacc4567, _mm_unpackhi_epi16(vprod01234567, vsignprod1x01234567));
626 const __m128i vsignprod1x89ABCDEF = _mm_cmpgt_epi16(_mm_setzero_si128(), vprod89ABCDEF);
627 vacc89AB = _mm_add_epi32(vacc89AB, _mm_unpacklo_epi16(vprod89ABCDEF, vsignprod1x89ABCDEF));
628 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_unpackhi_epi16(vprod89ABCDEF, vsignprod1x89ABCDEF));
629
630 const __m128i vi2x01234567 = _mm_loadl_epi64((const __m128i*) i2);
631 const __m128i vk2x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t)));
632 const __m128i vi2x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i2 + 8));
633 const __m128i vk2x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 40 * sizeof(int8_t)));
634 i2 += 16;
635
636 const __m128i vxi2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x01234567, vi2x01234567), 8);
637 const __m128i vxk2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x01234567, vk2x01234567), 8);
638 const __m128i vxi2x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x89ABCDEF, vi2x89ABCDEF), 8);
639 const __m128i vxk2x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x89ABCDEF, vk2x89ABCDEF), 8);
640
641 vprod01234567 = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
642 vprod89ABCDEF = _mm_mullo_epi16(vxi2x89ABCDEF, vxk2x89ABCDEF);
643
644
645 const __m128i vi3x01234567 = _mm_loadl_epi64((const __m128i*) i3);
646 const __m128i vk3x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t)));
647 const __m128i vi3x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i3 + 8));
648 const __m128i vk3x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 56 * sizeof(int8_t)));
649 i3 += 16;
650
651 const __m128i vxi3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x01234567, vi3x01234567), 8);
652 const __m128i vxk3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x01234567, vk3x01234567), 8);
653 const __m128i vxi3x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x89ABCDEF, vi3x89ABCDEF), 8);
654 const __m128i vxk3x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x89ABCDEF, vk3x89ABCDEF), 8);
655
656 vprod01234567 = _mm_add_epi16(vprod01234567, _mm_mullo_epi16(vxi3x01234567, vxk3x01234567));
657 vprod89ABCDEF = _mm_add_epi16(vprod89ABCDEF, _mm_mullo_epi16(vxi3x89ABCDEF, vxk3x89ABCDEF));
658
659 const __m128i vsignprod3x01234567 = _mm_cmpgt_epi16(_mm_setzero_si128(), vprod01234567);
660 vacc0123 = _mm_add_epi32(vacc0123, _mm_unpacklo_epi16(vprod01234567, vsignprod3x01234567));
661 vacc4567 = _mm_add_epi32(vacc4567, _mm_unpackhi_epi16(vprod01234567, vsignprod3x01234567));
662 const __m128i vsignprod3x89ABCDEF = _mm_cmpgt_epi16(_mm_setzero_si128(), vprod89ABCDEF);
663 vacc89AB = _mm_add_epi32(vacc89AB, _mm_unpacklo_epi16(vprod89ABCDEF, vsignprod3x89ABCDEF));
664 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_unpackhi_epi16(vprod89ABCDEF, vsignprod3x89ABCDEF));
665
666 const __m128i vi4x01234567 = _mm_loadl_epi64((const __m128i*) i4);
667 const __m128i vk4x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 64 * sizeof(int8_t)));
668 const __m128i vi4x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i4 + 8));
669 const __m128i vk4x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 72 * sizeof(int8_t)));
670 i4 += 16;
671
672 const __m128i vxi4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x01234567, vi4x01234567), 8);
673 const __m128i vxk4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x01234567, vk4x01234567), 8);
674 const __m128i vxi4x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x89ABCDEF, vi4x89ABCDEF), 8);
675 const __m128i vxk4x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x89ABCDEF, vk4x89ABCDEF), 8);
676
677 vprod01234567 = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
678 vprod89ABCDEF = _mm_mullo_epi16(vxi4x89ABCDEF, vxk4x89ABCDEF);
679
680
681 const __m128i vi5x01234567 = _mm_loadl_epi64((const __m128i*) i5);
682 const __m128i vk5x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 80 * sizeof(int8_t)));
683 const __m128i vi5x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i5 + 8));
684 const __m128i vk5x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 88 * sizeof(int8_t)));
685 i5 += 16;
686
687 const __m128i vxi5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x01234567, vi5x01234567), 8);
688 const __m128i vxk5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x01234567, vk5x01234567), 8);
689 const __m128i vxi5x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x89ABCDEF, vi5x89ABCDEF), 8);
690 const __m128i vxk5x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x89ABCDEF, vk5x89ABCDEF), 8);
691
692 vprod01234567 = _mm_add_epi16(vprod01234567, _mm_mullo_epi16(vxi5x01234567, vxk5x01234567));
693 vprod89ABCDEF = _mm_add_epi16(vprod89ABCDEF, _mm_mullo_epi16(vxi5x89ABCDEF, vxk5x89ABCDEF));
694
695 const __m128i vsignprod5x01234567 = _mm_cmpgt_epi16(_mm_setzero_si128(), vprod01234567);
696 vacc0123 = _mm_add_epi32(vacc0123, _mm_unpacklo_epi16(vprod01234567, vsignprod5x01234567));
697 vacc4567 = _mm_add_epi32(vacc4567, _mm_unpackhi_epi16(vprod01234567, vsignprod5x01234567));
698 const __m128i vsignprod5x89ABCDEF = _mm_cmpgt_epi16(_mm_setzero_si128(), vprod89ABCDEF);
699 vacc89AB = _mm_add_epi32(vacc89AB, _mm_unpacklo_epi16(vprod89ABCDEF, vsignprod5x89ABCDEF));
700 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_unpackhi_epi16(vprod89ABCDEF, vsignprod5x89ABCDEF));
701
702 const __m128i vi6x01234567 = _mm_loadl_epi64((const __m128i*) i6);
703 const __m128i vk6x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 96 * sizeof(int8_t)));
704 const __m128i vi6x89ABCDEF = _mm_loadl_epi64((const __m128i*) (i6 + 8));
705 const __m128i vk6x89ABCDEF = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 104 * sizeof(int8_t)));
706 i6 += 16;
707
708 const __m128i vxi6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x01234567, vi6x01234567), 8);
709 const __m128i vxk6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x01234567, vk6x01234567), 8);
710 const __m128i vxi6x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x89ABCDEF, vi6x89ABCDEF), 8);
711 const __m128i vxk6x89ABCDEF = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x89ABCDEF, vk6x89ABCDEF), 8);
712
713 vprod01234567 = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
714 vprod89ABCDEF = _mm_mullo_epi16(vxi6x89ABCDEF, vxk6x89ABCDEF);
715
716 const __m128i vsignprod6x01234567 = _mm_cmpgt_epi16(_mm_setzero_si128(), vprod01234567);
717 vacc0123 = _mm_add_epi32(vacc0123, _mm_unpacklo_epi16(vprod01234567, vsignprod6x01234567));
718 vacc4567 = _mm_add_epi32(vacc4567, _mm_unpackhi_epi16(vprod01234567, vsignprod6x01234567));
719 const __m128i vsignprod6x89ABCDEF = _mm_cmpgt_epi16(_mm_setzero_si128(), vprod89ABCDEF);
720 vacc89AB = _mm_add_epi32(vacc89AB, _mm_unpacklo_epi16(vprod89ABCDEF, vsignprod6x89ABCDEF));
721 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_unpackhi_epi16(vprod89ABCDEF, vsignprod6x89ABCDEF));
722
723 w = (const void*) ((uintptr_t) w + 112 * sizeof(int8_t));
724
725 __m128 vscaled0123 = _mm_cvtepi32_ps(vacc0123);
726 __m128 vscaled4567 = _mm_cvtepi32_ps(vacc4567);
727 __m128 vscaled89AB = _mm_cvtepi32_ps(vacc89AB);
728 __m128 vscaledCDEF = _mm_cvtepi32_ps(vaccCDEF);
729
730 vscaled0123 = _mm_mul_ps(vscaled0123, vscale);
731 vscaled4567 = _mm_mul_ps(vscaled4567, vscale);
732 vscaled89AB = _mm_mul_ps(vscaled89AB, vscale);
733 vscaledCDEF = _mm_mul_ps(vscaledCDEF, vscale);
734
735 vscaled0123 = _mm_min_ps(vscaled0123, voutput_max_less_zero_point);
736 vscaled4567 = _mm_min_ps(vscaled4567, voutput_max_less_zero_point);
737 vscaled89AB = _mm_min_ps(vscaled89AB, voutput_max_less_zero_point);
738 vscaledCDEF = _mm_min_ps(vscaledCDEF, voutput_max_less_zero_point);
739
740 vacc0123 = _mm_cvtps_epi32(vscaled0123);
741 vacc4567 = _mm_cvtps_epi32(vscaled4567);
742 vacc89AB = _mm_cvtps_epi32(vscaled89AB);
743 vaccCDEF = _mm_cvtps_epi32(vscaledCDEF);
744
745 __m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
746 __m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(vacc89AB, vaccCDEF), voutput_zero_point);
747
748 vout01234567 = _mm_max_epi16(vout01234567, voutput_min);
749 vout89ABCDEF = _mm_max_epi16(vout89ABCDEF, voutput_min);
750
751 __m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
752
753
754 _mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
755 output += 16;
756 }
757
758 if XNN_UNLIKELY(c != 0) {
759 do {
760 __m128i vacc0123 = _mm_loadu_si128((const __m128i*) b);
761 __m128i vacc4567 = _mm_loadu_si128((const __m128i*) (b + 4));
762 b += 8;
763
764
765 const __m128i vi0x01234567 = _mm_loadl_epi64((const __m128i*) i0);
766 const __m128i vk0x01234567 = _mm_loadl_epi64((const __m128i*) w);
767 i0 += 8;
768
769 const __m128i vxi0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi0x01234567, vi0x01234567), 8);
770 const __m128i vxk0x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk0x01234567, vk0x01234567), 8);
771
772 __m128i vprod01234567 = _mm_mullo_epi16(vxi0x01234567, vxk0x01234567);
773
774
775 const __m128i vi1x01234567 = _mm_loadl_epi64((const __m128i*) i1);
776 const __m128i vk1x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t)));
777 i1 += 8;
778
779 const __m128i vxi1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi1x01234567, vi1x01234567), 8);
780 const __m128i vxk1x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk1x01234567, vk1x01234567), 8);
781
782 vprod01234567 = _mm_add_epi16(vprod01234567, _mm_mullo_epi16(vxi1x01234567, vxk1x01234567));
783
784 const __m128i vsignprod1x01234567 = _mm_cmpgt_epi16(_mm_setzero_si128(), vprod01234567);
785 vacc0123 = _mm_add_epi32(vacc0123, _mm_unpacklo_epi16(vprod01234567, vsignprod1x01234567));
786 vacc4567 = _mm_add_epi32(vacc4567, _mm_unpackhi_epi16(vprod01234567, vsignprod1x01234567));
787
788 const __m128i vi2x01234567 = _mm_loadl_epi64((const __m128i*) i2);
789 const __m128i vk2x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t)));
790 i2 += 8;
791
792 const __m128i vxi2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi2x01234567, vi2x01234567), 8);
793 const __m128i vxk2x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk2x01234567, vk2x01234567), 8);
794
795 vprod01234567 = _mm_mullo_epi16(vxi2x01234567, vxk2x01234567);
796
797
798 const __m128i vi3x01234567 = _mm_loadl_epi64((const __m128i*) i3);
799 const __m128i vk3x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t)));
800 i3 += 8;
801
802 const __m128i vxi3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi3x01234567, vi3x01234567), 8);
803 const __m128i vxk3x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk3x01234567, vk3x01234567), 8);
804
805 vprod01234567 = _mm_add_epi16(vprod01234567, _mm_mullo_epi16(vxi3x01234567, vxk3x01234567));
806
807 const __m128i vsignprod3x01234567 = _mm_cmpgt_epi16(_mm_setzero_si128(), vprod01234567);
808 vacc0123 = _mm_add_epi32(vacc0123, _mm_unpacklo_epi16(vprod01234567, vsignprod3x01234567));
809 vacc4567 = _mm_add_epi32(vacc4567, _mm_unpackhi_epi16(vprod01234567, vsignprod3x01234567));
810
811 const __m128i vi4x01234567 = _mm_loadl_epi64((const __m128i*) i4);
812 const __m128i vk4x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t)));
813 i4 += 8;
814
815 const __m128i vxi4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi4x01234567, vi4x01234567), 8);
816 const __m128i vxk4x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk4x01234567, vk4x01234567), 8);
817
818 vprod01234567 = _mm_mullo_epi16(vxi4x01234567, vxk4x01234567);
819
820
821 const __m128i vi5x01234567 = _mm_loadl_epi64((const __m128i*) i5);
822 const __m128i vk5x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 40 * sizeof(int8_t)));
823 i5 += 8;
824
825 const __m128i vxi5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi5x01234567, vi5x01234567), 8);
826 const __m128i vxk5x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk5x01234567, vk5x01234567), 8);
827
828 vprod01234567 = _mm_add_epi16(vprod01234567, _mm_mullo_epi16(vxi5x01234567, vxk5x01234567));
829
830 const __m128i vsignprod5x01234567 = _mm_cmpgt_epi16(_mm_setzero_si128(), vprod01234567);
831 vacc0123 = _mm_add_epi32(vacc0123, _mm_unpacklo_epi16(vprod01234567, vsignprod5x01234567));
832 vacc4567 = _mm_add_epi32(vacc4567, _mm_unpackhi_epi16(vprod01234567, vsignprod5x01234567));
833
834 const __m128i vi6x01234567 = _mm_loadl_epi64((const __m128i*) i6);
835 const __m128i vk6x01234567 = _mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t)));
836 i6 += 8;
837
838 const __m128i vxi6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vi6x01234567, vi6x01234567), 8);
839 const __m128i vxk6x01234567 = _mm_srai_epi16(_mm_unpacklo_epi8(vk6x01234567, vk6x01234567), 8);
840
841 vprod01234567 = _mm_mullo_epi16(vxi6x01234567, vxk6x01234567);
842
843 const __m128i vsignprod6x01234567 = _mm_cmpgt_epi16(_mm_setzero_si128(), vprod01234567);
844 vacc0123 = _mm_add_epi32(vacc0123, _mm_unpacklo_epi16(vprod01234567, vsignprod6x01234567));
845 vacc4567 = _mm_add_epi32(vacc4567, _mm_unpackhi_epi16(vprod01234567, vsignprod6x01234567));
846
847 __m128 vscaled0123 = _mm_cvtepi32_ps(vacc0123);
848 __m128 vscaled4567 = _mm_cvtepi32_ps(vacc4567);
849
850 vscaled0123 = _mm_mul_ps(vscaled0123, vscale);
851 vscaled4567 = _mm_mul_ps(vscaled4567, vscale);
852
853 w = (void*) ((uintptr_t) w + 56);
854
855 vscaled0123 = _mm_min_ps(vscaled0123, voutput_max_less_zero_point);
856 vscaled4567 = _mm_min_ps(vscaled4567, voutput_max_less_zero_point);
857
858 vacc0123 = _mm_cvtps_epi32(vscaled0123);
859 vacc4567 = _mm_cvtps_epi32(vscaled4567);
860
861 __m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
862
863 vout01234567 = _mm_max_epi16(vout01234567, voutput_min);
864
865 __m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
866
867
868 if XNN_LIKELY(c >= 8) {
869 _mm_storel_epi64((__m128i*) output, vout0123456701234567);
870 output += 8;
871 c -= 8;
872 } else {
873 if (c & 4) {
874 unaligned_store_u32(output, (uint32_t) _mm_cvtsi128_si32(vout0123456701234567));
875 vout0123456701234567 = _mm_srli_epi64(vout0123456701234567, 32);
876 output += 4;
877 }
878 if (c & 2) {
879 unaligned_store_u16(output, (uint16_t) _mm_extract_epi16(vout0123456701234567, 0));
880 vout0123456701234567 = _mm_srli_epi32(vout0123456701234567, 16);
881 output += 2;
882 }
883 if (c & 1) {
884 *output = (int8_t) _mm_cvtsi128_si32(vout0123456701234567);
885 output += 1;
886 }
887 c = 0;
888 }
889 } while (c != 0);
890 }
891 }
892
893 input = (const int8_t**) ((uintptr_t) input + input_stride);
894 output = (int8_t*) ((uintptr_t) output + output_increment);
895 } while (--output_width != 0);
896 }