Debian Codegraph @

src/qs8-dwconv/gen/qs8-dwconv-25p32c-minmax-fp32-avx2-mul16-vpmovsx.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-dwconv/unipass-avx2-mul16-vpmovsx.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2020 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include <immintrin.h>
13
14 #include "xnnpack/dwconv.h"
15 #include "xnnpack/unaligned.h"
16
17
18 void xnn_qs8_dwconv_minmax_fp32_ukernel_25p32c__avx2_mul16_vpmovsx(
19 size_t channels,
20 size_t output_width,
21 const int8_t** input,
22 const void* weights,
23 int8_t* output,
24 intptr_t input_stride,
25 size_t output_increment,
26 size_t input_offset,
27 const int8_t* zero,
28 const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
29 {
30 assert(channels != 0);
31 assert(output_width != 0);
32
33 const __m256 vscale = _mm256_set1_ps(params->fp32_scalar.scale);
34 XNN_FORCE_REALIZATION(vscale);
35
36 const __m256 voutput_max_less_zero_point = _mm256_set1_ps((int32_t) params->fp32_scalar.output_max - (int32_t) params->fp32_scalar.output_zero_point);
37 const __m256i voutput_zero_point = _mm256_set1_epi16(params->fp32_scalar.output_zero_point);
38 const __m128i voutput_min = _mm_set1_epi8(params->fp32_scalar.output_min);
39 XNN_FORCE_REALIZATION(voutput_max_less_zero_point);
40 XNN_FORCE_REALIZATION(voutput_zero_point);
41 XNN_FORCE_REALIZATION(voutput_min);
42
43 do {
44 const int8_t* i0 = input[0];
45 assert(i0 != NULL);
46 if XNN_UNPREDICTABLE(i0 != zero) {
47 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
48 }
49 const int8_t* i1 = input[1];
50 assert(i1 != NULL);
51 if XNN_UNPREDICTABLE(i1 != zero) {
52 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
53 }
54 const int8_t* i2 = input[2];
55 assert(i2 != NULL);
56 if XNN_UNPREDICTABLE(i2 != zero) {
57 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
58 }
59 const int8_t* i3 = input[3];
60 assert(i3 != NULL);
61 if XNN_UNPREDICTABLE(i3 != zero) {
62 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
63 }
64 const int8_t* i4 = input[4];
65 assert(i4 != NULL);
66 if XNN_UNPREDICTABLE(i4 != zero) {
67 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
68 }
69 const int8_t* i5 = input[5];
70 assert(i5 != NULL);
71 if XNN_UNPREDICTABLE(i5 != zero) {
72 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
73 }
74 const int8_t* i6 = input[6];
75 assert(i6 != NULL);
76 if XNN_UNPREDICTABLE(i6 != zero) {
77 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
78 }
79 const int8_t* i7 = input[7];
80 assert(i7 != NULL);
81 if XNN_UNPREDICTABLE(i7 != zero) {
82 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
83 }
84 const int8_t* i8 = input[8];
85 assert(i8 != NULL);
86 if XNN_UNPREDICTABLE(i8 != zero) {
87 i8 = (const int8_t*) ((uintptr_t) i8 + input_offset);
88 }
89 const int8_t* i9 = input[9];
90 assert(i9 != NULL);
91 if XNN_UNPREDICTABLE(i9 != zero) {
92 i9 = (const int8_t*) ((uintptr_t) i9 + input_offset);
93 }
94 const int8_t* i10 = input[10];
95 assert(i10 != NULL);
96 if XNN_UNPREDICTABLE(i10 != zero) {
97 i10 = (const int8_t*) ((uintptr_t) i10 + input_offset);
98 }
99 const int8_t* i11 = input[11];
100 assert(i11 != NULL);
101 if XNN_UNPREDICTABLE(i11 != zero) {
102 i11 = (const int8_t*) ((uintptr_t) i11 + input_offset);
103 }
104 const int8_t* i12 = input[12];
105 assert(i12 != NULL);
106 if XNN_UNPREDICTABLE(i12 != zero) {
107 i12 = (const int8_t*) ((uintptr_t) i12 + input_offset);
108 }
109 const int8_t* i13 = input[13];
110 assert(i13 != NULL);
111 if XNN_UNPREDICTABLE(i13 != zero) {
112 i13 = (const int8_t*) ((uintptr_t) i13 + input_offset);
113 }
114 const int8_t* i14 = input[14];
115 assert(i14 != NULL);
116 if XNN_UNPREDICTABLE(i14 != zero) {
117 i14 = (const int8_t*) ((uintptr_t) i14 + input_offset);
118 }
119 const int8_t* i15 = input[15];
120 assert(i15 != NULL);
121 if XNN_UNPREDICTABLE(i15 != zero) {
122 i15 = (const int8_t*) ((uintptr_t) i15 + input_offset);
123 }
124 const int8_t* i16 = input[16];
125 assert(i16 != NULL);
126 if XNN_UNPREDICTABLE(i16 != zero) {
127 i16 = (const int8_t*) ((uintptr_t) i16 + input_offset);
128 }
129 const int8_t* i17 = input[17];
130 assert(i17 != NULL);
131 if XNN_UNPREDICTABLE(i17 != zero) {
132 i17 = (const int8_t*) ((uintptr_t) i17 + input_offset);
133 }
134 const int8_t* i18 = input[18];
135 assert(i18 != NULL);
136 if XNN_UNPREDICTABLE(i18 != zero) {
137 i18 = (const int8_t*) ((uintptr_t) i18 + input_offset);
138 }
139 const int8_t* i19 = input[19];
140 assert(i19 != NULL);
141 if XNN_UNPREDICTABLE(i19 != zero) {
142 i19 = (const int8_t*) ((uintptr_t) i19 + input_offset);
143 }
144 const int8_t* i20 = input[20];
145 assert(i20 != NULL);
146 if XNN_UNPREDICTABLE(i20 != zero) {
147 i20 = (const int8_t*) ((uintptr_t) i20 + input_offset);
148 }
149 const int8_t* i21 = input[21];
150 assert(i21 != NULL);
151 if XNN_UNPREDICTABLE(i21 != zero) {
152 i21 = (const int8_t*) ((uintptr_t) i21 + input_offset);
153 }
154 const int8_t* i22 = input[22];
155 assert(i22 != NULL);
156 if XNN_UNPREDICTABLE(i22 != zero) {
157 i22 = (const int8_t*) ((uintptr_t) i22 + input_offset);
158 }
159 const int8_t* i23 = input[23];
160 assert(i23 != NULL);
161 if XNN_UNPREDICTABLE(i23 != zero) {
162 i23 = (const int8_t*) ((uintptr_t) i23 + input_offset);
163 }
164 const int8_t* i24 = input[24];
165 assert(i24 != NULL);
166 if XNN_UNPREDICTABLE(i24 != zero) {
167 i24 = (const int8_t*) ((uintptr_t) i24 + input_offset);
168 }
169 input = (const int8_t**) ((uintptr_t) input + input_stride);
170
171 size_t c = channels;
172 const void* w = weights;
173 for (; c >= 32; c -= 32) {
174 __m256i vacc01234567 = _mm256_loadu_si256((const __m256i*) w);
175 __m256i vacc89ABCDEF = _mm256_loadu_si256((const __m256i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
176 __m256i vaccGHIJKLMN = _mm256_loadu_si256((const __m256i*) ((uintptr_t) w + 16 * sizeof(int32_t)));
177 __m256i vaccOPQRSTUV = _mm256_loadu_si256((const __m256i*) ((uintptr_t) w + 24 * sizeof(int32_t)));
178
179
180 const __m256i vi0x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i0));
181 const __m256i vk0x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 0 * sizeof(int8_t))));
182 const __m256i vi0xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i0 + 16)));
183 const __m256i vk0xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 16 * sizeof(int8_t))));
184 i0 += 32;
185
186 const __m256i vprod0x0123456789ABCDEF = _mm256_mullo_epi16(vi0x0123456789ABCDEF, vk0x0123456789ABCDEF);
187 const __m128i vprod0x89ABCDEF = _mm256_extracti128_si256(vprod0x0123456789ABCDEF, 1);
188 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod0x0123456789ABCDEF)));
189 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod0x89ABCDEF));
190 const __m256i vprod0xGHIJKLMNOPQRSTUV = _mm256_mullo_epi16(vi0xGHIJKLMNOPQRSTUV, vk0xGHIJKLMNOPQRSTUV);
191 const __m128i vprod0xOPQRSTUV = _mm256_extracti128_si256(vprod0xGHIJKLMNOPQRSTUV, 1);
192 vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod0xGHIJKLMNOPQRSTUV)));
193 vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod0xOPQRSTUV));
194
195 const __m256i vi1x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i1));
196 const __m256i vk1x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 32 * sizeof(int8_t))));
197 const __m256i vi1xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i1 + 16)));
198 const __m256i vk1xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 48 * sizeof(int8_t))));
199 i1 += 32;
200
201 const __m256i vprod1x0123456789ABCDEF = _mm256_mullo_epi16(vi1x0123456789ABCDEF, vk1x0123456789ABCDEF);
202 const __m128i vprod1x89ABCDEF = _mm256_extracti128_si256(vprod1x0123456789ABCDEF, 1);
203 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod1x0123456789ABCDEF)));
204 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod1x89ABCDEF));
205 const __m256i vprod1xGHIJKLMNOPQRSTUV = _mm256_mullo_epi16(vi1xGHIJKLMNOPQRSTUV, vk1xGHIJKLMNOPQRSTUV);
206 const __m128i vprod1xOPQRSTUV = _mm256_extracti128_si256(vprod1xGHIJKLMNOPQRSTUV, 1);
207 vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod1xGHIJKLMNOPQRSTUV)));
208 vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod1xOPQRSTUV));
209
210 const __m256i vi2x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i2));
211 const __m256i vk2x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 64 * sizeof(int8_t))));
212 const __m256i vi2xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i2 + 16)));
213 const __m256i vk2xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 80 * sizeof(int8_t))));
214 i2 += 32;
215
216 const __m256i vprod2x0123456789ABCDEF = _mm256_mullo_epi16(vi2x0123456789ABCDEF, vk2x0123456789ABCDEF);
217 const __m128i vprod2x89ABCDEF = _mm256_extracti128_si256(vprod2x0123456789ABCDEF, 1);
218 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod2x0123456789ABCDEF)));
219 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod2x89ABCDEF));
220 const __m256i vprod2xGHIJKLMNOPQRSTUV = _mm256_mullo_epi16(vi2xGHIJKLMNOPQRSTUV, vk2xGHIJKLMNOPQRSTUV);
221 const __m128i vprod2xOPQRSTUV = _mm256_extracti128_si256(vprod2xGHIJKLMNOPQRSTUV, 1);
222 vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod2xGHIJKLMNOPQRSTUV)));
223 vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod2xOPQRSTUV));
224
225 const __m256i vi3x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i3));
226 const __m256i vk3x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 96 * sizeof(int8_t))));
227 const __m256i vi3xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i3 + 16)));
228 const __m256i vk3xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 112 * sizeof(int8_t))));
229 i3 += 32;
230
231 const __m256i vprod3x0123456789ABCDEF = _mm256_mullo_epi16(vi3x0123456789ABCDEF, vk3x0123456789ABCDEF);
232 const __m128i vprod3x89ABCDEF = _mm256_extracti128_si256(vprod3x0123456789ABCDEF, 1);
233 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod3x0123456789ABCDEF)));
234 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod3x89ABCDEF));
235 const __m256i vprod3xGHIJKLMNOPQRSTUV = _mm256_mullo_epi16(vi3xGHIJKLMNOPQRSTUV, vk3xGHIJKLMNOPQRSTUV);
236 const __m128i vprod3xOPQRSTUV = _mm256_extracti128_si256(vprod3xGHIJKLMNOPQRSTUV, 1);
237 vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod3xGHIJKLMNOPQRSTUV)));
238 vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod3xOPQRSTUV));
239
240 const __m256i vi4x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i4));
241 const __m256i vk4x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 128 * sizeof(int8_t))));
242 const __m256i vi4xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i4 + 16)));
243 const __m256i vk4xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 144 * sizeof(int8_t))));
244 i4 += 32;
245
246 const __m256i vprod4x0123456789ABCDEF = _mm256_mullo_epi16(vi4x0123456789ABCDEF, vk4x0123456789ABCDEF);
247 const __m128i vprod4x89ABCDEF = _mm256_extracti128_si256(vprod4x0123456789ABCDEF, 1);
248 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod4x0123456789ABCDEF)));
249 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod4x89ABCDEF));
250 const __m256i vprod4xGHIJKLMNOPQRSTUV = _mm256_mullo_epi16(vi4xGHIJKLMNOPQRSTUV, vk4xGHIJKLMNOPQRSTUV);
251 const __m128i vprod4xOPQRSTUV = _mm256_extracti128_si256(vprod4xGHIJKLMNOPQRSTUV, 1);
252 vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod4xGHIJKLMNOPQRSTUV)));
253 vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod4xOPQRSTUV));
254
255 const __m256i vi5x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i5));
256 const __m256i vk5x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 160 * sizeof(int8_t))));
257 const __m256i vi5xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i5 + 16)));
258 const __m256i vk5xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 176 * sizeof(int8_t))));
259 i5 += 32;
260
261 const __m256i vprod5x0123456789ABCDEF = _mm256_mullo_epi16(vi5x0123456789ABCDEF, vk5x0123456789ABCDEF);
262 const __m128i vprod5x89ABCDEF = _mm256_extracti128_si256(vprod5x0123456789ABCDEF, 1);
263 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod5x0123456789ABCDEF)));
264 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod5x89ABCDEF));
265 const __m256i vprod5xGHIJKLMNOPQRSTUV = _mm256_mullo_epi16(vi5xGHIJKLMNOPQRSTUV, vk5xGHIJKLMNOPQRSTUV);
266 const __m128i vprod5xOPQRSTUV = _mm256_extracti128_si256(vprod5xGHIJKLMNOPQRSTUV, 1);
267 vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod5xGHIJKLMNOPQRSTUV)));
268 vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod5xOPQRSTUV));
269
270 const __m256i vi6x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i6));
271 const __m256i vk6x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 192 * sizeof(int8_t))));
272 const __m256i vi6xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i6 + 16)));
273 const __m256i vk6xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 208 * sizeof(int8_t))));
274 i6 += 32;
275
276 const __m256i vprod6x0123456789ABCDEF = _mm256_mullo_epi16(vi6x0123456789ABCDEF, vk6x0123456789ABCDEF);
277 const __m128i vprod6x89ABCDEF = _mm256_extracti128_si256(vprod6x0123456789ABCDEF, 1);
278 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod6x0123456789ABCDEF)));
279 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod6x89ABCDEF));
280 const __m256i vprod6xGHIJKLMNOPQRSTUV = _mm256_mullo_epi16(vi6xGHIJKLMNOPQRSTUV, vk6xGHIJKLMNOPQRSTUV);
281 const __m128i vprod6xOPQRSTUV = _mm256_extracti128_si256(vprod6xGHIJKLMNOPQRSTUV, 1);
282 vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod6xGHIJKLMNOPQRSTUV)));
283 vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod6xOPQRSTUV));
284
285 const __m256i vi7x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i7));
286 const __m256i vk7x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 224 * sizeof(int8_t))));
287 const __m256i vi7xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i7 + 16)));
288 const __m256i vk7xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 240 * sizeof(int8_t))));
289 i7 += 32;
290
291 const __m256i vprod7x0123456789ABCDEF = _mm256_mullo_epi16(vi7x0123456789ABCDEF, vk7x0123456789ABCDEF);
292 const __m128i vprod7x89ABCDEF = _mm256_extracti128_si256(vprod7x0123456789ABCDEF, 1);
293 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod7x0123456789ABCDEF)));
294 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod7x89ABCDEF));
295 const __m256i vprod7xGHIJKLMNOPQRSTUV = _mm256_mullo_epi16(vi7xGHIJKLMNOPQRSTUV, vk7xGHIJKLMNOPQRSTUV);
296 const __m128i vprod7xOPQRSTUV = _mm256_extracti128_si256(vprod7xGHIJKLMNOPQRSTUV, 1);
297 vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod7xGHIJKLMNOPQRSTUV)));
298 vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod7xOPQRSTUV));
299
300 const __m256i vi8x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i8));
301 const __m256i vk8x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 256 * sizeof(int8_t))));
302 const __m256i vi8xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i8 + 16)));
303 const __m256i vk8xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 272 * sizeof(int8_t))));
304 i8 += 32;
305
306 const __m256i vprod8x0123456789ABCDEF = _mm256_mullo_epi16(vi8x0123456789ABCDEF, vk8x0123456789ABCDEF);
307 const __m128i vprod8x89ABCDEF = _mm256_extracti128_si256(vprod8x0123456789ABCDEF, 1);
308 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod8x0123456789ABCDEF)));
309 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod8x89ABCDEF));
310 const __m256i vprod8xGHIJKLMNOPQRSTUV = _mm256_mullo_epi16(vi8xGHIJKLMNOPQRSTUV, vk8xGHIJKLMNOPQRSTUV);
311 const __m128i vprod8xOPQRSTUV = _mm256_extracti128_si256(vprod8xGHIJKLMNOPQRSTUV, 1);
312 vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod8xGHIJKLMNOPQRSTUV)));
313 vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod8xOPQRSTUV));
314
315 const __m256i vi9x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i9));
316 const __m256i vk9x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 288 * sizeof(int8_t))));
317 const __m256i vi9xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i9 + 16)));
318 const __m256i vk9xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 304 * sizeof(int8_t))));
319 i9 += 32;
320
321 const __m256i vprod9x0123456789ABCDEF = _mm256_mullo_epi16(vi9x0123456789ABCDEF, vk9x0123456789ABCDEF);
322 const __m128i vprod9x89ABCDEF = _mm256_extracti128_si256(vprod9x0123456789ABCDEF, 1);
323 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod9x0123456789ABCDEF)));
324 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod9x89ABCDEF));
325 const __m256i vprod9xGHIJKLMNOPQRSTUV = _mm256_mullo_epi16(vi9xGHIJKLMNOPQRSTUV, vk9xGHIJKLMNOPQRSTUV);
326 const __m128i vprod9xOPQRSTUV = _mm256_extracti128_si256(vprod9xGHIJKLMNOPQRSTUV, 1);
327 vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod9xGHIJKLMNOPQRSTUV)));
328 vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod9xOPQRSTUV));
329
330 const __m256i vi10x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i10));
331 const __m256i vk10x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 320 * sizeof(int8_t))));
332 const __m256i vi10xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i10 + 16)));
333 const __m256i vk10xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 336 * sizeof(int8_t))));
334 i10 += 32;
335
336 const __m256i vprod10x0123456789ABCDEF = _mm256_mullo_epi16(vi10x0123456789ABCDEF, vk10x0123456789ABCDEF);
337 const __m128i vprod10x89ABCDEF = _mm256_extracti128_si256(vprod10x0123456789ABCDEF, 1);
338 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod10x0123456789ABCDEF)));
339 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod10x89ABCDEF));
340 const __m256i vprod10xGHIJKLMNOPQRSTUV = _mm256_mullo_epi16(vi10xGHIJKLMNOPQRSTUV, vk10xGHIJKLMNOPQRSTUV);
341 const __m128i vprod10xOPQRSTUV = _mm256_extracti128_si256(vprod10xGHIJKLMNOPQRSTUV, 1);
342 vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod10xGHIJKLMNOPQRSTUV)));
343 vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod10xOPQRSTUV));
344
345 const __m256i vi11x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i11));
346 const __m256i vk11x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 352 * sizeof(int8_t))));
347 const __m256i vi11xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i11 + 16)));
348 const __m256i vk11xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 368 * sizeof(int8_t))));
349 i11 += 32;
350
351 const __m256i vprod11x0123456789ABCDEF = _mm256_mullo_epi16(vi11x0123456789ABCDEF, vk11x0123456789ABCDEF);
352 const __m128i vprod11x89ABCDEF = _mm256_extracti128_si256(vprod11x0123456789ABCDEF, 1);
353 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod11x0123456789ABCDEF)));
354 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod11x89ABCDEF));
355 const __m256i vprod11xGHIJKLMNOPQRSTUV = _mm256_mullo_epi16(vi11xGHIJKLMNOPQRSTUV, vk11xGHIJKLMNOPQRSTUV);
356 const __m128i vprod11xOPQRSTUV = _mm256_extracti128_si256(vprod11xGHIJKLMNOPQRSTUV, 1);
357 vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod11xGHIJKLMNOPQRSTUV)));
358 vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod11xOPQRSTUV));
359
360 const __m256i vi12x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i12));
361 const __m256i vk12x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 384 * sizeof(int8_t))));
362 const __m256i vi12xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i12 + 16)));
363 const __m256i vk12xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 400 * sizeof(int8_t))));
364 i12 += 32;
365
366 const __m256i vprod12x0123456789ABCDEF = _mm256_mullo_epi16(vi12x0123456789ABCDEF, vk12x0123456789ABCDEF);
367 const __m128i vprod12x89ABCDEF = _mm256_extracti128_si256(vprod12x0123456789ABCDEF, 1);
368 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod12x0123456789ABCDEF)));
369 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod12x89ABCDEF));
370 const __m256i vprod12xGHIJKLMNOPQRSTUV = _mm256_mullo_epi16(vi12xGHIJKLMNOPQRSTUV, vk12xGHIJKLMNOPQRSTUV);
371 const __m128i vprod12xOPQRSTUV = _mm256_extracti128_si256(vprod12xGHIJKLMNOPQRSTUV, 1);
372 vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod12xGHIJKLMNOPQRSTUV)));
373 vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod12xOPQRSTUV));
374
375 const __m256i vi13x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i13));
376 const __m256i vk13x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 416 * sizeof(int8_t))));
377 const __m256i vi13xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i13 + 16)));
378 const __m256i vk13xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 432 * sizeof(int8_t))));
379 i13 += 32;
380
381 const __m256i vprod13x0123456789ABCDEF = _mm256_mullo_epi16(vi13x0123456789ABCDEF, vk13x0123456789ABCDEF);
382 const __m128i vprod13x89ABCDEF = _mm256_extracti128_si256(vprod13x0123456789ABCDEF, 1);
383 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod13x0123456789ABCDEF)));
384 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod13x89ABCDEF));
385 const __m256i vprod13xGHIJKLMNOPQRSTUV = _mm256_mullo_epi16(vi13xGHIJKLMNOPQRSTUV, vk13xGHIJKLMNOPQRSTUV);
386 const __m128i vprod13xOPQRSTUV = _mm256_extracti128_si256(vprod13xGHIJKLMNOPQRSTUV, 1);
387 vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod13xGHIJKLMNOPQRSTUV)));
388 vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod13xOPQRSTUV));
389
390 const __m256i vi14x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i14));
391 const __m256i vk14x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 448 * sizeof(int8_t))));
392 const __m256i vi14xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i14 + 16)));
393 const __m256i vk14xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 464 * sizeof(int8_t))));
394 i14 += 32;
395
396 const __m256i vprod14x0123456789ABCDEF = _mm256_mullo_epi16(vi14x0123456789ABCDEF, vk14x0123456789ABCDEF);
397 const __m128i vprod14x89ABCDEF = _mm256_extracti128_si256(vprod14x0123456789ABCDEF, 1);
398 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod14x0123456789ABCDEF)));
399 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod14x89ABCDEF));
400 const __m256i vprod14xGHIJKLMNOPQRSTUV = _mm256_mullo_epi16(vi14xGHIJKLMNOPQRSTUV, vk14xGHIJKLMNOPQRSTUV);
401 const __m128i vprod14xOPQRSTUV = _mm256_extracti128_si256(vprod14xGHIJKLMNOPQRSTUV, 1);
402 vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod14xGHIJKLMNOPQRSTUV)));
403 vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod14xOPQRSTUV));
404
405 const __m256i vi15x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i15));
406 const __m256i vk15x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 480 * sizeof(int8_t))));
407 const __m256i vi15xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i15 + 16)));
408 const __m256i vk15xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 496 * sizeof(int8_t))));
409 i15 += 32;
410
411 const __m256i vprod15x0123456789ABCDEF = _mm256_mullo_epi16(vi15x0123456789ABCDEF, vk15x0123456789ABCDEF);
412 const __m128i vprod15x89ABCDEF = _mm256_extracti128_si256(vprod15x0123456789ABCDEF, 1);
413 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod15x0123456789ABCDEF)));
414 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod15x89ABCDEF));
415 const __m256i vprod15xGHIJKLMNOPQRSTUV = _mm256_mullo_epi16(vi15xGHIJKLMNOPQRSTUV, vk15xGHIJKLMNOPQRSTUV);
416 const __m128i vprod15xOPQRSTUV = _mm256_extracti128_si256(vprod15xGHIJKLMNOPQRSTUV, 1);
417 vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod15xGHIJKLMNOPQRSTUV)));
418 vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod15xOPQRSTUV));
419
420 const __m256i vi16x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i16));
421 const __m256i vk16x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 512 * sizeof(int8_t))));
422 const __m256i vi16xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i16 + 16)));
423 const __m256i vk16xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 528 * sizeof(int8_t))));
424 i16 += 32;
425
426 const __m256i vprod16x0123456789ABCDEF = _mm256_mullo_epi16(vi16x0123456789ABCDEF, vk16x0123456789ABCDEF);
427 const __m128i vprod16x89ABCDEF = _mm256_extracti128_si256(vprod16x0123456789ABCDEF, 1);
428 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod16x0123456789ABCDEF)));
429 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod16x89ABCDEF));
430 const __m256i vprod16xGHIJKLMNOPQRSTUV = _mm256_mullo_epi16(vi16xGHIJKLMNOPQRSTUV, vk16xGHIJKLMNOPQRSTUV);
431 const __m128i vprod16xOPQRSTUV = _mm256_extracti128_si256(vprod16xGHIJKLMNOPQRSTUV, 1);
432 vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod16xGHIJKLMNOPQRSTUV)));
433 vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod16xOPQRSTUV));
434
435 const __m256i vi17x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i17));
436 const __m256i vk17x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 544 * sizeof(int8_t))));
437 const __m256i vi17xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i17 + 16)));
438 const __m256i vk17xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 560 * sizeof(int8_t))));
439 i17 += 32;
440
441 const __m256i vprod17x0123456789ABCDEF = _mm256_mullo_epi16(vi17x0123456789ABCDEF, vk17x0123456789ABCDEF);
442 const __m128i vprod17x89ABCDEF = _mm256_extracti128_si256(vprod17x0123456789ABCDEF, 1);
443 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod17x0123456789ABCDEF)));
444 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod17x89ABCDEF));
445 const __m256i vprod17xGHIJKLMNOPQRSTUV = _mm256_mullo_epi16(vi17xGHIJKLMNOPQRSTUV, vk17xGHIJKLMNOPQRSTUV);
446 const __m128i vprod17xOPQRSTUV = _mm256_extracti128_si256(vprod17xGHIJKLMNOPQRSTUV, 1);
447 vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod17xGHIJKLMNOPQRSTUV)));
448 vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod17xOPQRSTUV));
449
450 const __m256i vi18x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i18));
451 const __m256i vk18x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 576 * sizeof(int8_t))));
452 const __m256i vi18xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i18 + 16)));
453 const __m256i vk18xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 592 * sizeof(int8_t))));
454 i18 += 32;
455
456 const __m256i vprod18x0123456789ABCDEF = _mm256_mullo_epi16(vi18x0123456789ABCDEF, vk18x0123456789ABCDEF);
457 const __m128i vprod18x89ABCDEF = _mm256_extracti128_si256(vprod18x0123456789ABCDEF, 1);
458 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod18x0123456789ABCDEF)));
459 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod18x89ABCDEF));
460 const __m256i vprod18xGHIJKLMNOPQRSTUV = _mm256_mullo_epi16(vi18xGHIJKLMNOPQRSTUV, vk18xGHIJKLMNOPQRSTUV);
461 const __m128i vprod18xOPQRSTUV = _mm256_extracti128_si256(vprod18xGHIJKLMNOPQRSTUV, 1);
462 vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod18xGHIJKLMNOPQRSTUV)));
463 vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod18xOPQRSTUV));
464
465 const __m256i vi19x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i19));
466 const __m256i vk19x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 608 * sizeof(int8_t))));
467 const __m256i vi19xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i19 + 16)));
468 const __m256i vk19xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 624 * sizeof(int8_t))));
469 i19 += 32;
470
471 const __m256i vprod19x0123456789ABCDEF = _mm256_mullo_epi16(vi19x0123456789ABCDEF, vk19x0123456789ABCDEF);
472 const __m128i vprod19x89ABCDEF = _mm256_extracti128_si256(vprod19x0123456789ABCDEF, 1);
473 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod19x0123456789ABCDEF)));
474 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod19x89ABCDEF));
475 const __m256i vprod19xGHIJKLMNOPQRSTUV = _mm256_mullo_epi16(vi19xGHIJKLMNOPQRSTUV, vk19xGHIJKLMNOPQRSTUV);
476 const __m128i vprod19xOPQRSTUV = _mm256_extracti128_si256(vprod19xGHIJKLMNOPQRSTUV, 1);
477 vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod19xGHIJKLMNOPQRSTUV)));
478 vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod19xOPQRSTUV));
479
480 const __m256i vi20x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i20));
481 const __m256i vk20x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 640 * sizeof(int8_t))));
482 const __m256i vi20xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i20 + 16)));
483 const __m256i vk20xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 656 * sizeof(int8_t))));
484 i20 += 32;
485
486 const __m256i vprod20x0123456789ABCDEF = _mm256_mullo_epi16(vi20x0123456789ABCDEF, vk20x0123456789ABCDEF);
487 const __m128i vprod20x89ABCDEF = _mm256_extracti128_si256(vprod20x0123456789ABCDEF, 1);
488 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod20x0123456789ABCDEF)));
489 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod20x89ABCDEF));
490 const __m256i vprod20xGHIJKLMNOPQRSTUV = _mm256_mullo_epi16(vi20xGHIJKLMNOPQRSTUV, vk20xGHIJKLMNOPQRSTUV);
491 const __m128i vprod20xOPQRSTUV = _mm256_extracti128_si256(vprod20xGHIJKLMNOPQRSTUV, 1);
492 vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod20xGHIJKLMNOPQRSTUV)));
493 vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod20xOPQRSTUV));
494
495 const __m256i vi21x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i21));
496 const __m256i vk21x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 672 * sizeof(int8_t))));
497 const __m256i vi21xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i21 + 16)));
498 const __m256i vk21xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 688 * sizeof(int8_t))));
499 i21 += 32;
500
501 const __m256i vprod21x0123456789ABCDEF = _mm256_mullo_epi16(vi21x0123456789ABCDEF, vk21x0123456789ABCDEF);
502 const __m128i vprod21x89ABCDEF = _mm256_extracti128_si256(vprod21x0123456789ABCDEF, 1);
503 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod21x0123456789ABCDEF)));
504 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod21x89ABCDEF));
505 const __m256i vprod21xGHIJKLMNOPQRSTUV = _mm256_mullo_epi16(vi21xGHIJKLMNOPQRSTUV, vk21xGHIJKLMNOPQRSTUV);
506 const __m128i vprod21xOPQRSTUV = _mm256_extracti128_si256(vprod21xGHIJKLMNOPQRSTUV, 1);
507 vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod21xGHIJKLMNOPQRSTUV)));
508 vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod21xOPQRSTUV));
509
510 const __m256i vi22x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i22));
511 const __m256i vk22x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 704 * sizeof(int8_t))));
512 const __m256i vi22xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i22 + 16)));
513 const __m256i vk22xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 720 * sizeof(int8_t))));
514 i22 += 32;
515
516 const __m256i vprod22x0123456789ABCDEF = _mm256_mullo_epi16(vi22x0123456789ABCDEF, vk22x0123456789ABCDEF);
517 const __m128i vprod22x89ABCDEF = _mm256_extracti128_si256(vprod22x0123456789ABCDEF, 1);
518 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod22x0123456789ABCDEF)));
519 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod22x89ABCDEF));
520 const __m256i vprod22xGHIJKLMNOPQRSTUV = _mm256_mullo_epi16(vi22xGHIJKLMNOPQRSTUV, vk22xGHIJKLMNOPQRSTUV);
521 const __m128i vprod22xOPQRSTUV = _mm256_extracti128_si256(vprod22xGHIJKLMNOPQRSTUV, 1);
522 vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod22xGHIJKLMNOPQRSTUV)));
523 vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod22xOPQRSTUV));
524
525 const __m256i vi23x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i23));
526 const __m256i vk23x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 736 * sizeof(int8_t))));
527 const __m256i vi23xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i23 + 16)));
528 const __m256i vk23xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 752 * sizeof(int8_t))));
529 i23 += 32;
530
531 const __m256i vprod23x0123456789ABCDEF = _mm256_mullo_epi16(vi23x0123456789ABCDEF, vk23x0123456789ABCDEF);
532 const __m128i vprod23x89ABCDEF = _mm256_extracti128_si256(vprod23x0123456789ABCDEF, 1);
533 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod23x0123456789ABCDEF)));
534 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod23x89ABCDEF));
535 const __m256i vprod23xGHIJKLMNOPQRSTUV = _mm256_mullo_epi16(vi23xGHIJKLMNOPQRSTUV, vk23xGHIJKLMNOPQRSTUV);
536 const __m128i vprod23xOPQRSTUV = _mm256_extracti128_si256(vprod23xGHIJKLMNOPQRSTUV, 1);
537 vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod23xGHIJKLMNOPQRSTUV)));
538 vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod23xOPQRSTUV));
539
540 const __m256i vi24x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i24));
541 const __m256i vk24x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 768 * sizeof(int8_t))));
542 const __m256i vi24xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i24 + 16)));
543 const __m256i vk24xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 784 * sizeof(int8_t))));
544 i24 += 32;
545
546 const __m256i vprod24x0123456789ABCDEF = _mm256_mullo_epi16(vi24x0123456789ABCDEF, vk24x0123456789ABCDEF);
547 const __m128i vprod24x89ABCDEF = _mm256_extracti128_si256(vprod24x0123456789ABCDEF, 1);
548 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod24x0123456789ABCDEF)));
549 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod24x89ABCDEF));
550 const __m256i vprod24xGHIJKLMNOPQRSTUV = _mm256_mullo_epi16(vi24xGHIJKLMNOPQRSTUV, vk24xGHIJKLMNOPQRSTUV);
551 const __m128i vprod24xOPQRSTUV = _mm256_extracti128_si256(vprod24xGHIJKLMNOPQRSTUV, 1);
552 vaccGHIJKLMN = _mm256_add_epi32(vaccGHIJKLMN, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod24xGHIJKLMNOPQRSTUV)));
553 vaccOPQRSTUV = _mm256_add_epi32(vaccOPQRSTUV, _mm256_cvtepi16_epi32(vprod24xOPQRSTUV));
554
555 w = (const void*) ((uintptr_t) w + 32 * sizeof(int32_t) + 800 * sizeof(int8_t));
556
557 __m256 vfpacc01234567 = _mm256_cvtepi32_ps(vacc01234567);
558 __m256 vfpacc89ABCDEF = _mm256_cvtepi32_ps(vacc89ABCDEF);
559 __m256 vfpaccGHIJKLMN = _mm256_cvtepi32_ps(vaccGHIJKLMN);
560 __m256 vfpaccOPQRSTUV = _mm256_cvtepi32_ps(vaccOPQRSTUV);
561
562 vfpacc01234567 = _mm256_mul_ps(vfpacc01234567, vscale);
563 vfpacc89ABCDEF = _mm256_mul_ps(vfpacc89ABCDEF, vscale);
564 vfpaccGHIJKLMN = _mm256_mul_ps(vfpaccGHIJKLMN, vscale);
565 vfpaccOPQRSTUV = _mm256_mul_ps(vfpaccOPQRSTUV, vscale);
566
567 vfpacc01234567 = _mm256_min_ps(vfpacc01234567, voutput_max_less_zero_point);
568 vfpacc89ABCDEF = _mm256_min_ps(vfpacc89ABCDEF, voutput_max_less_zero_point);
569 vfpaccGHIJKLMN = _mm256_min_ps(vfpaccGHIJKLMN, voutput_max_less_zero_point);
570 vfpaccOPQRSTUV = _mm256_min_ps(vfpaccOPQRSTUV, voutput_max_less_zero_point);
571
572 vacc01234567 = _mm256_cvtps_epi32(vfpacc01234567);
573 vacc89ABCDEF = _mm256_cvtps_epi32(vfpacc89ABCDEF);
574 vaccGHIJKLMN = _mm256_cvtps_epi32(vfpaccGHIJKLMN);
575 vaccOPQRSTUV = _mm256_cvtps_epi32(vfpaccOPQRSTUV);
576
577 const __m256i vout012389AB4567CDEF = _mm256_adds_epi16(_mm256_packs_epi32(vacc01234567, vacc89ABCDEF), voutput_zero_point);
578 const __m256i voutGHIJOPQRKLMNSTUV = _mm256_adds_epi16(_mm256_packs_epi32(vaccGHIJKLMN, vaccOPQRSTUV), voutput_zero_point);
579
580 __m128i vout0123456789ABCDEF = _mm_shuffle_epi32(_mm_packs_epi16(_mm256_castsi256_si128(vout012389AB4567CDEF), _mm256_extracti128_si256(vout012389AB4567CDEF, 1)), _MM_SHUFFLE(3, 1, 2, 0));
581 __m128i voutGHIJKLMNOPQRSTUV = _mm_shuffle_epi32(_mm_packs_epi16(_mm256_castsi256_si128(voutGHIJOPQRKLMNSTUV), _mm256_extracti128_si256(voutGHIJOPQRKLMNSTUV, 1)), _MM_SHUFFLE(3, 1, 2, 0));
582
583 vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
584 voutGHIJKLMNOPQRSTUV = _mm_max_epi8(voutGHIJKLMNOPQRSTUV, voutput_min);
585
586 _mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
587 _mm_storeu_si128((__m128i*) (output + 16), voutGHIJKLMNOPQRSTUV);
588 output += 32;
589 }
590 if XNN_UNLIKELY(c != 0) {
591 const int8_t* k = (const int8_t*) ((uintptr_t) w + 32 * sizeof(int32_t));
592 do {
593 __m256i vacc01234567 = _mm256_loadu_si256((const __m256i*) w);
594 __m256i vacc89ABCDEF = _mm256_loadu_si256((const __m256i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
595
596
597 const __m256i vi0x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i0));
598 const __m256i vk0x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) k));
599 i0 += 16;
600
601 const __m256i vprod0x0123456789ABCDEF = _mm256_mullo_epi16(vi0x0123456789ABCDEF, vk0x0123456789ABCDEF);
602 const __m128i vprod0x89ABCDEF = _mm256_extracti128_si256(vprod0x0123456789ABCDEF, 1);
603 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod0x0123456789ABCDEF)));
604 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod0x89ABCDEF));
605
606 const __m256i vi1x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i1));
607 const __m256i vk1x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (k + 32)));
608 i1 += 16;
609
610 const __m256i vprod1x0123456789ABCDEF = _mm256_mullo_epi16(vi1x0123456789ABCDEF, vk1x0123456789ABCDEF);
611 const __m128i vprod1x89ABCDEF = _mm256_extracti128_si256(vprod1x0123456789ABCDEF, 1);
612 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod1x0123456789ABCDEF)));
613 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod1x89ABCDEF));
614
615 const __m256i vi2x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i2));
616 const __m256i vk2x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (k + 64)));
617 i2 += 16;
618
619 const __m256i vprod2x0123456789ABCDEF = _mm256_mullo_epi16(vi2x0123456789ABCDEF, vk2x0123456789ABCDEF);
620 const __m128i vprod2x89ABCDEF = _mm256_extracti128_si256(vprod2x0123456789ABCDEF, 1);
621 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod2x0123456789ABCDEF)));
622 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod2x89ABCDEF));
623
624 const __m256i vi3x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i3));
625 const __m256i vk3x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (k + 96)));
626 i3 += 16;
627
628 const __m256i vprod3x0123456789ABCDEF = _mm256_mullo_epi16(vi3x0123456789ABCDEF, vk3x0123456789ABCDEF);
629 const __m128i vprod3x89ABCDEF = _mm256_extracti128_si256(vprod3x0123456789ABCDEF, 1);
630 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod3x0123456789ABCDEF)));
631 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod3x89ABCDEF));
632
633 const __m256i vi4x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i4));
634 const __m256i vk4x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (k + 128)));
635 i4 += 16;
636
637 const __m256i vprod4x0123456789ABCDEF = _mm256_mullo_epi16(vi4x0123456789ABCDEF, vk4x0123456789ABCDEF);
638 const __m128i vprod4x89ABCDEF = _mm256_extracti128_si256(vprod4x0123456789ABCDEF, 1);
639 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod4x0123456789ABCDEF)));
640 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod4x89ABCDEF));
641
642 const __m256i vi5x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i5));
643 const __m256i vk5x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (k + 160)));
644 i5 += 16;
645
646 const __m256i vprod5x0123456789ABCDEF = _mm256_mullo_epi16(vi5x0123456789ABCDEF, vk5x0123456789ABCDEF);
647 const __m128i vprod5x89ABCDEF = _mm256_extracti128_si256(vprod5x0123456789ABCDEF, 1);
648 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod5x0123456789ABCDEF)));
649 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod5x89ABCDEF));
650
651 const __m256i vi6x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i6));
652 const __m256i vk6x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (k + 192)));
653 i6 += 16;
654
655 const __m256i vprod6x0123456789ABCDEF = _mm256_mullo_epi16(vi6x0123456789ABCDEF, vk6x0123456789ABCDEF);
656 const __m128i vprod6x89ABCDEF = _mm256_extracti128_si256(vprod6x0123456789ABCDEF, 1);
657 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod6x0123456789ABCDEF)));
658 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod6x89ABCDEF));
659
660 const __m256i vi7x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i7));
661 const __m256i vk7x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (k + 224)));
662 i7 += 16;
663
664 const __m256i vprod7x0123456789ABCDEF = _mm256_mullo_epi16(vi7x0123456789ABCDEF, vk7x0123456789ABCDEF);
665 const __m128i vprod7x89ABCDEF = _mm256_extracti128_si256(vprod7x0123456789ABCDEF, 1);
666 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod7x0123456789ABCDEF)));
667 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod7x89ABCDEF));
668
669 const __m256i vi8x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i8));
670 const __m256i vk8x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (k + 256)));
671 i8 += 16;
672
673 const __m256i vprod8x0123456789ABCDEF = _mm256_mullo_epi16(vi8x0123456789ABCDEF, vk8x0123456789ABCDEF);
674 const __m128i vprod8x89ABCDEF = _mm256_extracti128_si256(vprod8x0123456789ABCDEF, 1);
675 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod8x0123456789ABCDEF)));
676 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod8x89ABCDEF));
677
678 const __m256i vi9x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i9));
679 const __m256i vk9x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (k + 288)));
680 i9 += 16;
681
682 const __m256i vprod9x0123456789ABCDEF = _mm256_mullo_epi16(vi9x0123456789ABCDEF, vk9x0123456789ABCDEF);
683 const __m128i vprod9x89ABCDEF = _mm256_extracti128_si256(vprod9x0123456789ABCDEF, 1);
684 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod9x0123456789ABCDEF)));
685 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod9x89ABCDEF));
686
687 const __m256i vi10x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i10));
688 const __m256i vk10x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (k + 320)));
689 i10 += 16;
690
691 const __m256i vprod10x0123456789ABCDEF = _mm256_mullo_epi16(vi10x0123456789ABCDEF, vk10x0123456789ABCDEF);
692 const __m128i vprod10x89ABCDEF = _mm256_extracti128_si256(vprod10x0123456789ABCDEF, 1);
693 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod10x0123456789ABCDEF)));
694 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod10x89ABCDEF));
695
696 const __m256i vi11x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i11));
697 const __m256i vk11x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (k + 352)));
698 i11 += 16;
699
700 const __m256i vprod11x0123456789ABCDEF = _mm256_mullo_epi16(vi11x0123456789ABCDEF, vk11x0123456789ABCDEF);
701 const __m128i vprod11x89ABCDEF = _mm256_extracti128_si256(vprod11x0123456789ABCDEF, 1);
702 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod11x0123456789ABCDEF)));
703 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod11x89ABCDEF));
704
705 const __m256i vi12x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i12));
706 const __m256i vk12x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (k + 384)));
707 i12 += 16;
708
709 const __m256i vprod12x0123456789ABCDEF = _mm256_mullo_epi16(vi12x0123456789ABCDEF, vk12x0123456789ABCDEF);
710 const __m128i vprod12x89ABCDEF = _mm256_extracti128_si256(vprod12x0123456789ABCDEF, 1);
711 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod12x0123456789ABCDEF)));
712 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod12x89ABCDEF));
713
714 const __m256i vi13x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i13));
715 const __m256i vk13x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (k + 416)));
716 i13 += 16;
717
718 const __m256i vprod13x0123456789ABCDEF = _mm256_mullo_epi16(vi13x0123456789ABCDEF, vk13x0123456789ABCDEF);
719 const __m128i vprod13x89ABCDEF = _mm256_extracti128_si256(vprod13x0123456789ABCDEF, 1);
720 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod13x0123456789ABCDEF)));
721 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod13x89ABCDEF));
722
723 const __m256i vi14x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i14));
724 const __m256i vk14x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (k + 448)));
725 i14 += 16;
726
727 const __m256i vprod14x0123456789ABCDEF = _mm256_mullo_epi16(vi14x0123456789ABCDEF, vk14x0123456789ABCDEF);
728 const __m128i vprod14x89ABCDEF = _mm256_extracti128_si256(vprod14x0123456789ABCDEF, 1);
729 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod14x0123456789ABCDEF)));
730 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod14x89ABCDEF));
731
732 const __m256i vi15x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i15));
733 const __m256i vk15x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (k + 480)));
734 i15 += 16;
735
736 const __m256i vprod15x0123456789ABCDEF = _mm256_mullo_epi16(vi15x0123456789ABCDEF, vk15x0123456789ABCDEF);
737 const __m128i vprod15x89ABCDEF = _mm256_extracti128_si256(vprod15x0123456789ABCDEF, 1);
738 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod15x0123456789ABCDEF)));
739 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod15x89ABCDEF));
740
741 const __m256i vi16x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i16));
742 const __m256i vk16x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (k + 512)));
743 i16 += 16;
744
745 const __m256i vprod16x0123456789ABCDEF = _mm256_mullo_epi16(vi16x0123456789ABCDEF, vk16x0123456789ABCDEF);
746 const __m128i vprod16x89ABCDEF = _mm256_extracti128_si256(vprod16x0123456789ABCDEF, 1);
747 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod16x0123456789ABCDEF)));
748 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod16x89ABCDEF));
749
750 const __m256i vi17x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i17));
751 const __m256i vk17x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (k + 544)));
752 i17 += 16;
753
754 const __m256i vprod17x0123456789ABCDEF = _mm256_mullo_epi16(vi17x0123456789ABCDEF, vk17x0123456789ABCDEF);
755 const __m128i vprod17x89ABCDEF = _mm256_extracti128_si256(vprod17x0123456789ABCDEF, 1);
756 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod17x0123456789ABCDEF)));
757 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod17x89ABCDEF));
758
759 const __m256i vi18x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i18));
760 const __m256i vk18x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (k + 576)));
761 i18 += 16;
762
763 const __m256i vprod18x0123456789ABCDEF = _mm256_mullo_epi16(vi18x0123456789ABCDEF, vk18x0123456789ABCDEF);
764 const __m128i vprod18x89ABCDEF = _mm256_extracti128_si256(vprod18x0123456789ABCDEF, 1);
765 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod18x0123456789ABCDEF)));
766 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod18x89ABCDEF));
767
768 const __m256i vi19x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i19));
769 const __m256i vk19x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (k + 608)));
770 i19 += 16;
771
772 const __m256i vprod19x0123456789ABCDEF = _mm256_mullo_epi16(vi19x0123456789ABCDEF, vk19x0123456789ABCDEF);
773 const __m128i vprod19x89ABCDEF = _mm256_extracti128_si256(vprod19x0123456789ABCDEF, 1);
774 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod19x0123456789ABCDEF)));
775 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod19x89ABCDEF));
776
777 const __m256i vi20x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i20));
778 const __m256i vk20x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (k + 640)));
779 i20 += 16;
780
781 const __m256i vprod20x0123456789ABCDEF = _mm256_mullo_epi16(vi20x0123456789ABCDEF, vk20x0123456789ABCDEF);
782 const __m128i vprod20x89ABCDEF = _mm256_extracti128_si256(vprod20x0123456789ABCDEF, 1);
783 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod20x0123456789ABCDEF)));
784 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod20x89ABCDEF));
785
786 const __m256i vi21x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i21));
787 const __m256i vk21x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (k + 672)));
788 i21 += 16;
789
790 const __m256i vprod21x0123456789ABCDEF = _mm256_mullo_epi16(vi21x0123456789ABCDEF, vk21x0123456789ABCDEF);
791 const __m128i vprod21x89ABCDEF = _mm256_extracti128_si256(vprod21x0123456789ABCDEF, 1);
792 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod21x0123456789ABCDEF)));
793 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod21x89ABCDEF));
794
795 const __m256i vi22x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i22));
796 const __m256i vk22x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (k + 704)));
797 i22 += 16;
798
799 const __m256i vprod22x0123456789ABCDEF = _mm256_mullo_epi16(vi22x0123456789ABCDEF, vk22x0123456789ABCDEF);
800 const __m128i vprod22x89ABCDEF = _mm256_extracti128_si256(vprod22x0123456789ABCDEF, 1);
801 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod22x0123456789ABCDEF)));
802 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod22x89ABCDEF));
803
804 const __m256i vi23x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i23));
805 const __m256i vk23x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (k + 736)));
806 i23 += 16;
807
808 const __m256i vprod23x0123456789ABCDEF = _mm256_mullo_epi16(vi23x0123456789ABCDEF, vk23x0123456789ABCDEF);
809 const __m128i vprod23x89ABCDEF = _mm256_extracti128_si256(vprod23x0123456789ABCDEF, 1);
810 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod23x0123456789ABCDEF)));
811 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod23x89ABCDEF));
812
813 const __m256i vi24x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i24));
814 const __m256i vk24x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (k + 768)));
815 i24 += 16;
816
817 const __m256i vprod24x0123456789ABCDEF = _mm256_mullo_epi16(vi24x0123456789ABCDEF, vk24x0123456789ABCDEF);
818 const __m128i vprod24x89ABCDEF = _mm256_extracti128_si256(vprod24x0123456789ABCDEF, 1);
819 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod24x0123456789ABCDEF)));
820 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod24x89ABCDEF));
821
822 k += 16;
823
824 __m256 vfpacc01234567 = _mm256_cvtepi32_ps(vacc01234567);
825 __m256 vfpacc89ABCDEF = _mm256_cvtepi32_ps(vacc89ABCDEF);
826
827 vfpacc01234567 = _mm256_mul_ps(vfpacc01234567, vscale);
828 vfpacc89ABCDEF = _mm256_mul_ps(vfpacc89ABCDEF, vscale);
829
830 vfpacc01234567 = _mm256_min_ps(vfpacc01234567, voutput_max_less_zero_point);
831 vfpacc89ABCDEF = _mm256_min_ps(vfpacc89ABCDEF, voutput_max_less_zero_point);
832
833 vacc01234567 = _mm256_cvtps_epi32(vfpacc01234567);
834 vacc89ABCDEF = _mm256_cvtps_epi32(vfpacc89ABCDEF);
835
836 w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t));
837
838 __m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc01234567), _mm256_extracti128_si256(vacc01234567, 1)), _mm256_castsi256_si128(voutput_zero_point));
839 __m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc89ABCDEF), _mm256_extracti128_si256(vacc89ABCDEF, 1)), _mm256_castsi256_si128(voutput_zero_point));
840
841 __m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
842 vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
843
844 if XNN_LIKELY(c >= 16) {
845 _mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
846 output += 16;
847 c -= 16;
848 } else {
849 if (c & 8) {
850 _mm_storel_epi64((__m128i*) output, vout0123456789ABCDEF);
851 vout0123456789ABCDEF = _mm_unpackhi_epi64(vout0123456789ABCDEF, vout0123456789ABCDEF);
852 output += 8;
853 }
854 if (c & 4) {
855 unaligned_store_u32(output, (uint32_t) _mm_cvtsi128_si32(vout0123456789ABCDEF));
856 vout0123456789ABCDEF = _mm_srli_epi64(vout0123456789ABCDEF, 32);
857 output += 4;
858 }
859 if (c & 2) {
860 unaligned_store_u16(output, (uint16_t) _mm_extract_epi16(vout0123456789ABCDEF, 0));
861 vout0123456789ABCDEF = _mm_srli_epi32(vout0123456789ABCDEF, 16);
862 output += 2;
863 }
864 if (c & 1) {
865 *output = (int8_t) _mm_extract_epi8(vout0123456789ABCDEF, 0);
866 output += 1;
867 }
868 c = 0;
869 }
870 } while (c != 0);
871 }
872
873 output = (int8_t*) ((uintptr_t) output + output_increment);
874 } while (--output_width != 0);
875 }