Debian Codegraph @

src/qs8-dwconv/gen/qs8-dwconv-8f8m9l16c8s8r-minmax-fp32-avx2-mul32.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-dwconv/multipass-avx2-mul32.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include <immintrin.h>
13
14 #include "xnnpack/common.h"
15 #include "xnnpack/dwconv.h"
16 #include "xnnpack/math.h"
17 #include "xnnpack/unaligned.h"
18
19
20 void xnn_qs8_dwconv_minmax_fp32_ukernel_8f8m9l16c8s8r__avx2_mul32(
21 size_t channels,
22 size_t output_width,
23 const int8_t** input,
24 const void* weights,
25 int8_t* output,
26 intptr_t input_stride,
27 size_t output_increment,
28 size_t input_offset,
29 const int8_t* zero,
30 size_t kernel_size,
31 int32_t* buffer,
32 const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
33 {
34 assert(channels != 0);
35 assert(output_width != 0);
36 assert(kernel_size > 8);
37
38
39 const __m256 vscale = _mm256_set1_ps(params->fp32_scalar.scale);
40 XNN_FORCE_REALIZATION(vscale);
41 const __m256 voutput_max_less_zero_point = _mm256_set1_ps((int32_t) params->fp32_scalar.output_max - (int32_t) params->fp32_scalar.output_zero_point);
42 const __m256i voutput_zero_point = _mm256_set1_epi16(params->fp32_scalar.output_zero_point);
43 const __m128i voutput_min = _mm_set1_epi8(params->fp32_scalar.output_min);
44 XNN_FORCE_REALIZATION(voutput_max_less_zero_point);
45 XNN_FORCE_REALIZATION(voutput_zero_point);
46 XNN_FORCE_REALIZATION(voutput_min);
47
48 do {
49 const void* w = weights;
50
51 // First pass to process 8 inputs.
52 {
53 int32_t* b = buffer;
54 const int8_t* i0 = input[0];
55 assert(i0 != NULL);
56 if XNN_UNPREDICTABLE(i0 != zero) {
57 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
58 }
59 const int8_t* i1 = input[1];
60 assert(i1 != NULL);
61 if XNN_UNPREDICTABLE(i1 != zero) {
62 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
63 }
64 const int8_t* i2 = input[2];
65 assert(i2 != NULL);
66 if XNN_UNPREDICTABLE(i2 != zero) {
67 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
68 }
69 const int8_t* i3 = input[3];
70 assert(i3 != NULL);
71 if XNN_UNPREDICTABLE(i3 != zero) {
72 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
73 }
74 const int8_t* i4 = input[4];
75 assert(i4 != NULL);
76 if XNN_UNPREDICTABLE(i4 != zero) {
77 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
78 }
79 const int8_t* i5 = input[5];
80 assert(i5 != NULL);
81 if XNN_UNPREDICTABLE(i5 != zero) {
82 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
83 }
84 const int8_t* i6 = input[6];
85 assert(i6 != NULL);
86 if XNN_UNPREDICTABLE(i6 != zero) {
87 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
88 }
89 const int8_t* i7 = input[7];
90 assert(i7 != NULL);
91 if XNN_UNPREDICTABLE(i7 != zero) {
92 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
93 }
94 input += 8;
95
96 size_t c = round_up_po2(channels, 8);
97
98 for (; c >= 16; c -= 16) {
99 __m256i vacc01234567 = _mm256_loadu_si256((const __m256i*) w);
100 __m256i vacc89ABCDEF = _mm256_loadu_si256((const __m256i*) ((const int32_t*) w + 8));
101
102
103 const __m256i vi0x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i0));
104 const __m256i vk0x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 0 * sizeof(int8_t))));
105 const __m256i vi0x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i0 + 8)));
106 const __m256i vk0x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 8 * sizeof(int8_t))));
107 i0 += 16;
108
109 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi0x01234567, vk0x01234567));
110 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi0x89ABCDEF, vk0x89ABCDEF));
111
112 const __m256i vi1x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i1));
113 const __m256i vk1x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 16 * sizeof(int8_t))));
114 const __m256i vi1x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i1 + 8)));
115 const __m256i vk1x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 24 * sizeof(int8_t))));
116 i1 += 16;
117
118 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi1x01234567, vk1x01234567));
119 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi1x89ABCDEF, vk1x89ABCDEF));
120
121 const __m256i vi2x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i2));
122 const __m256i vk2x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 32 * sizeof(int8_t))));
123 const __m256i vi2x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i2 + 8)));
124 const __m256i vk2x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 40 * sizeof(int8_t))));
125 i2 += 16;
126
127 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi2x01234567, vk2x01234567));
128 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi2x89ABCDEF, vk2x89ABCDEF));
129
130 const __m256i vi3x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i3));
131 const __m256i vk3x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 48 * sizeof(int8_t))));
132 const __m256i vi3x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i3 + 8)));
133 const __m256i vk3x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 56 * sizeof(int8_t))));
134 i3 += 16;
135
136 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi3x01234567, vk3x01234567));
137 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi3x89ABCDEF, vk3x89ABCDEF));
138
139 const __m256i vi4x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i4));
140 const __m256i vk4x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 64 * sizeof(int8_t))));
141 const __m256i vi4x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i4 + 8)));
142 const __m256i vk4x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 72 * sizeof(int8_t))));
143 i4 += 16;
144
145 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi4x01234567, vk4x01234567));
146 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi4x89ABCDEF, vk4x89ABCDEF));
147
148 const __m256i vi5x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i5));
149 const __m256i vk5x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 80 * sizeof(int8_t))));
150 const __m256i vi5x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i5 + 8)));
151 const __m256i vk5x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 88 * sizeof(int8_t))));
152 i5 += 16;
153
154 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi5x01234567, vk5x01234567));
155 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi5x89ABCDEF, vk5x89ABCDEF));
156
157 const __m256i vi6x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i6));
158 const __m256i vk6x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 96 * sizeof(int8_t))));
159 const __m256i vi6x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i6 + 8)));
160 const __m256i vk6x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 104 * sizeof(int8_t))));
161 i6 += 16;
162
163 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi6x01234567, vk6x01234567));
164 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi6x89ABCDEF, vk6x89ABCDEF));
165
166 const __m256i vi7x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i7));
167 const __m256i vk7x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 112 * sizeof(int8_t))));
168 const __m256i vi7x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i7 + 8)));
169 const __m256i vk7x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 120 * sizeof(int8_t))));
170 i7 += 16;
171
172 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi7x01234567, vk7x01234567));
173 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi7x89ABCDEF, vk7x89ABCDEF));
174
175 w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 128 * sizeof(int8_t));
176
177 _mm256_storeu_si256((__m256i*) b, vacc01234567);
178 _mm256_storeu_si256((__m256i*) (b + 8), vacc89ABCDEF);
179 b += 16;
180 }
181
182 if XNN_UNLIKELY(c != 0) {
183 do {
184 __m256i vacc01234567 = _mm256_loadu_si256((const __m256i*) w);
185
186 const __m256i vi0x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i0));
187 const __m256i vk0x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t))));
188 i0 += 8;
189
190 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi0x01234567, vk0x01234567));
191
192 const __m256i vi1x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i1));
193 const __m256i vk1x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 8 * sizeof(int8_t))));
194 i1 += 8;
195
196 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi1x01234567, vk1x01234567));
197
198 const __m256i vi2x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i2));
199 const __m256i vk2x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 16 * sizeof(int8_t))));
200 i2 += 8;
201
202 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi2x01234567, vk2x01234567));
203
204 const __m256i vi3x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i3));
205 const __m256i vk3x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 24 * sizeof(int8_t))));
206 i3 += 8;
207
208 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi3x01234567, vk3x01234567));
209
210 const __m256i vi4x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i4));
211 const __m256i vk4x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 32 * sizeof(int8_t))));
212 i4 += 8;
213
214 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi4x01234567, vk4x01234567));
215
216 const __m256i vi5x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i5));
217 const __m256i vk5x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 40 * sizeof(int8_t))));
218 i5 += 8;
219
220 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi5x01234567, vk5x01234567));
221
222 const __m256i vi6x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i6));
223 const __m256i vk6x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 48 * sizeof(int8_t))));
224 i6 += 8;
225
226 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi6x01234567, vk6x01234567));
227
228 const __m256i vi7x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i7));
229 const __m256i vk7x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 56 * sizeof(int8_t))));
230 i7 += 8;
231
232 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi7x01234567, vk7x01234567));
233
234 w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 64 * sizeof(int8_t));
235
236 _mm256_storeu_si256((__m256i*) b, vacc01234567);
237 b += 8;
238 c -= 8;
239 } while (c != 0);
240 }
241 }
242
243 // Middle pass to process 8 inputs in each iteration.
244 for (size_t ks = kernel_size - 8; ks > 9; ks -= 8) {
245 int32_t* b = buffer;
246 const int8_t* i0 = input[0];
247 assert(i0 != NULL);
248 if XNN_UNPREDICTABLE(i0 != zero) {
249 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
250 }
251 const int8_t* i1 = input[1];
252 assert(i1 != NULL);
253 if XNN_UNPREDICTABLE(i1 != zero) {
254 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
255 }
256 const int8_t* i2 = input[2];
257 assert(i2 != NULL);
258 if XNN_UNPREDICTABLE(i2 != zero) {
259 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
260 }
261 const int8_t* i3 = input[3];
262 assert(i3 != NULL);
263 if XNN_UNPREDICTABLE(i3 != zero) {
264 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
265 }
266 const int8_t* i4 = input[4];
267 assert(i4 != NULL);
268 if XNN_UNPREDICTABLE(i4 != zero) {
269 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
270 }
271 const int8_t* i5 = input[5];
272 assert(i5 != NULL);
273 if XNN_UNPREDICTABLE(i5 != zero) {
274 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
275 }
276 const int8_t* i6 = input[6];
277 assert(i6 != NULL);
278 if XNN_UNPREDICTABLE(i6 != zero) {
279 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
280 }
281 const int8_t* i7 = input[7];
282 assert(i7 != NULL);
283 if XNN_UNPREDICTABLE(i7 != zero) {
284 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
285 }
286 input += 8;
287
288 size_t c = round_up_po2(channels, 8);
289
290 for (; c >= 16; c -= 16) {
291 __m256i vacc01234567 = _mm256_loadu_si256((const __m256i*) b);
292 __m256i vacc89ABCDEF = _mm256_loadu_si256((const __m256i*) (b + 8));
293
294
295 const __m256i vi0x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i0));
296 const __m256i vk0x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 0 * sizeof(int8_t))));
297 const __m256i vi0x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i0 + 8)));
298 const __m256i vk0x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t))));
299 i0 += 16;
300
301 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi0x01234567, vk0x01234567));
302 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi0x89ABCDEF, vk0x89ABCDEF));
303
304 const __m256i vi1x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i1));
305 const __m256i vk1x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t))));
306 const __m256i vi1x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i1 + 8)));
307 const __m256i vk1x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t))));
308 i1 += 16;
309
310 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi1x01234567, vk1x01234567));
311 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi1x89ABCDEF, vk1x89ABCDEF));
312
313 const __m256i vi2x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i2));
314 const __m256i vk2x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t))));
315 const __m256i vi2x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i2 + 8)));
316 const __m256i vk2x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 40 * sizeof(int8_t))));
317 i2 += 16;
318
319 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi2x01234567, vk2x01234567));
320 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi2x89ABCDEF, vk2x89ABCDEF));
321
322 const __m256i vi3x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i3));
323 const __m256i vk3x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t))));
324 const __m256i vi3x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i3 + 8)));
325 const __m256i vk3x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 56 * sizeof(int8_t))));
326 i3 += 16;
327
328 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi3x01234567, vk3x01234567));
329 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi3x89ABCDEF, vk3x89ABCDEF));
330
331 const __m256i vi4x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i4));
332 const __m256i vk4x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 64 * sizeof(int8_t))));
333 const __m256i vi4x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i4 + 8)));
334 const __m256i vk4x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 72 * sizeof(int8_t))));
335 i4 += 16;
336
337 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi4x01234567, vk4x01234567));
338 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi4x89ABCDEF, vk4x89ABCDEF));
339
340 const __m256i vi5x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i5));
341 const __m256i vk5x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 80 * sizeof(int8_t))));
342 const __m256i vi5x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i5 + 8)));
343 const __m256i vk5x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 88 * sizeof(int8_t))));
344 i5 += 16;
345
346 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi5x01234567, vk5x01234567));
347 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi5x89ABCDEF, vk5x89ABCDEF));
348
349 const __m256i vi6x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i6));
350 const __m256i vk6x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 96 * sizeof(int8_t))));
351 const __m256i vi6x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i6 + 8)));
352 const __m256i vk6x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 104 * sizeof(int8_t))));
353 i6 += 16;
354
355 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi6x01234567, vk6x01234567));
356 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi6x89ABCDEF, vk6x89ABCDEF));
357
358 const __m256i vi7x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i7));
359 const __m256i vk7x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 112 * sizeof(int8_t))));
360 const __m256i vi7x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i7 + 8)));
361 const __m256i vk7x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 120 * sizeof(int8_t))));
362 i7 += 16;
363
364 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi7x01234567, vk7x01234567));
365 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi7x89ABCDEF, vk7x89ABCDEF));
366
367 w = (const void*) ((uintptr_t) w + 128 * sizeof(int8_t));
368
369 _mm256_storeu_si256((__m256i*) b, vacc01234567);
370 _mm256_storeu_si256((__m256i*) (b + 8), vacc89ABCDEF);
371 b += 16;
372 }
373
374 if XNN_UNLIKELY(c != 0) {
375 do {
376 __m256i vacc01234567 = _mm256_loadu_si256((const __m256i*) b);
377
378 const __m256i vi0x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i0));
379 const __m256i vk0x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w)));
380 i0 += 8;
381
382 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi0x01234567, vk0x01234567));
383
384 const __m256i vi1x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i1));
385 const __m256i vk1x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t))));
386 i1 += 8;
387
388 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi1x01234567, vk1x01234567));
389
390 const __m256i vi2x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i2));
391 const __m256i vk2x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t))));
392 i2 += 8;
393
394 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi2x01234567, vk2x01234567));
395
396 const __m256i vi3x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i3));
397 const __m256i vk3x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t))));
398 i3 += 8;
399
400 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi3x01234567, vk3x01234567));
401
402 const __m256i vi4x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i4));
403 const __m256i vk4x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t))));
404 i4 += 8;
405
406 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi4x01234567, vk4x01234567));
407
408 const __m256i vi5x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i5));
409 const __m256i vk5x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 40 * sizeof(int8_t))));
410 i5 += 8;
411
412 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi5x01234567, vk5x01234567));
413
414 const __m256i vi6x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i6));
415 const __m256i vk6x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t))));
416 i6 += 8;
417
418 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi6x01234567, vk6x01234567));
419
420 const __m256i vi7x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i7));
421 const __m256i vk7x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 56 * sizeof(int8_t))));
422 i7 += 8;
423
424 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi7x01234567, vk7x01234567));
425
426 w = (const void*) ((uintptr_t) w + 64 * sizeof(int8_t));
427
428 _mm256_storeu_si256((__m256i*) b, vacc01234567);
429 b += 8;
430 c -= 8;
431 } while (c != 0);
432 }
433 }
434
435 // Last pass to process up to 9 inputs.
436 {
437 const int32_t* b = buffer;
438 const int8_t* i0 = input[0];
439 assert(i0 != NULL);
440 if XNN_UNPREDICTABLE(i0 != zero) {
441 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
442 }
443 const int8_t* i1 = input[1];
444 assert(i1 != NULL);
445 if XNN_UNPREDICTABLE(i1 != zero) {
446 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
447 }
448 const int8_t* i2 = input[2];
449 assert(i2 != NULL);
450 if XNN_UNPREDICTABLE(i2 != zero) {
451 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
452 }
453 const int8_t* i3 = input[3];
454 assert(i3 != NULL);
455 if XNN_UNPREDICTABLE(i3 != zero) {
456 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
457 }
458 const int8_t* i4 = input[4];
459 assert(i4 != NULL);
460 if XNN_UNPREDICTABLE(i4 != zero) {
461 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
462 }
463 const int8_t* i5 = input[5];
464 assert(i5 != NULL);
465 if XNN_UNPREDICTABLE(i5 != zero) {
466 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
467 }
468 const int8_t* i6 = input[6];
469 assert(i6 != NULL);
470 if XNN_UNPREDICTABLE(i6 != zero) {
471 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
472 }
473 const int8_t* i7 = input[7];
474 assert(i7 != NULL);
475 if XNN_UNPREDICTABLE(i7 != zero) {
476 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
477 }
478 const int8_t* i8 = input[8];
479 assert(i8 != NULL);
480 if XNN_UNPREDICTABLE(i8 != zero) {
481 i8 = (const int8_t*) ((uintptr_t) i8 + input_offset);
482 }
483
484 size_t c = channels;
485
486 for (; c >= 16; c -= 16) {
487 __m256i vacc01234567 = _mm256_loadu_si256((const __m256i*) b);
488 __m256i vacc89ABCDEF = _mm256_loadu_si256((const __m256i*) (b + 8));
489 b += 16;
490
491
492 const __m256i vi0x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i0));
493 const __m256i vk0x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 0 * sizeof(int8_t))));
494 const __m256i vi0x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i0 + 8)));
495 const __m256i vk0x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t))));
496 i0 += 16;
497
498 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi0x01234567, vk0x01234567));
499 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi0x89ABCDEF, vk0x89ABCDEF));
500
501 const __m256i vi1x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i1));
502 const __m256i vk1x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t))));
503 const __m256i vi1x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i1 + 8)));
504 const __m256i vk1x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t))));
505 i1 += 16;
506
507 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi1x01234567, vk1x01234567));
508 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi1x89ABCDEF, vk1x89ABCDEF));
509
510 const __m256i vi2x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i2));
511 const __m256i vk2x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t))));
512 const __m256i vi2x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i2 + 8)));
513 const __m256i vk2x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 40 * sizeof(int8_t))));
514 i2 += 16;
515
516 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi2x01234567, vk2x01234567));
517 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi2x89ABCDEF, vk2x89ABCDEF));
518
519 const __m256i vi3x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i3));
520 const __m256i vk3x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t))));
521 const __m256i vi3x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i3 + 8)));
522 const __m256i vk3x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 56 * sizeof(int8_t))));
523 i3 += 16;
524
525 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi3x01234567, vk3x01234567));
526 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi3x89ABCDEF, vk3x89ABCDEF));
527
528 const __m256i vi4x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i4));
529 const __m256i vk4x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 64 * sizeof(int8_t))));
530 const __m256i vi4x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i4 + 8)));
531 const __m256i vk4x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 72 * sizeof(int8_t))));
532 i4 += 16;
533
534 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi4x01234567, vk4x01234567));
535 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi4x89ABCDEF, vk4x89ABCDEF));
536
537 const __m256i vi5x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i5));
538 const __m256i vk5x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 80 * sizeof(int8_t))));
539 const __m256i vi5x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i5 + 8)));
540 const __m256i vk5x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 88 * sizeof(int8_t))));
541 i5 += 16;
542
543 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi5x01234567, vk5x01234567));
544 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi5x89ABCDEF, vk5x89ABCDEF));
545
546 const __m256i vi6x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i6));
547 const __m256i vk6x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 96 * sizeof(int8_t))));
548 const __m256i vi6x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i6 + 8)));
549 const __m256i vk6x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 104 * sizeof(int8_t))));
550 i6 += 16;
551
552 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi6x01234567, vk6x01234567));
553 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi6x89ABCDEF, vk6x89ABCDEF));
554
555 const __m256i vi7x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i7));
556 const __m256i vk7x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 112 * sizeof(int8_t))));
557 const __m256i vi7x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i7 + 8)));
558 const __m256i vk7x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 120 * sizeof(int8_t))));
559 i7 += 16;
560
561 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi7x01234567, vk7x01234567));
562 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi7x89ABCDEF, vk7x89ABCDEF));
563
564 const __m256i vi8x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i8));
565 const __m256i vk8x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 128 * sizeof(int8_t))));
566 const __m256i vi8x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) (i8 + 8)));
567 const __m256i vk8x89ABCDEF = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 136 * sizeof(int8_t))));
568 i8 += 16;
569
570 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi8x01234567, vk8x01234567));
571 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_mullo_epi32(vi8x89ABCDEF, vk8x89ABCDEF));
572
573 w = (const void*) ((uintptr_t) w + 144 * sizeof(int8_t));
574
575 __m256 vscaled01234567 = _mm256_cvtepi32_ps(vacc01234567);
576 __m256 vscaled89ABCDEF = _mm256_cvtepi32_ps(vacc89ABCDEF);
577
578 vscaled01234567 = _mm256_mul_ps(vscaled01234567, vscale);
579 vscaled89ABCDEF = _mm256_mul_ps(vscaled89ABCDEF, vscale);
580
581 vscaled01234567 = _mm256_min_ps(vscaled01234567, voutput_max_less_zero_point);
582 vscaled89ABCDEF = _mm256_min_ps(vscaled89ABCDEF, voutput_max_less_zero_point);
583
584 vacc01234567 = _mm256_cvtps_epi32(vscaled01234567);
585 vacc89ABCDEF = _mm256_cvtps_epi32(vscaled89ABCDEF);
586
587 __m256i vout012389AB4567CDEF = _mm256_adds_epi16(_mm256_packs_epi32(vacc01234567, vacc89ABCDEF), voutput_zero_point);
588
589 __m128i vout0123456789ABCDEF = _mm_shuffle_epi32(_mm_packs_epi16(_mm256_castsi256_si128(vout012389AB4567CDEF), _mm256_extracti128_si256(vout012389AB4567CDEF, 1)), _MM_SHUFFLE(3, 1, 2, 0));
590
591 vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
592
593 _mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
594 output += 16;
595 }
596
597 if XNN_UNLIKELY(c != 0) {
598 do {
599 __m256i vacc01234567 = _mm256_loadu_si256((const __m256i*) b);
600 b += 8;
601
602 const __m256i vi0x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i0));
603 const __m256i vk0x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 0 * sizeof(int8_t))));
604 i0 += 8;
605
606 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi0x01234567, vk0x01234567));
607
608 const __m256i vi1x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i1));
609 const __m256i vk1x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t))));
610 i1 += 8;
611
612 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi1x01234567, vk1x01234567));
613
614 const __m256i vi2x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i2));
615 const __m256i vk2x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t))));
616 i2 += 8;
617
618 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi2x01234567, vk2x01234567));
619
620 const __m256i vi3x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i3));
621 const __m256i vk3x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t))));
622 i3 += 8;
623
624 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi3x01234567, vk3x01234567));
625
626 const __m256i vi4x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i4));
627 const __m256i vk4x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t))));
628 i4 += 8;
629
630 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi4x01234567, vk4x01234567));
631
632 const __m256i vi5x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i5));
633 const __m256i vk5x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 40 * sizeof(int8_t))));
634 i5 += 8;
635
636 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi5x01234567, vk5x01234567));
637
638 const __m256i vi6x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i6));
639 const __m256i vk6x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t))));
640 i6 += 8;
641
642 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi6x01234567, vk6x01234567));
643
644 const __m256i vi7x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i7));
645 const __m256i vk7x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 56 * sizeof(int8_t))));
646 i7 += 8;
647
648 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi7x01234567, vk7x01234567));
649
650 const __m256i vi8x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i8));
651 const __m256i vk8x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 64 * sizeof(int8_t))));
652 i8 += 8;
653
654 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi8x01234567, vk8x01234567));
655
656 __m256 vscaled01234567 = _mm256_cvtepi32_ps(vacc01234567);
657 vscaled01234567 = _mm256_mul_ps(vscaled01234567, vscale);
658 vscaled01234567 = _mm256_min_ps(vscaled01234567, voutput_max_less_zero_point);
659 vacc01234567 = _mm256_cvtps_epi32(vscaled01234567);
660
661 w = (void*) ((uintptr_t) w + 72 * sizeof(int8_t));
662
663 __m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc01234567), _mm256_extracti128_si256(vacc01234567, 1)), _mm256_castsi256_si128(voutput_zero_point));
664
665 __m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
666
667 vout0123456701234567 = _mm_max_epi8(vout0123456701234567, voutput_min);
668
669 if XNN_LIKELY(c >= 8) {
670 _mm_storel_epi64((__m128i*) output, vout0123456701234567);
671 output += 8;
672 c -= 8;
673 } else {
674 if (c & 4) {
675 unaligned_store_u32(output, (uint32_t) _mm_cvtsi128_si32(vout0123456701234567));
676 vout0123456701234567 = _mm_srli_epi64(vout0123456701234567, 32);
677 output += 4;
678 }
679 if (c & 2) {
680 unaligned_store_u16(output, (uint16_t) _mm_extract_epi16(vout0123456701234567, 0));
681 vout0123456701234567 = _mm_srli_epi32(vout0123456701234567, 16);
682 output += 2;
683 }
684 if (c & 1) {
685 *output = (int8_t) _mm_extract_epi8(vout0123456701234567, 0);
686 output += 1;
687 }
688 c = 0;
689 }
690 } while (c != 0);
691 }
692 }
693
694 input = (const int8_t**) ((uintptr_t) input + input_stride);
695 output = (int8_t*) ((uintptr_t) output + output_increment);
696 } while (--output_width != 0);
697 }