Debian Codegraph @

src/qs8-dwconv/gen/qs8-dwconv-8f8m9l8c8s8r-minmax-fp32-avx2-mul32.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-dwconv/multipass-avx2-mul32.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include <immintrin.h>
13
14 #include "xnnpack/common.h"
15 #include "xnnpack/dwconv.h"
16 #include "xnnpack/math.h"
17 #include "xnnpack/unaligned.h"
18
19
20 void xnn_qs8_dwconv_minmax_fp32_ukernel_8f8m9l8c8s8r__avx2_mul32(
21 size_t channels,
22 size_t output_width,
23 const int8_t** input,
24 const void* weights,
25 int8_t* output,
26 intptr_t input_stride,
27 size_t output_increment,
28 size_t input_offset,
29 const int8_t* zero,
30 size_t kernel_size,
31 int32_t* buffer,
32 const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
33 {
34 assert(channels != 0);
35 assert(output_width != 0);
36 assert(kernel_size > 8);
37
38
39 const __m256 vscale = _mm256_set1_ps(params->fp32_scalar.scale);
40 XNN_FORCE_REALIZATION(vscale);
41 const __m256 voutput_max_less_zero_point = _mm256_set1_ps((int32_t) params->fp32_scalar.output_max - (int32_t) params->fp32_scalar.output_zero_point);
42 const __m128i voutput_zero_point = _mm_set1_epi16(params->fp32_scalar.output_zero_point);
43 const __m128i voutput_min = _mm_set1_epi8(params->fp32_scalar.output_min);
44 XNN_FORCE_REALIZATION(voutput_max_less_zero_point);
45 XNN_FORCE_REALIZATION(voutput_zero_point);
46 XNN_FORCE_REALIZATION(voutput_min);
47
48 do {
49 const void* w = weights;
50
51 // First pass to process 8 inputs.
52 {
53 int32_t* b = buffer;
54 const int8_t* i0 = input[0];
55 assert(i0 != NULL);
56 if XNN_UNPREDICTABLE(i0 != zero) {
57 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
58 }
59 const int8_t* i1 = input[1];
60 assert(i1 != NULL);
61 if XNN_UNPREDICTABLE(i1 != zero) {
62 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
63 }
64 const int8_t* i2 = input[2];
65 assert(i2 != NULL);
66 if XNN_UNPREDICTABLE(i2 != zero) {
67 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
68 }
69 const int8_t* i3 = input[3];
70 assert(i3 != NULL);
71 if XNN_UNPREDICTABLE(i3 != zero) {
72 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
73 }
74 const int8_t* i4 = input[4];
75 assert(i4 != NULL);
76 if XNN_UNPREDICTABLE(i4 != zero) {
77 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
78 }
79 const int8_t* i5 = input[5];
80 assert(i5 != NULL);
81 if XNN_UNPREDICTABLE(i5 != zero) {
82 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
83 }
84 const int8_t* i6 = input[6];
85 assert(i6 != NULL);
86 if XNN_UNPREDICTABLE(i6 != zero) {
87 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
88 }
89 const int8_t* i7 = input[7];
90 assert(i7 != NULL);
91 if XNN_UNPREDICTABLE(i7 != zero) {
92 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
93 }
94 input += 8;
95
96 size_t c = round_up_po2(channels, 8);
97
98 for (; c >= 8; c -= 8) {
99 __m256i vacc01234567 = _mm256_loadu_si256((const __m256i*) w);
100
101
102 const __m256i vi0x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i0));
103 const __m256i vk0x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 0 * sizeof(int8_t))));
104 i0 += 8;
105
106 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi0x01234567, vk0x01234567));
107
108 const __m256i vi1x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i1));
109 const __m256i vk1x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 8 * sizeof(int8_t))));
110 i1 += 8;
111
112 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi1x01234567, vk1x01234567));
113
114 const __m256i vi2x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i2));
115 const __m256i vk2x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 16 * sizeof(int8_t))));
116 i2 += 8;
117
118 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi2x01234567, vk2x01234567));
119
120 const __m256i vi3x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i3));
121 const __m256i vk3x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 24 * sizeof(int8_t))));
122 i3 += 8;
123
124 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi3x01234567, vk3x01234567));
125
126 const __m256i vi4x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i4));
127 const __m256i vk4x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 32 * sizeof(int8_t))));
128 i4 += 8;
129
130 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi4x01234567, vk4x01234567));
131
132 const __m256i vi5x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i5));
133 const __m256i vk5x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 40 * sizeof(int8_t))));
134 i5 += 8;
135
136 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi5x01234567, vk5x01234567));
137
138 const __m256i vi6x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i6));
139 const __m256i vk6x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 48 * sizeof(int8_t))));
140 i6 += 8;
141
142 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi6x01234567, vk6x01234567));
143
144 const __m256i vi7x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i7));
145 const __m256i vk7x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int32_t) + 56 * sizeof(int8_t))));
146 i7 += 8;
147
148 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi7x01234567, vk7x01234567));
149
150 w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 64 * sizeof(int8_t));
151
152 _mm256_storeu_si256((__m256i*) b, vacc01234567);
153 b += 8;
154 }
155
156 assert(c == 0);
157 }
158
159 // Middle pass to process 8 inputs in each iteration.
160 for (size_t ks = kernel_size - 8; ks > 9; ks -= 8) {
161 int32_t* b = buffer;
162 const int8_t* i0 = input[0];
163 assert(i0 != NULL);
164 if XNN_UNPREDICTABLE(i0 != zero) {
165 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
166 }
167 const int8_t* i1 = input[1];
168 assert(i1 != NULL);
169 if XNN_UNPREDICTABLE(i1 != zero) {
170 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
171 }
172 const int8_t* i2 = input[2];
173 assert(i2 != NULL);
174 if XNN_UNPREDICTABLE(i2 != zero) {
175 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
176 }
177 const int8_t* i3 = input[3];
178 assert(i3 != NULL);
179 if XNN_UNPREDICTABLE(i3 != zero) {
180 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
181 }
182 const int8_t* i4 = input[4];
183 assert(i4 != NULL);
184 if XNN_UNPREDICTABLE(i4 != zero) {
185 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
186 }
187 const int8_t* i5 = input[5];
188 assert(i5 != NULL);
189 if XNN_UNPREDICTABLE(i5 != zero) {
190 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
191 }
192 const int8_t* i6 = input[6];
193 assert(i6 != NULL);
194 if XNN_UNPREDICTABLE(i6 != zero) {
195 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
196 }
197 const int8_t* i7 = input[7];
198 assert(i7 != NULL);
199 if XNN_UNPREDICTABLE(i7 != zero) {
200 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
201 }
202 input += 8;
203
204 size_t c = round_up_po2(channels, 8);
205
206 for (; c >= 8; c -= 8) {
207 __m256i vacc01234567 = _mm256_loadu_si256((const __m256i*) b);
208
209
210 const __m256i vi0x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i0));
211 const __m256i vk0x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 0 * sizeof(int8_t))));
212 i0 += 8;
213
214 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi0x01234567, vk0x01234567));
215
216 const __m256i vi1x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i1));
217 const __m256i vk1x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t))));
218 i1 += 8;
219
220 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi1x01234567, vk1x01234567));
221
222 const __m256i vi2x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i2));
223 const __m256i vk2x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t))));
224 i2 += 8;
225
226 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi2x01234567, vk2x01234567));
227
228 const __m256i vi3x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i3));
229 const __m256i vk3x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t))));
230 i3 += 8;
231
232 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi3x01234567, vk3x01234567));
233
234 const __m256i vi4x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i4));
235 const __m256i vk4x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t))));
236 i4 += 8;
237
238 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi4x01234567, vk4x01234567));
239
240 const __m256i vi5x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i5));
241 const __m256i vk5x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 40 * sizeof(int8_t))));
242 i5 += 8;
243
244 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi5x01234567, vk5x01234567));
245
246 const __m256i vi6x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i6));
247 const __m256i vk6x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t))));
248 i6 += 8;
249
250 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi6x01234567, vk6x01234567));
251
252 const __m256i vi7x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i7));
253 const __m256i vk7x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 56 * sizeof(int8_t))));
254 i7 += 8;
255
256 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi7x01234567, vk7x01234567));
257
258 w = (const void*) ((uintptr_t) w + 64 * sizeof(int8_t));
259
260 _mm256_storeu_si256((__m256i*) b, vacc01234567);
261 b += 8;
262 }
263
264 assert(c == 0);
265 }
266
267 // Last pass to process up to 9 inputs.
268 {
269 const int32_t* b = buffer;
270 const int8_t* i0 = input[0];
271 assert(i0 != NULL);
272 if XNN_UNPREDICTABLE(i0 != zero) {
273 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
274 }
275 const int8_t* i1 = input[1];
276 assert(i1 != NULL);
277 if XNN_UNPREDICTABLE(i1 != zero) {
278 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
279 }
280 const int8_t* i2 = input[2];
281 assert(i2 != NULL);
282 if XNN_UNPREDICTABLE(i2 != zero) {
283 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
284 }
285 const int8_t* i3 = input[3];
286 assert(i3 != NULL);
287 if XNN_UNPREDICTABLE(i3 != zero) {
288 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
289 }
290 const int8_t* i4 = input[4];
291 assert(i4 != NULL);
292 if XNN_UNPREDICTABLE(i4 != zero) {
293 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
294 }
295 const int8_t* i5 = input[5];
296 assert(i5 != NULL);
297 if XNN_UNPREDICTABLE(i5 != zero) {
298 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
299 }
300 const int8_t* i6 = input[6];
301 assert(i6 != NULL);
302 if XNN_UNPREDICTABLE(i6 != zero) {
303 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
304 }
305 const int8_t* i7 = input[7];
306 assert(i7 != NULL);
307 if XNN_UNPREDICTABLE(i7 != zero) {
308 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
309 }
310 const int8_t* i8 = input[8];
311 assert(i8 != NULL);
312 if XNN_UNPREDICTABLE(i8 != zero) {
313 i8 = (const int8_t*) ((uintptr_t) i8 + input_offset);
314 }
315
316 size_t c = channels;
317
318 for (; c >= 8; c -= 8) {
319 __m256i vacc01234567 = _mm256_loadu_si256((const __m256i*) b);
320 b += 8;
321
322
323 const __m256i vi0x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i0));
324 const __m256i vk0x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 0 * sizeof(int8_t))));
325 i0 += 8;
326
327 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi0x01234567, vk0x01234567));
328
329 const __m256i vi1x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i1));
330 const __m256i vk1x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t))));
331 i1 += 8;
332
333 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi1x01234567, vk1x01234567));
334
335 const __m256i vi2x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i2));
336 const __m256i vk2x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t))));
337 i2 += 8;
338
339 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi2x01234567, vk2x01234567));
340
341 const __m256i vi3x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i3));
342 const __m256i vk3x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t))));
343 i3 += 8;
344
345 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi3x01234567, vk3x01234567));
346
347 const __m256i vi4x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i4));
348 const __m256i vk4x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t))));
349 i4 += 8;
350
351 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi4x01234567, vk4x01234567));
352
353 const __m256i vi5x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i5));
354 const __m256i vk5x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 40 * sizeof(int8_t))));
355 i5 += 8;
356
357 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi5x01234567, vk5x01234567));
358
359 const __m256i vi6x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i6));
360 const __m256i vk6x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t))));
361 i6 += 8;
362
363 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi6x01234567, vk6x01234567));
364
365 const __m256i vi7x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i7));
366 const __m256i vk7x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 56 * sizeof(int8_t))));
367 i7 += 8;
368
369 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi7x01234567, vk7x01234567));
370
371 const __m256i vi8x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i8));
372 const __m256i vk8x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 64 * sizeof(int8_t))));
373 i8 += 8;
374
375 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi8x01234567, vk8x01234567));
376
377 w = (const void*) ((uintptr_t) w + 72 * sizeof(int8_t));
378
379 __m256 vscaled01234567 = _mm256_cvtepi32_ps(vacc01234567);
380
381 vscaled01234567 = _mm256_mul_ps(vscaled01234567, vscale);
382
383 vscaled01234567 = _mm256_min_ps(vscaled01234567, voutput_max_less_zero_point);
384
385 vacc01234567 = _mm256_cvtps_epi32(vscaled01234567);
386
387 __m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc01234567), _mm256_extracti128_si256(vacc01234567, 1)), voutput_zero_point);
388
389 __m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
390
391 vout0123456701234567 = _mm_max_epi8(vout0123456701234567, voutput_min);
392
393 _mm_storel_epi64((__m128i*) output, vout0123456701234567);
394 output += 8;
395 }
396
397 if XNN_UNLIKELY(c != 0) {
398 {
399 __m256i vacc01234567 = _mm256_loadu_si256((const __m256i*) b);
400 b += 8;
401
402 const __m256i vi0x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i0));
403 const __m256i vk0x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 0 * sizeof(int8_t))));
404
405 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi0x01234567, vk0x01234567));
406
407 const __m256i vi1x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i1));
408 const __m256i vk1x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 8 * sizeof(int8_t))));
409
410 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi1x01234567, vk1x01234567));
411
412 const __m256i vi2x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i2));
413 const __m256i vk2x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t))));
414
415 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi2x01234567, vk2x01234567));
416
417 const __m256i vi3x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i3));
418 const __m256i vk3x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 24 * sizeof(int8_t))));
419
420 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi3x01234567, vk3x01234567));
421
422 const __m256i vi4x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i4));
423 const __m256i vk4x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t))));
424
425 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi4x01234567, vk4x01234567));
426
427 const __m256i vi5x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i5));
428 const __m256i vk5x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 40 * sizeof(int8_t))));
429
430 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi5x01234567, vk5x01234567));
431
432 const __m256i vi6x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i6));
433 const __m256i vk6x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t))));
434
435 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi6x01234567, vk6x01234567));
436
437 const __m256i vi7x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i7));
438 const __m256i vk7x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 56 * sizeof(int8_t))));
439
440 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi7x01234567, vk7x01234567));
441
442 const __m256i vi8x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) i8));
443 const __m256i vk8x01234567 = _mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*) ((uintptr_t) w + 64 * sizeof(int8_t))));
444
445 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_mullo_epi32(vi8x01234567, vk8x01234567));
446
447 __m256 vscaled01234567 = _mm256_cvtepi32_ps(vacc01234567);
448 vscaled01234567 = _mm256_mul_ps(vscaled01234567, vscale);
449 vscaled01234567 = _mm256_min_ps(vscaled01234567, voutput_max_less_zero_point);
450 vacc01234567 = _mm256_cvtps_epi32(vscaled01234567);
451
452 __m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc01234567), _mm256_extracti128_si256(vacc01234567, 1)), voutput_zero_point);
453
454 __m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
455
456 vout0123456701234567 = _mm_max_epi8(vout0123456701234567, voutput_min);
457
458 if (c & 4) {
459 unaligned_store_u32(output, (uint32_t) _mm_cvtsi128_si32(vout0123456701234567));
460 vout0123456701234567 = _mm_srli_epi64(vout0123456701234567, 32);
461 output += 4;
462 }
463 if (c & 2) {
464 unaligned_store_u16(output, (uint16_t) _mm_extract_epi16(vout0123456701234567, 0));
465 vout0123456701234567 = _mm_srli_epi32(vout0123456701234567, 16);
466 output += 2;
467 }
468 if (c & 1) {
469 *output = (int8_t) _mm_extract_epi8(vout0123456701234567, 0);
470 output += 1;
471 }
472 }
473 }
474 }
475
476 input = (const int8_t**) ((uintptr_t) input + input_stride);
477 output = (int8_t*) ((uintptr_t) output + output_increment);
478 } while (--output_width != 0);
479 }