Debian Codegraph @

src/qs8-dwconv/gen/qs8-dwconv-6f6m7l16c16s1r-minmax-fp32-avx512skx-mul32.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-dwconv/multipass-avx512skx-mul32.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include <immintrin.h>
13
14 #include "xnnpack/dwconv.h"
15 #include "xnnpack/intrinsics-polyfill.h"
16
17
18 void xnn_qs8_dwconv_minmax_fp32_ukernel_6f6m7l16c16s1r__avx512skx_mul32(
19 size_t channels,
20 size_t output_width,
21 const int8_t** input,
22 const void* weights,
23 int8_t* output,
24 intptr_t input_stride,
25 size_t output_increment,
26 size_t input_offset,
27 const int8_t* zero,
28 size_t kernel_size,
29 int32_t* buffer,
30 const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
31 {
32 assert(channels != 0);
33 assert(output_width != 0);
34 assert(kernel_size > 6);
35
36
37 const __m512 vscale = _mm512_set1_ps(params->fp32_scalar.scale);
38 // XNN_FORCE_REALIZATION(vscale);
39 const __m512 voutput_max_less_zero_point = _mm512_set1_ps((int32_t) params->fp32_scalar.output_max - (int32_t) params->fp32_scalar.output_zero_point);
40 const __m256i voutput_zero_point = _mm256_set1_epi16((int16_t) params->fp32_scalar.output_zero_point);
41 const __m128i voutput_min = _mm_set1_epi8(params->fp32_scalar.output_min);
42 // XNN_FORCE_REALIZATION(voutput_max_less_zero_point);
43 // XNN_FORCE_REALIZATION(voutput_zero_point);
44 // XNN_FORCE_REALIZATION(voutput_min);
45
46 do {
47 const void* w = weights;
48
49 // First pass to process 6 inputs.
50 {
51 int32_t* b = buffer;
52 const int8_t* i0 = input[0];
53 assert(i0 != NULL);
54 if XNN_UNPREDICTABLE(i0 != zero) {
55 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
56 }
57 const int8_t* i1 = input[1];
58 assert(i1 != NULL);
59 if XNN_UNPREDICTABLE(i1 != zero) {
60 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
61 }
62 const int8_t* i2 = input[2];
63 assert(i2 != NULL);
64 if XNN_UNPREDICTABLE(i2 != zero) {
65 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
66 }
67 const int8_t* i3 = input[3];
68 assert(i3 != NULL);
69 if XNN_UNPREDICTABLE(i3 != zero) {
70 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
71 }
72 const int8_t* i4 = input[4];
73 assert(i4 != NULL);
74 if XNN_UNPREDICTABLE(i4 != zero) {
75 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
76 }
77 const int8_t* i5 = input[5];
78 assert(i5 != NULL);
79 if XNN_UNPREDICTABLE(i5 != zero) {
80 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
81 }
82 input += 6;
83
84 size_t c = channels;
85
86 for (; c >= 16; c -= 16) {
87 __m512i vacc0123456789ABCDEF = _mm512_loadu_si512(w);
88
89 const __m512i vi0x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i*) i0));
90 const __m512i vk0x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 0 * sizeof(int8_t))));
91 i0 += 16;
92
93 vacc0123456789ABCDEF = _mm512_add_epi32(vacc0123456789ABCDEF, _mm512_mullo_epi32(vi0x0123456789ABCDEF, vk0x0123456789ABCDEF));
94
95 const __m512i vi1x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i*) i1));
96 const __m512i vk1x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 16 * sizeof(int8_t))));
97 i1 += 16;
98
99 vacc0123456789ABCDEF = _mm512_add_epi32(vacc0123456789ABCDEF, _mm512_mullo_epi32(vi1x0123456789ABCDEF, vk1x0123456789ABCDEF));
100
101 const __m512i vi2x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i*) i2));
102 const __m512i vk2x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 32 * sizeof(int8_t))));
103 i2 += 16;
104
105 vacc0123456789ABCDEF = _mm512_add_epi32(vacc0123456789ABCDEF, _mm512_mullo_epi32(vi2x0123456789ABCDEF, vk2x0123456789ABCDEF));
106
107 const __m512i vi3x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i*) i3));
108 const __m512i vk3x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 48 * sizeof(int8_t))));
109 i3 += 16;
110
111 vacc0123456789ABCDEF = _mm512_add_epi32(vacc0123456789ABCDEF, _mm512_mullo_epi32(vi3x0123456789ABCDEF, vk3x0123456789ABCDEF));
112
113 const __m512i vi4x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i*) i4));
114 const __m512i vk4x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 64 * sizeof(int8_t))));
115 i4 += 16;
116
117 vacc0123456789ABCDEF = _mm512_add_epi32(vacc0123456789ABCDEF, _mm512_mullo_epi32(vi4x0123456789ABCDEF, vk4x0123456789ABCDEF));
118
119 const __m512i vi5x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i*) i5));
120 const __m512i vk5x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 80 * sizeof(int8_t))));
121 i5 += 16;
122
123 vacc0123456789ABCDEF = _mm512_add_epi32(vacc0123456789ABCDEF, _mm512_mullo_epi32(vi5x0123456789ABCDEF, vk5x0123456789ABCDEF));
124
125 w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 96 * sizeof(int8_t));
126
127 _mm512_storeu_si512((__m512i*) b, vacc0123456789ABCDEF);
128 b += 16;
129 }
130
131
132 if (c != 0) {
133 assert(c >= 1);
134 assert(c <= 15);
135 __m512i vacc0123456789ABCDEF = _mm512_loadu_si512(w);
136
137 const __m512i vi0x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i*) i0));
138 const __m512i vk0x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 0 * sizeof(int8_t))));
139 vacc0123456789ABCDEF = _mm512_add_epi32(vacc0123456789ABCDEF, _mm512_mullo_epi32(vi0x0123456789ABCDEF, vk0x0123456789ABCDEF));
140
141 const __m512i vi1x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i*) i1));
142 const __m512i vk1x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 16 * sizeof(int8_t))));
143 vacc0123456789ABCDEF = _mm512_add_epi32(vacc0123456789ABCDEF, _mm512_mullo_epi32(vi1x0123456789ABCDEF, vk1x0123456789ABCDEF));
144
145 const __m512i vi2x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i*) i2));
146 const __m512i vk2x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 32 * sizeof(int8_t))));
147 vacc0123456789ABCDEF = _mm512_add_epi32(vacc0123456789ABCDEF, _mm512_mullo_epi32(vi2x0123456789ABCDEF, vk2x0123456789ABCDEF));
148
149 const __m512i vi3x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i*) i3));
150 const __m512i vk3x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 48 * sizeof(int8_t))));
151 vacc0123456789ABCDEF = _mm512_add_epi32(vacc0123456789ABCDEF, _mm512_mullo_epi32(vi3x0123456789ABCDEF, vk3x0123456789ABCDEF));
152
153 const __m512i vi4x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i*) i4));
154 const __m512i vk4x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 64 * sizeof(int8_t))));
155 vacc0123456789ABCDEF = _mm512_add_epi32(vacc0123456789ABCDEF, _mm512_mullo_epi32(vi4x0123456789ABCDEF, vk4x0123456789ABCDEF));
156
157 const __m512i vi5x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i*) i5));
158 const __m512i vk5x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 80 * sizeof(int8_t))));
159 vacc0123456789ABCDEF = _mm512_add_epi32(vacc0123456789ABCDEF, _mm512_mullo_epi32(vi5x0123456789ABCDEF, vk5x0123456789ABCDEF));
160
161 w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 96 * sizeof(int8_t));
162
163 _mm512_storeu_si512((__m512i*) b, vacc0123456789ABCDEF);
164 }
165 }
166
167 // Middle pass to process 6 inputs in each iteration.
168 for (size_t ks = kernel_size - 6; ks > 7; ks -= 6) {
169 int32_t* b = buffer;
170 const int8_t* i0 = input[0];
171 assert(i0 != NULL);
172 if XNN_UNPREDICTABLE(i0 != zero) {
173 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
174 }
175 const int8_t* i1 = input[1];
176 assert(i1 != NULL);
177 if XNN_UNPREDICTABLE(i1 != zero) {
178 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
179 }
180 const int8_t* i2 = input[2];
181 assert(i2 != NULL);
182 if XNN_UNPREDICTABLE(i2 != zero) {
183 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
184 }
185 const int8_t* i3 = input[3];
186 assert(i3 != NULL);
187 if XNN_UNPREDICTABLE(i3 != zero) {
188 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
189 }
190 const int8_t* i4 = input[4];
191 assert(i4 != NULL);
192 if XNN_UNPREDICTABLE(i4 != zero) {
193 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
194 }
195 const int8_t* i5 = input[5];
196 assert(i5 != NULL);
197 if XNN_UNPREDICTABLE(i5 != zero) {
198 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
199 }
200 input += 6;
201
202 size_t c = channels;
203
204 for (; c >= 16; c -= 16) {
205 __m512i vacc0123456789ABCDEF = _mm512_loadu_si512(b);
206
207 const __m512i vi0x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i*) i0));
208 const __m512i vk0x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_load_si128((const __m128i*) ((uintptr_t) w + 0 * sizeof(int8_t))));
209 i0 += 16;
210
211 vacc0123456789ABCDEF = _mm512_add_epi32(vacc0123456789ABCDEF, _mm512_mullo_epi32(vi0x0123456789ABCDEF, vk0x0123456789ABCDEF));
212
213 const __m512i vi1x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i*) i1));
214 const __m512i vk1x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t))));
215 i1 += 16;
216
217 vacc0123456789ABCDEF = _mm512_add_epi32(vacc0123456789ABCDEF, _mm512_mullo_epi32(vi1x0123456789ABCDEF, vk1x0123456789ABCDEF));
218
219 const __m512i vi2x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i*) i2));
220 const __m512i vk2x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_load_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t))));
221 i2 += 16;
222
223 vacc0123456789ABCDEF = _mm512_add_epi32(vacc0123456789ABCDEF, _mm512_mullo_epi32(vi2x0123456789ABCDEF, vk2x0123456789ABCDEF));
224
225 const __m512i vi3x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i*) i3));
226 const __m512i vk3x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_load_si128((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t))));
227 i3 += 16;
228
229 vacc0123456789ABCDEF = _mm512_add_epi32(vacc0123456789ABCDEF, _mm512_mullo_epi32(vi3x0123456789ABCDEF, vk3x0123456789ABCDEF));
230
231 const __m512i vi4x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i*) i4));
232 const __m512i vk4x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_load_si128((const __m128i*) ((uintptr_t) w + 64 * sizeof(int8_t))));
233 i4 += 16;
234
235 vacc0123456789ABCDEF = _mm512_add_epi32(vacc0123456789ABCDEF, _mm512_mullo_epi32(vi4x0123456789ABCDEF, vk4x0123456789ABCDEF));
236
237 const __m512i vi5x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i*) i5));
238 const __m512i vk5x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_load_si128((const __m128i*) ((uintptr_t) w + 80 * sizeof(int8_t))));
239 i5 += 16;
240
241 vacc0123456789ABCDEF = _mm512_add_epi32(vacc0123456789ABCDEF, _mm512_mullo_epi32(vi5x0123456789ABCDEF, vk5x0123456789ABCDEF));
242
243 w = (const void*) ((uintptr_t) w + 96 * sizeof(int8_t));
244
245 _mm512_storeu_si512((__m512i*) b, vacc0123456789ABCDEF);
246 b += 16;
247 }
248
249
250 if (c != 0) {
251 assert(c >= 1);
252 assert(c <= 15);
253 __m512i vacc0123456789ABCDEF = _mm512_loadu_si512(b);
254
255 const __m512i vi0x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i*) i0));
256 const __m512i vk0x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 0 * sizeof(int8_t))));
257 vacc0123456789ABCDEF = _mm512_add_epi32(vacc0123456789ABCDEF, _mm512_mullo_epi32(vi0x0123456789ABCDEF, vk0x0123456789ABCDEF));
258
259 const __m512i vi1x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i*) i1));
260 const __m512i vk1x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t))));
261 vacc0123456789ABCDEF = _mm512_add_epi32(vacc0123456789ABCDEF, _mm512_mullo_epi32(vi1x0123456789ABCDEF, vk1x0123456789ABCDEF));
262
263 const __m512i vi2x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i*) i2));
264 const __m512i vk2x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t))));
265 vacc0123456789ABCDEF = _mm512_add_epi32(vacc0123456789ABCDEF, _mm512_mullo_epi32(vi2x0123456789ABCDEF, vk2x0123456789ABCDEF));
266
267 const __m512i vi3x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i*) i3));
268 const __m512i vk3x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t))));
269 vacc0123456789ABCDEF = _mm512_add_epi32(vacc0123456789ABCDEF, _mm512_mullo_epi32(vi3x0123456789ABCDEF, vk3x0123456789ABCDEF));
270
271 const __m512i vi4x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i*) i4));
272 const __m512i vk4x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 64 * sizeof(int8_t))));
273 vacc0123456789ABCDEF = _mm512_add_epi32(vacc0123456789ABCDEF, _mm512_mullo_epi32(vi4x0123456789ABCDEF, vk4x0123456789ABCDEF));
274
275 const __m512i vi5x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i*) i5));
276 const __m512i vk5x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 80 * sizeof(int8_t))));
277 vacc0123456789ABCDEF = _mm512_add_epi32(vacc0123456789ABCDEF, _mm512_mullo_epi32(vi5x0123456789ABCDEF, vk5x0123456789ABCDEF));
278
279 w = (const void*) ((uintptr_t) w + 96 * sizeof(int8_t));
280
281 _mm512_storeu_si512((__m512i*) b, vacc0123456789ABCDEF);
282 }
283 }
284
285 // Last pass to process up to 7 inputs.
286 {
287 const int32_t* b = buffer;
288 const int8_t* i0 = input[0];
289 assert(i0 != NULL);
290 if XNN_UNPREDICTABLE(i0 != zero) {
291 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
292 }
293 const int8_t* i1 = input[1];
294 assert(i1 != NULL);
295 if XNN_UNPREDICTABLE(i1 != zero) {
296 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
297 }
298 const int8_t* i2 = input[2];
299 assert(i2 != NULL);
300 if XNN_UNPREDICTABLE(i2 != zero) {
301 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
302 }
303 const int8_t* i3 = input[3];
304 assert(i3 != NULL);
305 if XNN_UNPREDICTABLE(i3 != zero) {
306 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
307 }
308 const int8_t* i4 = input[4];
309 assert(i4 != NULL);
310 if XNN_UNPREDICTABLE(i4 != zero) {
311 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
312 }
313 const int8_t* i5 = input[5];
314 assert(i5 != NULL);
315 if XNN_UNPREDICTABLE(i5 != zero) {
316 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
317 }
318 const int8_t* i6 = input[6];
319 assert(i6 != NULL);
320 if XNN_UNPREDICTABLE(i6 != zero) {
321 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
322 }
323
324 size_t c = channels;
325
326 for (; c >= 16; c -= 16) {
327 __m512i vacc0123456789ABCDEF = _mm512_loadu_si512(b);
328 b += 16;
329
330 const __m512i vi0x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i*) i0));
331 const __m512i vk0x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_load_si128((const __m128i*) ((uintptr_t) w + 0 * sizeof(int8_t))));
332 i0 += 16;
333
334 vacc0123456789ABCDEF = _mm512_add_epi32(vacc0123456789ABCDEF, _mm512_mullo_epi32(vi0x0123456789ABCDEF, vk0x0123456789ABCDEF));
335
336 const __m512i vi1x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i*) i1));
337 const __m512i vk1x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_load_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t))));
338 i1 += 16;
339
340 vacc0123456789ABCDEF = _mm512_add_epi32(vacc0123456789ABCDEF, _mm512_mullo_epi32(vi1x0123456789ABCDEF, vk1x0123456789ABCDEF));
341
342 const __m512i vi2x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i*) i2));
343 const __m512i vk2x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_load_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t))));
344 i2 += 16;
345
346 vacc0123456789ABCDEF = _mm512_add_epi32(vacc0123456789ABCDEF, _mm512_mullo_epi32(vi2x0123456789ABCDEF, vk2x0123456789ABCDEF));
347
348 const __m512i vi3x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i*) i3));
349 const __m512i vk3x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_load_si128((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t))));
350 i3 += 16;
351
352 vacc0123456789ABCDEF = _mm512_add_epi32(vacc0123456789ABCDEF, _mm512_mullo_epi32(vi3x0123456789ABCDEF, vk3x0123456789ABCDEF));
353
354 const __m512i vi4x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i*) i4));
355 const __m512i vk4x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_load_si128((const __m128i*) ((uintptr_t) w + 64 * sizeof(int8_t))));
356 i4 += 16;
357
358 vacc0123456789ABCDEF = _mm512_add_epi32(vacc0123456789ABCDEF, _mm512_mullo_epi32(vi4x0123456789ABCDEF, vk4x0123456789ABCDEF));
359
360 const __m512i vi5x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i*) i5));
361 const __m512i vk5x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_load_si128((const __m128i*) ((uintptr_t) w + 80 * sizeof(int8_t))));
362 i5 += 16;
363
364 vacc0123456789ABCDEF = _mm512_add_epi32(vacc0123456789ABCDEF, _mm512_mullo_epi32(vi5x0123456789ABCDEF, vk5x0123456789ABCDEF));
365
366 const __m512i vi6x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i*) i6));
367 const __m512i vk6x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_load_si128((const __m128i*) ((uintptr_t) w + 96 * sizeof(int8_t))));
368 i6 += 16;
369
370 vacc0123456789ABCDEF = _mm512_add_epi32(vacc0123456789ABCDEF, _mm512_mullo_epi32(vi6x0123456789ABCDEF, vk6x0123456789ABCDEF));
371
372 w = (const void*) ((uintptr_t) w + 112 * sizeof(int8_t));
373
374 __m512 vscaled0123456789ABCDEF = _mm512_cvtepi32_ps(vacc0123456789ABCDEF);
375
376 vscaled0123456789ABCDEF = _mm512_mul_ps(vscaled0123456789ABCDEF, vscale);
377
378 vscaled0123456789ABCDEF = _mm512_min_ps(vscaled0123456789ABCDEF, voutput_max_less_zero_point);
379
380 vacc0123456789ABCDEF = _mm512_cvtps_epi32(vscaled0123456789ABCDEF);
381
382 __m256i vout012389AB4567CDEF = _mm256_adds_epi16(_mm256_packs_epi32(_mm512_castsi512_si256(vacc0123456789ABCDEF), _mm512_extracti32x8_epi32(vacc0123456789ABCDEF, 1)), voutput_zero_point);
383
384 const __m128i vout012389AB = _mm256_castsi256_si128(vout012389AB4567CDEF);
385 const __m128i vout4567CDEF = _mm256_extracti128_si256(vout012389AB4567CDEF, 1);
386 __m128i vout0123456789ABCDEF = _mm_shuffle_epi32(_mm_packs_epi16(vout012389AB, vout4567CDEF), _MM_SHUFFLE(3, 1, 2, 0));
387
388 vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
389
390 _mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
391 output += 16;
392 }
393
394 if XNN_UNLIKELY(c != 0) {
395 // Prepare mask for valid 8-bit elements (depends on nc).
396 const __mmask16 vmask = _cvtu32_mask16((uint32_t) ((UINT32_C(1) << (c & 15)) - UINT32_C(1)));
397 {
398 __m512i vacc0123456789ABCDEF = _mm512_loadu_si512(b);
399 b += 16;
400
401 const __m512i vi0x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i*) i0));
402 const __m512i vk0x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 0 * sizeof(int8_t))));
403
404 vacc0123456789ABCDEF = _mm512_add_epi32(vacc0123456789ABCDEF, _mm512_mullo_epi32(vi0x0123456789ABCDEF, vk0x0123456789ABCDEF));
405
406 const __m512i vi1x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i*) i1));
407 const __m512i vk1x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t))));
408
409 vacc0123456789ABCDEF = _mm512_add_epi32(vacc0123456789ABCDEF, _mm512_mullo_epi32(vi1x0123456789ABCDEF, vk1x0123456789ABCDEF));
410
411 const __m512i vi2x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i*) i2));
412 const __m512i vk2x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t))));
413
414 vacc0123456789ABCDEF = _mm512_add_epi32(vacc0123456789ABCDEF, _mm512_mullo_epi32(vi2x0123456789ABCDEF, vk2x0123456789ABCDEF));
415
416 const __m512i vi3x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i*) i3));
417 const __m512i vk3x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t))));
418
419 vacc0123456789ABCDEF = _mm512_add_epi32(vacc0123456789ABCDEF, _mm512_mullo_epi32(vi3x0123456789ABCDEF, vk3x0123456789ABCDEF));
420
421 const __m512i vi4x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i*) i4));
422 const __m512i vk4x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 64 * sizeof(int8_t))));
423
424 vacc0123456789ABCDEF = _mm512_add_epi32(vacc0123456789ABCDEF, _mm512_mullo_epi32(vi4x0123456789ABCDEF, vk4x0123456789ABCDEF));
425
426 const __m512i vi5x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i*) i5));
427 const __m512i vk5x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 80 * sizeof(int8_t))));
428
429 vacc0123456789ABCDEF = _mm512_add_epi32(vacc0123456789ABCDEF, _mm512_mullo_epi32(vi5x0123456789ABCDEF, vk5x0123456789ABCDEF));
430
431 const __m512i vi6x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i*) i6));
432 const __m512i vk6x0123456789ABCDEF = _mm512_cvtepi8_epi32(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 96 * sizeof(int8_t))));
433
434 vacc0123456789ABCDEF = _mm512_add_epi32(vacc0123456789ABCDEF, _mm512_mullo_epi32(vi6x0123456789ABCDEF, vk6x0123456789ABCDEF));
435
436 __m512 vscaled0123456789ABCDEF = _mm512_cvtepi32_ps(vacc0123456789ABCDEF);
437 vscaled0123456789ABCDEF = _mm512_mul_ps(vscaled0123456789ABCDEF, vscale);
438 vscaled0123456789ABCDEF = _mm512_min_ps(vscaled0123456789ABCDEF, voutput_max_less_zero_point);
439 vacc0123456789ABCDEF = _mm512_cvtps_epi32(vscaled0123456789ABCDEF);
440
441
442 __m256i vout012389AB4567CDEF = _mm256_adds_epi16(_mm256_packs_epi32(_mm512_castsi512_si256(vacc0123456789ABCDEF), _mm512_extracti32x8_epi32(vacc0123456789ABCDEF, 1)), voutput_zero_point);
443
444 const __m128i vout012389AB = _mm256_castsi256_si128(vout012389AB4567CDEF);
445 const __m128i vout4567CDEF = _mm256_extracti128_si256(vout012389AB4567CDEF, 1);
446 __m128i vout0123456789ABCDEF = _mm_shuffle_epi32(_mm_packs_epi16(vout012389AB, vout4567CDEF), _MM_SHUFFLE(3, 1, 2, 0));
447 vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
448
449
450 _mm_mask_storeu_epi8(output, vmask, vout0123456789ABCDEF);
451 output = (int8_t*) ((uintptr_t) output + c);
452 }
453 }
454 }
455
456 input = (const int8_t**) ((uintptr_t) input + input_stride);
457 output = (int8_t*) ((uintptr_t) output + output_increment);
458 } while (--output_width != 0);
459 }