Debian Codegraph @

src/qs8-dwconv/gen/qs8-dwconv-5f5m5l8c4s4r-minmax-fp32-avx-mul32.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-dwconv/multipass-sse-mul32.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include <immintrin.h>
13
14 #include "xnnpack/dwconv.h"
15 #include "xnnpack/intrinsics-polyfill.h"
16 #include "xnnpack/math.h"
17 #include "xnnpack/unaligned.h"
18
19
20 void xnn_qs8_dwconv_minmax_fp32_ukernel_5f5m5l8c4s4r__avx_mul32(
21 size_t channels,
22 size_t output_width,
23 const int8_t** input,
24 const void* weights,
25 int8_t* output,
26 intptr_t input_stride,
27 size_t output_increment,
28 size_t input_offset,
29 const int8_t* zero,
30 size_t kernel_size,
31 int32_t* buffer,
32 const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
33 {
34 assert(channels != 0);
35 assert(output_width != 0);
36 assert(kernel_size > 5);
37
38
39 const __m128 vscale = _mm_set1_ps(params->fp32_scalar.scale);
40 XNN_FORCE_REALIZATION(vscale);
41
42 const __m128 voutput_max_less_zero_point = _mm_set1_ps((int32_t) params->fp32_scalar.output_max - (int32_t) params->fp32_scalar.output_zero_point);
43 const __m128i voutput_zero_point = _mm_set1_epi16(params->fp32_scalar.output_zero_point);
44 const __m128i voutput_min = _mm_set1_epi8(params->fp32_scalar.output_min);
45 XNN_FORCE_REALIZATION(voutput_max_less_zero_point);
46 XNN_FORCE_REALIZATION(voutput_zero_point);
47 XNN_FORCE_REALIZATION(voutput_min);
48
49 do {
50 const void* w = weights;
51
52 // First pass to process 5 inputs.
53 {
54 int32_t* b = buffer;
55 const int8_t* i0 = input[0];
56 assert(i0 != NULL);
57 if XNN_UNPREDICTABLE(i0 != zero) {
58 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
59 }
60 const int8_t* i1 = input[1];
61 assert(i1 != NULL);
62 if XNN_UNPREDICTABLE(i1 != zero) {
63 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
64 }
65 const int8_t* i2 = input[2];
66 assert(i2 != NULL);
67 if XNN_UNPREDICTABLE(i2 != zero) {
68 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
69 }
70 const int8_t* i3 = input[3];
71 assert(i3 != NULL);
72 if XNN_UNPREDICTABLE(i3 != zero) {
73 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
74 }
75 const int8_t* i4 = input[4];
76 assert(i4 != NULL);
77 if XNN_UNPREDICTABLE(i4 != zero) {
78 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
79 }
80 input += 5;
81
82 size_t c = round_up_po2(channels, 4);
83
84 for (; c >= 8; c -= 8) {
85 __m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
86 __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
87
88
89 const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i0)));
90 const __m128i vk0x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 8 * sizeof(int32_t) + 0 * sizeof(int8_t)))));
91 const __m128i vi0x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i0 + 4)));
92 const __m128i vk0x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 8 * sizeof(int32_t) + 4 * sizeof(int8_t)))));
93 i0 += 8;
94
95 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi0x0123, vk0x0123));
96 vacc4567 = _mm_add_epi32(vacc4567, _mm_mullo_epi32(vi0x4567, vk0x4567));
97
98 const __m128i vi1x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i1)));
99 const __m128i vk1x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 8 * sizeof(int32_t) + 8 * sizeof(int8_t)))));
100 const __m128i vi1x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i1 + 4)));
101 const __m128i vk1x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 8 * sizeof(int32_t) + 12 * sizeof(int8_t)))));
102 i1 += 8;
103
104 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi1x0123, vk1x0123));
105 vacc4567 = _mm_add_epi32(vacc4567, _mm_mullo_epi32(vi1x4567, vk1x4567));
106
107 const __m128i vi2x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i2)));
108 const __m128i vk2x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 8 * sizeof(int32_t) + 16 * sizeof(int8_t)))));
109 const __m128i vi2x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i2 + 4)));
110 const __m128i vk2x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 8 * sizeof(int32_t) + 20 * sizeof(int8_t)))));
111 i2 += 8;
112
113 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi2x0123, vk2x0123));
114 vacc4567 = _mm_add_epi32(vacc4567, _mm_mullo_epi32(vi2x4567, vk2x4567));
115
116 const __m128i vi3x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i3)));
117 const __m128i vk3x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 8 * sizeof(int32_t) + 24 * sizeof(int8_t)))));
118 const __m128i vi3x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i3 + 4)));
119 const __m128i vk3x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 8 * sizeof(int32_t) + 28 * sizeof(int8_t)))));
120 i3 += 8;
121
122 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi3x0123, vk3x0123));
123 vacc4567 = _mm_add_epi32(vacc4567, _mm_mullo_epi32(vi3x4567, vk3x4567));
124
125 const __m128i vi4x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i4)));
126 const __m128i vk4x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 8 * sizeof(int32_t) + 32 * sizeof(int8_t)))));
127 const __m128i vi4x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i4 + 4)));
128 const __m128i vk4x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 8 * sizeof(int32_t) + 36 * sizeof(int8_t)))));
129 i4 += 8;
130
131 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi4x0123, vk4x0123));
132 vacc4567 = _mm_add_epi32(vacc4567, _mm_mullo_epi32(vi4x4567, vk4x4567));
133
134 w = (const void*) ((uintptr_t) w + 8 * sizeof(int32_t) + 40 * sizeof(int8_t));
135
136 _mm_storeu_si128((__m128i*) b, vacc0123);
137 _mm_storeu_si128((__m128i*) (b + 4), vacc4567);
138 b += 8;
139 }
140
141 if XNN_UNLIKELY(c != 0) {
142 do {
143 __m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
144
145 const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i0)));
146 const __m128i vk0x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int32_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))));
147 i0 += 4;
148
149 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi0x0123, vk0x0123));
150
151 const __m128i vi1x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i1)));
152 const __m128i vk1x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int32_t*) ((uintptr_t) w + 4 * sizeof(int32_t) + 4 * sizeof(int8_t)))));
153 i1 += 4;
154
155 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi1x0123, vk1x0123));
156
157 const __m128i vi2x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i2)));
158 const __m128i vk2x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int32_t*) ((uintptr_t) w + 4 * sizeof(int32_t) + 8 * sizeof(int8_t)))));
159 i2 += 4;
160
161 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi2x0123, vk2x0123));
162
163 const __m128i vi3x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i3)));
164 const __m128i vk3x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int32_t*) ((uintptr_t) w + 4 * sizeof(int32_t) + 12 * sizeof(int8_t)))));
165 i3 += 4;
166
167 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi3x0123, vk3x0123));
168
169 const __m128i vi4x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i4)));
170 const __m128i vk4x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int32_t*) ((uintptr_t) w + 4 * sizeof(int32_t) + 16 * sizeof(int8_t)))));
171 i4 += 4;
172
173 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi4x0123, vk4x0123));
174
175 w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t) + 20 * sizeof(int8_t));
176
177 _mm_storeu_si128((__m128i*) b, vacc0123);
178 b += 4;
179 c -= 4;
180 } while (c != 0);
181 }
182 }
183
184 // Middle pass to process 5 inputs in each iteration.
185 for (size_t ks = kernel_size - 5; ks > 5; ks -= 5) {
186 int32_t* b = buffer;
187 const int8_t* i0 = input[0];
188 assert(i0 != NULL);
189 if XNN_UNPREDICTABLE(i0 != zero) {
190 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
191 }
192 const int8_t* i1 = input[1];
193 assert(i1 != NULL);
194 if XNN_UNPREDICTABLE(i1 != zero) {
195 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
196 }
197 const int8_t* i2 = input[2];
198 assert(i2 != NULL);
199 if XNN_UNPREDICTABLE(i2 != zero) {
200 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
201 }
202 const int8_t* i3 = input[3];
203 assert(i3 != NULL);
204 if XNN_UNPREDICTABLE(i3 != zero) {
205 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
206 }
207 const int8_t* i4 = input[4];
208 assert(i4 != NULL);
209 if XNN_UNPREDICTABLE(i4 != zero) {
210 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
211 }
212 input += 5;
213
214 size_t c = round_up_po2(channels, 4);
215
216 for (; c >= 8; c -= 8) {
217 __m128i vacc0123 = _mm_loadu_si128((const __m128i*) b);
218 __m128i vacc4567 = _mm_loadu_si128((const __m128i*) (b + 4));
219
220
221 const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i0)));
222 const __m128i vk0x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 0 * sizeof(int8_t)))));
223 const __m128i vi0x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i0 + 4)));
224 const __m128i vk0x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 4 * sizeof(int8_t)))));
225 i0 += 8;
226
227 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi0x0123, vk0x0123));
228 vacc4567 = _mm_add_epi32(vacc4567, _mm_mullo_epi32(vi0x4567, vk0x4567));
229
230 const __m128i vi1x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i1)));
231 const __m128i vk1x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 8 * sizeof(int8_t)))));
232 const __m128i vi1x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i1 + 4)));
233 const __m128i vk1x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 12 * sizeof(int8_t)))));
234 i1 += 8;
235
236 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi1x0123, vk1x0123));
237 vacc4567 = _mm_add_epi32(vacc4567, _mm_mullo_epi32(vi1x4567, vk1x4567));
238
239 const __m128i vi2x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i2)));
240 const __m128i vk2x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 16 * sizeof(int8_t)))));
241 const __m128i vi2x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i2 + 4)));
242 const __m128i vk2x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 20 * sizeof(int8_t)))));
243 i2 += 8;
244
245 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi2x0123, vk2x0123));
246 vacc4567 = _mm_add_epi32(vacc4567, _mm_mullo_epi32(vi2x4567, vk2x4567));
247
248 const __m128i vi3x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i3)));
249 const __m128i vk3x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 24 * sizeof(int8_t)))));
250 const __m128i vi3x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i3 + 4)));
251 const __m128i vk3x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 28 * sizeof(int8_t)))));
252 i3 += 8;
253
254 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi3x0123, vk3x0123));
255 vacc4567 = _mm_add_epi32(vacc4567, _mm_mullo_epi32(vi3x4567, vk3x4567));
256
257 const __m128i vi4x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i4)));
258 const __m128i vk4x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 32 * sizeof(int8_t)))));
259 const __m128i vi4x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i4 + 4)));
260 const __m128i vk4x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 36 * sizeof(int8_t)))));
261 i4 += 8;
262
263 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi4x0123, vk4x0123));
264 vacc4567 = _mm_add_epi32(vacc4567, _mm_mullo_epi32(vi4x4567, vk4x4567));
265
266 w = (const void*) ((uintptr_t) w + 40 * sizeof(int8_t));
267
268 _mm_storeu_si128((__m128i*) b, vacc0123);
269 _mm_storeu_si128((__m128i*) (b + 4), vacc4567);
270 b += 8;
271 }
272
273 if XNN_UNLIKELY(c != 0) {
274 do {
275 __m128i vacc0123 = _mm_loadu_si128((const __m128i*) b);
276
277 const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i0)));
278 const __m128i vk0x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int32_t*) ((uintptr_t) w))));
279 i0 += 4;
280
281 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi0x0123, vk0x0123));
282
283 const __m128i vi1x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i1)));
284 const __m128i vk1x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int32_t*) ((uintptr_t) w + 4 * sizeof(int8_t)))));
285 i1 += 4;
286
287 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi1x0123, vk1x0123));
288
289 const __m128i vi2x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i2)));
290 const __m128i vk2x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int32_t*) ((uintptr_t) w + 8 * sizeof(int8_t)))));
291 i2 += 4;
292
293 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi2x0123, vk2x0123));
294
295 const __m128i vi3x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i3)));
296 const __m128i vk3x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int32_t*) ((uintptr_t) w + 12 * sizeof(int8_t)))));
297 i3 += 4;
298
299 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi3x0123, vk3x0123));
300
301 const __m128i vi4x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i4)));
302 const __m128i vk4x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int32_t*) ((uintptr_t) w + 16 * sizeof(int8_t)))));
303 i4 += 4;
304
305 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi4x0123, vk4x0123));
306
307 w = (const void*) ((uintptr_t) w + 20 * sizeof(int8_t));
308
309 _mm_storeu_si128((__m128i*) b, vacc0123);
310 b += 4;
311 c -= 4;
312 } while (c != 0);
313 }
314 }
315
316 // Last pass to process up to 5 inputs.
317 {
318 const int32_t* b = buffer;
319 const int8_t* i0 = input[0];
320 assert(i0 != NULL);
321 if XNN_UNPREDICTABLE(i0 != zero) {
322 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
323 }
324 const int8_t* i1 = input[1];
325 assert(i1 != NULL);
326 if XNN_UNPREDICTABLE(i1 != zero) {
327 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
328 }
329 const int8_t* i2 = input[2];
330 assert(i2 != NULL);
331 if XNN_UNPREDICTABLE(i2 != zero) {
332 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
333 }
334 const int8_t* i3 = input[3];
335 assert(i3 != NULL);
336 if XNN_UNPREDICTABLE(i3 != zero) {
337 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
338 }
339 const int8_t* i4 = input[4];
340 assert(i4 != NULL);
341 if XNN_UNPREDICTABLE(i4 != zero) {
342 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
343 }
344
345 size_t c = channels;
346
347 for (; c >= 8; c -= 8) {
348 __m128i vacc0123 = _mm_loadu_si128((const __m128i*) b);
349 __m128i vacc4567 = _mm_loadu_si128((const __m128i*) (b + 4));
350 b += 8;
351
352
353 const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i0)));
354 const __m128i vk0x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 0 * sizeof(int8_t)))));
355 const __m128i vi0x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i0 + 4)));
356 const __m128i vk0x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 4 * sizeof(int8_t)))));
357 i0 += 8;
358
359 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi0x0123, vk0x0123));
360 vacc4567 = _mm_add_epi32(vacc4567, _mm_mullo_epi32(vi0x4567, vk0x4567));
361
362 const __m128i vi1x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i1)));
363 const __m128i vk1x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 8 * sizeof(int8_t)))));
364 const __m128i vi1x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i1 + 4)));
365 const __m128i vk1x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 12 * sizeof(int8_t)))));
366 i1 += 8;
367
368 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi1x0123, vk1x0123));
369 vacc4567 = _mm_add_epi32(vacc4567, _mm_mullo_epi32(vi1x4567, vk1x4567));
370
371 const __m128i vi2x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i2)));
372 const __m128i vk2x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 16 * sizeof(int8_t)))));
373 const __m128i vi2x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i2 + 4)));
374 const __m128i vk2x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 20 * sizeof(int8_t)))));
375 i2 += 8;
376
377 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi2x0123, vk2x0123));
378 vacc4567 = _mm_add_epi32(vacc4567, _mm_mullo_epi32(vi2x4567, vk2x4567));
379
380 const __m128i vi3x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i3)));
381 const __m128i vk3x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 24 * sizeof(int8_t)))));
382 const __m128i vi3x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i3 + 4)));
383 const __m128i vk3x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 28 * sizeof(int8_t)))));
384 i3 += 8;
385
386 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi3x0123, vk3x0123));
387 vacc4567 = _mm_add_epi32(vacc4567, _mm_mullo_epi32(vi3x4567, vk3x4567));
388
389 const __m128i vi4x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i4)));
390 const __m128i vk4x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 32 * sizeof(int8_t)))));
391 const __m128i vi4x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i4 + 4)));
392 const __m128i vk4x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 36 * sizeof(int8_t)))));
393 i4 += 8;
394
395 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi4x0123, vk4x0123));
396 vacc4567 = _mm_add_epi32(vacc4567, _mm_mullo_epi32(vi4x4567, vk4x4567));
397
398 w = (const void*) ((uintptr_t) w + 40 * sizeof(int8_t));
399
400 __m128 vscaled0123 = _mm_cvtepi32_ps(vacc0123);
401 __m128 vscaled4567 = _mm_cvtepi32_ps(vacc4567);
402
403 vscaled0123 = _mm_mul_ps(vscaled0123, vscale);
404 vscaled4567 = _mm_mul_ps(vscaled4567, vscale);
405
406 vscaled0123 = _mm_min_ps(vscaled0123, voutput_max_less_zero_point);
407 vscaled4567 = _mm_min_ps(vscaled4567, voutput_max_less_zero_point);
408
409 vacc0123 = _mm_cvtps_epi32(vscaled0123);
410 vacc4567 = _mm_cvtps_epi32(vscaled4567);
411
412 __m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
413
414 __m128i vout0123456701234567 = _mm_packs_epi16(vout01234567, vout01234567);
415 vout0123456701234567 = _mm_max_epi8(vout0123456701234567, voutput_min);
416
417 _mm_storel_epi64((__m128i*) output, vout0123456701234567);
418 output += 8;
419 }
420
421 if XNN_UNLIKELY(c != 0) {
422 do {
423 __m128i vacc0123 = _mm_loadu_si128((const __m128i*) b);
424 b += 4;
425
426 const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i0)));
427 const __m128i vk0x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 0 * sizeof(int8_t)))));
428 i0 += 4;
429
430 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi0x0123, vk0x0123));
431
432 const __m128i vi1x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i1)));
433 const __m128i vk1x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 4 * sizeof(int8_t)))));
434 i1 += 4;
435
436 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi1x0123, vk1x0123));
437
438 const __m128i vi2x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i2)));
439 const __m128i vk2x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 8 * sizeof(int8_t)))));
440 i2 += 4;
441
442 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi2x0123, vk2x0123));
443
444 const __m128i vi3x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i3)));
445 const __m128i vk3x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 12 * sizeof(int8_t)))));
446 i3 += 4;
447
448 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi3x0123, vk3x0123));
449
450 const __m128i vi4x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i4)));
451 const __m128i vk4x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 16 * sizeof(int8_t)))));
452 i4 += 4;
453
454 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi4x0123, vk4x0123));
455
456 __m128 vscaled0123 = _mm_cvtepi32_ps(vacc0123);
457 vscaled0123 = _mm_mul_ps(vscaled0123, vscale);
458 vscaled0123 = _mm_min_ps(vscaled0123, voutput_max_less_zero_point);
459 vacc0123 = _mm_cvtps_epi32(vscaled0123);
460
461 w = (void*) ((uintptr_t) w + 20 * sizeof(int8_t));
462
463 __m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
464
465 vout0123 = _mm_packs_epi16(vout0123, vout0123);
466 vout0123 = _mm_max_epi8(vout0123, voutput_min);
467
468 if XNN_LIKELY(c >= 4) {
469 _mm_storeu_si32(output, vout0123);
470 output += 4;
471 c -= 4;
472 } else {
473 if (c & 2) {
474 unaligned_store_u16(output, (uint16_t) _mm_extract_epi16(vout0123, 0));
475 vout0123 = _mm_srli_epi32(vout0123, 16);
476 output += 2;
477 }
478 if (c & 1) {
479 *output = (int8_t) _mm_extract_epi8(vout0123, 0);
480 output += 1;
481 }
482 c = 0;
483 }
484 } while (c != 0);
485 }
486 }
487
488 input = (const int8_t**) ((uintptr_t) input + input_stride);
489 output = (int8_t*) ((uintptr_t) output + output_increment);
490 } while (--output_width != 0);
491 }