Debian Codegraph @

src/qs8-dwconv/gen/qs8-dwconv-5f5m5l16c4s4r-minmax-fp32-avx-mul32.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-dwconv/multipass-sse-mul32.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include <immintrin.h>
13
14 #include "xnnpack/dwconv.h"
15 #include "xnnpack/intrinsics-polyfill.h"
16 #include "xnnpack/math.h"
17 #include "xnnpack/unaligned.h"
18
19
20 void xnn_qs8_dwconv_minmax_fp32_ukernel_5f5m5l16c4s4r__avx_mul32(
21 size_t channels,
22 size_t output_width,
23 const int8_t** input,
24 const void* weights,
25 int8_t* output,
26 intptr_t input_stride,
27 size_t output_increment,
28 size_t input_offset,
29 const int8_t* zero,
30 size_t kernel_size,
31 int32_t* buffer,
32 const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
33 {
34 assert(channels != 0);
35 assert(output_width != 0);
36 assert(kernel_size > 5);
37
38
39 const __m128 vscale = _mm_set1_ps(params->fp32_scalar.scale);
40 XNN_FORCE_REALIZATION(vscale);
41
42 const __m128 voutput_max_less_zero_point = _mm_set1_ps((int32_t) params->fp32_scalar.output_max - (int32_t) params->fp32_scalar.output_zero_point);
43 const __m128i voutput_zero_point = _mm_set1_epi16(params->fp32_scalar.output_zero_point);
44 const __m128i voutput_min = _mm_set1_epi8(params->fp32_scalar.output_min);
45 XNN_FORCE_REALIZATION(voutput_max_less_zero_point);
46 XNN_FORCE_REALIZATION(voutput_zero_point);
47 XNN_FORCE_REALIZATION(voutput_min);
48
49 do {
50 const void* w = weights;
51
52 // First pass to process 5 inputs.
53 {
54 int32_t* b = buffer;
55 const int8_t* i0 = input[0];
56 assert(i0 != NULL);
57 if XNN_UNPREDICTABLE(i0 != zero) {
58 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
59 }
60 const int8_t* i1 = input[1];
61 assert(i1 != NULL);
62 if XNN_UNPREDICTABLE(i1 != zero) {
63 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
64 }
65 const int8_t* i2 = input[2];
66 assert(i2 != NULL);
67 if XNN_UNPREDICTABLE(i2 != zero) {
68 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
69 }
70 const int8_t* i3 = input[3];
71 assert(i3 != NULL);
72 if XNN_UNPREDICTABLE(i3 != zero) {
73 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
74 }
75 const int8_t* i4 = input[4];
76 assert(i4 != NULL);
77 if XNN_UNPREDICTABLE(i4 != zero) {
78 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
79 }
80 input += 5;
81
82 size_t c = round_up_po2(channels, 4);
83
84 for (; c >= 16; c -= 16) {
85 __m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
86 __m128i vacc4567 = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 4));
87 __m128i vacc89AB = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 8));
88 __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) ((const int32_t*) w + 12));
89
90
91 const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i0)));
92 const __m128i vk0x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 16 * sizeof(int32_t) + 0 * sizeof(int8_t)))));
93 const __m128i vi0x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i0 + 4)));
94 const __m128i vk0x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 16 * sizeof(int32_t) + 4 * sizeof(int8_t)))));
95 const __m128i vi0x89AB = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i0 + 8)));
96 const __m128i vk0x89AB = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 16 * sizeof(int32_t) + 8 * sizeof(int8_t)))));
97 const __m128i vi0xCDEF = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i0 + 12)));
98 const __m128i vk0xCDEF = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 16 * sizeof(int32_t) + 12 * sizeof(int8_t)))));
99 i0 += 16;
100
101 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi0x0123, vk0x0123));
102 vacc4567 = _mm_add_epi32(vacc4567, _mm_mullo_epi32(vi0x4567, vk0x4567));
103 vacc89AB = _mm_add_epi32(vacc89AB, _mm_mullo_epi32(vi0x89AB, vk0x89AB));
104 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_mullo_epi32(vi0xCDEF, vk0xCDEF));
105
106 const __m128i vi1x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i1)));
107 const __m128i vk1x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 16 * sizeof(int32_t) + 16 * sizeof(int8_t)))));
108 const __m128i vi1x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i1 + 4)));
109 const __m128i vk1x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 16 * sizeof(int32_t) + 20 * sizeof(int8_t)))));
110 const __m128i vi1x89AB = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i1 + 8)));
111 const __m128i vk1x89AB = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 16 * sizeof(int32_t) + 24 * sizeof(int8_t)))));
112 const __m128i vi1xCDEF = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i1 + 12)));
113 const __m128i vk1xCDEF = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 16 * sizeof(int32_t) + 28 * sizeof(int8_t)))));
114 i1 += 16;
115
116 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi1x0123, vk1x0123));
117 vacc4567 = _mm_add_epi32(vacc4567, _mm_mullo_epi32(vi1x4567, vk1x4567));
118 vacc89AB = _mm_add_epi32(vacc89AB, _mm_mullo_epi32(vi1x89AB, vk1x89AB));
119 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_mullo_epi32(vi1xCDEF, vk1xCDEF));
120
121 const __m128i vi2x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i2)));
122 const __m128i vk2x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 16 * sizeof(int32_t) + 32 * sizeof(int8_t)))));
123 const __m128i vi2x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i2 + 4)));
124 const __m128i vk2x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 16 * sizeof(int32_t) + 36 * sizeof(int8_t)))));
125 const __m128i vi2x89AB = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i2 + 8)));
126 const __m128i vk2x89AB = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 16 * sizeof(int32_t) + 40 * sizeof(int8_t)))));
127 const __m128i vi2xCDEF = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i2 + 12)));
128 const __m128i vk2xCDEF = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 16 * sizeof(int32_t) + 44 * sizeof(int8_t)))));
129 i2 += 16;
130
131 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi2x0123, vk2x0123));
132 vacc4567 = _mm_add_epi32(vacc4567, _mm_mullo_epi32(vi2x4567, vk2x4567));
133 vacc89AB = _mm_add_epi32(vacc89AB, _mm_mullo_epi32(vi2x89AB, vk2x89AB));
134 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_mullo_epi32(vi2xCDEF, vk2xCDEF));
135
136 const __m128i vi3x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i3)));
137 const __m128i vk3x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 16 * sizeof(int32_t) + 48 * sizeof(int8_t)))));
138 const __m128i vi3x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i3 + 4)));
139 const __m128i vk3x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 16 * sizeof(int32_t) + 52 * sizeof(int8_t)))));
140 const __m128i vi3x89AB = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i3 + 8)));
141 const __m128i vk3x89AB = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 16 * sizeof(int32_t) + 56 * sizeof(int8_t)))));
142 const __m128i vi3xCDEF = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i3 + 12)));
143 const __m128i vk3xCDEF = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 16 * sizeof(int32_t) + 60 * sizeof(int8_t)))));
144 i3 += 16;
145
146 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi3x0123, vk3x0123));
147 vacc4567 = _mm_add_epi32(vacc4567, _mm_mullo_epi32(vi3x4567, vk3x4567));
148 vacc89AB = _mm_add_epi32(vacc89AB, _mm_mullo_epi32(vi3x89AB, vk3x89AB));
149 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_mullo_epi32(vi3xCDEF, vk3xCDEF));
150
151 const __m128i vi4x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i4)));
152 const __m128i vk4x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 16 * sizeof(int32_t) + 64 * sizeof(int8_t)))));
153 const __m128i vi4x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i4 + 4)));
154 const __m128i vk4x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 16 * sizeof(int32_t) + 68 * sizeof(int8_t)))));
155 const __m128i vi4x89AB = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i4 + 8)));
156 const __m128i vk4x89AB = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 16 * sizeof(int32_t) + 72 * sizeof(int8_t)))));
157 const __m128i vi4xCDEF = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i4 + 12)));
158 const __m128i vk4xCDEF = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 16 * sizeof(int32_t) + 76 * sizeof(int8_t)))));
159 i4 += 16;
160
161 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi4x0123, vk4x0123));
162 vacc4567 = _mm_add_epi32(vacc4567, _mm_mullo_epi32(vi4x4567, vk4x4567));
163 vacc89AB = _mm_add_epi32(vacc89AB, _mm_mullo_epi32(vi4x89AB, vk4x89AB));
164 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_mullo_epi32(vi4xCDEF, vk4xCDEF));
165
166 w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 80 * sizeof(int8_t));
167
168 _mm_storeu_si128((__m128i*) b, vacc0123);
169 _mm_storeu_si128((__m128i*) (b + 4), vacc4567);
170 _mm_storeu_si128((__m128i*) (b + 8), vacc89AB);
171 _mm_storeu_si128((__m128i*) (b + 12), vaccCDEF);
172 b += 16;
173 }
174
175 if XNN_UNLIKELY(c != 0) {
176 do {
177 __m128i vacc0123 = _mm_loadu_si128((const __m128i*) w);
178
179 const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i0)));
180 const __m128i vk0x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int32_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))));
181 i0 += 4;
182
183 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi0x0123, vk0x0123));
184
185 const __m128i vi1x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i1)));
186 const __m128i vk1x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int32_t*) ((uintptr_t) w + 4 * sizeof(int32_t) + 4 * sizeof(int8_t)))));
187 i1 += 4;
188
189 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi1x0123, vk1x0123));
190
191 const __m128i vi2x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i2)));
192 const __m128i vk2x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int32_t*) ((uintptr_t) w + 4 * sizeof(int32_t) + 8 * sizeof(int8_t)))));
193 i2 += 4;
194
195 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi2x0123, vk2x0123));
196
197 const __m128i vi3x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i3)));
198 const __m128i vk3x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int32_t*) ((uintptr_t) w + 4 * sizeof(int32_t) + 12 * sizeof(int8_t)))));
199 i3 += 4;
200
201 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi3x0123, vk3x0123));
202
203 const __m128i vi4x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i4)));
204 const __m128i vk4x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int32_t*) ((uintptr_t) w + 4 * sizeof(int32_t) + 16 * sizeof(int8_t)))));
205 i4 += 4;
206
207 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi4x0123, vk4x0123));
208
209 w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t) + 20 * sizeof(int8_t));
210
211 _mm_storeu_si128((__m128i*) b, vacc0123);
212 b += 4;
213 c -= 4;
214 } while (c != 0);
215 }
216 }
217
218 // Middle pass to process 5 inputs in each iteration.
219 for (size_t ks = kernel_size - 5; ks > 5; ks -= 5) {
220 int32_t* b = buffer;
221 const int8_t* i0 = input[0];
222 assert(i0 != NULL);
223 if XNN_UNPREDICTABLE(i0 != zero) {
224 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
225 }
226 const int8_t* i1 = input[1];
227 assert(i1 != NULL);
228 if XNN_UNPREDICTABLE(i1 != zero) {
229 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
230 }
231 const int8_t* i2 = input[2];
232 assert(i2 != NULL);
233 if XNN_UNPREDICTABLE(i2 != zero) {
234 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
235 }
236 const int8_t* i3 = input[3];
237 assert(i3 != NULL);
238 if XNN_UNPREDICTABLE(i3 != zero) {
239 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
240 }
241 const int8_t* i4 = input[4];
242 assert(i4 != NULL);
243 if XNN_UNPREDICTABLE(i4 != zero) {
244 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
245 }
246 input += 5;
247
248 size_t c = round_up_po2(channels, 4);
249
250 for (; c >= 16; c -= 16) {
251 __m128i vacc0123 = _mm_loadu_si128((const __m128i*) b);
252 __m128i vacc4567 = _mm_loadu_si128((const __m128i*) (b + 4));
253 __m128i vacc89AB = _mm_loadu_si128((const __m128i*) (b + 8));
254 __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) (b + 12));
255
256
257 const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i0)));
258 const __m128i vk0x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 0 * sizeof(int8_t)))));
259 const __m128i vi0x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i0 + 4)));
260 const __m128i vk0x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 4 * sizeof(int8_t)))));
261 const __m128i vi0x89AB = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i0 + 8)));
262 const __m128i vk0x89AB = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 8 * sizeof(int8_t)))));
263 const __m128i vi0xCDEF = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i0 + 12)));
264 const __m128i vk0xCDEF = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 12 * sizeof(int8_t)))));
265 i0 += 16;
266
267 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi0x0123, vk0x0123));
268 vacc4567 = _mm_add_epi32(vacc4567, _mm_mullo_epi32(vi0x4567, vk0x4567));
269 vacc89AB = _mm_add_epi32(vacc89AB, _mm_mullo_epi32(vi0x89AB, vk0x89AB));
270 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_mullo_epi32(vi0xCDEF, vk0xCDEF));
271
272 const __m128i vi1x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i1)));
273 const __m128i vk1x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 16 * sizeof(int8_t)))));
274 const __m128i vi1x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i1 + 4)));
275 const __m128i vk1x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 20 * sizeof(int8_t)))));
276 const __m128i vi1x89AB = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i1 + 8)));
277 const __m128i vk1x89AB = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 24 * sizeof(int8_t)))));
278 const __m128i vi1xCDEF = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i1 + 12)));
279 const __m128i vk1xCDEF = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 28 * sizeof(int8_t)))));
280 i1 += 16;
281
282 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi1x0123, vk1x0123));
283 vacc4567 = _mm_add_epi32(vacc4567, _mm_mullo_epi32(vi1x4567, vk1x4567));
284 vacc89AB = _mm_add_epi32(vacc89AB, _mm_mullo_epi32(vi1x89AB, vk1x89AB));
285 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_mullo_epi32(vi1xCDEF, vk1xCDEF));
286
287 const __m128i vi2x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i2)));
288 const __m128i vk2x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 32 * sizeof(int8_t)))));
289 const __m128i vi2x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i2 + 4)));
290 const __m128i vk2x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 36 * sizeof(int8_t)))));
291 const __m128i vi2x89AB = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i2 + 8)));
292 const __m128i vk2x89AB = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 40 * sizeof(int8_t)))));
293 const __m128i vi2xCDEF = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i2 + 12)));
294 const __m128i vk2xCDEF = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 44 * sizeof(int8_t)))));
295 i2 += 16;
296
297 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi2x0123, vk2x0123));
298 vacc4567 = _mm_add_epi32(vacc4567, _mm_mullo_epi32(vi2x4567, vk2x4567));
299 vacc89AB = _mm_add_epi32(vacc89AB, _mm_mullo_epi32(vi2x89AB, vk2x89AB));
300 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_mullo_epi32(vi2xCDEF, vk2xCDEF));
301
302 const __m128i vi3x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i3)));
303 const __m128i vk3x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 48 * sizeof(int8_t)))));
304 const __m128i vi3x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i3 + 4)));
305 const __m128i vk3x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 52 * sizeof(int8_t)))));
306 const __m128i vi3x89AB = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i3 + 8)));
307 const __m128i vk3x89AB = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 56 * sizeof(int8_t)))));
308 const __m128i vi3xCDEF = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i3 + 12)));
309 const __m128i vk3xCDEF = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 60 * sizeof(int8_t)))));
310 i3 += 16;
311
312 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi3x0123, vk3x0123));
313 vacc4567 = _mm_add_epi32(vacc4567, _mm_mullo_epi32(vi3x4567, vk3x4567));
314 vacc89AB = _mm_add_epi32(vacc89AB, _mm_mullo_epi32(vi3x89AB, vk3x89AB));
315 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_mullo_epi32(vi3xCDEF, vk3xCDEF));
316
317 const __m128i vi4x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i4)));
318 const __m128i vk4x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 64 * sizeof(int8_t)))));
319 const __m128i vi4x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i4 + 4)));
320 const __m128i vk4x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 68 * sizeof(int8_t)))));
321 const __m128i vi4x89AB = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i4 + 8)));
322 const __m128i vk4x89AB = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 72 * sizeof(int8_t)))));
323 const __m128i vi4xCDEF = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i4 + 12)));
324 const __m128i vk4xCDEF = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 76 * sizeof(int8_t)))));
325 i4 += 16;
326
327 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi4x0123, vk4x0123));
328 vacc4567 = _mm_add_epi32(vacc4567, _mm_mullo_epi32(vi4x4567, vk4x4567));
329 vacc89AB = _mm_add_epi32(vacc89AB, _mm_mullo_epi32(vi4x89AB, vk4x89AB));
330 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_mullo_epi32(vi4xCDEF, vk4xCDEF));
331
332 w = (const void*) ((uintptr_t) w + 80 * sizeof(int8_t));
333
334 _mm_storeu_si128((__m128i*) b, vacc0123);
335 _mm_storeu_si128((__m128i*) (b + 4), vacc4567);
336 _mm_storeu_si128((__m128i*) (b + 8), vacc89AB);
337 _mm_storeu_si128((__m128i*) (b + 12), vaccCDEF);
338 b += 16;
339 }
340
341 if XNN_UNLIKELY(c != 0) {
342 do {
343 __m128i vacc0123 = _mm_loadu_si128((const __m128i*) b);
344
345 const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i0)));
346 const __m128i vk0x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int32_t*) ((uintptr_t) w))));
347 i0 += 4;
348
349 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi0x0123, vk0x0123));
350
351 const __m128i vi1x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i1)));
352 const __m128i vk1x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int32_t*) ((uintptr_t) w + 4 * sizeof(int8_t)))));
353 i1 += 4;
354
355 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi1x0123, vk1x0123));
356
357 const __m128i vi2x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i2)));
358 const __m128i vk2x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int32_t*) ((uintptr_t) w + 8 * sizeof(int8_t)))));
359 i2 += 4;
360
361 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi2x0123, vk2x0123));
362
363 const __m128i vi3x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i3)));
364 const __m128i vk3x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int32_t*) ((uintptr_t) w + 12 * sizeof(int8_t)))));
365 i3 += 4;
366
367 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi3x0123, vk3x0123));
368
369 const __m128i vi4x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i4)));
370 const __m128i vk4x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int32_t*) ((uintptr_t) w + 16 * sizeof(int8_t)))));
371 i4 += 4;
372
373 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi4x0123, vk4x0123));
374
375 w = (const void*) ((uintptr_t) w + 20 * sizeof(int8_t));
376
377 _mm_storeu_si128((__m128i*) b, vacc0123);
378 b += 4;
379 c -= 4;
380 } while (c != 0);
381 }
382 }
383
384 // Last pass to process up to 5 inputs.
385 {
386 const int32_t* b = buffer;
387 const int8_t* i0 = input[0];
388 assert(i0 != NULL);
389 if XNN_UNPREDICTABLE(i0 != zero) {
390 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
391 }
392 const int8_t* i1 = input[1];
393 assert(i1 != NULL);
394 if XNN_UNPREDICTABLE(i1 != zero) {
395 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
396 }
397 const int8_t* i2 = input[2];
398 assert(i2 != NULL);
399 if XNN_UNPREDICTABLE(i2 != zero) {
400 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
401 }
402 const int8_t* i3 = input[3];
403 assert(i3 != NULL);
404 if XNN_UNPREDICTABLE(i3 != zero) {
405 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
406 }
407 const int8_t* i4 = input[4];
408 assert(i4 != NULL);
409 if XNN_UNPREDICTABLE(i4 != zero) {
410 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
411 }
412
413 size_t c = channels;
414
415 for (; c >= 16; c -= 16) {
416 __m128i vacc0123 = _mm_loadu_si128((const __m128i*) b);
417 __m128i vacc4567 = _mm_loadu_si128((const __m128i*) (b + 4));
418 __m128i vacc89AB = _mm_loadu_si128((const __m128i*) (b + 8));
419 __m128i vaccCDEF = _mm_loadu_si128((const __m128i*) (b + 12));
420 b += 16;
421
422
423 const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i0)));
424 const __m128i vk0x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 0 * sizeof(int8_t)))));
425 const __m128i vi0x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i0 + 4)));
426 const __m128i vk0x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 4 * sizeof(int8_t)))));
427 const __m128i vi0x89AB = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i0 + 8)));
428 const __m128i vk0x89AB = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 8 * sizeof(int8_t)))));
429 const __m128i vi0xCDEF = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i0 + 12)));
430 const __m128i vk0xCDEF = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 12 * sizeof(int8_t)))));
431 i0 += 16;
432
433 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi0x0123, vk0x0123));
434 vacc4567 = _mm_add_epi32(vacc4567, _mm_mullo_epi32(vi0x4567, vk0x4567));
435 vacc89AB = _mm_add_epi32(vacc89AB, _mm_mullo_epi32(vi0x89AB, vk0x89AB));
436 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_mullo_epi32(vi0xCDEF, vk0xCDEF));
437
438 const __m128i vi1x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i1)));
439 const __m128i vk1x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 16 * sizeof(int8_t)))));
440 const __m128i vi1x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i1 + 4)));
441 const __m128i vk1x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 20 * sizeof(int8_t)))));
442 const __m128i vi1x89AB = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i1 + 8)));
443 const __m128i vk1x89AB = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 24 * sizeof(int8_t)))));
444 const __m128i vi1xCDEF = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i1 + 12)));
445 const __m128i vk1xCDEF = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 28 * sizeof(int8_t)))));
446 i1 += 16;
447
448 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi1x0123, vk1x0123));
449 vacc4567 = _mm_add_epi32(vacc4567, _mm_mullo_epi32(vi1x4567, vk1x4567));
450 vacc89AB = _mm_add_epi32(vacc89AB, _mm_mullo_epi32(vi1x89AB, vk1x89AB));
451 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_mullo_epi32(vi1xCDEF, vk1xCDEF));
452
453 const __m128i vi2x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i2)));
454 const __m128i vk2x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 32 * sizeof(int8_t)))));
455 const __m128i vi2x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i2 + 4)));
456 const __m128i vk2x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 36 * sizeof(int8_t)))));
457 const __m128i vi2x89AB = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i2 + 8)));
458 const __m128i vk2x89AB = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 40 * sizeof(int8_t)))));
459 const __m128i vi2xCDEF = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i2 + 12)));
460 const __m128i vk2xCDEF = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 44 * sizeof(int8_t)))));
461 i2 += 16;
462
463 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi2x0123, vk2x0123));
464 vacc4567 = _mm_add_epi32(vacc4567, _mm_mullo_epi32(vi2x4567, vk2x4567));
465 vacc89AB = _mm_add_epi32(vacc89AB, _mm_mullo_epi32(vi2x89AB, vk2x89AB));
466 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_mullo_epi32(vi2xCDEF, vk2xCDEF));
467
468 const __m128i vi3x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i3)));
469 const __m128i vk3x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 48 * sizeof(int8_t)))));
470 const __m128i vi3x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i3 + 4)));
471 const __m128i vk3x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 52 * sizeof(int8_t)))));
472 const __m128i vi3x89AB = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i3 + 8)));
473 const __m128i vk3x89AB = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 56 * sizeof(int8_t)))));
474 const __m128i vi3xCDEF = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i3 + 12)));
475 const __m128i vk3xCDEF = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 60 * sizeof(int8_t)))));
476 i3 += 16;
477
478 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi3x0123, vk3x0123));
479 vacc4567 = _mm_add_epi32(vacc4567, _mm_mullo_epi32(vi3x4567, vk3x4567));
480 vacc89AB = _mm_add_epi32(vacc89AB, _mm_mullo_epi32(vi3x89AB, vk3x89AB));
481 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_mullo_epi32(vi3xCDEF, vk3xCDEF));
482
483 const __m128i vi4x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i4)));
484 const __m128i vk4x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 64 * sizeof(int8_t)))));
485 const __m128i vi4x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i4 + 4)));
486 const __m128i vk4x4567 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 68 * sizeof(int8_t)))));
487 const __m128i vi4x89AB = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i4 + 8)));
488 const __m128i vk4x89AB = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 72 * sizeof(int8_t)))));
489 const __m128i vi4xCDEF = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i4 + 12)));
490 const __m128i vk4xCDEF = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 76 * sizeof(int8_t)))));
491 i4 += 16;
492
493 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi4x0123, vk4x0123));
494 vacc4567 = _mm_add_epi32(vacc4567, _mm_mullo_epi32(vi4x4567, vk4x4567));
495 vacc89AB = _mm_add_epi32(vacc89AB, _mm_mullo_epi32(vi4x89AB, vk4x89AB));
496 vaccCDEF = _mm_add_epi32(vaccCDEF, _mm_mullo_epi32(vi4xCDEF, vk4xCDEF));
497
498 w = (const void*) ((uintptr_t) w + 80 * sizeof(int8_t));
499
500 __m128 vscaled0123 = _mm_cvtepi32_ps(vacc0123);
501 __m128 vscaled4567 = _mm_cvtepi32_ps(vacc4567);
502 __m128 vscaled89AB = _mm_cvtepi32_ps(vacc89AB);
503 __m128 vscaledCDEF = _mm_cvtepi32_ps(vaccCDEF);
504
505 vscaled0123 = _mm_mul_ps(vscaled0123, vscale);
506 vscaled4567 = _mm_mul_ps(vscaled4567, vscale);
507 vscaled89AB = _mm_mul_ps(vscaled89AB, vscale);
508 vscaledCDEF = _mm_mul_ps(vscaledCDEF, vscale);
509
510 vscaled0123 = _mm_min_ps(vscaled0123, voutput_max_less_zero_point);
511 vscaled4567 = _mm_min_ps(vscaled4567, voutput_max_less_zero_point);
512 vscaled89AB = _mm_min_ps(vscaled89AB, voutput_max_less_zero_point);
513 vscaledCDEF = _mm_min_ps(vscaledCDEF, voutput_max_less_zero_point);
514
515 vacc0123 = _mm_cvtps_epi32(vscaled0123);
516 vacc4567 = _mm_cvtps_epi32(vscaled4567);
517 vacc89AB = _mm_cvtps_epi32(vscaled89AB);
518 vaccCDEF = _mm_cvtps_epi32(vscaledCDEF);
519
520 __m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc4567), voutput_zero_point);
521 __m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(vacc89AB, vaccCDEF), voutput_zero_point);
522
523 __m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
524 vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
525
526 _mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
527 output += 16;
528 }
529
530 if XNN_UNLIKELY(c != 0) {
531 do {
532 __m128i vacc0123 = _mm_loadu_si128((const __m128i*) b);
533 b += 4;
534
535 const __m128i vi0x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i0)));
536 const __m128i vk0x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 0 * sizeof(int8_t)))));
537 i0 += 4;
538
539 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi0x0123, vk0x0123));
540
541 const __m128i vi1x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i1)));
542 const __m128i vk1x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 4 * sizeof(int8_t)))));
543 i1 += 4;
544
545 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi1x0123, vk1x0123));
546
547 const __m128i vi2x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i2)));
548 const __m128i vk2x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 8 * sizeof(int8_t)))));
549 i2 += 4;
550
551 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi2x0123, vk2x0123));
552
553 const __m128i vi3x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i3)));
554 const __m128i vk3x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 12 * sizeof(int8_t)))));
555 i3 += 4;
556
557 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi3x0123, vk3x0123));
558
559 const __m128i vi4x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128((int) unaligned_load_s32(i4)));
560 const __m128i vk4x0123 = _mm_cvtepi8_epi32(_mm_cvtsi32_si128(*((const int*) ((uintptr_t) w + 16 * sizeof(int8_t)))));
561 i4 += 4;
562
563 vacc0123 = _mm_add_epi32(vacc0123, _mm_mullo_epi32(vi4x0123, vk4x0123));
564
565 __m128 vscaled0123 = _mm_cvtepi32_ps(vacc0123);
566 vscaled0123 = _mm_mul_ps(vscaled0123, vscale);
567 vscaled0123 = _mm_min_ps(vscaled0123, voutput_max_less_zero_point);
568 vacc0123 = _mm_cvtps_epi32(vscaled0123);
569
570 w = (void*) ((uintptr_t) w + 20 * sizeof(int8_t));
571
572 __m128i vout0123 = _mm_adds_epi16(_mm_packs_epi32(vacc0123, vacc0123), voutput_zero_point);
573
574 vout0123 = _mm_packs_epi16(vout0123, vout0123);
575 vout0123 = _mm_max_epi8(vout0123, voutput_min);
576
577 if XNN_LIKELY(c >= 4) {
578 _mm_storeu_si32(output, vout0123);
579 output += 4;
580 c -= 4;
581 } else {
582 if (c & 2) {
583 unaligned_store_u16(output, (uint16_t) _mm_extract_epi16(vout0123, 0));
584 vout0123 = _mm_srli_epi32(vout0123, 16);
585 output += 2;
586 }
587 if (c & 1) {
588 *output = (int8_t) _mm_extract_epi8(vout0123, 0);
589 output += 1;
590 }
591 c = 0;
592 }
593 } while (c != 0);
594 }
595 }
596
597 input = (const int8_t**) ((uintptr_t) input + input_stride);
598 output = (int8_t*) ((uintptr_t) output + output_increment);
599 } while (--output_width != 0);
600 }