Debian Codegraph @

src/qs8-dwconv/gen/qs8-dwconv-8f8m9l16c16s16r-minmax-fp32-avx2-mul16-vpmovsx.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-dwconv/multipass-avx2-mul16-vpmovsx.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include <immintrin.h>
13
14 #include "xnnpack/dwconv.h"
15 #include "xnnpack/math.h"
16 #include "xnnpack/unaligned.h"
17
18
19 void xnn_qs8_dwconv_minmax_fp32_ukernel_8f8m9l16c16s16r__avx2_mul16_vpmovsx(
20 size_t channels,
21 size_t output_width,
22 const int8_t** input,
23 const void* weights,
24 int8_t* output,
25 intptr_t input_stride,
26 size_t output_increment,
27 size_t input_offset,
28 const int8_t* zero,
29 size_t kernel_size,
30 int32_t* buffer,
31 const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
32 {
33 assert(channels != 0);
34 assert(output_width != 0);
35 assert(kernel_size > 8);
36
37 const __m256 vscale = _mm256_set1_ps(params->fp32_scalar.scale);
38 XNN_FORCE_REALIZATION(vscale);
39 const __m256 voutput_max_less_zero_point = _mm256_set1_ps((int32_t) params->fp32_scalar.output_max - (int32_t) params->fp32_scalar.output_zero_point);
40 const __m256i voutput_zero_point = _mm256_set1_epi16(params->fp32_scalar.output_zero_point);
41 const __m128i voutput_min = _mm_set1_epi8(params->fp32_scalar.output_min);
42 XNN_FORCE_REALIZATION(voutput_max_less_zero_point);
43 XNN_FORCE_REALIZATION(voutput_zero_point);
44 XNN_FORCE_REALIZATION(voutput_min);
45
46 do {
47 const void* w = weights;
48
49 // First pass to process 8 inputs.
50 {
51 int32_t* b = buffer;
52
53 const int8_t* i0 = input[0];
54 assert(i0 != NULL);
55 if XNN_UNPREDICTABLE(i0 != zero) {
56 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
57 }
58 const int8_t* i1 = input[1];
59 assert(i1 != NULL);
60 if XNN_UNPREDICTABLE(i1 != zero) {
61 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
62 }
63 const int8_t* i2 = input[2];
64 assert(i2 != NULL);
65 if XNN_UNPREDICTABLE(i2 != zero) {
66 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
67 }
68 const int8_t* i3 = input[3];
69 assert(i3 != NULL);
70 if XNN_UNPREDICTABLE(i3 != zero) {
71 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
72 }
73 const int8_t* i4 = input[4];
74 assert(i4 != NULL);
75 if XNN_UNPREDICTABLE(i4 != zero) {
76 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
77 }
78 const int8_t* i5 = input[5];
79 assert(i5 != NULL);
80 if XNN_UNPREDICTABLE(i5 != zero) {
81 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
82 }
83 const int8_t* i6 = input[6];
84 assert(i6 != NULL);
85 if XNN_UNPREDICTABLE(i6 != zero) {
86 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
87 }
88 const int8_t* i7 = input[7];
89 assert(i7 != NULL);
90 if XNN_UNPREDICTABLE(i7 != zero) {
91 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
92 }
93 input += 8;
94
95 size_t c = round_up_po2(channels, 16);
96
97 for (; c >= 16; c -= 16) {
98 __m256i vacc01234567 = _mm256_loadu_si256((const __m256i*) w);
99 __m256i vacc89ABCDEF = _mm256_loadu_si256((const __m256i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
100
101
102 const __m256i vi0x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i0));
103 const __m256i vk0x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 0 * sizeof(int8_t))));
104 i0 += 16;
105
106 const __m256i vprod0x0123456789ABCDEF = _mm256_mullo_epi16(vi0x0123456789ABCDEF, vk0x0123456789ABCDEF);
107 const __m128i vprod0x89ABCDEF = _mm256_extracti128_si256(vprod0x0123456789ABCDEF, 1);
108 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod0x0123456789ABCDEF)));
109 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod0x89ABCDEF));
110
111 const __m256i vi1x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i1));
112 const __m256i vk1x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 16 * sizeof(int8_t))));
113 i1 += 16;
114
115 const __m256i vprod1x0123456789ABCDEF = _mm256_mullo_epi16(vi1x0123456789ABCDEF, vk1x0123456789ABCDEF);
116 const __m128i vprod1x89ABCDEF = _mm256_extracti128_si256(vprod1x0123456789ABCDEF, 1);
117 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod1x0123456789ABCDEF)));
118 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod1x89ABCDEF));
119
120 const __m256i vi2x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i2));
121 const __m256i vk2x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 32 * sizeof(int8_t))));
122 i2 += 16;
123
124 const __m256i vprod2x0123456789ABCDEF = _mm256_mullo_epi16(vi2x0123456789ABCDEF, vk2x0123456789ABCDEF);
125 const __m128i vprod2x89ABCDEF = _mm256_extracti128_si256(vprod2x0123456789ABCDEF, 1);
126 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod2x0123456789ABCDEF)));
127 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod2x89ABCDEF));
128
129 const __m256i vi3x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i3));
130 const __m256i vk3x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 48 * sizeof(int8_t))));
131 i3 += 16;
132
133 const __m256i vprod3x0123456789ABCDEF = _mm256_mullo_epi16(vi3x0123456789ABCDEF, vk3x0123456789ABCDEF);
134 const __m128i vprod3x89ABCDEF = _mm256_extracti128_si256(vprod3x0123456789ABCDEF, 1);
135 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod3x0123456789ABCDEF)));
136 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod3x89ABCDEF));
137
138 const __m256i vi4x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i4));
139 const __m256i vk4x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 64 * sizeof(int8_t))));
140 i4 += 16;
141
142 const __m256i vprod4x0123456789ABCDEF = _mm256_mullo_epi16(vi4x0123456789ABCDEF, vk4x0123456789ABCDEF);
143 const __m128i vprod4x89ABCDEF = _mm256_extracti128_si256(vprod4x0123456789ABCDEF, 1);
144 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod4x0123456789ABCDEF)));
145 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod4x89ABCDEF));
146
147 const __m256i vi5x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i5));
148 const __m256i vk5x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 80 * sizeof(int8_t))));
149 i5 += 16;
150
151 const __m256i vprod5x0123456789ABCDEF = _mm256_mullo_epi16(vi5x0123456789ABCDEF, vk5x0123456789ABCDEF);
152 const __m128i vprod5x89ABCDEF = _mm256_extracti128_si256(vprod5x0123456789ABCDEF, 1);
153 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod5x0123456789ABCDEF)));
154 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod5x89ABCDEF));
155
156 const __m256i vi6x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i6));
157 const __m256i vk6x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 96 * sizeof(int8_t))));
158 i6 += 16;
159
160 const __m256i vprod6x0123456789ABCDEF = _mm256_mullo_epi16(vi6x0123456789ABCDEF, vk6x0123456789ABCDEF);
161 const __m128i vprod6x89ABCDEF = _mm256_extracti128_si256(vprod6x0123456789ABCDEF, 1);
162 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod6x0123456789ABCDEF)));
163 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod6x89ABCDEF));
164
165 const __m256i vi7x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i7));
166 const __m256i vk7x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 112 * sizeof(int8_t))));
167 i7 += 16;
168
169 const __m256i vprod7x0123456789ABCDEF = _mm256_mullo_epi16(vi7x0123456789ABCDEF, vk7x0123456789ABCDEF);
170 const __m128i vprod7x89ABCDEF = _mm256_extracti128_si256(vprod7x0123456789ABCDEF, 1);
171 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod7x0123456789ABCDEF)));
172 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod7x89ABCDEF));
173
174 w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 128 * sizeof(int8_t));
175
176 _mm256_storeu_si256((__m256i*) b, vacc01234567);
177 _mm256_storeu_si256((__m256i*) (b + 8), vacc89ABCDEF);
178 b += 16;
179 }
180
181 assert(c == 0);
182 }
183
184 // Middle pass to process 8 inputs in each iteration.
185 for (size_t ks = kernel_size - 8; ks > 9; ks -= 8) {
186 int32_t* b = buffer;
187
188 const int8_t* i0 = input[0];
189 assert(i0 != NULL);
190 if XNN_UNPREDICTABLE(i0 != zero) {
191 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
192 }
193 const int8_t* i1 = input[1];
194 assert(i1 != NULL);
195 if XNN_UNPREDICTABLE(i1 != zero) {
196 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
197 }
198 const int8_t* i2 = input[2];
199 assert(i2 != NULL);
200 if XNN_UNPREDICTABLE(i2 != zero) {
201 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
202 }
203 const int8_t* i3 = input[3];
204 assert(i3 != NULL);
205 if XNN_UNPREDICTABLE(i3 != zero) {
206 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
207 }
208 const int8_t* i4 = input[4];
209 assert(i4 != NULL);
210 if XNN_UNPREDICTABLE(i4 != zero) {
211 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
212 }
213 const int8_t* i5 = input[5];
214 assert(i5 != NULL);
215 if XNN_UNPREDICTABLE(i5 != zero) {
216 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
217 }
218 const int8_t* i6 = input[6];
219 assert(i6 != NULL);
220 if XNN_UNPREDICTABLE(i6 != zero) {
221 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
222 }
223 const int8_t* i7 = input[7];
224 assert(i7 != NULL);
225 if XNN_UNPREDICTABLE(i7 != zero) {
226 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
227 }
228 input += 8;
229
230 size_t c = round_up_po2(channels, 16);
231
232 for (; c >= 16; c -= 16) {
233 __m256i vacc01234567 = _mm256_loadu_si256((const __m256i*) b);
234 __m256i vacc89ABCDEF = _mm256_loadu_si256((const __m256i*) (b + 8));
235
236
237 const __m256i vi0x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i0));
238 const __m256i vk0x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 0 * sizeof(int8_t))));
239 i0 += 16;
240
241 const __m256i vprod0x0123456789ABCDEF = _mm256_mullo_epi16(vi0x0123456789ABCDEF, vk0x0123456789ABCDEF);
242 const __m128i vprod0x89ABCDEF = _mm256_extracti128_si256(vprod0x0123456789ABCDEF, 1);
243 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod0x0123456789ABCDEF)));
244 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod0x89ABCDEF));
245
246 const __m256i vi1x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i1));
247 const __m256i vk1x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t))));
248 i1 += 16;
249
250 const __m256i vprod1x0123456789ABCDEF = _mm256_mullo_epi16(vi1x0123456789ABCDEF, vk1x0123456789ABCDEF);
251 const __m128i vprod1x89ABCDEF = _mm256_extracti128_si256(vprod1x0123456789ABCDEF, 1);
252 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod1x0123456789ABCDEF)));
253 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod1x89ABCDEF));
254
255 const __m256i vi2x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i2));
256 const __m256i vk2x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t))));
257 i2 += 16;
258
259 const __m256i vprod2x0123456789ABCDEF = _mm256_mullo_epi16(vi2x0123456789ABCDEF, vk2x0123456789ABCDEF);
260 const __m128i vprod2x89ABCDEF = _mm256_extracti128_si256(vprod2x0123456789ABCDEF, 1);
261 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod2x0123456789ABCDEF)));
262 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod2x89ABCDEF));
263
264 const __m256i vi3x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i3));
265 const __m256i vk3x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t))));
266 i3 += 16;
267
268 const __m256i vprod3x0123456789ABCDEF = _mm256_mullo_epi16(vi3x0123456789ABCDEF, vk3x0123456789ABCDEF);
269 const __m128i vprod3x89ABCDEF = _mm256_extracti128_si256(vprod3x0123456789ABCDEF, 1);
270 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod3x0123456789ABCDEF)));
271 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod3x89ABCDEF));
272
273 const __m256i vi4x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i4));
274 const __m256i vk4x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 64 * sizeof(int8_t))));
275 i4 += 16;
276
277 const __m256i vprod4x0123456789ABCDEF = _mm256_mullo_epi16(vi4x0123456789ABCDEF, vk4x0123456789ABCDEF);
278 const __m128i vprod4x89ABCDEF = _mm256_extracti128_si256(vprod4x0123456789ABCDEF, 1);
279 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod4x0123456789ABCDEF)));
280 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod4x89ABCDEF));
281
282 const __m256i vi5x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i5));
283 const __m256i vk5x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 80 * sizeof(int8_t))));
284 i5 += 16;
285
286 const __m256i vprod5x0123456789ABCDEF = _mm256_mullo_epi16(vi5x0123456789ABCDEF, vk5x0123456789ABCDEF);
287 const __m128i vprod5x89ABCDEF = _mm256_extracti128_si256(vprod5x0123456789ABCDEF, 1);
288 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod5x0123456789ABCDEF)));
289 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod5x89ABCDEF));
290
291 const __m256i vi6x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i6));
292 const __m256i vk6x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 96 * sizeof(int8_t))));
293 i6 += 16;
294
295 const __m256i vprod6x0123456789ABCDEF = _mm256_mullo_epi16(vi6x0123456789ABCDEF, vk6x0123456789ABCDEF);
296 const __m128i vprod6x89ABCDEF = _mm256_extracti128_si256(vprod6x0123456789ABCDEF, 1);
297 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod6x0123456789ABCDEF)));
298 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod6x89ABCDEF));
299
300 const __m256i vi7x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i7));
301 const __m256i vk7x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 112 * sizeof(int8_t))));
302 i7 += 16;
303
304 const __m256i vprod7x0123456789ABCDEF = _mm256_mullo_epi16(vi7x0123456789ABCDEF, vk7x0123456789ABCDEF);
305 const __m128i vprod7x89ABCDEF = _mm256_extracti128_si256(vprod7x0123456789ABCDEF, 1);
306 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod7x0123456789ABCDEF)));
307 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod7x89ABCDEF));
308
309 w = (const void*) ((uintptr_t) w + 128 * sizeof(int8_t));
310
311 _mm256_storeu_si256((__m256i*) b, vacc01234567);
312 _mm256_storeu_si256((__m256i*) (b + 8), vacc89ABCDEF);
313 b += 16;
314 }
315
316 assert(c == 0);
317 }
318
319
320 // Last pass to process up to 9 inputs.
321 {
322 const int32_t* b = buffer;
323
324 const int8_t* i0 = input[0];
325 assert(i0 != NULL);
326 if XNN_UNPREDICTABLE(i0 != zero) {
327 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
328 }
329 const int8_t* i1 = input[1];
330 assert(i1 != NULL);
331 if XNN_UNPREDICTABLE(i1 != zero) {
332 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
333 }
334 const int8_t* i2 = input[2];
335 assert(i2 != NULL);
336 if XNN_UNPREDICTABLE(i2 != zero) {
337 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
338 }
339 const int8_t* i3 = input[3];
340 assert(i3 != NULL);
341 if XNN_UNPREDICTABLE(i3 != zero) {
342 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
343 }
344 const int8_t* i4 = input[4];
345 assert(i4 != NULL);
346 if XNN_UNPREDICTABLE(i4 != zero) {
347 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
348 }
349 const int8_t* i5 = input[5];
350 assert(i5 != NULL);
351 if XNN_UNPREDICTABLE(i5 != zero) {
352 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
353 }
354 const int8_t* i6 = input[6];
355 assert(i6 != NULL);
356 if XNN_UNPREDICTABLE(i6 != zero) {
357 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
358 }
359 const int8_t* i7 = input[7];
360 assert(i7 != NULL);
361 if XNN_UNPREDICTABLE(i7 != zero) {
362 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
363 }
364 const int8_t* i8 = input[8];
365 assert(i8 != NULL);
366 if XNN_UNPREDICTABLE(i8 != zero) {
367 i8 = (const int8_t*) ((uintptr_t) i8 + input_offset);
368 }
369
370 size_t c = channels;
371
372 for (; c >= 16; c -= 16) {
373 __m256i vacc01234567 = _mm256_loadu_si256((const __m256i*) b);
374 __m256i vacc89ABCDEF = _mm256_loadu_si256((const __m256i*) (b + 8));
375 b += 16;
376
377
378 const __m256i vi0x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i0));
379 const __m256i vk0x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 0 * sizeof(int8_t))));
380 i0 += 16;
381
382 const __m256i vprod0x0123456789ABCDEF = _mm256_mullo_epi16(vi0x0123456789ABCDEF, vk0x0123456789ABCDEF);
383 const __m128i vprod0x89ABCDEF = _mm256_extracti128_si256(vprod0x0123456789ABCDEF, 1);
384 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod0x0123456789ABCDEF)));
385 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod0x89ABCDEF));
386
387 const __m256i vi1x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i1));
388 const __m256i vk1x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t))));
389 i1 += 16;
390
391 const __m256i vprod1x0123456789ABCDEF = _mm256_mullo_epi16(vi1x0123456789ABCDEF, vk1x0123456789ABCDEF);
392 const __m128i vprod1x89ABCDEF = _mm256_extracti128_si256(vprod1x0123456789ABCDEF, 1);
393 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod1x0123456789ABCDEF)));
394 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod1x89ABCDEF));
395
396 const __m256i vi2x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i2));
397 const __m256i vk2x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t))));
398 i2 += 16;
399
400 const __m256i vprod2x0123456789ABCDEF = _mm256_mullo_epi16(vi2x0123456789ABCDEF, vk2x0123456789ABCDEF);
401 const __m128i vprod2x89ABCDEF = _mm256_extracti128_si256(vprod2x0123456789ABCDEF, 1);
402 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod2x0123456789ABCDEF)));
403 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod2x89ABCDEF));
404
405 const __m256i vi3x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i3));
406 const __m256i vk3x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t))));
407 i3 += 16;
408
409 const __m256i vprod3x0123456789ABCDEF = _mm256_mullo_epi16(vi3x0123456789ABCDEF, vk3x0123456789ABCDEF);
410 const __m128i vprod3x89ABCDEF = _mm256_extracti128_si256(vprod3x0123456789ABCDEF, 1);
411 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod3x0123456789ABCDEF)));
412 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod3x89ABCDEF));
413
414 const __m256i vi4x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i4));
415 const __m256i vk4x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 64 * sizeof(int8_t))));
416 i4 += 16;
417
418 const __m256i vprod4x0123456789ABCDEF = _mm256_mullo_epi16(vi4x0123456789ABCDEF, vk4x0123456789ABCDEF);
419 const __m128i vprod4x89ABCDEF = _mm256_extracti128_si256(vprod4x0123456789ABCDEF, 1);
420 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod4x0123456789ABCDEF)));
421 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod4x89ABCDEF));
422
423 const __m256i vi5x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i5));
424 const __m256i vk5x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 80 * sizeof(int8_t))));
425 i5 += 16;
426
427 const __m256i vprod5x0123456789ABCDEF = _mm256_mullo_epi16(vi5x0123456789ABCDEF, vk5x0123456789ABCDEF);
428 const __m128i vprod5x89ABCDEF = _mm256_extracti128_si256(vprod5x0123456789ABCDEF, 1);
429 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod5x0123456789ABCDEF)));
430 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod5x89ABCDEF));
431
432 const __m256i vi6x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i6));
433 const __m256i vk6x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 96 * sizeof(int8_t))));
434 i6 += 16;
435
436 const __m256i vprod6x0123456789ABCDEF = _mm256_mullo_epi16(vi6x0123456789ABCDEF, vk6x0123456789ABCDEF);
437 const __m128i vprod6x89ABCDEF = _mm256_extracti128_si256(vprod6x0123456789ABCDEF, 1);
438 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod6x0123456789ABCDEF)));
439 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod6x89ABCDEF));
440
441 const __m256i vi7x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i7));
442 const __m256i vk7x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 112 * sizeof(int8_t))));
443 i7 += 16;
444
445 const __m256i vprod7x0123456789ABCDEF = _mm256_mullo_epi16(vi7x0123456789ABCDEF, vk7x0123456789ABCDEF);
446 const __m128i vprod7x89ABCDEF = _mm256_extracti128_si256(vprod7x0123456789ABCDEF, 1);
447 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod7x0123456789ABCDEF)));
448 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod7x89ABCDEF));
449
450 const __m256i vi8x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i8));
451 const __m256i vk8x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 128 * sizeof(int8_t))));
452 i8 += 16;
453
454 const __m256i vprod8x0123456789ABCDEF = _mm256_mullo_epi16(vi8x0123456789ABCDEF, vk8x0123456789ABCDEF);
455 const __m128i vprod8x89ABCDEF = _mm256_extracti128_si256(vprod8x0123456789ABCDEF, 1);
456 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod8x0123456789ABCDEF)));
457 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod8x89ABCDEF));
458
459 w = (const void*) ((uintptr_t) w + 144 * sizeof(int8_t));
460
461 __m256 vfpacc01234567 = _mm256_cvtepi32_ps(vacc01234567);
462 __m256 vfpacc89ABCDEF = _mm256_cvtepi32_ps(vacc89ABCDEF);
463
464 vfpacc01234567 = _mm256_mul_ps(vfpacc01234567, vscale);
465 vfpacc89ABCDEF = _mm256_mul_ps(vfpacc89ABCDEF, vscale);
466
467 vfpacc01234567 = _mm256_min_ps(vfpacc01234567, voutput_max_less_zero_point);
468 vfpacc89ABCDEF = _mm256_min_ps(vfpacc89ABCDEF, voutput_max_less_zero_point);
469
470 vacc01234567 = _mm256_cvtps_epi32(vfpacc01234567);
471 vacc89ABCDEF = _mm256_cvtps_epi32(vfpacc89ABCDEF);
472
473 const __m256i vout012389AB4567CDEF = _mm256_adds_epi16(_mm256_packs_epi32(vacc01234567, vacc89ABCDEF), voutput_zero_point);
474
475 __m128i vout0123456789ABCDEF = _mm_shuffle_epi32(_mm_packs_epi16(_mm256_castsi256_si128(vout012389AB4567CDEF), _mm256_extracti128_si256(vout012389AB4567CDEF, 1)), _MM_SHUFFLE(3, 1, 2, 0));
476
477 vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
478
479 _mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
480 output += 16;
481 }
482
483 if XNN_UNLIKELY(c != 0) {
484 {
485 __m256i vacc01234567 = _mm256_loadu_si256((const __m256i*) b);
486 __m256i vacc89ABCDEF = _mm256_loadu_si256((const __m256i*) (b + 8));
487 b += 16;
488
489
490 const __m256i vi0x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i0));
491 const __m256i vk0x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w)));
492
493 const __m256i vprod0x0123456789ABCDEF = _mm256_mullo_epi16(vi0x0123456789ABCDEF, vk0x0123456789ABCDEF);
494 const __m128i vprod0x89ABCDEF = _mm256_extracti128_si256(vprod0x0123456789ABCDEF, 1);
495 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod0x0123456789ABCDEF)));
496 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod0x89ABCDEF));
497
498 const __m256i vi1x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i1));
499 const __m256i vk1x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t))));
500
501 const __m256i vprod1x0123456789ABCDEF = _mm256_mullo_epi16(vi1x0123456789ABCDEF, vk1x0123456789ABCDEF);
502 const __m128i vprod1x89ABCDEF = _mm256_extracti128_si256(vprod1x0123456789ABCDEF, 1);
503 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod1x0123456789ABCDEF)));
504 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod1x89ABCDEF));
505
506 const __m256i vi2x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i2));
507 const __m256i vk2x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t))));
508
509 const __m256i vprod2x0123456789ABCDEF = _mm256_mullo_epi16(vi2x0123456789ABCDEF, vk2x0123456789ABCDEF);
510 const __m128i vprod2x89ABCDEF = _mm256_extracti128_si256(vprod2x0123456789ABCDEF, 1);
511 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod2x0123456789ABCDEF)));
512 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod2x89ABCDEF));
513
514 const __m256i vi3x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i3));
515 const __m256i vk3x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t))));
516
517 const __m256i vprod3x0123456789ABCDEF = _mm256_mullo_epi16(vi3x0123456789ABCDEF, vk3x0123456789ABCDEF);
518 const __m128i vprod3x89ABCDEF = _mm256_extracti128_si256(vprod3x0123456789ABCDEF, 1);
519 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod3x0123456789ABCDEF)));
520 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod3x89ABCDEF));
521
522 const __m256i vi4x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i4));
523 const __m256i vk4x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 64 * sizeof(int8_t))));
524
525 const __m256i vprod4x0123456789ABCDEF = _mm256_mullo_epi16(vi4x0123456789ABCDEF, vk4x0123456789ABCDEF);
526 const __m128i vprod4x89ABCDEF = _mm256_extracti128_si256(vprod4x0123456789ABCDEF, 1);
527 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod4x0123456789ABCDEF)));
528 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod4x89ABCDEF));
529
530 const __m256i vi5x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i5));
531 const __m256i vk5x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 80 * sizeof(int8_t))));
532
533 const __m256i vprod5x0123456789ABCDEF = _mm256_mullo_epi16(vi5x0123456789ABCDEF, vk5x0123456789ABCDEF);
534 const __m128i vprod5x89ABCDEF = _mm256_extracti128_si256(vprod5x0123456789ABCDEF, 1);
535 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod5x0123456789ABCDEF)));
536 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod5x89ABCDEF));
537
538 const __m256i vi6x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i6));
539 const __m256i vk6x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 96 * sizeof(int8_t))));
540
541 const __m256i vprod6x0123456789ABCDEF = _mm256_mullo_epi16(vi6x0123456789ABCDEF, vk6x0123456789ABCDEF);
542 const __m128i vprod6x89ABCDEF = _mm256_extracti128_si256(vprod6x0123456789ABCDEF, 1);
543 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod6x0123456789ABCDEF)));
544 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod6x89ABCDEF));
545
546 const __m256i vi7x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i7));
547 const __m256i vk7x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 112 * sizeof(int8_t))));
548
549 const __m256i vprod7x0123456789ABCDEF = _mm256_mullo_epi16(vi7x0123456789ABCDEF, vk7x0123456789ABCDEF);
550 const __m128i vprod7x89ABCDEF = _mm256_extracti128_si256(vprod7x0123456789ABCDEF, 1);
551 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod7x0123456789ABCDEF)));
552 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod7x89ABCDEF));
553
554 const __m256i vi8x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i8));
555 const __m256i vk8x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 128 * sizeof(int8_t))));
556
557 const __m256i vprod8x0123456789ABCDEF = _mm256_mullo_epi16(vi8x0123456789ABCDEF, vk8x0123456789ABCDEF);
558 const __m128i vprod8x89ABCDEF = _mm256_extracti128_si256(vprod8x0123456789ABCDEF, 1);
559 vacc01234567 = _mm256_add_epi32(vacc01234567, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(vprod8x0123456789ABCDEF)));
560 vacc89ABCDEF = _mm256_add_epi32(vacc89ABCDEF, _mm256_cvtepi16_epi32(vprod8x89ABCDEF));
561
562 __m256 vfpacc01234567 = _mm256_cvtepi32_ps(vacc01234567);
563 __m256 vfpacc89ABCDEF = _mm256_cvtepi32_ps(vacc89ABCDEF);
564
565 vfpacc01234567 = _mm256_mul_ps(vfpacc01234567, vscale);
566 vfpacc89ABCDEF = _mm256_mul_ps(vfpacc89ABCDEF, vscale);
567
568 vfpacc01234567 = _mm256_min_ps(vfpacc01234567, voutput_max_less_zero_point);
569 vfpacc89ABCDEF = _mm256_min_ps(vfpacc89ABCDEF, voutput_max_less_zero_point);
570
571 vacc01234567 = _mm256_cvtps_epi32(vfpacc01234567);
572 vacc89ABCDEF = _mm256_cvtps_epi32(vfpacc89ABCDEF);
573
574 w = (void*) ((uintptr_t) w + 144 * sizeof(int8_t));
575
576 __m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc01234567), _mm256_extracti128_si256(vacc01234567, 1)), _mm256_castsi256_si128(voutput_zero_point));
577 __m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc89ABCDEF), _mm256_extracti128_si256(vacc89ABCDEF, 1)), _mm256_castsi256_si128(voutput_zero_point));
578
579 __m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
580 vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
581
582 if (c & 8) {
583 _mm_storel_epi64((__m128i*) output, vout0123456789ABCDEF);
584 vout0123456789ABCDEF = _mm_unpackhi_epi64(vout0123456789ABCDEF, vout0123456789ABCDEF);
585 output += 8;
586 }
587 if (c & 4) {
588 unaligned_store_u32(output, (uint32_t) _mm_cvtsi128_si32(vout0123456789ABCDEF));
589 vout0123456789ABCDEF = _mm_srli_epi64(vout0123456789ABCDEF, 32);
590 output += 4;
591 }
592 if (c & 2) {
593 unaligned_store_u16(output, (uint16_t) _mm_extract_epi16(vout0123456789ABCDEF, 0));
594 vout0123456789ABCDEF = _mm_srli_epi32(vout0123456789ABCDEF, 16);
595 output += 2;
596 }
597 if (c & 1) {
598 *output = (int8_t) _mm_extract_epi8(vout0123456789ABCDEF, 0);
599 output += 1;
600 }
601 }
602 }
603 }
604
605
606 input = (const int8_t**) ((uintptr_t) input + input_stride);
607 output = (int8_t*) ((uintptr_t) output + output_increment);
608 } while (--output_width != 0);
609 }