Debian Codegraph @

src/qs8-dwconv/gen/qs8-dwconv-6f6m7l16c16s16r-minmax-fp32-avx2-mul16-add16-vpunpck.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-dwconv/multipass-avx2-mul16-vpunpck.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include <immintrin.h>
13
14 #include "xnnpack/dwconv.h"
15 #include "xnnpack/math.h"
16 #include "xnnpack/unaligned.h"
17
18
19 void xnn_qs8_dwconv_minmax_fp32_ukernel_6f6m7l16c16s16r__avx2_mul16_add16_vpunpck(
20 size_t channels,
21 size_t output_width,
22 const int8_t** input,
23 const void* weights,
24 int8_t* output,
25 intptr_t input_stride,
26 size_t output_increment,
27 size_t input_offset,
28 const int8_t* zero,
29 size_t kernel_size,
30 int32_t* buffer,
31 const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
32 {
33 assert(channels != 0);
34 assert(output_width != 0);
35 assert(kernel_size > 6);
36
37 const __m256 vscale = _mm256_set1_ps(params->fp32_scalar.scale);
38 XNN_FORCE_REALIZATION(vscale);
39 const __m256 voutput_max_less_zero_point = _mm256_set1_ps((int32_t) params->fp32_scalar.output_max - (int32_t) params->fp32_scalar.output_zero_point);
40 const __m256i voutput_zero_point = _mm256_set1_epi16(params->fp32_scalar.output_zero_point);
41 const __m128i voutput_min = _mm_set1_epi8(params->fp32_scalar.output_min);
42 XNN_FORCE_REALIZATION(voutput_max_less_zero_point);
43 XNN_FORCE_REALIZATION(voutput_zero_point);
44 XNN_FORCE_REALIZATION(voutput_min);
45
46 do {
47 const void* w = weights;
48
49 // First pass to process 6 inputs.
50 {
51 int32_t* b = buffer;
52
53 const int8_t* i0 = input[0];
54 assert(i0 != NULL);
55 if XNN_UNPREDICTABLE(i0 != zero) {
56 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
57 }
58 const int8_t* i1 = input[1];
59 assert(i1 != NULL);
60 if XNN_UNPREDICTABLE(i1 != zero) {
61 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
62 }
63 const int8_t* i2 = input[2];
64 assert(i2 != NULL);
65 if XNN_UNPREDICTABLE(i2 != zero) {
66 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
67 }
68 const int8_t* i3 = input[3];
69 assert(i3 != NULL);
70 if XNN_UNPREDICTABLE(i3 != zero) {
71 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
72 }
73 const int8_t* i4 = input[4];
74 assert(i4 != NULL);
75 if XNN_UNPREDICTABLE(i4 != zero) {
76 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
77 }
78 const int8_t* i5 = input[5];
79 assert(i5 != NULL);
80 if XNN_UNPREDICTABLE(i5 != zero) {
81 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
82 }
83 input += 6;
84
85 size_t c = round_up_po2(channels, 16);
86
87 for (; c >= 16; c -= 16) {
88 __m256i vacc01234567 = _mm256_loadu_si256((const __m256i*) w);
89 __m256i vacc89ABCDEF = _mm256_loadu_si256((const __m256i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
90
91 __m256i vacc012389AB = _mm256_inserti128_si256(vacc01234567, _mm256_castsi256_si128(vacc89ABCDEF), 1);
92 __m256i vacc4567CDEF = _mm256_permute2x128_si256(vacc01234567, vacc89ABCDEF, 0x31);
93
94
95 const __m256i vi0x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i0));
96 const __m256i vk0x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 0 * sizeof(int8_t))));
97 i0 += 16;
98
99 __m256i vacc0123456789ABCDEF = _mm256_mullo_epi16(vi0x0123456789ABCDEF, vk0x0123456789ABCDEF);
100
101
102 const __m256i vi1x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i1));
103 const __m256i vk1x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 16 * sizeof(int8_t))));
104 i1 += 16;
105
106 vacc0123456789ABCDEF = _mm256_add_epi16(vacc0123456789ABCDEF, _mm256_mullo_epi16(vi1x0123456789ABCDEF, vk1x0123456789ABCDEF));
107
108 __m256i vsignacc0123456789ABCDEF = _mm256_srai_epi16(vacc0123456789ABCDEF, 15);
109 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vacc0123456789ABCDEF, vsignacc0123456789ABCDEF));
110 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vacc0123456789ABCDEF, vsignacc0123456789ABCDEF));
111
112 const __m256i vi2x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i2));
113 const __m256i vk2x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 32 * sizeof(int8_t))));
114 i2 += 16;
115
116 vacc0123456789ABCDEF = _mm256_mullo_epi16(vi2x0123456789ABCDEF, vk2x0123456789ABCDEF);
117
118
119 const __m256i vi3x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i3));
120 const __m256i vk3x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 48 * sizeof(int8_t))));
121 i3 += 16;
122
123 vacc0123456789ABCDEF = _mm256_add_epi16(vacc0123456789ABCDEF, _mm256_mullo_epi16(vi3x0123456789ABCDEF, vk3x0123456789ABCDEF));
124
125 vsignacc0123456789ABCDEF = _mm256_srai_epi16(vacc0123456789ABCDEF, 15);
126 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vacc0123456789ABCDEF, vsignacc0123456789ABCDEF));
127 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vacc0123456789ABCDEF, vsignacc0123456789ABCDEF));
128
129 const __m256i vi4x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i4));
130 const __m256i vk4x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 64 * sizeof(int8_t))));
131 i4 += 16;
132
133 vacc0123456789ABCDEF = _mm256_mullo_epi16(vi4x0123456789ABCDEF, vk4x0123456789ABCDEF);
134
135
136 const __m256i vi5x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i5));
137 const __m256i vk5x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 80 * sizeof(int8_t))));
138 i5 += 16;
139
140 vacc0123456789ABCDEF = _mm256_add_epi16(vacc0123456789ABCDEF, _mm256_mullo_epi16(vi5x0123456789ABCDEF, vk5x0123456789ABCDEF));
141
142 vsignacc0123456789ABCDEF = _mm256_srai_epi16(vacc0123456789ABCDEF, 15);
143 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vacc0123456789ABCDEF, vsignacc0123456789ABCDEF));
144 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vacc0123456789ABCDEF, vsignacc0123456789ABCDEF));
145
146 w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 96 * sizeof(int8_t));
147
148 vacc01234567 = _mm256_inserti128_si256(vacc012389AB, _mm256_castsi256_si128(vacc4567CDEF), 1);
149 vacc89ABCDEF = _mm256_permute2x128_si256(vacc012389AB, vacc4567CDEF, 0x31);
150
151 _mm256_storeu_si256((__m256i*) b, vacc01234567);
152 _mm256_storeu_si256((__m256i*) (b + 8), vacc89ABCDEF);
153 b += 16;
154 }
155
156 assert(c == 0);
157 }
158
159 // Middle pass to process 6 inputs in each iteration.
160 for (size_t ks = kernel_size - 6; ks > 7; ks -= 6) {
161 int32_t* b = buffer;
162
163 const int8_t* i0 = input[0];
164 assert(i0 != NULL);
165 if XNN_UNPREDICTABLE(i0 != zero) {
166 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
167 }
168 const int8_t* i1 = input[1];
169 assert(i1 != NULL);
170 if XNN_UNPREDICTABLE(i1 != zero) {
171 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
172 }
173 const int8_t* i2 = input[2];
174 assert(i2 != NULL);
175 if XNN_UNPREDICTABLE(i2 != zero) {
176 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
177 }
178 const int8_t* i3 = input[3];
179 assert(i3 != NULL);
180 if XNN_UNPREDICTABLE(i3 != zero) {
181 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
182 }
183 const int8_t* i4 = input[4];
184 assert(i4 != NULL);
185 if XNN_UNPREDICTABLE(i4 != zero) {
186 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
187 }
188 const int8_t* i5 = input[5];
189 assert(i5 != NULL);
190 if XNN_UNPREDICTABLE(i5 != zero) {
191 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
192 }
193 input += 6;
194
195 size_t c = round_up_po2(channels, 16);
196
197 for (; c >= 16; c -= 16) {
198 __m256i vacc01234567 = _mm256_loadu_si256((const __m256i*) b);
199 __m256i vacc89ABCDEF = _mm256_loadu_si256((const __m256i*) (b + 8));
200
201 __m256i vacc012389AB = _mm256_inserti128_si256(vacc01234567, _mm256_castsi256_si128(vacc89ABCDEF), 1);
202 __m256i vacc4567CDEF = _mm256_permute2x128_si256(vacc01234567, vacc89ABCDEF, 0x31);
203
204
205 const __m256i vi0x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i0));
206 const __m256i vk0x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 0 * sizeof(int8_t))));
207 i0 += 16;
208
209 __m256i vacc0123456789ABCDEF = _mm256_mullo_epi16(vi0x0123456789ABCDEF, vk0x0123456789ABCDEF);
210
211
212 const __m256i vi1x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i1));
213 const __m256i vk1x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t))));
214 i1 += 16;
215
216 vacc0123456789ABCDEF = _mm256_add_epi16(vacc0123456789ABCDEF, _mm256_mullo_epi16(vi1x0123456789ABCDEF, vk1x0123456789ABCDEF));
217
218 __m256i vsignacc0123456789ABCDEF = _mm256_srai_epi16(vacc0123456789ABCDEF, 15);
219 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vacc0123456789ABCDEF, vsignacc0123456789ABCDEF));
220 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vacc0123456789ABCDEF, vsignacc0123456789ABCDEF));
221
222 const __m256i vi2x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i2));
223 const __m256i vk2x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t))));
224 i2 += 16;
225
226 vacc0123456789ABCDEF = _mm256_mullo_epi16(vi2x0123456789ABCDEF, vk2x0123456789ABCDEF);
227
228
229 const __m256i vi3x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i3));
230 const __m256i vk3x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t))));
231 i3 += 16;
232
233 vacc0123456789ABCDEF = _mm256_add_epi16(vacc0123456789ABCDEF, _mm256_mullo_epi16(vi3x0123456789ABCDEF, vk3x0123456789ABCDEF));
234
235 vsignacc0123456789ABCDEF = _mm256_srai_epi16(vacc0123456789ABCDEF, 15);
236 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vacc0123456789ABCDEF, vsignacc0123456789ABCDEF));
237 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vacc0123456789ABCDEF, vsignacc0123456789ABCDEF));
238
239 const __m256i vi4x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i4));
240 const __m256i vk4x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 64 * sizeof(int8_t))));
241 i4 += 16;
242
243 vacc0123456789ABCDEF = _mm256_mullo_epi16(vi4x0123456789ABCDEF, vk4x0123456789ABCDEF);
244
245
246 const __m256i vi5x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i5));
247 const __m256i vk5x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 80 * sizeof(int8_t))));
248 i5 += 16;
249
250 vacc0123456789ABCDEF = _mm256_add_epi16(vacc0123456789ABCDEF, _mm256_mullo_epi16(vi5x0123456789ABCDEF, vk5x0123456789ABCDEF));
251
252 vsignacc0123456789ABCDEF = _mm256_srai_epi16(vacc0123456789ABCDEF, 15);
253 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vacc0123456789ABCDEF, vsignacc0123456789ABCDEF));
254 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vacc0123456789ABCDEF, vsignacc0123456789ABCDEF));
255
256 w = (const void*) ((uintptr_t) w + 96 * sizeof(int8_t));
257
258 vacc01234567 = _mm256_inserti128_si256(vacc012389AB, _mm256_castsi256_si128(vacc4567CDEF), 1);
259 vacc89ABCDEF = _mm256_permute2x128_si256(vacc012389AB, vacc4567CDEF, 0x31);
260
261 _mm256_storeu_si256((__m256i*) b, vacc01234567);
262 _mm256_storeu_si256((__m256i*) (b + 8), vacc89ABCDEF);
263 b += 16;
264 }
265
266 assert(c == 0);
267 }
268
269
270 // Last pass to process up to 7 inputs.
271 {
272 const int32_t* b = buffer;
273
274 const int8_t* i0 = input[0];
275 assert(i0 != NULL);
276 if XNN_UNPREDICTABLE(i0 != zero) {
277 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
278 }
279 const int8_t* i1 = input[1];
280 assert(i1 != NULL);
281 if XNN_UNPREDICTABLE(i1 != zero) {
282 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
283 }
284 const int8_t* i2 = input[2];
285 assert(i2 != NULL);
286 if XNN_UNPREDICTABLE(i2 != zero) {
287 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
288 }
289 const int8_t* i3 = input[3];
290 assert(i3 != NULL);
291 if XNN_UNPREDICTABLE(i3 != zero) {
292 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
293 }
294 const int8_t* i4 = input[4];
295 assert(i4 != NULL);
296 if XNN_UNPREDICTABLE(i4 != zero) {
297 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
298 }
299 const int8_t* i5 = input[5];
300 assert(i5 != NULL);
301 if XNN_UNPREDICTABLE(i5 != zero) {
302 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
303 }
304 const int8_t* i6 = input[6];
305 assert(i6 != NULL);
306 if XNN_UNPREDICTABLE(i6 != zero) {
307 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
308 }
309
310 size_t c = channels;
311
312 for (; c >= 16; c -= 16) {
313 __m256i vacc01234567 = _mm256_loadu_si256((const __m256i*) b);
314 __m256i vacc89ABCDEF = _mm256_loadu_si256((const __m256i*) (b + 8));
315 b += 16;
316
317 __m256i vacc012389AB = _mm256_inserti128_si256(vacc01234567, _mm256_castsi256_si128(vacc89ABCDEF), 1);
318 __m256i vacc4567CDEF = _mm256_permute2x128_si256(vacc01234567, vacc89ABCDEF, 0x31);
319
320
321 const __m256i vi0x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i0));
322 const __m256i vk0x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 0 * sizeof(int8_t))));
323 i0 += 16;
324
325 __m256i vacc0123456789ABCDEF = _mm256_mullo_epi16(vi0x0123456789ABCDEF, vk0x0123456789ABCDEF);
326
327
328 const __m256i vi1x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i1));
329 const __m256i vk1x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t))));
330 i1 += 16;
331
332 vacc0123456789ABCDEF = _mm256_add_epi16(vacc0123456789ABCDEF, _mm256_mullo_epi16(vi1x0123456789ABCDEF, vk1x0123456789ABCDEF));
333
334 __m256i vsignacc0123456789ABCDEF = _mm256_srai_epi16(vacc0123456789ABCDEF, 15);
335 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vacc0123456789ABCDEF, vsignacc0123456789ABCDEF));
336 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vacc0123456789ABCDEF, vsignacc0123456789ABCDEF));
337
338 const __m256i vi2x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i2));
339 const __m256i vk2x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t))));
340 i2 += 16;
341
342 vacc0123456789ABCDEF = _mm256_mullo_epi16(vi2x0123456789ABCDEF, vk2x0123456789ABCDEF);
343
344
345 const __m256i vi3x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i3));
346 const __m256i vk3x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t))));
347 i3 += 16;
348
349 vacc0123456789ABCDEF = _mm256_add_epi16(vacc0123456789ABCDEF, _mm256_mullo_epi16(vi3x0123456789ABCDEF, vk3x0123456789ABCDEF));
350
351 vsignacc0123456789ABCDEF = _mm256_srai_epi16(vacc0123456789ABCDEF, 15);
352 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vacc0123456789ABCDEF, vsignacc0123456789ABCDEF));
353 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vacc0123456789ABCDEF, vsignacc0123456789ABCDEF));
354
355 const __m256i vi4x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i4));
356 const __m256i vk4x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 64 * sizeof(int8_t))));
357 i4 += 16;
358
359 vacc0123456789ABCDEF = _mm256_mullo_epi16(vi4x0123456789ABCDEF, vk4x0123456789ABCDEF);
360
361
362 const __m256i vi5x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i5));
363 const __m256i vk5x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 80 * sizeof(int8_t))));
364 i5 += 16;
365
366 vacc0123456789ABCDEF = _mm256_add_epi16(vacc0123456789ABCDEF, _mm256_mullo_epi16(vi5x0123456789ABCDEF, vk5x0123456789ABCDEF));
367
368 vsignacc0123456789ABCDEF = _mm256_srai_epi16(vacc0123456789ABCDEF, 15);
369 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vacc0123456789ABCDEF, vsignacc0123456789ABCDEF));
370 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vacc0123456789ABCDEF, vsignacc0123456789ABCDEF));
371
372 const __m256i vi6x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i6));
373 const __m256i vk6x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 96 * sizeof(int8_t))));
374 i6 += 16;
375
376 vacc0123456789ABCDEF = _mm256_mullo_epi16(vi6x0123456789ABCDEF, vk6x0123456789ABCDEF);
377
378 vsignacc0123456789ABCDEF = _mm256_srai_epi16(vacc0123456789ABCDEF, 15);
379 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vacc0123456789ABCDEF, vsignacc0123456789ABCDEF));
380 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vacc0123456789ABCDEF, vsignacc0123456789ABCDEF));
381
382 vacc01234567 = _mm256_inserti128_si256(vacc012389AB, _mm256_castsi256_si128(vacc4567CDEF), 1);
383 vacc89ABCDEF = _mm256_permute2x128_si256(vacc012389AB, vacc4567CDEF, 0x31);
384
385 w = (const void*) ((uintptr_t) w + 112 * sizeof(int8_t));
386
387 __m256 vfpacc01234567 = _mm256_cvtepi32_ps(vacc01234567);
388 __m256 vfpacc89ABCDEF = _mm256_cvtepi32_ps(vacc89ABCDEF);
389
390 vfpacc01234567 = _mm256_mul_ps(vfpacc01234567, vscale);
391 vfpacc89ABCDEF = _mm256_mul_ps(vfpacc89ABCDEF, vscale);
392
393 vfpacc01234567 = _mm256_min_ps(vfpacc01234567, voutput_max_less_zero_point);
394 vfpacc89ABCDEF = _mm256_min_ps(vfpacc89ABCDEF, voutput_max_less_zero_point);
395
396 vacc01234567 = _mm256_cvtps_epi32(vfpacc01234567);
397 vacc89ABCDEF = _mm256_cvtps_epi32(vfpacc89ABCDEF);
398
399 const __m256i vout012389AB4567CDEF = _mm256_adds_epi16(_mm256_packs_epi32(vacc01234567, vacc89ABCDEF), voutput_zero_point);
400
401 __m128i vout0123456789ABCDEF = _mm_shuffle_epi32(_mm_packs_epi16(_mm256_castsi256_si128(vout012389AB4567CDEF), _mm256_extracti128_si256(vout012389AB4567CDEF, 1)), _MM_SHUFFLE(3, 1, 2, 0));
402
403 vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
404
405 _mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
406 output += 16;
407 }
408
409 if XNN_UNLIKELY(c != 0) {
410 {
411 __m256i vacc01234567 = _mm256_loadu_si256((const __m256i*) b);
412 __m256i vacc89ABCDEF = _mm256_loadu_si256((const __m256i*) (b + 8));
413 b += 16;
414
415 __m256i vacc012389AB = _mm256_inserti128_si256(vacc01234567, _mm256_castsi256_si128(vacc89ABCDEF), 1);
416 __m256i vacc4567CDEF = _mm256_permute2x128_si256(vacc01234567, vacc89ABCDEF, 0x31);
417
418
419 const __m256i vi0x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i0));
420 const __m256i vk0x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w)));
421
422 const __m256i vprod0x0123456789ABCDEFlo = _mm256_mullo_epi16(vi0x0123456789ABCDEF, vk0x0123456789ABCDEF);
423 const __m256i vprod0x0123456789ABCDEFhi = _mm256_srai_epi16(vprod0x0123456789ABCDEFlo, 15);
424
425 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vprod0x0123456789ABCDEFlo, vprod0x0123456789ABCDEFhi));
426 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vprod0x0123456789ABCDEFlo, vprod0x0123456789ABCDEFhi));
427
428 const __m256i vi1x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i1));
429 const __m256i vk1x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t))));
430
431 const __m256i vprod1x0123456789ABCDEFlo = _mm256_mullo_epi16(vi1x0123456789ABCDEF, vk1x0123456789ABCDEF);
432 const __m256i vprod1x0123456789ABCDEFhi = _mm256_srai_epi16(vprod1x0123456789ABCDEFlo, 15);
433
434 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vprod1x0123456789ABCDEFlo, vprod1x0123456789ABCDEFhi));
435 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vprod1x0123456789ABCDEFlo, vprod1x0123456789ABCDEFhi));
436
437 const __m256i vi2x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i2));
438 const __m256i vk2x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t))));
439
440 const __m256i vprod2x0123456789ABCDEFlo = _mm256_mullo_epi16(vi2x0123456789ABCDEF, vk2x0123456789ABCDEF);
441 const __m256i vprod2x0123456789ABCDEFhi = _mm256_srai_epi16(vprod2x0123456789ABCDEFlo, 15);
442
443 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vprod2x0123456789ABCDEFlo, vprod2x0123456789ABCDEFhi));
444 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vprod2x0123456789ABCDEFlo, vprod2x0123456789ABCDEFhi));
445
446 const __m256i vi3x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i3));
447 const __m256i vk3x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t))));
448
449 const __m256i vprod3x0123456789ABCDEFlo = _mm256_mullo_epi16(vi3x0123456789ABCDEF, vk3x0123456789ABCDEF);
450 const __m256i vprod3x0123456789ABCDEFhi = _mm256_srai_epi16(vprod3x0123456789ABCDEFlo, 15);
451
452 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vprod3x0123456789ABCDEFlo, vprod3x0123456789ABCDEFhi));
453 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vprod3x0123456789ABCDEFlo, vprod3x0123456789ABCDEFhi));
454
455 const __m256i vi4x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i4));
456 const __m256i vk4x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 64 * sizeof(int8_t))));
457
458 const __m256i vprod4x0123456789ABCDEFlo = _mm256_mullo_epi16(vi4x0123456789ABCDEF, vk4x0123456789ABCDEF);
459 const __m256i vprod4x0123456789ABCDEFhi = _mm256_srai_epi16(vprod4x0123456789ABCDEFlo, 15);
460
461 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vprod4x0123456789ABCDEFlo, vprod4x0123456789ABCDEFhi));
462 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vprod4x0123456789ABCDEFlo, vprod4x0123456789ABCDEFhi));
463
464 const __m256i vi5x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i5));
465 const __m256i vk5x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 80 * sizeof(int8_t))));
466
467 const __m256i vprod5x0123456789ABCDEFlo = _mm256_mullo_epi16(vi5x0123456789ABCDEF, vk5x0123456789ABCDEF);
468 const __m256i vprod5x0123456789ABCDEFhi = _mm256_srai_epi16(vprod5x0123456789ABCDEFlo, 15);
469
470 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vprod5x0123456789ABCDEFlo, vprod5x0123456789ABCDEFhi));
471 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vprod5x0123456789ABCDEFlo, vprod5x0123456789ABCDEFhi));
472
473 const __m256i vi6x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i6));
474 const __m256i vk6x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 96 * sizeof(int8_t))));
475
476 const __m256i vprod6x0123456789ABCDEFlo = _mm256_mullo_epi16(vi6x0123456789ABCDEF, vk6x0123456789ABCDEF);
477 const __m256i vprod6x0123456789ABCDEFhi = _mm256_srai_epi16(vprod6x0123456789ABCDEFlo, 15);
478
479 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vprod6x0123456789ABCDEFlo, vprod6x0123456789ABCDEFhi));
480 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vprod6x0123456789ABCDEFlo, vprod6x0123456789ABCDEFhi));
481
482 vacc01234567 = _mm256_inserti128_si256(vacc012389AB, _mm256_castsi256_si128(vacc4567CDEF), 1);
483 vacc89ABCDEF = _mm256_permute2x128_si256(vacc012389AB, vacc4567CDEF, 0x31);
484
485 __m256 vfpacc01234567 = _mm256_cvtepi32_ps(vacc01234567);
486 __m256 vfpacc89ABCDEF = _mm256_cvtepi32_ps(vacc89ABCDEF);
487
488 vfpacc01234567 = _mm256_mul_ps(vfpacc01234567, vscale);
489 vfpacc89ABCDEF = _mm256_mul_ps(vfpacc89ABCDEF, vscale);
490
491 vfpacc01234567 = _mm256_min_ps(vfpacc01234567, voutput_max_less_zero_point);
492 vfpacc89ABCDEF = _mm256_min_ps(vfpacc89ABCDEF, voutput_max_less_zero_point);
493
494 vacc01234567 = _mm256_cvtps_epi32(vfpacc01234567);
495 vacc89ABCDEF = _mm256_cvtps_epi32(vfpacc89ABCDEF);
496
497 w = (void*) ((uintptr_t) w + 112 * sizeof(int8_t));
498
499 __m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc01234567), _mm256_extracti128_si256(vacc01234567, 1)), _mm256_castsi256_si128(voutput_zero_point));
500 __m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc89ABCDEF), _mm256_extracti128_si256(vacc89ABCDEF, 1)), _mm256_castsi256_si128(voutput_zero_point));
501
502 __m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
503 vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
504
505 if (c & 8) {
506 _mm_storel_epi64((__m128i*) output, vout0123456789ABCDEF);
507 vout0123456789ABCDEF = _mm_unpackhi_epi64(vout0123456789ABCDEF, vout0123456789ABCDEF);
508 output += 8;
509 }
510 if (c & 4) {
511 unaligned_store_u32(output, (uint32_t) _mm_cvtsi128_si32(vout0123456789ABCDEF));
512 vout0123456789ABCDEF = _mm_srli_epi64(vout0123456789ABCDEF, 32);
513 output += 4;
514 }
515 if (c & 2) {
516 unaligned_store_u16(output, (uint16_t) _mm_extract_epi16(vout0123456789ABCDEF, 0));
517 vout0123456789ABCDEF = _mm_srli_epi32(vout0123456789ABCDEF, 16);
518 output += 2;
519 }
520 if (c & 1) {
521 *output = (int8_t) _mm_extract_epi8(vout0123456789ABCDEF, 0);
522 output += 1;
523 }
524 }
525 }
526 }
527
528
529 input = (const int8_t**) ((uintptr_t) input + input_stride);
530 output = (int8_t*) ((uintptr_t) output + output_increment);
531 } while (--output_width != 0);
532 }