Debian Codegraph @

src/qs8-dwconv/gen/qs8-dwconv-8f8m9l32c16s16r-minmax-fp32-avx2-mul16-vpunpck.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-dwconv/multipass-avx2-mul16-vpunpck.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include <immintrin.h>
13
14 #include "xnnpack/dwconv.h"
15 #include "xnnpack/math.h"
16 #include "xnnpack/unaligned.h"
17
18
19 void xnn_qs8_dwconv_minmax_fp32_ukernel_8f8m9l32c16s16r__avx2_mul16_vpunpck(
20 size_t channels,
21 size_t output_width,
22 const int8_t** input,
23 const void* weights,
24 int8_t* output,
25 intptr_t input_stride,
26 size_t output_increment,
27 size_t input_offset,
28 const int8_t* zero,
29 size_t kernel_size,
30 int32_t* buffer,
31 const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
32 {
33 assert(channels != 0);
34 assert(output_width != 0);
35 assert(kernel_size > 8);
36
37 const __m256 vscale = _mm256_set1_ps(params->fp32_scalar.scale);
38 XNN_FORCE_REALIZATION(vscale);
39 const __m256 voutput_max_less_zero_point = _mm256_set1_ps((int32_t) params->fp32_scalar.output_max - (int32_t) params->fp32_scalar.output_zero_point);
40 const __m256i voutput_zero_point = _mm256_set1_epi16(params->fp32_scalar.output_zero_point);
41 const __m128i voutput_min = _mm_set1_epi8(params->fp32_scalar.output_min);
42 XNN_FORCE_REALIZATION(voutput_max_less_zero_point);
43 XNN_FORCE_REALIZATION(voutput_zero_point);
44 XNN_FORCE_REALIZATION(voutput_min);
45
46 do {
47 const void* w = weights;
48
49 // First pass to process 8 inputs.
50 {
51 int32_t* b = buffer;
52
53 const int8_t* i0 = input[0];
54 assert(i0 != NULL);
55 if XNN_UNPREDICTABLE(i0 != zero) {
56 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
57 }
58 const int8_t* i1 = input[1];
59 assert(i1 != NULL);
60 if XNN_UNPREDICTABLE(i1 != zero) {
61 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
62 }
63 const int8_t* i2 = input[2];
64 assert(i2 != NULL);
65 if XNN_UNPREDICTABLE(i2 != zero) {
66 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
67 }
68 const int8_t* i3 = input[3];
69 assert(i3 != NULL);
70 if XNN_UNPREDICTABLE(i3 != zero) {
71 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
72 }
73 const int8_t* i4 = input[4];
74 assert(i4 != NULL);
75 if XNN_UNPREDICTABLE(i4 != zero) {
76 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
77 }
78 const int8_t* i5 = input[5];
79 assert(i5 != NULL);
80 if XNN_UNPREDICTABLE(i5 != zero) {
81 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
82 }
83 const int8_t* i6 = input[6];
84 assert(i6 != NULL);
85 if XNN_UNPREDICTABLE(i6 != zero) {
86 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
87 }
88 const int8_t* i7 = input[7];
89 assert(i7 != NULL);
90 if XNN_UNPREDICTABLE(i7 != zero) {
91 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
92 }
93 input += 8;
94
95 size_t c = round_up_po2(channels, 16);
96
97 for (; c >= 32; c -= 32) {
98 __m256i vacc01234567 = _mm256_loadu_si256((const __m256i*) w);
99 __m256i vacc89ABCDEF = _mm256_loadu_si256((const __m256i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
100 __m256i vaccGHIJKLMN = _mm256_loadu_si256((const __m256i*) ((uintptr_t) w + 16 * sizeof(int32_t)));
101 __m256i vaccOPQRSTUV = _mm256_loadu_si256((const __m256i*) ((uintptr_t) w + 24 * sizeof(int32_t)));
102
103 __m256i vacc012389AB = _mm256_inserti128_si256(vacc01234567, _mm256_castsi256_si128(vacc89ABCDEF), 1);
104 __m256i vacc4567CDEF = _mm256_permute2x128_si256(vacc01234567, vacc89ABCDEF, 0x31);
105 __m256i vaccGHIJOPQR = _mm256_inserti128_si256(vaccGHIJKLMN, _mm256_castsi256_si128(vaccOPQRSTUV), 1);
106 __m256i vaccKLMNSTUV = _mm256_permute2x128_si256(vaccGHIJKLMN, vaccOPQRSTUV, 0x31);
107
108
109 const __m256i vi0x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i0));
110 const __m256i vk0x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 0 * sizeof(int8_t))));
111 const __m256i vi0xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i0 + 16)));
112 const __m256i vk0xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 16 * sizeof(int8_t))));
113 i0 += 32;
114
115 const __m256i vprod0x0123456789ABCDEFlo = _mm256_mullo_epi16(vi0x0123456789ABCDEF, vk0x0123456789ABCDEF);
116 const __m256i vprod0x0123456789ABCDEFhi = _mm256_srai_epi16(vprod0x0123456789ABCDEFlo, 15);
117 const __m256i vprod0xGHIJKLMNOPQRSTUVlo = _mm256_mullo_epi16(vi0xGHIJKLMNOPQRSTUV, vk0xGHIJKLMNOPQRSTUV);
118 const __m256i vprod0xGHIJKLMNOPQRSTUVhi = _mm256_srai_epi16(vprod0xGHIJKLMNOPQRSTUVlo, 15);
119
120 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vprod0x0123456789ABCDEFlo, vprod0x0123456789ABCDEFhi));
121 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vprod0x0123456789ABCDEFlo, vprod0x0123456789ABCDEFhi));
122 vaccGHIJOPQR = _mm256_add_epi32(vaccGHIJOPQR, _mm256_unpacklo_epi16(vprod0xGHIJKLMNOPQRSTUVlo, vprod0xGHIJKLMNOPQRSTUVhi));
123 vaccKLMNSTUV = _mm256_add_epi32(vaccKLMNSTUV, _mm256_unpackhi_epi16(vprod0xGHIJKLMNOPQRSTUVlo, vprod0xGHIJKLMNOPQRSTUVhi));
124
125 const __m256i vi1x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i1));
126 const __m256i vk1x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 32 * sizeof(int8_t))));
127 const __m256i vi1xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i1 + 16)));
128 const __m256i vk1xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 48 * sizeof(int8_t))));
129 i1 += 32;
130
131 const __m256i vprod1x0123456789ABCDEFlo = _mm256_mullo_epi16(vi1x0123456789ABCDEF, vk1x0123456789ABCDEF);
132 const __m256i vprod1x0123456789ABCDEFhi = _mm256_srai_epi16(vprod1x0123456789ABCDEFlo, 15);
133 const __m256i vprod1xGHIJKLMNOPQRSTUVlo = _mm256_mullo_epi16(vi1xGHIJKLMNOPQRSTUV, vk1xGHIJKLMNOPQRSTUV);
134 const __m256i vprod1xGHIJKLMNOPQRSTUVhi = _mm256_srai_epi16(vprod1xGHIJKLMNOPQRSTUVlo, 15);
135
136 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vprod1x0123456789ABCDEFlo, vprod1x0123456789ABCDEFhi));
137 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vprod1x0123456789ABCDEFlo, vprod1x0123456789ABCDEFhi));
138 vaccGHIJOPQR = _mm256_add_epi32(vaccGHIJOPQR, _mm256_unpacklo_epi16(vprod1xGHIJKLMNOPQRSTUVlo, vprod1xGHIJKLMNOPQRSTUVhi));
139 vaccKLMNSTUV = _mm256_add_epi32(vaccKLMNSTUV, _mm256_unpackhi_epi16(vprod1xGHIJKLMNOPQRSTUVlo, vprod1xGHIJKLMNOPQRSTUVhi));
140
141 const __m256i vi2x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i2));
142 const __m256i vk2x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 64 * sizeof(int8_t))));
143 const __m256i vi2xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i2 + 16)));
144 const __m256i vk2xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 80 * sizeof(int8_t))));
145 i2 += 32;
146
147 const __m256i vprod2x0123456789ABCDEFlo = _mm256_mullo_epi16(vi2x0123456789ABCDEF, vk2x0123456789ABCDEF);
148 const __m256i vprod2x0123456789ABCDEFhi = _mm256_srai_epi16(vprod2x0123456789ABCDEFlo, 15);
149 const __m256i vprod2xGHIJKLMNOPQRSTUVlo = _mm256_mullo_epi16(vi2xGHIJKLMNOPQRSTUV, vk2xGHIJKLMNOPQRSTUV);
150 const __m256i vprod2xGHIJKLMNOPQRSTUVhi = _mm256_srai_epi16(vprod2xGHIJKLMNOPQRSTUVlo, 15);
151
152 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vprod2x0123456789ABCDEFlo, vprod2x0123456789ABCDEFhi));
153 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vprod2x0123456789ABCDEFlo, vprod2x0123456789ABCDEFhi));
154 vaccGHIJOPQR = _mm256_add_epi32(vaccGHIJOPQR, _mm256_unpacklo_epi16(vprod2xGHIJKLMNOPQRSTUVlo, vprod2xGHIJKLMNOPQRSTUVhi));
155 vaccKLMNSTUV = _mm256_add_epi32(vaccKLMNSTUV, _mm256_unpackhi_epi16(vprod2xGHIJKLMNOPQRSTUVlo, vprod2xGHIJKLMNOPQRSTUVhi));
156
157 const __m256i vi3x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i3));
158 const __m256i vk3x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 96 * sizeof(int8_t))));
159 const __m256i vi3xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i3 + 16)));
160 const __m256i vk3xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 112 * sizeof(int8_t))));
161 i3 += 32;
162
163 const __m256i vprod3x0123456789ABCDEFlo = _mm256_mullo_epi16(vi3x0123456789ABCDEF, vk3x0123456789ABCDEF);
164 const __m256i vprod3x0123456789ABCDEFhi = _mm256_srai_epi16(vprod3x0123456789ABCDEFlo, 15);
165 const __m256i vprod3xGHIJKLMNOPQRSTUVlo = _mm256_mullo_epi16(vi3xGHIJKLMNOPQRSTUV, vk3xGHIJKLMNOPQRSTUV);
166 const __m256i vprod3xGHIJKLMNOPQRSTUVhi = _mm256_srai_epi16(vprod3xGHIJKLMNOPQRSTUVlo, 15);
167
168 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vprod3x0123456789ABCDEFlo, vprod3x0123456789ABCDEFhi));
169 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vprod3x0123456789ABCDEFlo, vprod3x0123456789ABCDEFhi));
170 vaccGHIJOPQR = _mm256_add_epi32(vaccGHIJOPQR, _mm256_unpacklo_epi16(vprod3xGHIJKLMNOPQRSTUVlo, vprod3xGHIJKLMNOPQRSTUVhi));
171 vaccKLMNSTUV = _mm256_add_epi32(vaccKLMNSTUV, _mm256_unpackhi_epi16(vprod3xGHIJKLMNOPQRSTUVlo, vprod3xGHIJKLMNOPQRSTUVhi));
172
173 const __m256i vi4x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i4));
174 const __m256i vk4x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 128 * sizeof(int8_t))));
175 const __m256i vi4xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i4 + 16)));
176 const __m256i vk4xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 144 * sizeof(int8_t))));
177 i4 += 32;
178
179 const __m256i vprod4x0123456789ABCDEFlo = _mm256_mullo_epi16(vi4x0123456789ABCDEF, vk4x0123456789ABCDEF);
180 const __m256i vprod4x0123456789ABCDEFhi = _mm256_srai_epi16(vprod4x0123456789ABCDEFlo, 15);
181 const __m256i vprod4xGHIJKLMNOPQRSTUVlo = _mm256_mullo_epi16(vi4xGHIJKLMNOPQRSTUV, vk4xGHIJKLMNOPQRSTUV);
182 const __m256i vprod4xGHIJKLMNOPQRSTUVhi = _mm256_srai_epi16(vprod4xGHIJKLMNOPQRSTUVlo, 15);
183
184 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vprod4x0123456789ABCDEFlo, vprod4x0123456789ABCDEFhi));
185 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vprod4x0123456789ABCDEFlo, vprod4x0123456789ABCDEFhi));
186 vaccGHIJOPQR = _mm256_add_epi32(vaccGHIJOPQR, _mm256_unpacklo_epi16(vprod4xGHIJKLMNOPQRSTUVlo, vprod4xGHIJKLMNOPQRSTUVhi));
187 vaccKLMNSTUV = _mm256_add_epi32(vaccKLMNSTUV, _mm256_unpackhi_epi16(vprod4xGHIJKLMNOPQRSTUVlo, vprod4xGHIJKLMNOPQRSTUVhi));
188
189 const __m256i vi5x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i5));
190 const __m256i vk5x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 160 * sizeof(int8_t))));
191 const __m256i vi5xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i5 + 16)));
192 const __m256i vk5xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 176 * sizeof(int8_t))));
193 i5 += 32;
194
195 const __m256i vprod5x0123456789ABCDEFlo = _mm256_mullo_epi16(vi5x0123456789ABCDEF, vk5x0123456789ABCDEF);
196 const __m256i vprod5x0123456789ABCDEFhi = _mm256_srai_epi16(vprod5x0123456789ABCDEFlo, 15);
197 const __m256i vprod5xGHIJKLMNOPQRSTUVlo = _mm256_mullo_epi16(vi5xGHIJKLMNOPQRSTUV, vk5xGHIJKLMNOPQRSTUV);
198 const __m256i vprod5xGHIJKLMNOPQRSTUVhi = _mm256_srai_epi16(vprod5xGHIJKLMNOPQRSTUVlo, 15);
199
200 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vprod5x0123456789ABCDEFlo, vprod5x0123456789ABCDEFhi));
201 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vprod5x0123456789ABCDEFlo, vprod5x0123456789ABCDEFhi));
202 vaccGHIJOPQR = _mm256_add_epi32(vaccGHIJOPQR, _mm256_unpacklo_epi16(vprod5xGHIJKLMNOPQRSTUVlo, vprod5xGHIJKLMNOPQRSTUVhi));
203 vaccKLMNSTUV = _mm256_add_epi32(vaccKLMNSTUV, _mm256_unpackhi_epi16(vprod5xGHIJKLMNOPQRSTUVlo, vprod5xGHIJKLMNOPQRSTUVhi));
204
205 const __m256i vi6x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i6));
206 const __m256i vk6x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 192 * sizeof(int8_t))));
207 const __m256i vi6xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i6 + 16)));
208 const __m256i vk6xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 208 * sizeof(int8_t))));
209 i6 += 32;
210
211 const __m256i vprod6x0123456789ABCDEFlo = _mm256_mullo_epi16(vi6x0123456789ABCDEF, vk6x0123456789ABCDEF);
212 const __m256i vprod6x0123456789ABCDEFhi = _mm256_srai_epi16(vprod6x0123456789ABCDEFlo, 15);
213 const __m256i vprod6xGHIJKLMNOPQRSTUVlo = _mm256_mullo_epi16(vi6xGHIJKLMNOPQRSTUV, vk6xGHIJKLMNOPQRSTUV);
214 const __m256i vprod6xGHIJKLMNOPQRSTUVhi = _mm256_srai_epi16(vprod6xGHIJKLMNOPQRSTUVlo, 15);
215
216 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vprod6x0123456789ABCDEFlo, vprod6x0123456789ABCDEFhi));
217 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vprod6x0123456789ABCDEFlo, vprod6x0123456789ABCDEFhi));
218 vaccGHIJOPQR = _mm256_add_epi32(vaccGHIJOPQR, _mm256_unpacklo_epi16(vprod6xGHIJKLMNOPQRSTUVlo, vprod6xGHIJKLMNOPQRSTUVhi));
219 vaccKLMNSTUV = _mm256_add_epi32(vaccKLMNSTUV, _mm256_unpackhi_epi16(vprod6xGHIJKLMNOPQRSTUVlo, vprod6xGHIJKLMNOPQRSTUVhi));
220
221 const __m256i vi7x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i7));
222 const __m256i vk7x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 224 * sizeof(int8_t))));
223 const __m256i vi7xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i7 + 16)));
224 const __m256i vk7xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int32_t) + 240 * sizeof(int8_t))));
225 i7 += 32;
226
227 const __m256i vprod7x0123456789ABCDEFlo = _mm256_mullo_epi16(vi7x0123456789ABCDEF, vk7x0123456789ABCDEF);
228 const __m256i vprod7x0123456789ABCDEFhi = _mm256_srai_epi16(vprod7x0123456789ABCDEFlo, 15);
229 const __m256i vprod7xGHIJKLMNOPQRSTUVlo = _mm256_mullo_epi16(vi7xGHIJKLMNOPQRSTUV, vk7xGHIJKLMNOPQRSTUV);
230 const __m256i vprod7xGHIJKLMNOPQRSTUVhi = _mm256_srai_epi16(vprod7xGHIJKLMNOPQRSTUVlo, 15);
231
232 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vprod7x0123456789ABCDEFlo, vprod7x0123456789ABCDEFhi));
233 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vprod7x0123456789ABCDEFlo, vprod7x0123456789ABCDEFhi));
234 vaccGHIJOPQR = _mm256_add_epi32(vaccGHIJOPQR, _mm256_unpacklo_epi16(vprod7xGHIJKLMNOPQRSTUVlo, vprod7xGHIJKLMNOPQRSTUVhi));
235 vaccKLMNSTUV = _mm256_add_epi32(vaccKLMNSTUV, _mm256_unpackhi_epi16(vprod7xGHIJKLMNOPQRSTUVlo, vprod7xGHIJKLMNOPQRSTUVhi));
236
237 w = (const void*) ((uintptr_t) w + 32 * sizeof(int32_t) + 256 * sizeof(int8_t));
238
239 vacc01234567 = _mm256_inserti128_si256(vacc012389AB, _mm256_castsi256_si128(vacc4567CDEF), 1);
240 vacc89ABCDEF = _mm256_permute2x128_si256(vacc012389AB, vacc4567CDEF, 0x31);
241 vaccGHIJKLMN = _mm256_inserti128_si256(vaccGHIJOPQR, _mm256_castsi256_si128(vaccKLMNSTUV), 1);
242 vaccOPQRSTUV = _mm256_permute2x128_si256(vaccGHIJOPQR, vaccKLMNSTUV, 0x31);
243
244 _mm256_storeu_si256((__m256i*) b, vacc01234567);
245 _mm256_storeu_si256((__m256i*) (b + 8), vacc89ABCDEF);
246 _mm256_storeu_si256((__m256i*) (b + 16), vaccGHIJKLMN);
247 _mm256_storeu_si256((__m256i*) (b + 24), vaccOPQRSTUV);
248 b += 32;
249 }
250
251 if XNN_UNLIKELY(c != 0) {
252 do {
253 __m256i vacc01234567 = _mm256_loadu_si256((const __m256i*) w);
254 __m256i vacc89ABCDEF = _mm256_loadu_si256((const __m256i*) ((uintptr_t) w + 8 * sizeof(int32_t)));
255
256 __m256i vacc012389AB = _mm256_inserti128_si256(vacc01234567, _mm256_castsi256_si128(vacc89ABCDEF), 1);
257 __m256i vacc4567CDEF = _mm256_permute2x128_si256(vacc01234567, vacc89ABCDEF, 0x31);
258
259
260 const __m256i vi0x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i0));
261 const __m256i vk0x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t))));
262 i0 += 16;
263
264 const __m256i vprod0x0123456789ABCDEFlo = _mm256_mullo_epi16(vi0x0123456789ABCDEF, vk0x0123456789ABCDEF);
265 const __m256i vprod0x0123456789ABCDEFhi = _mm256_srai_epi16(vprod0x0123456789ABCDEFlo, 15);
266
267 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vprod0x0123456789ABCDEFlo, vprod0x0123456789ABCDEFhi));
268 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vprod0x0123456789ABCDEFlo, vprod0x0123456789ABCDEFhi));
269
270 const __m256i vi1x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i1));
271 const __m256i vk1x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 16 * sizeof(int8_t))));
272 i1 += 16;
273
274 const __m256i vprod1x0123456789ABCDEFlo = _mm256_mullo_epi16(vi1x0123456789ABCDEF, vk1x0123456789ABCDEF);
275 const __m256i vprod1x0123456789ABCDEFhi = _mm256_srai_epi16(vprod1x0123456789ABCDEFlo, 15);
276
277 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vprod1x0123456789ABCDEFlo, vprod1x0123456789ABCDEFhi));
278 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vprod1x0123456789ABCDEFlo, vprod1x0123456789ABCDEFhi));
279
280 const __m256i vi2x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i2));
281 const __m256i vk2x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 32 * sizeof(int8_t))));
282 i2 += 16;
283
284 const __m256i vprod2x0123456789ABCDEFlo = _mm256_mullo_epi16(vi2x0123456789ABCDEF, vk2x0123456789ABCDEF);
285 const __m256i vprod2x0123456789ABCDEFhi = _mm256_srai_epi16(vprod2x0123456789ABCDEFlo, 15);
286
287 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vprod2x0123456789ABCDEFlo, vprod2x0123456789ABCDEFhi));
288 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vprod2x0123456789ABCDEFlo, vprod2x0123456789ABCDEFhi));
289
290 const __m256i vi3x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i3));
291 const __m256i vk3x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 48 * sizeof(int8_t))));
292 i3 += 16;
293
294 const __m256i vprod3x0123456789ABCDEFlo = _mm256_mullo_epi16(vi3x0123456789ABCDEF, vk3x0123456789ABCDEF);
295 const __m256i vprod3x0123456789ABCDEFhi = _mm256_srai_epi16(vprod3x0123456789ABCDEFlo, 15);
296
297 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vprod3x0123456789ABCDEFlo, vprod3x0123456789ABCDEFhi));
298 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vprod3x0123456789ABCDEFlo, vprod3x0123456789ABCDEFhi));
299
300 const __m256i vi4x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i4));
301 const __m256i vk4x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 64 * sizeof(int8_t))));
302 i4 += 16;
303
304 const __m256i vprod4x0123456789ABCDEFlo = _mm256_mullo_epi16(vi4x0123456789ABCDEF, vk4x0123456789ABCDEF);
305 const __m256i vprod4x0123456789ABCDEFhi = _mm256_srai_epi16(vprod4x0123456789ABCDEFlo, 15);
306
307 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vprod4x0123456789ABCDEFlo, vprod4x0123456789ABCDEFhi));
308 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vprod4x0123456789ABCDEFlo, vprod4x0123456789ABCDEFhi));
309
310 const __m256i vi5x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i5));
311 const __m256i vk5x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 80 * sizeof(int8_t))));
312 i5 += 16;
313
314 const __m256i vprod5x0123456789ABCDEFlo = _mm256_mullo_epi16(vi5x0123456789ABCDEF, vk5x0123456789ABCDEF);
315 const __m256i vprod5x0123456789ABCDEFhi = _mm256_srai_epi16(vprod5x0123456789ABCDEFlo, 15);
316
317 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vprod5x0123456789ABCDEFlo, vprod5x0123456789ABCDEFhi));
318 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vprod5x0123456789ABCDEFlo, vprod5x0123456789ABCDEFhi));
319
320 const __m256i vi6x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i6));
321 const __m256i vk6x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 96 * sizeof(int8_t))));
322 i6 += 16;
323
324 const __m256i vprod6x0123456789ABCDEFlo = _mm256_mullo_epi16(vi6x0123456789ABCDEF, vk6x0123456789ABCDEF);
325 const __m256i vprod6x0123456789ABCDEFhi = _mm256_srai_epi16(vprod6x0123456789ABCDEFlo, 15);
326
327 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vprod6x0123456789ABCDEFlo, vprod6x0123456789ABCDEFhi));
328 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vprod6x0123456789ABCDEFlo, vprod6x0123456789ABCDEFhi));
329
330 const __m256i vi7x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i7));
331 const __m256i vk7x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int32_t) + 112 * sizeof(int8_t))));
332 i7 += 16;
333
334 const __m256i vprod7x0123456789ABCDEFlo = _mm256_mullo_epi16(vi7x0123456789ABCDEF, vk7x0123456789ABCDEF);
335 const __m256i vprod7x0123456789ABCDEFhi = _mm256_srai_epi16(vprod7x0123456789ABCDEFlo, 15);
336
337 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vprod7x0123456789ABCDEFlo, vprod7x0123456789ABCDEFhi));
338 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vprod7x0123456789ABCDEFlo, vprod7x0123456789ABCDEFhi));
339
340 vacc01234567 = _mm256_inserti128_si256(vacc012389AB, _mm256_castsi256_si128(vacc4567CDEF), 1);
341 vacc89ABCDEF = _mm256_permute2x128_si256(vacc012389AB, vacc4567CDEF, 0x31);
342
343 w = (const void*) ((uintptr_t) w + 16 * sizeof(int32_t) + 128 * sizeof(int8_t));
344
345 _mm256_storeu_si256((__m256i*) b, vacc01234567);
346 _mm256_storeu_si256((__m256i*) (b + 8), vacc89ABCDEF);
347 b += 16;
348 c -= 16;
349 } while (c != 0);
350
351 }
352 }
353
354 // Middle pass to process 8 inputs in each iteration.
355 for (size_t ks = kernel_size - 8; ks > 9; ks -= 8) {
356 int32_t* b = buffer;
357
358 const int8_t* i0 = input[0];
359 assert(i0 != NULL);
360 if XNN_UNPREDICTABLE(i0 != zero) {
361 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
362 }
363 const int8_t* i1 = input[1];
364 assert(i1 != NULL);
365 if XNN_UNPREDICTABLE(i1 != zero) {
366 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
367 }
368 const int8_t* i2 = input[2];
369 assert(i2 != NULL);
370 if XNN_UNPREDICTABLE(i2 != zero) {
371 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
372 }
373 const int8_t* i3 = input[3];
374 assert(i3 != NULL);
375 if XNN_UNPREDICTABLE(i3 != zero) {
376 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
377 }
378 const int8_t* i4 = input[4];
379 assert(i4 != NULL);
380 if XNN_UNPREDICTABLE(i4 != zero) {
381 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
382 }
383 const int8_t* i5 = input[5];
384 assert(i5 != NULL);
385 if XNN_UNPREDICTABLE(i5 != zero) {
386 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
387 }
388 const int8_t* i6 = input[6];
389 assert(i6 != NULL);
390 if XNN_UNPREDICTABLE(i6 != zero) {
391 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
392 }
393 const int8_t* i7 = input[7];
394 assert(i7 != NULL);
395 if XNN_UNPREDICTABLE(i7 != zero) {
396 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
397 }
398 input += 8;
399
400 size_t c = round_up_po2(channels, 16);
401
402 for (; c >= 32; c -= 32) {
403 __m256i vacc01234567 = _mm256_loadu_si256((const __m256i*) b);
404 __m256i vacc89ABCDEF = _mm256_loadu_si256((const __m256i*) (b + 8));
405 __m256i vaccGHIJKLMN = _mm256_loadu_si256((const __m256i*) (b + 16));
406 __m256i vaccOPQRSTUV = _mm256_loadu_si256((const __m256i*) (b + 24));
407
408 __m256i vacc012389AB = _mm256_inserti128_si256(vacc01234567, _mm256_castsi256_si128(vacc89ABCDEF), 1);
409 __m256i vacc4567CDEF = _mm256_permute2x128_si256(vacc01234567, vacc89ABCDEF, 0x31);
410 __m256i vaccGHIJOPQR = _mm256_inserti128_si256(vaccGHIJKLMN, _mm256_castsi256_si128(vaccOPQRSTUV), 1);
411 __m256i vaccKLMNSTUV = _mm256_permute2x128_si256(vaccGHIJKLMN, vaccOPQRSTUV, 0x31);
412
413
414 const __m256i vi0x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i0));
415 const __m256i vk0x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 0 * sizeof(int8_t))));
416 const __m256i vi0xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i0 + 16)));
417 const __m256i vk0xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t))));
418 i0 += 32;
419
420 const __m256i vprod0x0123456789ABCDEFlo = _mm256_mullo_epi16(vi0x0123456789ABCDEF, vk0x0123456789ABCDEF);
421 const __m256i vprod0x0123456789ABCDEFhi = _mm256_srai_epi16(vprod0x0123456789ABCDEFlo, 15);
422 const __m256i vprod0xGHIJKLMNOPQRSTUVlo = _mm256_mullo_epi16(vi0xGHIJKLMNOPQRSTUV, vk0xGHIJKLMNOPQRSTUV);
423 const __m256i vprod0xGHIJKLMNOPQRSTUVhi = _mm256_srai_epi16(vprod0xGHIJKLMNOPQRSTUVlo, 15);
424
425 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vprod0x0123456789ABCDEFlo, vprod0x0123456789ABCDEFhi));
426 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vprod0x0123456789ABCDEFlo, vprod0x0123456789ABCDEFhi));
427 vaccGHIJOPQR = _mm256_add_epi32(vaccGHIJOPQR, _mm256_unpacklo_epi16(vprod0xGHIJKLMNOPQRSTUVlo, vprod0xGHIJKLMNOPQRSTUVhi));
428 vaccKLMNSTUV = _mm256_add_epi32(vaccKLMNSTUV, _mm256_unpackhi_epi16(vprod0xGHIJKLMNOPQRSTUVlo, vprod0xGHIJKLMNOPQRSTUVhi));
429
430 const __m256i vi1x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i1));
431 const __m256i vk1x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t))));
432 const __m256i vi1xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i1 + 16)));
433 const __m256i vk1xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t))));
434 i1 += 32;
435
436 const __m256i vprod1x0123456789ABCDEFlo = _mm256_mullo_epi16(vi1x0123456789ABCDEF, vk1x0123456789ABCDEF);
437 const __m256i vprod1x0123456789ABCDEFhi = _mm256_srai_epi16(vprod1x0123456789ABCDEFlo, 15);
438 const __m256i vprod1xGHIJKLMNOPQRSTUVlo = _mm256_mullo_epi16(vi1xGHIJKLMNOPQRSTUV, vk1xGHIJKLMNOPQRSTUV);
439 const __m256i vprod1xGHIJKLMNOPQRSTUVhi = _mm256_srai_epi16(vprod1xGHIJKLMNOPQRSTUVlo, 15);
440
441 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vprod1x0123456789ABCDEFlo, vprod1x0123456789ABCDEFhi));
442 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vprod1x0123456789ABCDEFlo, vprod1x0123456789ABCDEFhi));
443 vaccGHIJOPQR = _mm256_add_epi32(vaccGHIJOPQR, _mm256_unpacklo_epi16(vprod1xGHIJKLMNOPQRSTUVlo, vprod1xGHIJKLMNOPQRSTUVhi));
444 vaccKLMNSTUV = _mm256_add_epi32(vaccKLMNSTUV, _mm256_unpackhi_epi16(vprod1xGHIJKLMNOPQRSTUVlo, vprod1xGHIJKLMNOPQRSTUVhi));
445
446 const __m256i vi2x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i2));
447 const __m256i vk2x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 64 * sizeof(int8_t))));
448 const __m256i vi2xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i2 + 16)));
449 const __m256i vk2xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 80 * sizeof(int8_t))));
450 i2 += 32;
451
452 const __m256i vprod2x0123456789ABCDEFlo = _mm256_mullo_epi16(vi2x0123456789ABCDEF, vk2x0123456789ABCDEF);
453 const __m256i vprod2x0123456789ABCDEFhi = _mm256_srai_epi16(vprod2x0123456789ABCDEFlo, 15);
454 const __m256i vprod2xGHIJKLMNOPQRSTUVlo = _mm256_mullo_epi16(vi2xGHIJKLMNOPQRSTUV, vk2xGHIJKLMNOPQRSTUV);
455 const __m256i vprod2xGHIJKLMNOPQRSTUVhi = _mm256_srai_epi16(vprod2xGHIJKLMNOPQRSTUVlo, 15);
456
457 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vprod2x0123456789ABCDEFlo, vprod2x0123456789ABCDEFhi));
458 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vprod2x0123456789ABCDEFlo, vprod2x0123456789ABCDEFhi));
459 vaccGHIJOPQR = _mm256_add_epi32(vaccGHIJOPQR, _mm256_unpacklo_epi16(vprod2xGHIJKLMNOPQRSTUVlo, vprod2xGHIJKLMNOPQRSTUVhi));
460 vaccKLMNSTUV = _mm256_add_epi32(vaccKLMNSTUV, _mm256_unpackhi_epi16(vprod2xGHIJKLMNOPQRSTUVlo, vprod2xGHIJKLMNOPQRSTUVhi));
461
462 const __m256i vi3x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i3));
463 const __m256i vk3x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 96 * sizeof(int8_t))));
464 const __m256i vi3xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i3 + 16)));
465 const __m256i vk3xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 112 * sizeof(int8_t))));
466 i3 += 32;
467
468 const __m256i vprod3x0123456789ABCDEFlo = _mm256_mullo_epi16(vi3x0123456789ABCDEF, vk3x0123456789ABCDEF);
469 const __m256i vprod3x0123456789ABCDEFhi = _mm256_srai_epi16(vprod3x0123456789ABCDEFlo, 15);
470 const __m256i vprod3xGHIJKLMNOPQRSTUVlo = _mm256_mullo_epi16(vi3xGHIJKLMNOPQRSTUV, vk3xGHIJKLMNOPQRSTUV);
471 const __m256i vprod3xGHIJKLMNOPQRSTUVhi = _mm256_srai_epi16(vprod3xGHIJKLMNOPQRSTUVlo, 15);
472
473 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vprod3x0123456789ABCDEFlo, vprod3x0123456789ABCDEFhi));
474 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vprod3x0123456789ABCDEFlo, vprod3x0123456789ABCDEFhi));
475 vaccGHIJOPQR = _mm256_add_epi32(vaccGHIJOPQR, _mm256_unpacklo_epi16(vprod3xGHIJKLMNOPQRSTUVlo, vprod3xGHIJKLMNOPQRSTUVhi));
476 vaccKLMNSTUV = _mm256_add_epi32(vaccKLMNSTUV, _mm256_unpackhi_epi16(vprod3xGHIJKLMNOPQRSTUVlo, vprod3xGHIJKLMNOPQRSTUVhi));
477
478 const __m256i vi4x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i4));
479 const __m256i vk4x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 128 * sizeof(int8_t))));
480 const __m256i vi4xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i4 + 16)));
481 const __m256i vk4xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 144 * sizeof(int8_t))));
482 i4 += 32;
483
484 const __m256i vprod4x0123456789ABCDEFlo = _mm256_mullo_epi16(vi4x0123456789ABCDEF, vk4x0123456789ABCDEF);
485 const __m256i vprod4x0123456789ABCDEFhi = _mm256_srai_epi16(vprod4x0123456789ABCDEFlo, 15);
486 const __m256i vprod4xGHIJKLMNOPQRSTUVlo = _mm256_mullo_epi16(vi4xGHIJKLMNOPQRSTUV, vk4xGHIJKLMNOPQRSTUV);
487 const __m256i vprod4xGHIJKLMNOPQRSTUVhi = _mm256_srai_epi16(vprod4xGHIJKLMNOPQRSTUVlo, 15);
488
489 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vprod4x0123456789ABCDEFlo, vprod4x0123456789ABCDEFhi));
490 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vprod4x0123456789ABCDEFlo, vprod4x0123456789ABCDEFhi));
491 vaccGHIJOPQR = _mm256_add_epi32(vaccGHIJOPQR, _mm256_unpacklo_epi16(vprod4xGHIJKLMNOPQRSTUVlo, vprod4xGHIJKLMNOPQRSTUVhi));
492 vaccKLMNSTUV = _mm256_add_epi32(vaccKLMNSTUV, _mm256_unpackhi_epi16(vprod4xGHIJKLMNOPQRSTUVlo, vprod4xGHIJKLMNOPQRSTUVhi));
493
494 const __m256i vi5x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i5));
495 const __m256i vk5x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 160 * sizeof(int8_t))));
496 const __m256i vi5xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i5 + 16)));
497 const __m256i vk5xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 176 * sizeof(int8_t))));
498 i5 += 32;
499
500 const __m256i vprod5x0123456789ABCDEFlo = _mm256_mullo_epi16(vi5x0123456789ABCDEF, vk5x0123456789ABCDEF);
501 const __m256i vprod5x0123456789ABCDEFhi = _mm256_srai_epi16(vprod5x0123456789ABCDEFlo, 15);
502 const __m256i vprod5xGHIJKLMNOPQRSTUVlo = _mm256_mullo_epi16(vi5xGHIJKLMNOPQRSTUV, vk5xGHIJKLMNOPQRSTUV);
503 const __m256i vprod5xGHIJKLMNOPQRSTUVhi = _mm256_srai_epi16(vprod5xGHIJKLMNOPQRSTUVlo, 15);
504
505 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vprod5x0123456789ABCDEFlo, vprod5x0123456789ABCDEFhi));
506 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vprod5x0123456789ABCDEFlo, vprod5x0123456789ABCDEFhi));
507 vaccGHIJOPQR = _mm256_add_epi32(vaccGHIJOPQR, _mm256_unpacklo_epi16(vprod5xGHIJKLMNOPQRSTUVlo, vprod5xGHIJKLMNOPQRSTUVhi));
508 vaccKLMNSTUV = _mm256_add_epi32(vaccKLMNSTUV, _mm256_unpackhi_epi16(vprod5xGHIJKLMNOPQRSTUVlo, vprod5xGHIJKLMNOPQRSTUVhi));
509
510 const __m256i vi6x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i6));
511 const __m256i vk6x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 192 * sizeof(int8_t))));
512 const __m256i vi6xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i6 + 16)));
513 const __m256i vk6xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 208 * sizeof(int8_t))));
514 i6 += 32;
515
516 const __m256i vprod6x0123456789ABCDEFlo = _mm256_mullo_epi16(vi6x0123456789ABCDEF, vk6x0123456789ABCDEF);
517 const __m256i vprod6x0123456789ABCDEFhi = _mm256_srai_epi16(vprod6x0123456789ABCDEFlo, 15);
518 const __m256i vprod6xGHIJKLMNOPQRSTUVlo = _mm256_mullo_epi16(vi6xGHIJKLMNOPQRSTUV, vk6xGHIJKLMNOPQRSTUV);
519 const __m256i vprod6xGHIJKLMNOPQRSTUVhi = _mm256_srai_epi16(vprod6xGHIJKLMNOPQRSTUVlo, 15);
520
521 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vprod6x0123456789ABCDEFlo, vprod6x0123456789ABCDEFhi));
522 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vprod6x0123456789ABCDEFlo, vprod6x0123456789ABCDEFhi));
523 vaccGHIJOPQR = _mm256_add_epi32(vaccGHIJOPQR, _mm256_unpacklo_epi16(vprod6xGHIJKLMNOPQRSTUVlo, vprod6xGHIJKLMNOPQRSTUVhi));
524 vaccKLMNSTUV = _mm256_add_epi32(vaccKLMNSTUV, _mm256_unpackhi_epi16(vprod6xGHIJKLMNOPQRSTUVlo, vprod6xGHIJKLMNOPQRSTUVhi));
525
526 const __m256i vi7x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i7));
527 const __m256i vk7x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 224 * sizeof(int8_t))));
528 const __m256i vi7xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i7 + 16)));
529 const __m256i vk7xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 240 * sizeof(int8_t))));
530 i7 += 32;
531
532 const __m256i vprod7x0123456789ABCDEFlo = _mm256_mullo_epi16(vi7x0123456789ABCDEF, vk7x0123456789ABCDEF);
533 const __m256i vprod7x0123456789ABCDEFhi = _mm256_srai_epi16(vprod7x0123456789ABCDEFlo, 15);
534 const __m256i vprod7xGHIJKLMNOPQRSTUVlo = _mm256_mullo_epi16(vi7xGHIJKLMNOPQRSTUV, vk7xGHIJKLMNOPQRSTUV);
535 const __m256i vprod7xGHIJKLMNOPQRSTUVhi = _mm256_srai_epi16(vprod7xGHIJKLMNOPQRSTUVlo, 15);
536
537 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vprod7x0123456789ABCDEFlo, vprod7x0123456789ABCDEFhi));
538 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vprod7x0123456789ABCDEFlo, vprod7x0123456789ABCDEFhi));
539 vaccGHIJOPQR = _mm256_add_epi32(vaccGHIJOPQR, _mm256_unpacklo_epi16(vprod7xGHIJKLMNOPQRSTUVlo, vprod7xGHIJKLMNOPQRSTUVhi));
540 vaccKLMNSTUV = _mm256_add_epi32(vaccKLMNSTUV, _mm256_unpackhi_epi16(vprod7xGHIJKLMNOPQRSTUVlo, vprod7xGHIJKLMNOPQRSTUVhi));
541
542 w = (const void*) ((uintptr_t) w + 256 * sizeof(int8_t));
543
544 vacc01234567 = _mm256_inserti128_si256(vacc012389AB, _mm256_castsi256_si128(vacc4567CDEF), 1);
545 vacc89ABCDEF = _mm256_permute2x128_si256(vacc012389AB, vacc4567CDEF, 0x31);
546 vaccGHIJKLMN = _mm256_inserti128_si256(vaccGHIJOPQR, _mm256_castsi256_si128(vaccKLMNSTUV), 1);
547 vaccOPQRSTUV = _mm256_permute2x128_si256(vaccGHIJOPQR, vaccKLMNSTUV, 0x31);
548
549 _mm256_storeu_si256((__m256i*) b, vacc01234567);
550 _mm256_storeu_si256((__m256i*) (b + 8), vacc89ABCDEF);
551 _mm256_storeu_si256((__m256i*) (b + 16), vaccGHIJKLMN);
552 _mm256_storeu_si256((__m256i*) (b + 24), vaccOPQRSTUV);
553 b += 32;
554 }
555
556 if XNN_UNLIKELY(c != 0) {
557 do {
558 __m256i vacc01234567 = _mm256_loadu_si256((const __m256i*) b);
559 __m256i vacc89ABCDEF = _mm256_loadu_si256((const __m256i*) (b + 8));
560
561 __m256i vacc012389AB = _mm256_inserti128_si256(vacc01234567, _mm256_castsi256_si128(vacc89ABCDEF), 1);
562 __m256i vacc4567CDEF = _mm256_permute2x128_si256(vacc01234567, vacc89ABCDEF, 0x31);
563
564
565 const __m256i vi0x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i0));
566 const __m256i vk0x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w)));
567 i0 += 16;
568 const __m256i vprod0x0123456789ABCDEFlo = _mm256_mullo_epi16(vi0x0123456789ABCDEF, vk0x0123456789ABCDEF);
569 const __m256i vprod0x0123456789ABCDEFhi = _mm256_srai_epi16(vprod0x0123456789ABCDEFlo, 15);
570
571 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vprod0x0123456789ABCDEFlo, vprod0x0123456789ABCDEFhi));
572 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vprod0x0123456789ABCDEFlo, vprod0x0123456789ABCDEFhi));
573
574 const __m256i vi1x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i1));
575 const __m256i vk1x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t))));
576 i1 += 16;
577 const __m256i vprod1x0123456789ABCDEFlo = _mm256_mullo_epi16(vi1x0123456789ABCDEF, vk1x0123456789ABCDEF);
578 const __m256i vprod1x0123456789ABCDEFhi = _mm256_srai_epi16(vprod1x0123456789ABCDEFlo, 15);
579
580 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vprod1x0123456789ABCDEFlo, vprod1x0123456789ABCDEFhi));
581 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vprod1x0123456789ABCDEFlo, vprod1x0123456789ABCDEFhi));
582
583 const __m256i vi2x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i2));
584 const __m256i vk2x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t))));
585 i2 += 16;
586 const __m256i vprod2x0123456789ABCDEFlo = _mm256_mullo_epi16(vi2x0123456789ABCDEF, vk2x0123456789ABCDEF);
587 const __m256i vprod2x0123456789ABCDEFhi = _mm256_srai_epi16(vprod2x0123456789ABCDEFlo, 15);
588
589 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vprod2x0123456789ABCDEFlo, vprod2x0123456789ABCDEFhi));
590 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vprod2x0123456789ABCDEFlo, vprod2x0123456789ABCDEFhi));
591
592 const __m256i vi3x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i3));
593 const __m256i vk3x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t))));
594 i3 += 16;
595 const __m256i vprod3x0123456789ABCDEFlo = _mm256_mullo_epi16(vi3x0123456789ABCDEF, vk3x0123456789ABCDEF);
596 const __m256i vprod3x0123456789ABCDEFhi = _mm256_srai_epi16(vprod3x0123456789ABCDEFlo, 15);
597
598 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vprod3x0123456789ABCDEFlo, vprod3x0123456789ABCDEFhi));
599 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vprod3x0123456789ABCDEFlo, vprod3x0123456789ABCDEFhi));
600
601 const __m256i vi4x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i4));
602 const __m256i vk4x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 64 * sizeof(int8_t))));
603 i4 += 16;
604 const __m256i vprod4x0123456789ABCDEFlo = _mm256_mullo_epi16(vi4x0123456789ABCDEF, vk4x0123456789ABCDEF);
605 const __m256i vprod4x0123456789ABCDEFhi = _mm256_srai_epi16(vprod4x0123456789ABCDEFlo, 15);
606
607 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vprod4x0123456789ABCDEFlo, vprod4x0123456789ABCDEFhi));
608 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vprod4x0123456789ABCDEFlo, vprod4x0123456789ABCDEFhi));
609
610 const __m256i vi5x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i5));
611 const __m256i vk5x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 80 * sizeof(int8_t))));
612 i5 += 16;
613 const __m256i vprod5x0123456789ABCDEFlo = _mm256_mullo_epi16(vi5x0123456789ABCDEF, vk5x0123456789ABCDEF);
614 const __m256i vprod5x0123456789ABCDEFhi = _mm256_srai_epi16(vprod5x0123456789ABCDEFlo, 15);
615
616 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vprod5x0123456789ABCDEFlo, vprod5x0123456789ABCDEFhi));
617 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vprod5x0123456789ABCDEFlo, vprod5x0123456789ABCDEFhi));
618
619 const __m256i vi6x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i6));
620 const __m256i vk6x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 96 * sizeof(int8_t))));
621 i6 += 16;
622 const __m256i vprod6x0123456789ABCDEFlo = _mm256_mullo_epi16(vi6x0123456789ABCDEF, vk6x0123456789ABCDEF);
623 const __m256i vprod6x0123456789ABCDEFhi = _mm256_srai_epi16(vprod6x0123456789ABCDEFlo, 15);
624
625 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vprod6x0123456789ABCDEFlo, vprod6x0123456789ABCDEFhi));
626 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vprod6x0123456789ABCDEFlo, vprod6x0123456789ABCDEFhi));
627
628 const __m256i vi7x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i7));
629 const __m256i vk7x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 112 * sizeof(int8_t))));
630 i7 += 16;
631 const __m256i vprod7x0123456789ABCDEFlo = _mm256_mullo_epi16(vi7x0123456789ABCDEF, vk7x0123456789ABCDEF);
632 const __m256i vprod7x0123456789ABCDEFhi = _mm256_srai_epi16(vprod7x0123456789ABCDEFlo, 15);
633
634 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vprod7x0123456789ABCDEFlo, vprod7x0123456789ABCDEFhi));
635 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vprod7x0123456789ABCDEFlo, vprod7x0123456789ABCDEFhi));
636
637 vacc01234567 = _mm256_inserti128_si256(vacc012389AB, _mm256_castsi256_si128(vacc4567CDEF), 1);
638 vacc89ABCDEF = _mm256_permute2x128_si256(vacc012389AB, vacc4567CDEF, 0x31);
639
640 w = (const void*) ((uintptr_t) w + 128 * sizeof(int8_t));
641
642 _mm256_storeu_si256((__m256i*) b, vacc01234567);
643 _mm256_storeu_si256((__m256i*) (b + 8), vacc89ABCDEF);
644 b += 16;
645 c -= 16;
646 } while (c != 0);
647 }
648 }
649
650
651 // Last pass to process up to 9 inputs.
652 {
653 const int32_t* b = buffer;
654
655 const int8_t* i0 = input[0];
656 assert(i0 != NULL);
657 if XNN_UNPREDICTABLE(i0 != zero) {
658 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
659 }
660 const int8_t* i1 = input[1];
661 assert(i1 != NULL);
662 if XNN_UNPREDICTABLE(i1 != zero) {
663 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
664 }
665 const int8_t* i2 = input[2];
666 assert(i2 != NULL);
667 if XNN_UNPREDICTABLE(i2 != zero) {
668 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
669 }
670 const int8_t* i3 = input[3];
671 assert(i3 != NULL);
672 if XNN_UNPREDICTABLE(i3 != zero) {
673 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
674 }
675 const int8_t* i4 = input[4];
676 assert(i4 != NULL);
677 if XNN_UNPREDICTABLE(i4 != zero) {
678 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
679 }
680 const int8_t* i5 = input[5];
681 assert(i5 != NULL);
682 if XNN_UNPREDICTABLE(i5 != zero) {
683 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
684 }
685 const int8_t* i6 = input[6];
686 assert(i6 != NULL);
687 if XNN_UNPREDICTABLE(i6 != zero) {
688 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
689 }
690 const int8_t* i7 = input[7];
691 assert(i7 != NULL);
692 if XNN_UNPREDICTABLE(i7 != zero) {
693 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
694 }
695 const int8_t* i8 = input[8];
696 assert(i8 != NULL);
697 if XNN_UNPREDICTABLE(i8 != zero) {
698 i8 = (const int8_t*) ((uintptr_t) i8 + input_offset);
699 }
700
701 size_t c = channels;
702
703 for (; c >= 32; c -= 32) {
704 __m256i vacc01234567 = _mm256_loadu_si256((const __m256i*) b);
705 __m256i vacc89ABCDEF = _mm256_loadu_si256((const __m256i*) (b + 8));
706 __m256i vaccGHIJKLMN = _mm256_loadu_si256((const __m256i*) (b + 16));
707 __m256i vaccOPQRSTUV = _mm256_loadu_si256((const __m256i*) (b + 24));
708 b += 32;
709
710 __m256i vacc012389AB = _mm256_inserti128_si256(vacc01234567, _mm256_castsi256_si128(vacc89ABCDEF), 1);
711 __m256i vacc4567CDEF = _mm256_permute2x128_si256(vacc01234567, vacc89ABCDEF, 0x31);
712 __m256i vaccGHIJOPQR = _mm256_inserti128_si256(vaccGHIJKLMN, _mm256_castsi256_si128(vaccOPQRSTUV), 1);
713 __m256i vaccKLMNSTUV = _mm256_permute2x128_si256(vaccGHIJKLMN, vaccOPQRSTUV, 0x31);
714
715
716 const __m256i vi0x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i0));
717 const __m256i vk0x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 0 * sizeof(int8_t))));
718 const __m256i vi0xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i0 + 16)));
719 const __m256i vk0xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t))));
720 i0 += 32;
721
722 const __m256i vprod0x0123456789ABCDEFlo = _mm256_mullo_epi16(vi0x0123456789ABCDEF, vk0x0123456789ABCDEF);
723 const __m256i vprod0x0123456789ABCDEFhi = _mm256_srai_epi16(vprod0x0123456789ABCDEFlo, 15);
724 const __m256i vprod0xGHIJKLMNOPQRSTUVlo = _mm256_mullo_epi16(vi0xGHIJKLMNOPQRSTUV, vk0xGHIJKLMNOPQRSTUV);
725 const __m256i vprod0xGHIJKLMNOPQRSTUVhi = _mm256_srai_epi16(vprod0xGHIJKLMNOPQRSTUVlo, 15);
726
727 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vprod0x0123456789ABCDEFlo, vprod0x0123456789ABCDEFhi));
728 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vprod0x0123456789ABCDEFlo, vprod0x0123456789ABCDEFhi));
729 vaccGHIJOPQR = _mm256_add_epi32(vaccGHIJOPQR, _mm256_unpacklo_epi16(vprod0xGHIJKLMNOPQRSTUVlo, vprod0xGHIJKLMNOPQRSTUVhi));
730 vaccKLMNSTUV = _mm256_add_epi32(vaccKLMNSTUV, _mm256_unpackhi_epi16(vprod0xGHIJKLMNOPQRSTUVlo, vprod0xGHIJKLMNOPQRSTUVhi));
731
732 const __m256i vi1x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i1));
733 const __m256i vk1x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t))));
734 const __m256i vi1xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i1 + 16)));
735 const __m256i vk1xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t))));
736 i1 += 32;
737
738 const __m256i vprod1x0123456789ABCDEFlo = _mm256_mullo_epi16(vi1x0123456789ABCDEF, vk1x0123456789ABCDEF);
739 const __m256i vprod1x0123456789ABCDEFhi = _mm256_srai_epi16(vprod1x0123456789ABCDEFlo, 15);
740 const __m256i vprod1xGHIJKLMNOPQRSTUVlo = _mm256_mullo_epi16(vi1xGHIJKLMNOPQRSTUV, vk1xGHIJKLMNOPQRSTUV);
741 const __m256i vprod1xGHIJKLMNOPQRSTUVhi = _mm256_srai_epi16(vprod1xGHIJKLMNOPQRSTUVlo, 15);
742
743 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vprod1x0123456789ABCDEFlo, vprod1x0123456789ABCDEFhi));
744 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vprod1x0123456789ABCDEFlo, vprod1x0123456789ABCDEFhi));
745 vaccGHIJOPQR = _mm256_add_epi32(vaccGHIJOPQR, _mm256_unpacklo_epi16(vprod1xGHIJKLMNOPQRSTUVlo, vprod1xGHIJKLMNOPQRSTUVhi));
746 vaccKLMNSTUV = _mm256_add_epi32(vaccKLMNSTUV, _mm256_unpackhi_epi16(vprod1xGHIJKLMNOPQRSTUVlo, vprod1xGHIJKLMNOPQRSTUVhi));
747
748 const __m256i vi2x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i2));
749 const __m256i vk2x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 64 * sizeof(int8_t))));
750 const __m256i vi2xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i2 + 16)));
751 const __m256i vk2xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 80 * sizeof(int8_t))));
752 i2 += 32;
753
754 const __m256i vprod2x0123456789ABCDEFlo = _mm256_mullo_epi16(vi2x0123456789ABCDEF, vk2x0123456789ABCDEF);
755 const __m256i vprod2x0123456789ABCDEFhi = _mm256_srai_epi16(vprod2x0123456789ABCDEFlo, 15);
756 const __m256i vprod2xGHIJKLMNOPQRSTUVlo = _mm256_mullo_epi16(vi2xGHIJKLMNOPQRSTUV, vk2xGHIJKLMNOPQRSTUV);
757 const __m256i vprod2xGHIJKLMNOPQRSTUVhi = _mm256_srai_epi16(vprod2xGHIJKLMNOPQRSTUVlo, 15);
758
759 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vprod2x0123456789ABCDEFlo, vprod2x0123456789ABCDEFhi));
760 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vprod2x0123456789ABCDEFlo, vprod2x0123456789ABCDEFhi));
761 vaccGHIJOPQR = _mm256_add_epi32(vaccGHIJOPQR, _mm256_unpacklo_epi16(vprod2xGHIJKLMNOPQRSTUVlo, vprod2xGHIJKLMNOPQRSTUVhi));
762 vaccKLMNSTUV = _mm256_add_epi32(vaccKLMNSTUV, _mm256_unpackhi_epi16(vprod2xGHIJKLMNOPQRSTUVlo, vprod2xGHIJKLMNOPQRSTUVhi));
763
764 const __m256i vi3x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i3));
765 const __m256i vk3x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 96 * sizeof(int8_t))));
766 const __m256i vi3xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i3 + 16)));
767 const __m256i vk3xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 112 * sizeof(int8_t))));
768 i3 += 32;
769
770 const __m256i vprod3x0123456789ABCDEFlo = _mm256_mullo_epi16(vi3x0123456789ABCDEF, vk3x0123456789ABCDEF);
771 const __m256i vprod3x0123456789ABCDEFhi = _mm256_srai_epi16(vprod3x0123456789ABCDEFlo, 15);
772 const __m256i vprod3xGHIJKLMNOPQRSTUVlo = _mm256_mullo_epi16(vi3xGHIJKLMNOPQRSTUV, vk3xGHIJKLMNOPQRSTUV);
773 const __m256i vprod3xGHIJKLMNOPQRSTUVhi = _mm256_srai_epi16(vprod3xGHIJKLMNOPQRSTUVlo, 15);
774
775 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vprod3x0123456789ABCDEFlo, vprod3x0123456789ABCDEFhi));
776 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vprod3x0123456789ABCDEFlo, vprod3x0123456789ABCDEFhi));
777 vaccGHIJOPQR = _mm256_add_epi32(vaccGHIJOPQR, _mm256_unpacklo_epi16(vprod3xGHIJKLMNOPQRSTUVlo, vprod3xGHIJKLMNOPQRSTUVhi));
778 vaccKLMNSTUV = _mm256_add_epi32(vaccKLMNSTUV, _mm256_unpackhi_epi16(vprod3xGHIJKLMNOPQRSTUVlo, vprod3xGHIJKLMNOPQRSTUVhi));
779
780 const __m256i vi4x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i4));
781 const __m256i vk4x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 128 * sizeof(int8_t))));
782 const __m256i vi4xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i4 + 16)));
783 const __m256i vk4xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 144 * sizeof(int8_t))));
784 i4 += 32;
785
786 const __m256i vprod4x0123456789ABCDEFlo = _mm256_mullo_epi16(vi4x0123456789ABCDEF, vk4x0123456789ABCDEF);
787 const __m256i vprod4x0123456789ABCDEFhi = _mm256_srai_epi16(vprod4x0123456789ABCDEFlo, 15);
788 const __m256i vprod4xGHIJKLMNOPQRSTUVlo = _mm256_mullo_epi16(vi4xGHIJKLMNOPQRSTUV, vk4xGHIJKLMNOPQRSTUV);
789 const __m256i vprod4xGHIJKLMNOPQRSTUVhi = _mm256_srai_epi16(vprod4xGHIJKLMNOPQRSTUVlo, 15);
790
791 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vprod4x0123456789ABCDEFlo, vprod4x0123456789ABCDEFhi));
792 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vprod4x0123456789ABCDEFlo, vprod4x0123456789ABCDEFhi));
793 vaccGHIJOPQR = _mm256_add_epi32(vaccGHIJOPQR, _mm256_unpacklo_epi16(vprod4xGHIJKLMNOPQRSTUVlo, vprod4xGHIJKLMNOPQRSTUVhi));
794 vaccKLMNSTUV = _mm256_add_epi32(vaccKLMNSTUV, _mm256_unpackhi_epi16(vprod4xGHIJKLMNOPQRSTUVlo, vprod4xGHIJKLMNOPQRSTUVhi));
795
796 const __m256i vi5x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i5));
797 const __m256i vk5x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 160 * sizeof(int8_t))));
798 const __m256i vi5xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i5 + 16)));
799 const __m256i vk5xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 176 * sizeof(int8_t))));
800 i5 += 32;
801
802 const __m256i vprod5x0123456789ABCDEFlo = _mm256_mullo_epi16(vi5x0123456789ABCDEF, vk5x0123456789ABCDEF);
803 const __m256i vprod5x0123456789ABCDEFhi = _mm256_srai_epi16(vprod5x0123456789ABCDEFlo, 15);
804 const __m256i vprod5xGHIJKLMNOPQRSTUVlo = _mm256_mullo_epi16(vi5xGHIJKLMNOPQRSTUV, vk5xGHIJKLMNOPQRSTUV);
805 const __m256i vprod5xGHIJKLMNOPQRSTUVhi = _mm256_srai_epi16(vprod5xGHIJKLMNOPQRSTUVlo, 15);
806
807 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vprod5x0123456789ABCDEFlo, vprod5x0123456789ABCDEFhi));
808 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vprod5x0123456789ABCDEFlo, vprod5x0123456789ABCDEFhi));
809 vaccGHIJOPQR = _mm256_add_epi32(vaccGHIJOPQR, _mm256_unpacklo_epi16(vprod5xGHIJKLMNOPQRSTUVlo, vprod5xGHIJKLMNOPQRSTUVhi));
810 vaccKLMNSTUV = _mm256_add_epi32(vaccKLMNSTUV, _mm256_unpackhi_epi16(vprod5xGHIJKLMNOPQRSTUVlo, vprod5xGHIJKLMNOPQRSTUVhi));
811
812 const __m256i vi6x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i6));
813 const __m256i vk6x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 192 * sizeof(int8_t))));
814 const __m256i vi6xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i6 + 16)));
815 const __m256i vk6xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 208 * sizeof(int8_t))));
816 i6 += 32;
817
818 const __m256i vprod6x0123456789ABCDEFlo = _mm256_mullo_epi16(vi6x0123456789ABCDEF, vk6x0123456789ABCDEF);
819 const __m256i vprod6x0123456789ABCDEFhi = _mm256_srai_epi16(vprod6x0123456789ABCDEFlo, 15);
820 const __m256i vprod6xGHIJKLMNOPQRSTUVlo = _mm256_mullo_epi16(vi6xGHIJKLMNOPQRSTUV, vk6xGHIJKLMNOPQRSTUV);
821 const __m256i vprod6xGHIJKLMNOPQRSTUVhi = _mm256_srai_epi16(vprod6xGHIJKLMNOPQRSTUVlo, 15);
822
823 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vprod6x0123456789ABCDEFlo, vprod6x0123456789ABCDEFhi));
824 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vprod6x0123456789ABCDEFlo, vprod6x0123456789ABCDEFhi));
825 vaccGHIJOPQR = _mm256_add_epi32(vaccGHIJOPQR, _mm256_unpacklo_epi16(vprod6xGHIJKLMNOPQRSTUVlo, vprod6xGHIJKLMNOPQRSTUVhi));
826 vaccKLMNSTUV = _mm256_add_epi32(vaccKLMNSTUV, _mm256_unpackhi_epi16(vprod6xGHIJKLMNOPQRSTUVlo, vprod6xGHIJKLMNOPQRSTUVhi));
827
828 const __m256i vi7x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i7));
829 const __m256i vk7x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 224 * sizeof(int8_t))));
830 const __m256i vi7xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i7 + 16)));
831 const __m256i vk7xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 240 * sizeof(int8_t))));
832 i7 += 32;
833
834 const __m256i vprod7x0123456789ABCDEFlo = _mm256_mullo_epi16(vi7x0123456789ABCDEF, vk7x0123456789ABCDEF);
835 const __m256i vprod7x0123456789ABCDEFhi = _mm256_srai_epi16(vprod7x0123456789ABCDEFlo, 15);
836 const __m256i vprod7xGHIJKLMNOPQRSTUVlo = _mm256_mullo_epi16(vi7xGHIJKLMNOPQRSTUV, vk7xGHIJKLMNOPQRSTUV);
837 const __m256i vprod7xGHIJKLMNOPQRSTUVhi = _mm256_srai_epi16(vprod7xGHIJKLMNOPQRSTUVlo, 15);
838
839 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vprod7x0123456789ABCDEFlo, vprod7x0123456789ABCDEFhi));
840 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vprod7x0123456789ABCDEFlo, vprod7x0123456789ABCDEFhi));
841 vaccGHIJOPQR = _mm256_add_epi32(vaccGHIJOPQR, _mm256_unpacklo_epi16(vprod7xGHIJKLMNOPQRSTUVlo, vprod7xGHIJKLMNOPQRSTUVhi));
842 vaccKLMNSTUV = _mm256_add_epi32(vaccKLMNSTUV, _mm256_unpackhi_epi16(vprod7xGHIJKLMNOPQRSTUVlo, vprod7xGHIJKLMNOPQRSTUVhi));
843
844 const __m256i vi8x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i8));
845 const __m256i vk8x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 256 * sizeof(int8_t))));
846 const __m256i vi8xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) (i8 + 16)));
847 const __m256i vk8xGHIJKLMNOPQRSTUV = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 272 * sizeof(int8_t))));
848 i8 += 32;
849
850 const __m256i vprod8x0123456789ABCDEFlo = _mm256_mullo_epi16(vi8x0123456789ABCDEF, vk8x0123456789ABCDEF);
851 const __m256i vprod8x0123456789ABCDEFhi = _mm256_srai_epi16(vprod8x0123456789ABCDEFlo, 15);
852 const __m256i vprod8xGHIJKLMNOPQRSTUVlo = _mm256_mullo_epi16(vi8xGHIJKLMNOPQRSTUV, vk8xGHIJKLMNOPQRSTUV);
853 const __m256i vprod8xGHIJKLMNOPQRSTUVhi = _mm256_srai_epi16(vprod8xGHIJKLMNOPQRSTUVlo, 15);
854
855 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vprod8x0123456789ABCDEFlo, vprod8x0123456789ABCDEFhi));
856 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vprod8x0123456789ABCDEFlo, vprod8x0123456789ABCDEFhi));
857 vaccGHIJOPQR = _mm256_add_epi32(vaccGHIJOPQR, _mm256_unpacklo_epi16(vprod8xGHIJKLMNOPQRSTUVlo, vprod8xGHIJKLMNOPQRSTUVhi));
858 vaccKLMNSTUV = _mm256_add_epi32(vaccKLMNSTUV, _mm256_unpackhi_epi16(vprod8xGHIJKLMNOPQRSTUVlo, vprod8xGHIJKLMNOPQRSTUVhi));
859
860 vacc01234567 = _mm256_inserti128_si256(vacc012389AB, _mm256_castsi256_si128(vacc4567CDEF), 1);
861 vacc89ABCDEF = _mm256_permute2x128_si256(vacc012389AB, vacc4567CDEF, 0x31);
862 vaccGHIJKLMN = _mm256_inserti128_si256(vaccGHIJOPQR, _mm256_castsi256_si128(vaccKLMNSTUV), 1);
863 vaccOPQRSTUV = _mm256_permute2x128_si256(vaccGHIJOPQR, vaccKLMNSTUV, 0x31);
864
865 w = (const void*) ((uintptr_t) w + 288 * sizeof(int8_t));
866
867 __m256 vfpacc01234567 = _mm256_cvtepi32_ps(vacc01234567);
868 __m256 vfpacc89ABCDEF = _mm256_cvtepi32_ps(vacc89ABCDEF);
869 __m256 vfpaccGHIJKLMN = _mm256_cvtepi32_ps(vaccGHIJKLMN);
870 __m256 vfpaccOPQRSTUV = _mm256_cvtepi32_ps(vaccOPQRSTUV);
871
872 vfpacc01234567 = _mm256_mul_ps(vfpacc01234567, vscale);
873 vfpacc89ABCDEF = _mm256_mul_ps(vfpacc89ABCDEF, vscale);
874 vfpaccGHIJKLMN = _mm256_mul_ps(vfpaccGHIJKLMN, vscale);
875 vfpaccOPQRSTUV = _mm256_mul_ps(vfpaccOPQRSTUV, vscale);
876
877 vfpacc01234567 = _mm256_min_ps(vfpacc01234567, voutput_max_less_zero_point);
878 vfpacc89ABCDEF = _mm256_min_ps(vfpacc89ABCDEF, voutput_max_less_zero_point);
879 vfpaccGHIJKLMN = _mm256_min_ps(vfpaccGHIJKLMN, voutput_max_less_zero_point);
880 vfpaccOPQRSTUV = _mm256_min_ps(vfpaccOPQRSTUV, voutput_max_less_zero_point);
881
882 vacc01234567 = _mm256_cvtps_epi32(vfpacc01234567);
883 vacc89ABCDEF = _mm256_cvtps_epi32(vfpacc89ABCDEF);
884 vaccGHIJKLMN = _mm256_cvtps_epi32(vfpaccGHIJKLMN);
885 vaccOPQRSTUV = _mm256_cvtps_epi32(vfpaccOPQRSTUV);
886
887 const __m256i vout012389AB4567CDEF = _mm256_adds_epi16(_mm256_packs_epi32(vacc01234567, vacc89ABCDEF), voutput_zero_point);
888 const __m256i voutGHIJOPQRKLMNSTUV = _mm256_adds_epi16(_mm256_packs_epi32(vaccGHIJKLMN, vaccOPQRSTUV), voutput_zero_point);
889
890 __m128i vout0123456789ABCDEF = _mm_shuffle_epi32(_mm_packs_epi16(_mm256_castsi256_si128(vout012389AB4567CDEF), _mm256_extracti128_si256(vout012389AB4567CDEF, 1)), _MM_SHUFFLE(3, 1, 2, 0));
891 __m128i voutGHIJKLMNOPQRSTUV = _mm_shuffle_epi32(_mm_packs_epi16(_mm256_castsi256_si128(voutGHIJOPQRKLMNSTUV), _mm256_extracti128_si256(voutGHIJOPQRKLMNSTUV, 1)), _MM_SHUFFLE(3, 1, 2, 0));
892
893 vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
894 voutGHIJKLMNOPQRSTUV = _mm_max_epi8(voutGHIJKLMNOPQRSTUV, voutput_min);
895
896 _mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
897 _mm_storeu_si128((__m128i*) (output + 16), voutGHIJKLMNOPQRSTUV);
898 output += 32;
899 }
900
901 if XNN_UNLIKELY(c != 0) {
902 do {
903 __m256i vacc01234567 = _mm256_loadu_si256((const __m256i*) b);
904 __m256i vacc89ABCDEF = _mm256_loadu_si256((const __m256i*) (b + 8));
905 b += 16;
906
907 __m256i vacc012389AB = _mm256_inserti128_si256(vacc01234567, _mm256_castsi256_si128(vacc89ABCDEF), 1);
908 __m256i vacc4567CDEF = _mm256_permute2x128_si256(vacc01234567, vacc89ABCDEF, 0x31);
909
910
911 const __m256i vi0x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i0));
912 const __m256i vk0x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w)));
913 i0 += 16;
914
915 const __m256i vprod0x0123456789ABCDEFlo = _mm256_mullo_epi16(vi0x0123456789ABCDEF, vk0x0123456789ABCDEF);
916 const __m256i vprod0x0123456789ABCDEFhi = _mm256_srai_epi16(vprod0x0123456789ABCDEFlo, 15);
917
918 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vprod0x0123456789ABCDEFlo, vprod0x0123456789ABCDEFhi));
919 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vprod0x0123456789ABCDEFlo, vprod0x0123456789ABCDEFhi));
920
921 const __m256i vi1x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i1));
922 const __m256i vk1x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 16 * sizeof(int8_t))));
923 i1 += 16;
924
925 const __m256i vprod1x0123456789ABCDEFlo = _mm256_mullo_epi16(vi1x0123456789ABCDEF, vk1x0123456789ABCDEF);
926 const __m256i vprod1x0123456789ABCDEFhi = _mm256_srai_epi16(vprod1x0123456789ABCDEFlo, 15);
927
928 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vprod1x0123456789ABCDEFlo, vprod1x0123456789ABCDEFhi));
929 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vprod1x0123456789ABCDEFlo, vprod1x0123456789ABCDEFhi));
930
931 const __m256i vi2x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i2));
932 const __m256i vk2x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 32 * sizeof(int8_t))));
933 i2 += 16;
934
935 const __m256i vprod2x0123456789ABCDEFlo = _mm256_mullo_epi16(vi2x0123456789ABCDEF, vk2x0123456789ABCDEF);
936 const __m256i vprod2x0123456789ABCDEFhi = _mm256_srai_epi16(vprod2x0123456789ABCDEFlo, 15);
937
938 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vprod2x0123456789ABCDEFlo, vprod2x0123456789ABCDEFhi));
939 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vprod2x0123456789ABCDEFlo, vprod2x0123456789ABCDEFhi));
940
941 const __m256i vi3x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i3));
942 const __m256i vk3x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 48 * sizeof(int8_t))));
943 i3 += 16;
944
945 const __m256i vprod3x0123456789ABCDEFlo = _mm256_mullo_epi16(vi3x0123456789ABCDEF, vk3x0123456789ABCDEF);
946 const __m256i vprod3x0123456789ABCDEFhi = _mm256_srai_epi16(vprod3x0123456789ABCDEFlo, 15);
947
948 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vprod3x0123456789ABCDEFlo, vprod3x0123456789ABCDEFhi));
949 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vprod3x0123456789ABCDEFlo, vprod3x0123456789ABCDEFhi));
950
951 const __m256i vi4x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i4));
952 const __m256i vk4x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 64 * sizeof(int8_t))));
953 i4 += 16;
954
955 const __m256i vprod4x0123456789ABCDEFlo = _mm256_mullo_epi16(vi4x0123456789ABCDEF, vk4x0123456789ABCDEF);
956 const __m256i vprod4x0123456789ABCDEFhi = _mm256_srai_epi16(vprod4x0123456789ABCDEFlo, 15);
957
958 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vprod4x0123456789ABCDEFlo, vprod4x0123456789ABCDEFhi));
959 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vprod4x0123456789ABCDEFlo, vprod4x0123456789ABCDEFhi));
960
961 const __m256i vi5x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i5));
962 const __m256i vk5x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 80 * sizeof(int8_t))));
963 i5 += 16;
964
965 const __m256i vprod5x0123456789ABCDEFlo = _mm256_mullo_epi16(vi5x0123456789ABCDEF, vk5x0123456789ABCDEF);
966 const __m256i vprod5x0123456789ABCDEFhi = _mm256_srai_epi16(vprod5x0123456789ABCDEFlo, 15);
967
968 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vprod5x0123456789ABCDEFlo, vprod5x0123456789ABCDEFhi));
969 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vprod5x0123456789ABCDEFlo, vprod5x0123456789ABCDEFhi));
970
971 const __m256i vi6x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i6));
972 const __m256i vk6x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 96 * sizeof(int8_t))));
973 i6 += 16;
974
975 const __m256i vprod6x0123456789ABCDEFlo = _mm256_mullo_epi16(vi6x0123456789ABCDEF, vk6x0123456789ABCDEF);
976 const __m256i vprod6x0123456789ABCDEFhi = _mm256_srai_epi16(vprod6x0123456789ABCDEFlo, 15);
977
978 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vprod6x0123456789ABCDEFlo, vprod6x0123456789ABCDEFhi));
979 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vprod6x0123456789ABCDEFlo, vprod6x0123456789ABCDEFhi));
980
981 const __m256i vi7x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i7));
982 const __m256i vk7x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 112 * sizeof(int8_t))));
983 i7 += 16;
984
985 const __m256i vprod7x0123456789ABCDEFlo = _mm256_mullo_epi16(vi7x0123456789ABCDEF, vk7x0123456789ABCDEF);
986 const __m256i vprod7x0123456789ABCDEFhi = _mm256_srai_epi16(vprod7x0123456789ABCDEFlo, 15);
987
988 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vprod7x0123456789ABCDEFlo, vprod7x0123456789ABCDEFhi));
989 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vprod7x0123456789ABCDEFlo, vprod7x0123456789ABCDEFhi));
990
991 const __m256i vi8x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) i8));
992 const __m256i vk8x0123456789ABCDEF = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*) ((uintptr_t) w + 128 * sizeof(int8_t))));
993 i8 += 16;
994
995 const __m256i vprod8x0123456789ABCDEFlo = _mm256_mullo_epi16(vi8x0123456789ABCDEF, vk8x0123456789ABCDEF);
996 const __m256i vprod8x0123456789ABCDEFhi = _mm256_srai_epi16(vprod8x0123456789ABCDEFlo, 15);
997
998 vacc012389AB = _mm256_add_epi32(vacc012389AB, _mm256_unpacklo_epi16(vprod8x0123456789ABCDEFlo, vprod8x0123456789ABCDEFhi));
999 vacc4567CDEF = _mm256_add_epi32(vacc4567CDEF, _mm256_unpackhi_epi16(vprod8x0123456789ABCDEFlo, vprod8x0123456789ABCDEFhi));
1000
1001 vacc01234567 = _mm256_inserti128_si256(vacc012389AB, _mm256_castsi256_si128(vacc4567CDEF), 1);
1002 vacc89ABCDEF = _mm256_permute2x128_si256(vacc012389AB, vacc4567CDEF, 0x31);
1003
1004 __m256 vfpacc01234567 = _mm256_cvtepi32_ps(vacc01234567);
1005 __m256 vfpacc89ABCDEF = _mm256_cvtepi32_ps(vacc89ABCDEF);
1006
1007 vfpacc01234567 = _mm256_mul_ps(vfpacc01234567, vscale);
1008 vfpacc89ABCDEF = _mm256_mul_ps(vfpacc89ABCDEF, vscale);
1009
1010 vfpacc01234567 = _mm256_min_ps(vfpacc01234567, voutput_max_less_zero_point);
1011 vfpacc89ABCDEF = _mm256_min_ps(vfpacc89ABCDEF, voutput_max_less_zero_point);
1012
1013 vacc01234567 = _mm256_cvtps_epi32(vfpacc01234567);
1014 vacc89ABCDEF = _mm256_cvtps_epi32(vfpacc89ABCDEF);
1015
1016 w = (void*) ((uintptr_t) w + 144 * sizeof(int8_t));
1017
1018 __m128i vout01234567 = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc01234567), _mm256_extracti128_si256(vacc01234567, 1)), _mm256_castsi256_si128(voutput_zero_point));
1019 __m128i vout89ABCDEF = _mm_adds_epi16(_mm_packs_epi32(_mm256_castsi256_si128(vacc89ABCDEF), _mm256_extracti128_si256(vacc89ABCDEF, 1)), _mm256_castsi256_si128(voutput_zero_point));
1020
1021 __m128i vout0123456789ABCDEF = _mm_packs_epi16(vout01234567, vout89ABCDEF);
1022 vout0123456789ABCDEF = _mm_max_epi8(vout0123456789ABCDEF, voutput_min);
1023
1024 if XNN_LIKELY(c >= 16) {
1025 _mm_storeu_si128((__m128i*) output, vout0123456789ABCDEF);
1026 output += 16;
1027 c -= 16;
1028 } else {
1029 if (c & 8) {
1030 _mm_storel_epi64((__m128i*) output, vout0123456789ABCDEF);
1031 vout0123456789ABCDEF = _mm_unpackhi_epi64(vout0123456789ABCDEF, vout0123456789ABCDEF);
1032 output += 8;
1033 }
1034 if (c & 4) {
1035 unaligned_store_u32(output, (uint32_t) _mm_cvtsi128_si32(vout0123456789ABCDEF));
1036 vout0123456789ABCDEF = _mm_srli_epi64(vout0123456789ABCDEF, 32);
1037 output += 4;
1038 }
1039 if (c & 2) {
1040 unaligned_store_u16(output, (uint16_t) _mm_extract_epi16(vout0123456789ABCDEF, 0));
1041 vout0123456789ABCDEF = _mm_srli_epi32(vout0123456789ABCDEF, 16);
1042 output += 2;
1043 }
1044 if (c & 1) {
1045 *output = (int8_t) _mm_extract_epi8(vout0123456789ABCDEF, 0);
1046 output += 1;
1047 }
1048 c = 0;
1049 }
1050 } while (c != 0);
1051 }
1052 }
1053
1054
1055 input = (const int8_t**) ((uintptr_t) input + input_stride);
1056 output = (int8_t*) ((uintptr_t) output + output_increment);
1057 } while (--output_width != 0);
1058 }