Debian Codegraph @

src/qs8-dwconv/gen/qs8-dwconv-8f8m9l4c1s1r-minmax-fp32-scalar-imagic.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-dwconv/multipass-scalar.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11 #include <stddef.h>
12 #include <stdint.h>
13
14 #include "xnnpack/common.h"
15 #include "xnnpack/dwconv.h"
16 #include "xnnpack/math.h"
17 #include "xnnpack/microparams.h"
18 #include "xnnpack/unaligned.h"
19
20
21 void xnn_qs8_dwconv_minmax_fp32_ukernel_8f8m9l4c1s1r__scalar_imagic(
22 size_t channels,
23 size_t output_width,
24 const int8_t** input,
25 const void* weights,
26 int8_t* output,
27 intptr_t input_stride,
28 size_t output_increment,
29 size_t input_offset,
30 const int8_t* zero,
31 size_t kernel_size,
32 int32_t* buffer,
33 const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)])
34 {
35 assert(channels != 0);
36 assert(output_width != 0);
37 assert(kernel_size > 8);
38
39 const float vscale = params->fp32_scalar.scale;
40 const float vmagic_bias = 12582912.0f;
41 const int32_t output_min_less_zero_point = (int32_t) params->fp32_scalar.output_min - (int32_t) params->fp32_scalar.output_zero_point;
42 const int32_t output_max_less_zero_point = (int32_t) params->fp32_scalar.output_max - (int32_t) params->fp32_scalar.output_zero_point;
43 const int32_t vmagic_min = (int32_t) float_as_uint32(12582912.0f + output_min_less_zero_point);
44 const int32_t vmagic_max = (int32_t) float_as_uint32(12582912.0f + output_max_less_zero_point);
45 const int32_t vmagic_bias_less_zero_point = INT32_C(0x4B400000) - (int32_t) params->fp32_scalar.output_zero_point;
46 do {
47 const void* w = weights;
48
49 // First pass to process 8 inputs.
50 {
51 int32_t* b = buffer;
52 const int8_t* i0 = input[0];
53 assert(i0 != NULL);
54 if XNN_UNPREDICTABLE(i0 != zero) {
55 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
56 }
57 const int8_t* i1 = input[1];
58 assert(i1 != NULL);
59 if XNN_UNPREDICTABLE(i1 != zero) {
60 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
61 }
62 const int8_t* i2 = input[2];
63 assert(i2 != NULL);
64 if XNN_UNPREDICTABLE(i2 != zero) {
65 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
66 }
67 const int8_t* i3 = input[3];
68 assert(i3 != NULL);
69 if XNN_UNPREDICTABLE(i3 != zero) {
70 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
71 }
72 const int8_t* i4 = input[4];
73 assert(i4 != NULL);
74 if XNN_UNPREDICTABLE(i4 != zero) {
75 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
76 }
77 const int8_t* i5 = input[5];
78 assert(i5 != NULL);
79 if XNN_UNPREDICTABLE(i5 != zero) {
80 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
81 }
82 const int8_t* i6 = input[6];
83 assert(i6 != NULL);
84 if XNN_UNPREDICTABLE(i6 != zero) {
85 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
86 }
87 const int8_t* i7 = input[7];
88 assert(i7 != NULL);
89 if XNN_UNPREDICTABLE(i7 != zero) {
90 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
91 }
92 input += 8;
93
94 size_t c = channels;
95 for (; c >= 4; c -= 4) {
96 int32_t vacc0 = ((const int32_t*) w)[0];
97 int32_t vacc1 = ((const int32_t*) w)[1];
98 int32_t vacc2 = ((const int32_t*) w)[2];
99 int32_t vacc3 = ((const int32_t*) w)[3];
100
101 const int32_t vi0x0 = (int32_t) i0[0];
102 const int32_t vi0x1 = (int32_t) i0[1];
103 const int32_t vi0x2 = (int32_t) i0[2];
104 const int32_t vi0x3 = (int32_t) i0[3];
105 i0 += 4;
106
107 const int32_t vk0x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[0];
108 const int32_t vk0x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[1];
109 const int32_t vk0x2 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[2];
110 const int32_t vk0x3 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[3];
111
112 vacc0 += vi0x0 * vk0x0;
113 vacc1 += vi0x1 * vk0x1;
114 vacc2 += vi0x2 * vk0x2;
115 vacc3 += vi0x3 * vk0x3;
116
117 const int32_t vi1x0 = (int32_t) i1[0];
118 const int32_t vi1x1 = (int32_t) i1[1];
119 const int32_t vi1x2 = (int32_t) i1[2];
120 const int32_t vi1x3 = (int32_t) i1[3];
121 i1 += 4;
122
123 const int32_t vk1x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[4];
124 const int32_t vk1x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[5];
125 const int32_t vk1x2 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[6];
126 const int32_t vk1x3 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[7];
127
128 vacc0 += vi1x0 * vk1x0;
129 vacc1 += vi1x1 * vk1x1;
130 vacc2 += vi1x2 * vk1x2;
131 vacc3 += vi1x3 * vk1x3;
132
133 const int32_t vi2x0 = (int32_t) i2[0];
134 const int32_t vi2x1 = (int32_t) i2[1];
135 const int32_t vi2x2 = (int32_t) i2[2];
136 const int32_t vi2x3 = (int32_t) i2[3];
137 i2 += 4;
138
139 const int32_t vk2x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[8];
140 const int32_t vk2x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[9];
141 const int32_t vk2x2 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[10];
142 const int32_t vk2x3 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[11];
143
144 vacc0 += vi2x0 * vk2x0;
145 vacc1 += vi2x1 * vk2x1;
146 vacc2 += vi2x2 * vk2x2;
147 vacc3 += vi2x3 * vk2x3;
148
149 const int32_t vi3x0 = (int32_t) i3[0];
150 const int32_t vi3x1 = (int32_t) i3[1];
151 const int32_t vi3x2 = (int32_t) i3[2];
152 const int32_t vi3x3 = (int32_t) i3[3];
153 i3 += 4;
154
155 const int32_t vk3x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[12];
156 const int32_t vk3x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[13];
157 const int32_t vk3x2 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[14];
158 const int32_t vk3x3 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[15];
159
160 vacc0 += vi3x0 * vk3x0;
161 vacc1 += vi3x1 * vk3x1;
162 vacc2 += vi3x2 * vk3x2;
163 vacc3 += vi3x3 * vk3x3;
164
165 const int32_t vi4x0 = (int32_t) i4[0];
166 const int32_t vi4x1 = (int32_t) i4[1];
167 const int32_t vi4x2 = (int32_t) i4[2];
168 const int32_t vi4x3 = (int32_t) i4[3];
169 i4 += 4;
170
171 const int32_t vk4x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[16];
172 const int32_t vk4x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[17];
173 const int32_t vk4x2 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[18];
174 const int32_t vk4x3 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[19];
175
176 vacc0 += vi4x0 * vk4x0;
177 vacc1 += vi4x1 * vk4x1;
178 vacc2 += vi4x2 * vk4x2;
179 vacc3 += vi4x3 * vk4x3;
180
181 const int32_t vi5x0 = (int32_t) i5[0];
182 const int32_t vi5x1 = (int32_t) i5[1];
183 const int32_t vi5x2 = (int32_t) i5[2];
184 const int32_t vi5x3 = (int32_t) i5[3];
185 i5 += 4;
186
187 const int32_t vk5x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[20];
188 const int32_t vk5x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[21];
189 const int32_t vk5x2 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[22];
190 const int32_t vk5x3 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[23];
191
192 vacc0 += vi5x0 * vk5x0;
193 vacc1 += vi5x1 * vk5x1;
194 vacc2 += vi5x2 * vk5x2;
195 vacc3 += vi5x3 * vk5x3;
196
197 const int32_t vi6x0 = (int32_t) i6[0];
198 const int32_t vi6x1 = (int32_t) i6[1];
199 const int32_t vi6x2 = (int32_t) i6[2];
200 const int32_t vi6x3 = (int32_t) i6[3];
201 i6 += 4;
202
203 const int32_t vk6x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[24];
204 const int32_t vk6x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[25];
205 const int32_t vk6x2 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[26];
206 const int32_t vk6x3 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[27];
207
208 vacc0 += vi6x0 * vk6x0;
209 vacc1 += vi6x1 * vk6x1;
210 vacc2 += vi6x2 * vk6x2;
211 vacc3 += vi6x3 * vk6x3;
212
213 const int32_t vi7x0 = (int32_t) i7[0];
214 const int32_t vi7x1 = (int32_t) i7[1];
215 const int32_t vi7x2 = (int32_t) i7[2];
216 const int32_t vi7x3 = (int32_t) i7[3];
217 i7 += 4;
218
219 const int32_t vk7x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[28];
220 const int32_t vk7x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[29];
221 const int32_t vk7x2 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[30];
222 const int32_t vk7x3 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[31];
223
224 vacc0 += vi7x0 * vk7x0;
225 vacc1 += vi7x1 * vk7x1;
226 vacc2 += vi7x2 * vk7x2;
227 vacc3 += vi7x3 * vk7x3;
228
229 w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t) + 32 * sizeof(int8_t));
230 b[0] = vacc0;
231 b[1] = vacc1;
232 b[2] = vacc2;
233 b[3] = vacc3;
234 b += 4;
235 }
236 if XNN_UNLIKELY(c != 0) {
237 do {
238 int32_t vacc = unaligned_load_s32(w);
239 const int32_t vi0 = (int32_t) *i0++;
240 const int32_t vk0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 1 * sizeof(int32_t)))[0];
241 vacc += vi0 * vk0;
242 const int32_t vi1 = (int32_t) *i1++;
243 const int32_t vk1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 1 * sizeof(int32_t)))[1];
244 vacc += vi1 * vk1;
245 const int32_t vi2 = (int32_t) *i2++;
246 const int32_t vk2 = (int32_t) ((const int8_t*) ((uintptr_t) w + 1 * sizeof(int32_t)))[2];
247 vacc += vi2 * vk2;
248 const int32_t vi3 = (int32_t) *i3++;
249 const int32_t vk3 = (int32_t) ((const int8_t*) ((uintptr_t) w + 1 * sizeof(int32_t)))[3];
250 vacc += vi3 * vk3;
251 const int32_t vi4 = (int32_t) *i4++;
252 const int32_t vk4 = (int32_t) ((const int8_t*) ((uintptr_t) w + 1 * sizeof(int32_t)))[4];
253 vacc += vi4 * vk4;
254 const int32_t vi5 = (int32_t) *i5++;
255 const int32_t vk5 = (int32_t) ((const int8_t*) ((uintptr_t) w + 1 * sizeof(int32_t)))[5];
256 vacc += vi5 * vk5;
257 const int32_t vi6 = (int32_t) *i6++;
258 const int32_t vk6 = (int32_t) ((const int8_t*) ((uintptr_t) w + 1 * sizeof(int32_t)))[6];
259 vacc += vi6 * vk6;
260 const int32_t vi7 = (int32_t) *i7++;
261 const int32_t vk7 = (int32_t) ((const int8_t*) ((uintptr_t) w + 1 * sizeof(int32_t)))[7];
262 vacc += vi7 * vk7;
263
264 w = (const void*) ((uintptr_t) w + 1 * sizeof(int32_t) + 8 * sizeof(int8_t));
265 *b++ = vacc;
266 } while (--c != 0);
267 }
268 }
269
270 // Middle pass to process 8 inputs in each iteration.
271 for (size_t ks = kernel_size - 8; ks > 9; ks -= 8) {
272 int32_t* b = buffer;
273 const int8_t* i0 = input[0];
274 assert(i0 != NULL);
275 if XNN_UNPREDICTABLE(i0 != zero) {
276 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
277 }
278 const int8_t* i1 = input[1];
279 assert(i1 != NULL);
280 if XNN_UNPREDICTABLE(i1 != zero) {
281 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
282 }
283 const int8_t* i2 = input[2];
284 assert(i2 != NULL);
285 if XNN_UNPREDICTABLE(i2 != zero) {
286 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
287 }
288 const int8_t* i3 = input[3];
289 assert(i3 != NULL);
290 if XNN_UNPREDICTABLE(i3 != zero) {
291 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
292 }
293 const int8_t* i4 = input[4];
294 assert(i4 != NULL);
295 if XNN_UNPREDICTABLE(i4 != zero) {
296 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
297 }
298 const int8_t* i5 = input[5];
299 assert(i5 != NULL);
300 if XNN_UNPREDICTABLE(i5 != zero) {
301 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
302 }
303 const int8_t* i6 = input[6];
304 assert(i6 != NULL);
305 if XNN_UNPREDICTABLE(i6 != zero) {
306 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
307 }
308 const int8_t* i7 = input[7];
309 assert(i7 != NULL);
310 if XNN_UNPREDICTABLE(i7 != zero) {
311 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
312 }
313 input += 8;
314
315 size_t c = channels;
316 for (; c >= 4; c -= 4) {
317 int32_t vacc0 = b[0];
318 int32_t vacc1 = b[1];
319 int32_t vacc2 = b[2];
320 int32_t vacc3 = b[3];
321
322 const int32_t vi0x0 = (int32_t) i0[0];
323 const int32_t vi0x1 = (int32_t) i0[1];
324 const int32_t vi0x2 = (int32_t) i0[2];
325 const int32_t vi0x3 = (int32_t) i0[3];
326 i0 += 4;
327
328 const int32_t vk0x0 = (int32_t) ((const int8_t*) w)[0];
329 const int32_t vk0x1 = (int32_t) ((const int8_t*) w)[1];
330 const int32_t vk0x2 = (int32_t) ((const int8_t*) w)[2];
331 const int32_t vk0x3 = (int32_t) ((const int8_t*) w)[3];
332
333 vacc0 += vi0x0 * vk0x0;
334 vacc1 += vi0x1 * vk0x1;
335 vacc2 += vi0x2 * vk0x2;
336 vacc3 += vi0x3 * vk0x3;
337
338 const int32_t vi1x0 = (int32_t) i1[0];
339 const int32_t vi1x1 = (int32_t) i1[1];
340 const int32_t vi1x2 = (int32_t) i1[2];
341 const int32_t vi1x3 = (int32_t) i1[3];
342 i1 += 4;
343
344 const int32_t vk1x0 = (int32_t) ((const int8_t*) w)[4];
345 const int32_t vk1x1 = (int32_t) ((const int8_t*) w)[5];
346 const int32_t vk1x2 = (int32_t) ((const int8_t*) w)[6];
347 const int32_t vk1x3 = (int32_t) ((const int8_t*) w)[7];
348
349 vacc0 += vi1x0 * vk1x0;
350 vacc1 += vi1x1 * vk1x1;
351 vacc2 += vi1x2 * vk1x2;
352 vacc3 += vi1x3 * vk1x3;
353
354 const int32_t vi2x0 = (int32_t) i2[0];
355 const int32_t vi2x1 = (int32_t) i2[1];
356 const int32_t vi2x2 = (int32_t) i2[2];
357 const int32_t vi2x3 = (int32_t) i2[3];
358 i2 += 4;
359
360 const int32_t vk2x0 = (int32_t) ((const int8_t*) w)[8];
361 const int32_t vk2x1 = (int32_t) ((const int8_t*) w)[9];
362 const int32_t vk2x2 = (int32_t) ((const int8_t*) w)[10];
363 const int32_t vk2x3 = (int32_t) ((const int8_t*) w)[11];
364
365 vacc0 += vi2x0 * vk2x0;
366 vacc1 += vi2x1 * vk2x1;
367 vacc2 += vi2x2 * vk2x2;
368 vacc3 += vi2x3 * vk2x3;
369
370 const int32_t vi3x0 = (int32_t) i3[0];
371 const int32_t vi3x1 = (int32_t) i3[1];
372 const int32_t vi3x2 = (int32_t) i3[2];
373 const int32_t vi3x3 = (int32_t) i3[3];
374 i3 += 4;
375
376 const int32_t vk3x0 = (int32_t) ((const int8_t*) w)[12];
377 const int32_t vk3x1 = (int32_t) ((const int8_t*) w)[13];
378 const int32_t vk3x2 = (int32_t) ((const int8_t*) w)[14];
379 const int32_t vk3x3 = (int32_t) ((const int8_t*) w)[15];
380
381 vacc0 += vi3x0 * vk3x0;
382 vacc1 += vi3x1 * vk3x1;
383 vacc2 += vi3x2 * vk3x2;
384 vacc3 += vi3x3 * vk3x3;
385
386 const int32_t vi4x0 = (int32_t) i4[0];
387 const int32_t vi4x1 = (int32_t) i4[1];
388 const int32_t vi4x2 = (int32_t) i4[2];
389 const int32_t vi4x3 = (int32_t) i4[3];
390 i4 += 4;
391
392 const int32_t vk4x0 = (int32_t) ((const int8_t*) w)[16];
393 const int32_t vk4x1 = (int32_t) ((const int8_t*) w)[17];
394 const int32_t vk4x2 = (int32_t) ((const int8_t*) w)[18];
395 const int32_t vk4x3 = (int32_t) ((const int8_t*) w)[19];
396
397 vacc0 += vi4x0 * vk4x0;
398 vacc1 += vi4x1 * vk4x1;
399 vacc2 += vi4x2 * vk4x2;
400 vacc3 += vi4x3 * vk4x3;
401
402 const int32_t vi5x0 = (int32_t) i5[0];
403 const int32_t vi5x1 = (int32_t) i5[1];
404 const int32_t vi5x2 = (int32_t) i5[2];
405 const int32_t vi5x3 = (int32_t) i5[3];
406 i5 += 4;
407
408 const int32_t vk5x0 = (int32_t) ((const int8_t*) w)[20];
409 const int32_t vk5x1 = (int32_t) ((const int8_t*) w)[21];
410 const int32_t vk5x2 = (int32_t) ((const int8_t*) w)[22];
411 const int32_t vk5x3 = (int32_t) ((const int8_t*) w)[23];
412
413 vacc0 += vi5x0 * vk5x0;
414 vacc1 += vi5x1 * vk5x1;
415 vacc2 += vi5x2 * vk5x2;
416 vacc3 += vi5x3 * vk5x3;
417
418 const int32_t vi6x0 = (int32_t) i6[0];
419 const int32_t vi6x1 = (int32_t) i6[1];
420 const int32_t vi6x2 = (int32_t) i6[2];
421 const int32_t vi6x3 = (int32_t) i6[3];
422 i6 += 4;
423
424 const int32_t vk6x0 = (int32_t) ((const int8_t*) w)[24];
425 const int32_t vk6x1 = (int32_t) ((const int8_t*) w)[25];
426 const int32_t vk6x2 = (int32_t) ((const int8_t*) w)[26];
427 const int32_t vk6x3 = (int32_t) ((const int8_t*) w)[27];
428
429 vacc0 += vi6x0 * vk6x0;
430 vacc1 += vi6x1 * vk6x1;
431 vacc2 += vi6x2 * vk6x2;
432 vacc3 += vi6x3 * vk6x3;
433
434 const int32_t vi7x0 = (int32_t) i7[0];
435 const int32_t vi7x1 = (int32_t) i7[1];
436 const int32_t vi7x2 = (int32_t) i7[2];
437 const int32_t vi7x3 = (int32_t) i7[3];
438 i7 += 4;
439
440 const int32_t vk7x0 = (int32_t) ((const int8_t*) w)[28];
441 const int32_t vk7x1 = (int32_t) ((const int8_t*) w)[29];
442 const int32_t vk7x2 = (int32_t) ((const int8_t*) w)[30];
443 const int32_t vk7x3 = (int32_t) ((const int8_t*) w)[31];
444
445 vacc0 += vi7x0 * vk7x0;
446 vacc1 += vi7x1 * vk7x1;
447 vacc2 += vi7x2 * vk7x2;
448 vacc3 += vi7x3 * vk7x3;
449
450 w = (const void*) ((uintptr_t) w + 32 * sizeof(int8_t));
451 b[0] = vacc0;
452 b[1] = vacc1;
453 b[2] = vacc2;
454 b[3] = vacc3;
455 b += 4;
456 }
457 if XNN_UNLIKELY(c != 0) {
458 do {
459 int32_t vacc = *b;
460 const int32_t vi0 = (int32_t) *i0++;
461 const int32_t vk0 = (int32_t) ((const int8_t*) w)[0];
462 vacc += vi0 * vk0;
463 const int32_t vi1 = (int32_t) *i1++;
464 const int32_t vk1 = (int32_t) ((const int8_t*) w)[1];
465 vacc += vi1 * vk1;
466 const int32_t vi2 = (int32_t) *i2++;
467 const int32_t vk2 = (int32_t) ((const int8_t*) w)[2];
468 vacc += vi2 * vk2;
469 const int32_t vi3 = (int32_t) *i3++;
470 const int32_t vk3 = (int32_t) ((const int8_t*) w)[3];
471 vacc += vi3 * vk3;
472 const int32_t vi4 = (int32_t) *i4++;
473 const int32_t vk4 = (int32_t) ((const int8_t*) w)[4];
474 vacc += vi4 * vk4;
475 const int32_t vi5 = (int32_t) *i5++;
476 const int32_t vk5 = (int32_t) ((const int8_t*) w)[5];
477 vacc += vi5 * vk5;
478 const int32_t vi6 = (int32_t) *i6++;
479 const int32_t vk6 = (int32_t) ((const int8_t*) w)[6];
480 vacc += vi6 * vk6;
481 const int32_t vi7 = (int32_t) *i7++;
482 const int32_t vk7 = (int32_t) ((const int8_t*) w)[7];
483 vacc += vi7 * vk7;
484
485 w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
486 *b++ = vacc;
487 } while (--c != 0);
488 }
489 }
490
491 // Last pass to process up to 9 inputs.
492 {
493 const int32_t* b = buffer;
494 const int8_t* i0 = input[0];
495 assert(i0 != NULL);
496 if XNN_UNPREDICTABLE(i0 != zero) {
497 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
498 }
499 const int8_t* i1 = input[1];
500 assert(i1 != NULL);
501 if XNN_UNPREDICTABLE(i1 != zero) {
502 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
503 }
504 const int8_t* i2 = input[2];
505 assert(i2 != NULL);
506 if XNN_UNPREDICTABLE(i2 != zero) {
507 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
508 }
509 const int8_t* i3 = input[3];
510 assert(i3 != NULL);
511 if XNN_UNPREDICTABLE(i3 != zero) {
512 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
513 }
514 const int8_t* i4 = input[4];
515 assert(i4 != NULL);
516 if XNN_UNPREDICTABLE(i4 != zero) {
517 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
518 }
519 const int8_t* i5 = input[5];
520 assert(i5 != NULL);
521 if XNN_UNPREDICTABLE(i5 != zero) {
522 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
523 }
524 const int8_t* i6 = input[6];
525 assert(i6 != NULL);
526 if XNN_UNPREDICTABLE(i6 != zero) {
527 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
528 }
529 const int8_t* i7 = input[7];
530 assert(i7 != NULL);
531 if XNN_UNPREDICTABLE(i7 != zero) {
532 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
533 }
534 const int8_t* i8 = input[8];
535 assert(i8 != NULL);
536 if XNN_UNPREDICTABLE(i8 != zero) {
537 i8 = (const int8_t*) ((uintptr_t) i8 + input_offset);
538 }
539
540 size_t c = channels;
541 for (; c >= 4; c -= 4) {
542 int32_t vacc0 = b[0];
543 int32_t vacc1 = b[1];
544 int32_t vacc2 = b[2];
545 int32_t vacc3 = b[3];
546 b += 4;
547
548 const int32_t vi0x0 = (int32_t) i0[0];
549 const int32_t vi0x1 = (int32_t) i0[1];
550 const int32_t vi0x2 = (int32_t) i0[2];
551 const int32_t vi0x3 = (int32_t) i0[3];
552 i0 += 4;
553
554 const int32_t vk0x0 = (int32_t) ((const int8_t*) w)[0];
555 const int32_t vk0x1 = (int32_t) ((const int8_t*) w)[1];
556 const int32_t vk0x2 = (int32_t) ((const int8_t*) w)[2];
557 const int32_t vk0x3 = (int32_t) ((const int8_t*) w)[3];
558
559 vacc0 += vi0x0 * vk0x0;
560 vacc1 += vi0x1 * vk0x1;
561 vacc2 += vi0x2 * vk0x2;
562 vacc3 += vi0x3 * vk0x3;
563
564 const int32_t vi1x0 = (int32_t) i1[0];
565 const int32_t vi1x1 = (int32_t) i1[1];
566 const int32_t vi1x2 = (int32_t) i1[2];
567 const int32_t vi1x3 = (int32_t) i1[3];
568 i1 += 4;
569
570 const int32_t vk1x0 = (int32_t) ((const int8_t*) w)[4];
571 const int32_t vk1x1 = (int32_t) ((const int8_t*) w)[5];
572 const int32_t vk1x2 = (int32_t) ((const int8_t*) w)[6];
573 const int32_t vk1x3 = (int32_t) ((const int8_t*) w)[7];
574
575 vacc0 += vi1x0 * vk1x0;
576 vacc1 += vi1x1 * vk1x1;
577 vacc2 += vi1x2 * vk1x2;
578 vacc3 += vi1x3 * vk1x3;
579
580 const int32_t vi2x0 = (int32_t) i2[0];
581 const int32_t vi2x1 = (int32_t) i2[1];
582 const int32_t vi2x2 = (int32_t) i2[2];
583 const int32_t vi2x3 = (int32_t) i2[3];
584 i2 += 4;
585
586 const int32_t vk2x0 = (int32_t) ((const int8_t*) w)[8];
587 const int32_t vk2x1 = (int32_t) ((const int8_t*) w)[9];
588 const int32_t vk2x2 = (int32_t) ((const int8_t*) w)[10];
589 const int32_t vk2x3 = (int32_t) ((const int8_t*) w)[11];
590
591 vacc0 += vi2x0 * vk2x0;
592 vacc1 += vi2x1 * vk2x1;
593 vacc2 += vi2x2 * vk2x2;
594 vacc3 += vi2x3 * vk2x3;
595
596 const int32_t vi3x0 = (int32_t) i3[0];
597 const int32_t vi3x1 = (int32_t) i3[1];
598 const int32_t vi3x2 = (int32_t) i3[2];
599 const int32_t vi3x3 = (int32_t) i3[3];
600 i3 += 4;
601
602 const int32_t vk3x0 = (int32_t) ((const int8_t*) w)[12];
603 const int32_t vk3x1 = (int32_t) ((const int8_t*) w)[13];
604 const int32_t vk3x2 = (int32_t) ((const int8_t*) w)[14];
605 const int32_t vk3x3 = (int32_t) ((const int8_t*) w)[15];
606
607 vacc0 += vi3x0 * vk3x0;
608 vacc1 += vi3x1 * vk3x1;
609 vacc2 += vi3x2 * vk3x2;
610 vacc3 += vi3x3 * vk3x3;
611
612 const int32_t vi4x0 = (int32_t) i4[0];
613 const int32_t vi4x1 = (int32_t) i4[1];
614 const int32_t vi4x2 = (int32_t) i4[2];
615 const int32_t vi4x3 = (int32_t) i4[3];
616 i4 += 4;
617
618 const int32_t vk4x0 = (int32_t) ((const int8_t*) w)[16];
619 const int32_t vk4x1 = (int32_t) ((const int8_t*) w)[17];
620 const int32_t vk4x2 = (int32_t) ((const int8_t*) w)[18];
621 const int32_t vk4x3 = (int32_t) ((const int8_t*) w)[19];
622
623 vacc0 += vi4x0 * vk4x0;
624 vacc1 += vi4x1 * vk4x1;
625 vacc2 += vi4x2 * vk4x2;
626 vacc3 += vi4x3 * vk4x3;
627
628 const int32_t vi5x0 = (int32_t) i5[0];
629 const int32_t vi5x1 = (int32_t) i5[1];
630 const int32_t vi5x2 = (int32_t) i5[2];
631 const int32_t vi5x3 = (int32_t) i5[3];
632 i5 += 4;
633
634 const int32_t vk5x0 = (int32_t) ((const int8_t*) w)[20];
635 const int32_t vk5x1 = (int32_t) ((const int8_t*) w)[21];
636 const int32_t vk5x2 = (int32_t) ((const int8_t*) w)[22];
637 const int32_t vk5x3 = (int32_t) ((const int8_t*) w)[23];
638
639 vacc0 += vi5x0 * vk5x0;
640 vacc1 += vi5x1 * vk5x1;
641 vacc2 += vi5x2 * vk5x2;
642 vacc3 += vi5x3 * vk5x3;
643
644 const int32_t vi6x0 = (int32_t) i6[0];
645 const int32_t vi6x1 = (int32_t) i6[1];
646 const int32_t vi6x2 = (int32_t) i6[2];
647 const int32_t vi6x3 = (int32_t) i6[3];
648 i6 += 4;
649
650 const int32_t vk6x0 = (int32_t) ((const int8_t*) w)[24];
651 const int32_t vk6x1 = (int32_t) ((const int8_t*) w)[25];
652 const int32_t vk6x2 = (int32_t) ((const int8_t*) w)[26];
653 const int32_t vk6x3 = (int32_t) ((const int8_t*) w)[27];
654
655 vacc0 += vi6x0 * vk6x0;
656 vacc1 += vi6x1 * vk6x1;
657 vacc2 += vi6x2 * vk6x2;
658 vacc3 += vi6x3 * vk6x3;
659
660 const int32_t vi7x0 = (int32_t) i7[0];
661 const int32_t vi7x1 = (int32_t) i7[1];
662 const int32_t vi7x2 = (int32_t) i7[2];
663 const int32_t vi7x3 = (int32_t) i7[3];
664 i7 += 4;
665
666 const int32_t vk7x0 = (int32_t) ((const int8_t*) w)[28];
667 const int32_t vk7x1 = (int32_t) ((const int8_t*) w)[29];
668 const int32_t vk7x2 = (int32_t) ((const int8_t*) w)[30];
669 const int32_t vk7x3 = (int32_t) ((const int8_t*) w)[31];
670
671 vacc0 += vi7x0 * vk7x0;
672 vacc1 += vi7x1 * vk7x1;
673 vacc2 += vi7x2 * vk7x2;
674 vacc3 += vi7x3 * vk7x3;
675
676 const int32_t vi8x0 = (int32_t) i8[0];
677 const int32_t vi8x1 = (int32_t) i8[1];
678 const int32_t vi8x2 = (int32_t) i8[2];
679 const int32_t vi8x3 = (int32_t) i8[3];
680 i8 += 4;
681
682 const int32_t vk8x0 = (int32_t) ((const int8_t*) w)[32];
683 const int32_t vk8x1 = (int32_t) ((const int8_t*) w)[33];
684 const int32_t vk8x2 = (int32_t) ((const int8_t*) w)[34];
685 const int32_t vk8x3 = (int32_t) ((const int8_t*) w)[35];
686
687 vacc0 += vi8x0 * vk8x0;
688 vacc1 += vi8x1 * vk8x1;
689 vacc2 += vi8x2 * vk8x2;
690 vacc3 += vi8x3 * vk8x3;
691
692 w = (const void*) ((uintptr_t) w + 36 * sizeof(int8_t));
693
694 float vfpacc0 = (float) vacc0;
695 float vfpacc1 = (float) vacc1;
696 float vfpacc2 = (float) vacc2;
697 float vfpacc3 = (float) vacc3;
698
699 vfpacc0 *= vscale;
700 vfpacc1 *= vscale;
701 vfpacc2 *= vscale;
702 vfpacc3 *= vscale;
703
704 vfpacc0 += vmagic_bias;
705 vfpacc1 += vmagic_bias;
706 vfpacc2 += vmagic_bias;
707 vfpacc3 += vmagic_bias;
708
709 int32_t vout0 = (int32_t) float_as_uint32(vfpacc0);
710 int32_t vout1 = (int32_t) float_as_uint32(vfpacc1);
711 int32_t vout2 = (int32_t) float_as_uint32(vfpacc2);
712 int32_t vout3 = (int32_t) float_as_uint32(vfpacc3);
713
714 vout0 = math_max_s32(vout0, vmagic_min);
715 vout1 = math_max_s32(vout1, vmagic_min);
716 vout2 = math_max_s32(vout2, vmagic_min);
717 vout3 = math_max_s32(vout3, vmagic_min);
718
719 vout0 = math_min_s32(vout0, vmagic_max);
720 vout1 = math_min_s32(vout1, vmagic_max);
721 vout2 = math_min_s32(vout2, vmagic_max);
722 vout3 = math_min_s32(vout3, vmagic_max);
723
724 vout0 -= vmagic_bias_less_zero_point;
725 vout1 -= vmagic_bias_less_zero_point;
726 vout2 -= vmagic_bias_less_zero_point;
727 vout3 -= vmagic_bias_less_zero_point;
728
729 output[0] = (int8_t) vout0;
730 output[1] = (int8_t) vout1;
731 output[2] = (int8_t) vout2;
732 output[3] = (int8_t) vout3;
733 output += 4;
734 }
735 if XNN_UNLIKELY(c != 0) {
736 do {
737 int32_t vacc = *b++;
738 const int32_t vi0 = (int32_t) *i0++;
739 const int32_t vk0 = (int32_t) ((const int8_t*) w)[0];
740 vacc += vi0 * vk0;
741 const int32_t vi1 = (int32_t) *i1++;
742 const int32_t vk1 = (int32_t) ((const int8_t*) w)[1];
743 vacc += vi1 * vk1;
744 const int32_t vi2 = (int32_t) *i2++;
745 const int32_t vk2 = (int32_t) ((const int8_t*) w)[2];
746 vacc += vi2 * vk2;
747 const int32_t vi3 = (int32_t) *i3++;
748 const int32_t vk3 = (int32_t) ((const int8_t*) w)[3];
749 vacc += vi3 * vk3;
750 const int32_t vi4 = (int32_t) *i4++;
751 const int32_t vk4 = (int32_t) ((const int8_t*) w)[4];
752 vacc += vi4 * vk4;
753 const int32_t vi5 = (int32_t) *i5++;
754 const int32_t vk5 = (int32_t) ((const int8_t*) w)[5];
755 vacc += vi5 * vk5;
756 const int32_t vi6 = (int32_t) *i6++;
757 const int32_t vk6 = (int32_t) ((const int8_t*) w)[6];
758 vacc += vi6 * vk6;
759 const int32_t vi7 = (int32_t) *i7++;
760 const int32_t vk7 = (int32_t) ((const int8_t*) w)[7];
761 vacc += vi7 * vk7;
762 const int32_t vi8 = (int32_t) *i8++;
763 const int32_t vk8 = (int32_t) ((const int8_t*) w)[8];
764 vacc += vi8 * vk8;
765
766 w = (const void*) ((uintptr_t) w + 9 * sizeof(int8_t));
767
768 float vfpacc = (float) vacc * vscale;
769
770 vfpacc += vmagic_bias;
771 int32_t vout = (int32_t) float_as_uint32(vfpacc);
772 vout = math_max_s32(vout, vmagic_min);
773 vout = math_min_s32(vout, vmagic_max);
774 vout -= vmagic_bias_less_zero_point;
775
776 *output++ = (int8_t) vout;
777 } while (--c != 0);
778 }
779 }
780
781 input = (const int8_t**) ((uintptr_t) input + input_stride);
782 output = (int8_t*) ((uintptr_t) output + output_increment);
783 } while (--output_width != 0);
784 }