Debian Codegraph @

src/qs8-dwconv/gen/qs8-dwconv-6f6m7l4c1s1r-minmax-fp32-scalar-imagic.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-dwconv/multipass-scalar.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11 #include <stddef.h>
12 #include <stdint.h>
13
14 #include "xnnpack/common.h"
15 #include "xnnpack/dwconv.h"
16 #include "xnnpack/math.h"
17 #include "xnnpack/microparams.h"
18 #include "xnnpack/unaligned.h"
19
20
21 void xnn_qs8_dwconv_minmax_fp32_ukernel_6f6m7l4c1s1r__scalar_imagic(
22 size_t channels,
23 size_t output_width,
24 const int8_t** input,
25 const void* weights,
26 int8_t* output,
27 intptr_t input_stride,
28 size_t output_increment,
29 size_t input_offset,
30 const int8_t* zero,
31 size_t kernel_size,
32 int32_t* buffer,
33 const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)])
34 {
35 assert(channels != 0);
36 assert(output_width != 0);
37 assert(kernel_size > 6);
38
39 const float vscale = params->fp32_scalar.scale;
40 const float vmagic_bias = 12582912.0f;
41 const int32_t output_min_less_zero_point = (int32_t) params->fp32_scalar.output_min - (int32_t) params->fp32_scalar.output_zero_point;
42 const int32_t output_max_less_zero_point = (int32_t) params->fp32_scalar.output_max - (int32_t) params->fp32_scalar.output_zero_point;
43 const int32_t vmagic_min = (int32_t) float_as_uint32(12582912.0f + output_min_less_zero_point);
44 const int32_t vmagic_max = (int32_t) float_as_uint32(12582912.0f + output_max_less_zero_point);
45 const int32_t vmagic_bias_less_zero_point = INT32_C(0x4B400000) - (int32_t) params->fp32_scalar.output_zero_point;
46 do {
47 const void* w = weights;
48
49 // First pass to process 6 inputs.
50 {
51 int32_t* b = buffer;
52 const int8_t* i0 = input[0];
53 assert(i0 != NULL);
54 if XNN_UNPREDICTABLE(i0 != zero) {
55 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
56 }
57 const int8_t* i1 = input[1];
58 assert(i1 != NULL);
59 if XNN_UNPREDICTABLE(i1 != zero) {
60 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
61 }
62 const int8_t* i2 = input[2];
63 assert(i2 != NULL);
64 if XNN_UNPREDICTABLE(i2 != zero) {
65 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
66 }
67 const int8_t* i3 = input[3];
68 assert(i3 != NULL);
69 if XNN_UNPREDICTABLE(i3 != zero) {
70 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
71 }
72 const int8_t* i4 = input[4];
73 assert(i4 != NULL);
74 if XNN_UNPREDICTABLE(i4 != zero) {
75 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
76 }
77 const int8_t* i5 = input[5];
78 assert(i5 != NULL);
79 if XNN_UNPREDICTABLE(i5 != zero) {
80 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
81 }
82 input += 6;
83
84 size_t c = channels;
85 for (; c >= 4; c -= 4) {
86 int32_t vacc0 = ((const int32_t*) w)[0];
87 int32_t vacc1 = ((const int32_t*) w)[1];
88 int32_t vacc2 = ((const int32_t*) w)[2];
89 int32_t vacc3 = ((const int32_t*) w)[3];
90
91 const int32_t vi0x0 = (int32_t) i0[0];
92 const int32_t vi0x1 = (int32_t) i0[1];
93 const int32_t vi0x2 = (int32_t) i0[2];
94 const int32_t vi0x3 = (int32_t) i0[3];
95 i0 += 4;
96
97 const int32_t vk0x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[0];
98 const int32_t vk0x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[1];
99 const int32_t vk0x2 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[2];
100 const int32_t vk0x3 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[3];
101
102 vacc0 += vi0x0 * vk0x0;
103 vacc1 += vi0x1 * vk0x1;
104 vacc2 += vi0x2 * vk0x2;
105 vacc3 += vi0x3 * vk0x3;
106
107 const int32_t vi1x0 = (int32_t) i1[0];
108 const int32_t vi1x1 = (int32_t) i1[1];
109 const int32_t vi1x2 = (int32_t) i1[2];
110 const int32_t vi1x3 = (int32_t) i1[3];
111 i1 += 4;
112
113 const int32_t vk1x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[4];
114 const int32_t vk1x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[5];
115 const int32_t vk1x2 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[6];
116 const int32_t vk1x3 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[7];
117
118 vacc0 += vi1x0 * vk1x0;
119 vacc1 += vi1x1 * vk1x1;
120 vacc2 += vi1x2 * vk1x2;
121 vacc3 += vi1x3 * vk1x3;
122
123 const int32_t vi2x0 = (int32_t) i2[0];
124 const int32_t vi2x1 = (int32_t) i2[1];
125 const int32_t vi2x2 = (int32_t) i2[2];
126 const int32_t vi2x3 = (int32_t) i2[3];
127 i2 += 4;
128
129 const int32_t vk2x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[8];
130 const int32_t vk2x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[9];
131 const int32_t vk2x2 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[10];
132 const int32_t vk2x3 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[11];
133
134 vacc0 += vi2x0 * vk2x0;
135 vacc1 += vi2x1 * vk2x1;
136 vacc2 += vi2x2 * vk2x2;
137 vacc3 += vi2x3 * vk2x3;
138
139 const int32_t vi3x0 = (int32_t) i3[0];
140 const int32_t vi3x1 = (int32_t) i3[1];
141 const int32_t vi3x2 = (int32_t) i3[2];
142 const int32_t vi3x3 = (int32_t) i3[3];
143 i3 += 4;
144
145 const int32_t vk3x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[12];
146 const int32_t vk3x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[13];
147 const int32_t vk3x2 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[14];
148 const int32_t vk3x3 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[15];
149
150 vacc0 += vi3x0 * vk3x0;
151 vacc1 += vi3x1 * vk3x1;
152 vacc2 += vi3x2 * vk3x2;
153 vacc3 += vi3x3 * vk3x3;
154
155 const int32_t vi4x0 = (int32_t) i4[0];
156 const int32_t vi4x1 = (int32_t) i4[1];
157 const int32_t vi4x2 = (int32_t) i4[2];
158 const int32_t vi4x3 = (int32_t) i4[3];
159 i4 += 4;
160
161 const int32_t vk4x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[16];
162 const int32_t vk4x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[17];
163 const int32_t vk4x2 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[18];
164 const int32_t vk4x3 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[19];
165
166 vacc0 += vi4x0 * vk4x0;
167 vacc1 += vi4x1 * vk4x1;
168 vacc2 += vi4x2 * vk4x2;
169 vacc3 += vi4x3 * vk4x3;
170
171 const int32_t vi5x0 = (int32_t) i5[0];
172 const int32_t vi5x1 = (int32_t) i5[1];
173 const int32_t vi5x2 = (int32_t) i5[2];
174 const int32_t vi5x3 = (int32_t) i5[3];
175 i5 += 4;
176
177 const int32_t vk5x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[20];
178 const int32_t vk5x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[21];
179 const int32_t vk5x2 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[22];
180 const int32_t vk5x3 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[23];
181
182 vacc0 += vi5x0 * vk5x0;
183 vacc1 += vi5x1 * vk5x1;
184 vacc2 += vi5x2 * vk5x2;
185 vacc3 += vi5x3 * vk5x3;
186
187 w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t) + 24 * sizeof(int8_t));
188 b[0] = vacc0;
189 b[1] = vacc1;
190 b[2] = vacc2;
191 b[3] = vacc3;
192 b += 4;
193 }
194 if XNN_UNLIKELY(c != 0) {
195 do {
196 int32_t vacc = unaligned_load_s32(w);
197 const int32_t vi0 = (int32_t) *i0++;
198 const int32_t vk0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 1 * sizeof(int32_t)))[0];
199 vacc += vi0 * vk0;
200 const int32_t vi1 = (int32_t) *i1++;
201 const int32_t vk1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 1 * sizeof(int32_t)))[1];
202 vacc += vi1 * vk1;
203 const int32_t vi2 = (int32_t) *i2++;
204 const int32_t vk2 = (int32_t) ((const int8_t*) ((uintptr_t) w + 1 * sizeof(int32_t)))[2];
205 vacc += vi2 * vk2;
206 const int32_t vi3 = (int32_t) *i3++;
207 const int32_t vk3 = (int32_t) ((const int8_t*) ((uintptr_t) w + 1 * sizeof(int32_t)))[3];
208 vacc += vi3 * vk3;
209 const int32_t vi4 = (int32_t) *i4++;
210 const int32_t vk4 = (int32_t) ((const int8_t*) ((uintptr_t) w + 1 * sizeof(int32_t)))[4];
211 vacc += vi4 * vk4;
212 const int32_t vi5 = (int32_t) *i5++;
213 const int32_t vk5 = (int32_t) ((const int8_t*) ((uintptr_t) w + 1 * sizeof(int32_t)))[5];
214 vacc += vi5 * vk5;
215
216 w = (const void*) ((uintptr_t) w + 1 * sizeof(int32_t) + 6 * sizeof(int8_t));
217 *b++ = vacc;
218 } while (--c != 0);
219 }
220 }
221
222 // Middle pass to process 6 inputs in each iteration.
223 for (size_t ks = kernel_size - 6; ks > 7; ks -= 6) {
224 int32_t* b = buffer;
225 const int8_t* i0 = input[0];
226 assert(i0 != NULL);
227 if XNN_UNPREDICTABLE(i0 != zero) {
228 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
229 }
230 const int8_t* i1 = input[1];
231 assert(i1 != NULL);
232 if XNN_UNPREDICTABLE(i1 != zero) {
233 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
234 }
235 const int8_t* i2 = input[2];
236 assert(i2 != NULL);
237 if XNN_UNPREDICTABLE(i2 != zero) {
238 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
239 }
240 const int8_t* i3 = input[3];
241 assert(i3 != NULL);
242 if XNN_UNPREDICTABLE(i3 != zero) {
243 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
244 }
245 const int8_t* i4 = input[4];
246 assert(i4 != NULL);
247 if XNN_UNPREDICTABLE(i4 != zero) {
248 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
249 }
250 const int8_t* i5 = input[5];
251 assert(i5 != NULL);
252 if XNN_UNPREDICTABLE(i5 != zero) {
253 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
254 }
255 input += 6;
256
257 size_t c = channels;
258 for (; c >= 4; c -= 4) {
259 int32_t vacc0 = b[0];
260 int32_t vacc1 = b[1];
261 int32_t vacc2 = b[2];
262 int32_t vacc3 = b[3];
263
264 const int32_t vi0x0 = (int32_t) i0[0];
265 const int32_t vi0x1 = (int32_t) i0[1];
266 const int32_t vi0x2 = (int32_t) i0[2];
267 const int32_t vi0x3 = (int32_t) i0[3];
268 i0 += 4;
269
270 const int32_t vk0x0 = (int32_t) ((const int8_t*) w)[0];
271 const int32_t vk0x1 = (int32_t) ((const int8_t*) w)[1];
272 const int32_t vk0x2 = (int32_t) ((const int8_t*) w)[2];
273 const int32_t vk0x3 = (int32_t) ((const int8_t*) w)[3];
274
275 vacc0 += vi0x0 * vk0x0;
276 vacc1 += vi0x1 * vk0x1;
277 vacc2 += vi0x2 * vk0x2;
278 vacc3 += vi0x3 * vk0x3;
279
280 const int32_t vi1x0 = (int32_t) i1[0];
281 const int32_t vi1x1 = (int32_t) i1[1];
282 const int32_t vi1x2 = (int32_t) i1[2];
283 const int32_t vi1x3 = (int32_t) i1[3];
284 i1 += 4;
285
286 const int32_t vk1x0 = (int32_t) ((const int8_t*) w)[4];
287 const int32_t vk1x1 = (int32_t) ((const int8_t*) w)[5];
288 const int32_t vk1x2 = (int32_t) ((const int8_t*) w)[6];
289 const int32_t vk1x3 = (int32_t) ((const int8_t*) w)[7];
290
291 vacc0 += vi1x0 * vk1x0;
292 vacc1 += vi1x1 * vk1x1;
293 vacc2 += vi1x2 * vk1x2;
294 vacc3 += vi1x3 * vk1x3;
295
296 const int32_t vi2x0 = (int32_t) i2[0];
297 const int32_t vi2x1 = (int32_t) i2[1];
298 const int32_t vi2x2 = (int32_t) i2[2];
299 const int32_t vi2x3 = (int32_t) i2[3];
300 i2 += 4;
301
302 const int32_t vk2x0 = (int32_t) ((const int8_t*) w)[8];
303 const int32_t vk2x1 = (int32_t) ((const int8_t*) w)[9];
304 const int32_t vk2x2 = (int32_t) ((const int8_t*) w)[10];
305 const int32_t vk2x3 = (int32_t) ((const int8_t*) w)[11];
306
307 vacc0 += vi2x0 * vk2x0;
308 vacc1 += vi2x1 * vk2x1;
309 vacc2 += vi2x2 * vk2x2;
310 vacc3 += vi2x3 * vk2x3;
311
312 const int32_t vi3x0 = (int32_t) i3[0];
313 const int32_t vi3x1 = (int32_t) i3[1];
314 const int32_t vi3x2 = (int32_t) i3[2];
315 const int32_t vi3x3 = (int32_t) i3[3];
316 i3 += 4;
317
318 const int32_t vk3x0 = (int32_t) ((const int8_t*) w)[12];
319 const int32_t vk3x1 = (int32_t) ((const int8_t*) w)[13];
320 const int32_t vk3x2 = (int32_t) ((const int8_t*) w)[14];
321 const int32_t vk3x3 = (int32_t) ((const int8_t*) w)[15];
322
323 vacc0 += vi3x0 * vk3x0;
324 vacc1 += vi3x1 * vk3x1;
325 vacc2 += vi3x2 * vk3x2;
326 vacc3 += vi3x3 * vk3x3;
327
328 const int32_t vi4x0 = (int32_t) i4[0];
329 const int32_t vi4x1 = (int32_t) i4[1];
330 const int32_t vi4x2 = (int32_t) i4[2];
331 const int32_t vi4x3 = (int32_t) i4[3];
332 i4 += 4;
333
334 const int32_t vk4x0 = (int32_t) ((const int8_t*) w)[16];
335 const int32_t vk4x1 = (int32_t) ((const int8_t*) w)[17];
336 const int32_t vk4x2 = (int32_t) ((const int8_t*) w)[18];
337 const int32_t vk4x3 = (int32_t) ((const int8_t*) w)[19];
338
339 vacc0 += vi4x0 * vk4x0;
340 vacc1 += vi4x1 * vk4x1;
341 vacc2 += vi4x2 * vk4x2;
342 vacc3 += vi4x3 * vk4x3;
343
344 const int32_t vi5x0 = (int32_t) i5[0];
345 const int32_t vi5x1 = (int32_t) i5[1];
346 const int32_t vi5x2 = (int32_t) i5[2];
347 const int32_t vi5x3 = (int32_t) i5[3];
348 i5 += 4;
349
350 const int32_t vk5x0 = (int32_t) ((const int8_t*) w)[20];
351 const int32_t vk5x1 = (int32_t) ((const int8_t*) w)[21];
352 const int32_t vk5x2 = (int32_t) ((const int8_t*) w)[22];
353 const int32_t vk5x3 = (int32_t) ((const int8_t*) w)[23];
354
355 vacc0 += vi5x0 * vk5x0;
356 vacc1 += vi5x1 * vk5x1;
357 vacc2 += vi5x2 * vk5x2;
358 vacc3 += vi5x3 * vk5x3;
359
360 w = (const void*) ((uintptr_t) w + 24 * sizeof(int8_t));
361 b[0] = vacc0;
362 b[1] = vacc1;
363 b[2] = vacc2;
364 b[3] = vacc3;
365 b += 4;
366 }
367 if XNN_UNLIKELY(c != 0) {
368 do {
369 int32_t vacc = *b;
370 const int32_t vi0 = (int32_t) *i0++;
371 const int32_t vk0 = (int32_t) ((const int8_t*) w)[0];
372 vacc += vi0 * vk0;
373 const int32_t vi1 = (int32_t) *i1++;
374 const int32_t vk1 = (int32_t) ((const int8_t*) w)[1];
375 vacc += vi1 * vk1;
376 const int32_t vi2 = (int32_t) *i2++;
377 const int32_t vk2 = (int32_t) ((const int8_t*) w)[2];
378 vacc += vi2 * vk2;
379 const int32_t vi3 = (int32_t) *i3++;
380 const int32_t vk3 = (int32_t) ((const int8_t*) w)[3];
381 vacc += vi3 * vk3;
382 const int32_t vi4 = (int32_t) *i4++;
383 const int32_t vk4 = (int32_t) ((const int8_t*) w)[4];
384 vacc += vi4 * vk4;
385 const int32_t vi5 = (int32_t) *i5++;
386 const int32_t vk5 = (int32_t) ((const int8_t*) w)[5];
387 vacc += vi5 * vk5;
388
389 w = (const void*) ((uintptr_t) w + 6 * sizeof(int8_t));
390 *b++ = vacc;
391 } while (--c != 0);
392 }
393 }
394
395 // Last pass to process up to 7 inputs.
396 {
397 const int32_t* b = buffer;
398 const int8_t* i0 = input[0];
399 assert(i0 != NULL);
400 if XNN_UNPREDICTABLE(i0 != zero) {
401 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
402 }
403 const int8_t* i1 = input[1];
404 assert(i1 != NULL);
405 if XNN_UNPREDICTABLE(i1 != zero) {
406 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
407 }
408 const int8_t* i2 = input[2];
409 assert(i2 != NULL);
410 if XNN_UNPREDICTABLE(i2 != zero) {
411 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
412 }
413 const int8_t* i3 = input[3];
414 assert(i3 != NULL);
415 if XNN_UNPREDICTABLE(i3 != zero) {
416 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
417 }
418 const int8_t* i4 = input[4];
419 assert(i4 != NULL);
420 if XNN_UNPREDICTABLE(i4 != zero) {
421 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
422 }
423 const int8_t* i5 = input[5];
424 assert(i5 != NULL);
425 if XNN_UNPREDICTABLE(i5 != zero) {
426 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
427 }
428 const int8_t* i6 = input[6];
429 assert(i6 != NULL);
430 if XNN_UNPREDICTABLE(i6 != zero) {
431 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
432 }
433
434 size_t c = channels;
435 for (; c >= 4; c -= 4) {
436 int32_t vacc0 = b[0];
437 int32_t vacc1 = b[1];
438 int32_t vacc2 = b[2];
439 int32_t vacc3 = b[3];
440 b += 4;
441
442 const int32_t vi0x0 = (int32_t) i0[0];
443 const int32_t vi0x1 = (int32_t) i0[1];
444 const int32_t vi0x2 = (int32_t) i0[2];
445 const int32_t vi0x3 = (int32_t) i0[3];
446 i0 += 4;
447
448 const int32_t vk0x0 = (int32_t) ((const int8_t*) w)[0];
449 const int32_t vk0x1 = (int32_t) ((const int8_t*) w)[1];
450 const int32_t vk0x2 = (int32_t) ((const int8_t*) w)[2];
451 const int32_t vk0x3 = (int32_t) ((const int8_t*) w)[3];
452
453 vacc0 += vi0x0 * vk0x0;
454 vacc1 += vi0x1 * vk0x1;
455 vacc2 += vi0x2 * vk0x2;
456 vacc3 += vi0x3 * vk0x3;
457
458 const int32_t vi1x0 = (int32_t) i1[0];
459 const int32_t vi1x1 = (int32_t) i1[1];
460 const int32_t vi1x2 = (int32_t) i1[2];
461 const int32_t vi1x3 = (int32_t) i1[3];
462 i1 += 4;
463
464 const int32_t vk1x0 = (int32_t) ((const int8_t*) w)[4];
465 const int32_t vk1x1 = (int32_t) ((const int8_t*) w)[5];
466 const int32_t vk1x2 = (int32_t) ((const int8_t*) w)[6];
467 const int32_t vk1x3 = (int32_t) ((const int8_t*) w)[7];
468
469 vacc0 += vi1x0 * vk1x0;
470 vacc1 += vi1x1 * vk1x1;
471 vacc2 += vi1x2 * vk1x2;
472 vacc3 += vi1x3 * vk1x3;
473
474 const int32_t vi2x0 = (int32_t) i2[0];
475 const int32_t vi2x1 = (int32_t) i2[1];
476 const int32_t vi2x2 = (int32_t) i2[2];
477 const int32_t vi2x3 = (int32_t) i2[3];
478 i2 += 4;
479
480 const int32_t vk2x0 = (int32_t) ((const int8_t*) w)[8];
481 const int32_t vk2x1 = (int32_t) ((const int8_t*) w)[9];
482 const int32_t vk2x2 = (int32_t) ((const int8_t*) w)[10];
483 const int32_t vk2x3 = (int32_t) ((const int8_t*) w)[11];
484
485 vacc0 += vi2x0 * vk2x0;
486 vacc1 += vi2x1 * vk2x1;
487 vacc2 += vi2x2 * vk2x2;
488 vacc3 += vi2x3 * vk2x3;
489
490 const int32_t vi3x0 = (int32_t) i3[0];
491 const int32_t vi3x1 = (int32_t) i3[1];
492 const int32_t vi3x2 = (int32_t) i3[2];
493 const int32_t vi3x3 = (int32_t) i3[3];
494 i3 += 4;
495
496 const int32_t vk3x0 = (int32_t) ((const int8_t*) w)[12];
497 const int32_t vk3x1 = (int32_t) ((const int8_t*) w)[13];
498 const int32_t vk3x2 = (int32_t) ((const int8_t*) w)[14];
499 const int32_t vk3x3 = (int32_t) ((const int8_t*) w)[15];
500
501 vacc0 += vi3x0 * vk3x0;
502 vacc1 += vi3x1 * vk3x1;
503 vacc2 += vi3x2 * vk3x2;
504 vacc3 += vi3x3 * vk3x3;
505
506 const int32_t vi4x0 = (int32_t) i4[0];
507 const int32_t vi4x1 = (int32_t) i4[1];
508 const int32_t vi4x2 = (int32_t) i4[2];
509 const int32_t vi4x3 = (int32_t) i4[3];
510 i4 += 4;
511
512 const int32_t vk4x0 = (int32_t) ((const int8_t*) w)[16];
513 const int32_t vk4x1 = (int32_t) ((const int8_t*) w)[17];
514 const int32_t vk4x2 = (int32_t) ((const int8_t*) w)[18];
515 const int32_t vk4x3 = (int32_t) ((const int8_t*) w)[19];
516
517 vacc0 += vi4x0 * vk4x0;
518 vacc1 += vi4x1 * vk4x1;
519 vacc2 += vi4x2 * vk4x2;
520 vacc3 += vi4x3 * vk4x3;
521
522 const int32_t vi5x0 = (int32_t) i5[0];
523 const int32_t vi5x1 = (int32_t) i5[1];
524 const int32_t vi5x2 = (int32_t) i5[2];
525 const int32_t vi5x3 = (int32_t) i5[3];
526 i5 += 4;
527
528 const int32_t vk5x0 = (int32_t) ((const int8_t*) w)[20];
529 const int32_t vk5x1 = (int32_t) ((const int8_t*) w)[21];
530 const int32_t vk5x2 = (int32_t) ((const int8_t*) w)[22];
531 const int32_t vk5x3 = (int32_t) ((const int8_t*) w)[23];
532
533 vacc0 += vi5x0 * vk5x0;
534 vacc1 += vi5x1 * vk5x1;
535 vacc2 += vi5x2 * vk5x2;
536 vacc3 += vi5x3 * vk5x3;
537
538 const int32_t vi6x0 = (int32_t) i6[0];
539 const int32_t vi6x1 = (int32_t) i6[1];
540 const int32_t vi6x2 = (int32_t) i6[2];
541 const int32_t vi6x3 = (int32_t) i6[3];
542 i6 += 4;
543
544 const int32_t vk6x0 = (int32_t) ((const int8_t*) w)[24];
545 const int32_t vk6x1 = (int32_t) ((const int8_t*) w)[25];
546 const int32_t vk6x2 = (int32_t) ((const int8_t*) w)[26];
547 const int32_t vk6x3 = (int32_t) ((const int8_t*) w)[27];
548
549 vacc0 += vi6x0 * vk6x0;
550 vacc1 += vi6x1 * vk6x1;
551 vacc2 += vi6x2 * vk6x2;
552 vacc3 += vi6x3 * vk6x3;
553
554 w = (const void*) ((uintptr_t) w + 28 * sizeof(int8_t));
555
556 float vfpacc0 = (float) vacc0;
557 float vfpacc1 = (float) vacc1;
558 float vfpacc2 = (float) vacc2;
559 float vfpacc3 = (float) vacc3;
560
561 vfpacc0 *= vscale;
562 vfpacc1 *= vscale;
563 vfpacc2 *= vscale;
564 vfpacc3 *= vscale;
565
566 vfpacc0 += vmagic_bias;
567 vfpacc1 += vmagic_bias;
568 vfpacc2 += vmagic_bias;
569 vfpacc3 += vmagic_bias;
570
571 int32_t vout0 = (int32_t) float_as_uint32(vfpacc0);
572 int32_t vout1 = (int32_t) float_as_uint32(vfpacc1);
573 int32_t vout2 = (int32_t) float_as_uint32(vfpacc2);
574 int32_t vout3 = (int32_t) float_as_uint32(vfpacc3);
575
576 vout0 = math_max_s32(vout0, vmagic_min);
577 vout1 = math_max_s32(vout1, vmagic_min);
578 vout2 = math_max_s32(vout2, vmagic_min);
579 vout3 = math_max_s32(vout3, vmagic_min);
580
581 vout0 = math_min_s32(vout0, vmagic_max);
582 vout1 = math_min_s32(vout1, vmagic_max);
583 vout2 = math_min_s32(vout2, vmagic_max);
584 vout3 = math_min_s32(vout3, vmagic_max);
585
586 vout0 -= vmagic_bias_less_zero_point;
587 vout1 -= vmagic_bias_less_zero_point;
588 vout2 -= vmagic_bias_less_zero_point;
589 vout3 -= vmagic_bias_less_zero_point;
590
591 output[0] = (int8_t) vout0;
592 output[1] = (int8_t) vout1;
593 output[2] = (int8_t) vout2;
594 output[3] = (int8_t) vout3;
595 output += 4;
596 }
597 if XNN_UNLIKELY(c != 0) {
598 do {
599 int32_t vacc = *b++;
600 const int32_t vi0 = (int32_t) *i0++;
601 const int32_t vk0 = (int32_t) ((const int8_t*) w)[0];
602 vacc += vi0 * vk0;
603 const int32_t vi1 = (int32_t) *i1++;
604 const int32_t vk1 = (int32_t) ((const int8_t*) w)[1];
605 vacc += vi1 * vk1;
606 const int32_t vi2 = (int32_t) *i2++;
607 const int32_t vk2 = (int32_t) ((const int8_t*) w)[2];
608 vacc += vi2 * vk2;
609 const int32_t vi3 = (int32_t) *i3++;
610 const int32_t vk3 = (int32_t) ((const int8_t*) w)[3];
611 vacc += vi3 * vk3;
612 const int32_t vi4 = (int32_t) *i4++;
613 const int32_t vk4 = (int32_t) ((const int8_t*) w)[4];
614 vacc += vi4 * vk4;
615 const int32_t vi5 = (int32_t) *i5++;
616 const int32_t vk5 = (int32_t) ((const int8_t*) w)[5];
617 vacc += vi5 * vk5;
618 const int32_t vi6 = (int32_t) *i6++;
619 const int32_t vk6 = (int32_t) ((const int8_t*) w)[6];
620 vacc += vi6 * vk6;
621
622 w = (const void*) ((uintptr_t) w + 7 * sizeof(int8_t));
623
624 float vfpacc = (float) vacc * vscale;
625
626 vfpacc += vmagic_bias;
627 int32_t vout = (int32_t) float_as_uint32(vfpacc);
628 vout = math_max_s32(vout, vmagic_min);
629 vout = math_min_s32(vout, vmagic_max);
630 vout -= vmagic_bias_less_zero_point;
631
632 *output++ = (int8_t) vout;
633 } while (--c != 0);
634 }
635 }
636
637 input = (const int8_t**) ((uintptr_t) input + input_stride);
638 output = (int8_t*) ((uintptr_t) output + output_increment);
639 } while (--output_width != 0);
640 }