Debian Codegraph @

src/qs8-dwconv/gen/qs8-dwconv-5f5m5l4c1s1r-minmax-fp32-scalar-imagic.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-dwconv/multipass-scalar.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11 #include <stddef.h>
12 #include <stdint.h>
13
14 #include "xnnpack/common.h"
15 #include "xnnpack/dwconv.h"
16 #include "xnnpack/math.h"
17 #include "xnnpack/microparams.h"
18 #include "xnnpack/unaligned.h"
19
20
21 void xnn_qs8_dwconv_minmax_fp32_ukernel_5f5m5l4c1s1r__scalar_imagic(
22 size_t channels,
23 size_t output_width,
24 const int8_t** input,
25 const void* weights,
26 int8_t* output,
27 intptr_t input_stride,
28 size_t output_increment,
29 size_t input_offset,
30 const int8_t* zero,
31 size_t kernel_size,
32 int32_t* buffer,
33 const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)])
34 {
35 assert(channels != 0);
36 assert(output_width != 0);
37 assert(kernel_size > 5);
38
39 const float vscale = params->fp32_scalar.scale;
40 const float vmagic_bias = 12582912.0f;
41 const int32_t output_min_less_zero_point = (int32_t) params->fp32_scalar.output_min - (int32_t) params->fp32_scalar.output_zero_point;
42 const int32_t output_max_less_zero_point = (int32_t) params->fp32_scalar.output_max - (int32_t) params->fp32_scalar.output_zero_point;
43 const int32_t vmagic_min = (int32_t) float_as_uint32(12582912.0f + output_min_less_zero_point);
44 const int32_t vmagic_max = (int32_t) float_as_uint32(12582912.0f + output_max_less_zero_point);
45 const int32_t vmagic_bias_less_zero_point = INT32_C(0x4B400000) - (int32_t) params->fp32_scalar.output_zero_point;
46 do {
47 const void* w = weights;
48
49 // First pass to process 5 inputs.
50 {
51 int32_t* b = buffer;
52 const int8_t* i0 = input[0];
53 assert(i0 != NULL);
54 if XNN_UNPREDICTABLE(i0 != zero) {
55 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
56 }
57 const int8_t* i1 = input[1];
58 assert(i1 != NULL);
59 if XNN_UNPREDICTABLE(i1 != zero) {
60 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
61 }
62 const int8_t* i2 = input[2];
63 assert(i2 != NULL);
64 if XNN_UNPREDICTABLE(i2 != zero) {
65 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
66 }
67 const int8_t* i3 = input[3];
68 assert(i3 != NULL);
69 if XNN_UNPREDICTABLE(i3 != zero) {
70 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
71 }
72 const int8_t* i4 = input[4];
73 assert(i4 != NULL);
74 if XNN_UNPREDICTABLE(i4 != zero) {
75 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
76 }
77 input += 5;
78
79 size_t c = channels;
80 for (; c >= 4; c -= 4) {
81 int32_t vacc0 = ((const int32_t*) w)[0];
82 int32_t vacc1 = ((const int32_t*) w)[1];
83 int32_t vacc2 = ((const int32_t*) w)[2];
84 int32_t vacc3 = ((const int32_t*) w)[3];
85
86 const int32_t vi0x0 = (int32_t) i0[0];
87 const int32_t vi0x1 = (int32_t) i0[1];
88 const int32_t vi0x2 = (int32_t) i0[2];
89 const int32_t vi0x3 = (int32_t) i0[3];
90 i0 += 4;
91
92 const int32_t vk0x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[0];
93 const int32_t vk0x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[1];
94 const int32_t vk0x2 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[2];
95 const int32_t vk0x3 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[3];
96
97 vacc0 += vi0x0 * vk0x0;
98 vacc1 += vi0x1 * vk0x1;
99 vacc2 += vi0x2 * vk0x2;
100 vacc3 += vi0x3 * vk0x3;
101
102 const int32_t vi1x0 = (int32_t) i1[0];
103 const int32_t vi1x1 = (int32_t) i1[1];
104 const int32_t vi1x2 = (int32_t) i1[2];
105 const int32_t vi1x3 = (int32_t) i1[3];
106 i1 += 4;
107
108 const int32_t vk1x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[4];
109 const int32_t vk1x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[5];
110 const int32_t vk1x2 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[6];
111 const int32_t vk1x3 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[7];
112
113 vacc0 += vi1x0 * vk1x0;
114 vacc1 += vi1x1 * vk1x1;
115 vacc2 += vi1x2 * vk1x2;
116 vacc3 += vi1x3 * vk1x3;
117
118 const int32_t vi2x0 = (int32_t) i2[0];
119 const int32_t vi2x1 = (int32_t) i2[1];
120 const int32_t vi2x2 = (int32_t) i2[2];
121 const int32_t vi2x3 = (int32_t) i2[3];
122 i2 += 4;
123
124 const int32_t vk2x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[8];
125 const int32_t vk2x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[9];
126 const int32_t vk2x2 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[10];
127 const int32_t vk2x3 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[11];
128
129 vacc0 += vi2x0 * vk2x0;
130 vacc1 += vi2x1 * vk2x1;
131 vacc2 += vi2x2 * vk2x2;
132 vacc3 += vi2x3 * vk2x3;
133
134 const int32_t vi3x0 = (int32_t) i3[0];
135 const int32_t vi3x1 = (int32_t) i3[1];
136 const int32_t vi3x2 = (int32_t) i3[2];
137 const int32_t vi3x3 = (int32_t) i3[3];
138 i3 += 4;
139
140 const int32_t vk3x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[12];
141 const int32_t vk3x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[13];
142 const int32_t vk3x2 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[14];
143 const int32_t vk3x3 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[15];
144
145 vacc0 += vi3x0 * vk3x0;
146 vacc1 += vi3x1 * vk3x1;
147 vacc2 += vi3x2 * vk3x2;
148 vacc3 += vi3x3 * vk3x3;
149
150 const int32_t vi4x0 = (int32_t) i4[0];
151 const int32_t vi4x1 = (int32_t) i4[1];
152 const int32_t vi4x2 = (int32_t) i4[2];
153 const int32_t vi4x3 = (int32_t) i4[3];
154 i4 += 4;
155
156 const int32_t vk4x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[16];
157 const int32_t vk4x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[17];
158 const int32_t vk4x2 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[18];
159 const int32_t vk4x3 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[19];
160
161 vacc0 += vi4x0 * vk4x0;
162 vacc1 += vi4x1 * vk4x1;
163 vacc2 += vi4x2 * vk4x2;
164 vacc3 += vi4x3 * vk4x3;
165
166 w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t) + 20 * sizeof(int8_t));
167 b[0] = vacc0;
168 b[1] = vacc1;
169 b[2] = vacc2;
170 b[3] = vacc3;
171 b += 4;
172 }
173 if XNN_UNLIKELY(c != 0) {
174 do {
175 int32_t vacc = unaligned_load_s32(w);
176 const int32_t vi0 = (int32_t) *i0++;
177 const int32_t vk0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 1 * sizeof(int32_t)))[0];
178 vacc += vi0 * vk0;
179 const int32_t vi1 = (int32_t) *i1++;
180 const int32_t vk1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 1 * sizeof(int32_t)))[1];
181 vacc += vi1 * vk1;
182 const int32_t vi2 = (int32_t) *i2++;
183 const int32_t vk2 = (int32_t) ((const int8_t*) ((uintptr_t) w + 1 * sizeof(int32_t)))[2];
184 vacc += vi2 * vk2;
185 const int32_t vi3 = (int32_t) *i3++;
186 const int32_t vk3 = (int32_t) ((const int8_t*) ((uintptr_t) w + 1 * sizeof(int32_t)))[3];
187 vacc += vi3 * vk3;
188 const int32_t vi4 = (int32_t) *i4++;
189 const int32_t vk4 = (int32_t) ((const int8_t*) ((uintptr_t) w + 1 * sizeof(int32_t)))[4];
190 vacc += vi4 * vk4;
191
192 w = (const void*) ((uintptr_t) w + 1 * sizeof(int32_t) + 5 * sizeof(int8_t));
193 *b++ = vacc;
194 } while (--c != 0);
195 }
196 }
197
198 // Middle pass to process 5 inputs in each iteration.
199 for (size_t ks = kernel_size - 5; ks > 5; ks -= 5) {
200 int32_t* b = buffer;
201 const int8_t* i0 = input[0];
202 assert(i0 != NULL);
203 if XNN_UNPREDICTABLE(i0 != zero) {
204 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
205 }
206 const int8_t* i1 = input[1];
207 assert(i1 != NULL);
208 if XNN_UNPREDICTABLE(i1 != zero) {
209 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
210 }
211 const int8_t* i2 = input[2];
212 assert(i2 != NULL);
213 if XNN_UNPREDICTABLE(i2 != zero) {
214 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
215 }
216 const int8_t* i3 = input[3];
217 assert(i3 != NULL);
218 if XNN_UNPREDICTABLE(i3 != zero) {
219 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
220 }
221 const int8_t* i4 = input[4];
222 assert(i4 != NULL);
223 if XNN_UNPREDICTABLE(i4 != zero) {
224 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
225 }
226 input += 5;
227
228 size_t c = channels;
229 for (; c >= 4; c -= 4) {
230 int32_t vacc0 = b[0];
231 int32_t vacc1 = b[1];
232 int32_t vacc2 = b[2];
233 int32_t vacc3 = b[3];
234
235 const int32_t vi0x0 = (int32_t) i0[0];
236 const int32_t vi0x1 = (int32_t) i0[1];
237 const int32_t vi0x2 = (int32_t) i0[2];
238 const int32_t vi0x3 = (int32_t) i0[3];
239 i0 += 4;
240
241 const int32_t vk0x0 = (int32_t) ((const int8_t*) w)[0];
242 const int32_t vk0x1 = (int32_t) ((const int8_t*) w)[1];
243 const int32_t vk0x2 = (int32_t) ((const int8_t*) w)[2];
244 const int32_t vk0x3 = (int32_t) ((const int8_t*) w)[3];
245
246 vacc0 += vi0x0 * vk0x0;
247 vacc1 += vi0x1 * vk0x1;
248 vacc2 += vi0x2 * vk0x2;
249 vacc3 += vi0x3 * vk0x3;
250
251 const int32_t vi1x0 = (int32_t) i1[0];
252 const int32_t vi1x1 = (int32_t) i1[1];
253 const int32_t vi1x2 = (int32_t) i1[2];
254 const int32_t vi1x3 = (int32_t) i1[3];
255 i1 += 4;
256
257 const int32_t vk1x0 = (int32_t) ((const int8_t*) w)[4];
258 const int32_t vk1x1 = (int32_t) ((const int8_t*) w)[5];
259 const int32_t vk1x2 = (int32_t) ((const int8_t*) w)[6];
260 const int32_t vk1x3 = (int32_t) ((const int8_t*) w)[7];
261
262 vacc0 += vi1x0 * vk1x0;
263 vacc1 += vi1x1 * vk1x1;
264 vacc2 += vi1x2 * vk1x2;
265 vacc3 += vi1x3 * vk1x3;
266
267 const int32_t vi2x0 = (int32_t) i2[0];
268 const int32_t vi2x1 = (int32_t) i2[1];
269 const int32_t vi2x2 = (int32_t) i2[2];
270 const int32_t vi2x3 = (int32_t) i2[3];
271 i2 += 4;
272
273 const int32_t vk2x0 = (int32_t) ((const int8_t*) w)[8];
274 const int32_t vk2x1 = (int32_t) ((const int8_t*) w)[9];
275 const int32_t vk2x2 = (int32_t) ((const int8_t*) w)[10];
276 const int32_t vk2x3 = (int32_t) ((const int8_t*) w)[11];
277
278 vacc0 += vi2x0 * vk2x0;
279 vacc1 += vi2x1 * vk2x1;
280 vacc2 += vi2x2 * vk2x2;
281 vacc3 += vi2x3 * vk2x3;
282
283 const int32_t vi3x0 = (int32_t) i3[0];
284 const int32_t vi3x1 = (int32_t) i3[1];
285 const int32_t vi3x2 = (int32_t) i3[2];
286 const int32_t vi3x3 = (int32_t) i3[3];
287 i3 += 4;
288
289 const int32_t vk3x0 = (int32_t) ((const int8_t*) w)[12];
290 const int32_t vk3x1 = (int32_t) ((const int8_t*) w)[13];
291 const int32_t vk3x2 = (int32_t) ((const int8_t*) w)[14];
292 const int32_t vk3x3 = (int32_t) ((const int8_t*) w)[15];
293
294 vacc0 += vi3x0 * vk3x0;
295 vacc1 += vi3x1 * vk3x1;
296 vacc2 += vi3x2 * vk3x2;
297 vacc3 += vi3x3 * vk3x3;
298
299 const int32_t vi4x0 = (int32_t) i4[0];
300 const int32_t vi4x1 = (int32_t) i4[1];
301 const int32_t vi4x2 = (int32_t) i4[2];
302 const int32_t vi4x3 = (int32_t) i4[3];
303 i4 += 4;
304
305 const int32_t vk4x0 = (int32_t) ((const int8_t*) w)[16];
306 const int32_t vk4x1 = (int32_t) ((const int8_t*) w)[17];
307 const int32_t vk4x2 = (int32_t) ((const int8_t*) w)[18];
308 const int32_t vk4x3 = (int32_t) ((const int8_t*) w)[19];
309
310 vacc0 += vi4x0 * vk4x0;
311 vacc1 += vi4x1 * vk4x1;
312 vacc2 += vi4x2 * vk4x2;
313 vacc3 += vi4x3 * vk4x3;
314
315 w = (const void*) ((uintptr_t) w + 20 * sizeof(int8_t));
316 b[0] = vacc0;
317 b[1] = vacc1;
318 b[2] = vacc2;
319 b[3] = vacc3;
320 b += 4;
321 }
322 if XNN_UNLIKELY(c != 0) {
323 do {
324 int32_t vacc = *b;
325 const int32_t vi0 = (int32_t) *i0++;
326 const int32_t vk0 = (int32_t) ((const int8_t*) w)[0];
327 vacc += vi0 * vk0;
328 const int32_t vi1 = (int32_t) *i1++;
329 const int32_t vk1 = (int32_t) ((const int8_t*) w)[1];
330 vacc += vi1 * vk1;
331 const int32_t vi2 = (int32_t) *i2++;
332 const int32_t vk2 = (int32_t) ((const int8_t*) w)[2];
333 vacc += vi2 * vk2;
334 const int32_t vi3 = (int32_t) *i3++;
335 const int32_t vk3 = (int32_t) ((const int8_t*) w)[3];
336 vacc += vi3 * vk3;
337 const int32_t vi4 = (int32_t) *i4++;
338 const int32_t vk4 = (int32_t) ((const int8_t*) w)[4];
339 vacc += vi4 * vk4;
340
341 w = (const void*) ((uintptr_t) w + 5 * sizeof(int8_t));
342 *b++ = vacc;
343 } while (--c != 0);
344 }
345 }
346
347 // Last pass to process up to 5 inputs.
348 {
349 const int32_t* b = buffer;
350 const int8_t* i0 = input[0];
351 assert(i0 != NULL);
352 if XNN_UNPREDICTABLE(i0 != zero) {
353 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
354 }
355 const int8_t* i1 = input[1];
356 assert(i1 != NULL);
357 if XNN_UNPREDICTABLE(i1 != zero) {
358 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
359 }
360 const int8_t* i2 = input[2];
361 assert(i2 != NULL);
362 if XNN_UNPREDICTABLE(i2 != zero) {
363 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
364 }
365 const int8_t* i3 = input[3];
366 assert(i3 != NULL);
367 if XNN_UNPREDICTABLE(i3 != zero) {
368 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
369 }
370 const int8_t* i4 = input[4];
371 assert(i4 != NULL);
372 if XNN_UNPREDICTABLE(i4 != zero) {
373 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
374 }
375
376 size_t c = channels;
377 for (; c >= 4; c -= 4) {
378 int32_t vacc0 = b[0];
379 int32_t vacc1 = b[1];
380 int32_t vacc2 = b[2];
381 int32_t vacc3 = b[3];
382 b += 4;
383
384 const int32_t vi0x0 = (int32_t) i0[0];
385 const int32_t vi0x1 = (int32_t) i0[1];
386 const int32_t vi0x2 = (int32_t) i0[2];
387 const int32_t vi0x3 = (int32_t) i0[3];
388 i0 += 4;
389
390 const int32_t vk0x0 = (int32_t) ((const int8_t*) w)[0];
391 const int32_t vk0x1 = (int32_t) ((const int8_t*) w)[1];
392 const int32_t vk0x2 = (int32_t) ((const int8_t*) w)[2];
393 const int32_t vk0x3 = (int32_t) ((const int8_t*) w)[3];
394
395 vacc0 += vi0x0 * vk0x0;
396 vacc1 += vi0x1 * vk0x1;
397 vacc2 += vi0x2 * vk0x2;
398 vacc3 += vi0x3 * vk0x3;
399
400 const int32_t vi1x0 = (int32_t) i1[0];
401 const int32_t vi1x1 = (int32_t) i1[1];
402 const int32_t vi1x2 = (int32_t) i1[2];
403 const int32_t vi1x3 = (int32_t) i1[3];
404 i1 += 4;
405
406 const int32_t vk1x0 = (int32_t) ((const int8_t*) w)[4];
407 const int32_t vk1x1 = (int32_t) ((const int8_t*) w)[5];
408 const int32_t vk1x2 = (int32_t) ((const int8_t*) w)[6];
409 const int32_t vk1x3 = (int32_t) ((const int8_t*) w)[7];
410
411 vacc0 += vi1x0 * vk1x0;
412 vacc1 += vi1x1 * vk1x1;
413 vacc2 += vi1x2 * vk1x2;
414 vacc3 += vi1x3 * vk1x3;
415
416 const int32_t vi2x0 = (int32_t) i2[0];
417 const int32_t vi2x1 = (int32_t) i2[1];
418 const int32_t vi2x2 = (int32_t) i2[2];
419 const int32_t vi2x3 = (int32_t) i2[3];
420 i2 += 4;
421
422 const int32_t vk2x0 = (int32_t) ((const int8_t*) w)[8];
423 const int32_t vk2x1 = (int32_t) ((const int8_t*) w)[9];
424 const int32_t vk2x2 = (int32_t) ((const int8_t*) w)[10];
425 const int32_t vk2x3 = (int32_t) ((const int8_t*) w)[11];
426
427 vacc0 += vi2x0 * vk2x0;
428 vacc1 += vi2x1 * vk2x1;
429 vacc2 += vi2x2 * vk2x2;
430 vacc3 += vi2x3 * vk2x3;
431
432 const int32_t vi3x0 = (int32_t) i3[0];
433 const int32_t vi3x1 = (int32_t) i3[1];
434 const int32_t vi3x2 = (int32_t) i3[2];
435 const int32_t vi3x3 = (int32_t) i3[3];
436 i3 += 4;
437
438 const int32_t vk3x0 = (int32_t) ((const int8_t*) w)[12];
439 const int32_t vk3x1 = (int32_t) ((const int8_t*) w)[13];
440 const int32_t vk3x2 = (int32_t) ((const int8_t*) w)[14];
441 const int32_t vk3x3 = (int32_t) ((const int8_t*) w)[15];
442
443 vacc0 += vi3x0 * vk3x0;
444 vacc1 += vi3x1 * vk3x1;
445 vacc2 += vi3x2 * vk3x2;
446 vacc3 += vi3x3 * vk3x3;
447
448 const int32_t vi4x0 = (int32_t) i4[0];
449 const int32_t vi4x1 = (int32_t) i4[1];
450 const int32_t vi4x2 = (int32_t) i4[2];
451 const int32_t vi4x3 = (int32_t) i4[3];
452 i4 += 4;
453
454 const int32_t vk4x0 = (int32_t) ((const int8_t*) w)[16];
455 const int32_t vk4x1 = (int32_t) ((const int8_t*) w)[17];
456 const int32_t vk4x2 = (int32_t) ((const int8_t*) w)[18];
457 const int32_t vk4x3 = (int32_t) ((const int8_t*) w)[19];
458
459 vacc0 += vi4x0 * vk4x0;
460 vacc1 += vi4x1 * vk4x1;
461 vacc2 += vi4x2 * vk4x2;
462 vacc3 += vi4x3 * vk4x3;
463
464 w = (const void*) ((uintptr_t) w + 20 * sizeof(int8_t));
465
466 float vfpacc0 = (float) vacc0;
467 float vfpacc1 = (float) vacc1;
468 float vfpacc2 = (float) vacc2;
469 float vfpacc3 = (float) vacc3;
470
471 vfpacc0 *= vscale;
472 vfpacc1 *= vscale;
473 vfpacc2 *= vscale;
474 vfpacc3 *= vscale;
475
476 vfpacc0 += vmagic_bias;
477 vfpacc1 += vmagic_bias;
478 vfpacc2 += vmagic_bias;
479 vfpacc3 += vmagic_bias;
480
481 int32_t vout0 = (int32_t) float_as_uint32(vfpacc0);
482 int32_t vout1 = (int32_t) float_as_uint32(vfpacc1);
483 int32_t vout2 = (int32_t) float_as_uint32(vfpacc2);
484 int32_t vout3 = (int32_t) float_as_uint32(vfpacc3);
485
486 vout0 = math_max_s32(vout0, vmagic_min);
487 vout1 = math_max_s32(vout1, vmagic_min);
488 vout2 = math_max_s32(vout2, vmagic_min);
489 vout3 = math_max_s32(vout3, vmagic_min);
490
491 vout0 = math_min_s32(vout0, vmagic_max);
492 vout1 = math_min_s32(vout1, vmagic_max);
493 vout2 = math_min_s32(vout2, vmagic_max);
494 vout3 = math_min_s32(vout3, vmagic_max);
495
496 vout0 -= vmagic_bias_less_zero_point;
497 vout1 -= vmagic_bias_less_zero_point;
498 vout2 -= vmagic_bias_less_zero_point;
499 vout3 -= vmagic_bias_less_zero_point;
500
501 output[0] = (int8_t) vout0;
502 output[1] = (int8_t) vout1;
503 output[2] = (int8_t) vout2;
504 output[3] = (int8_t) vout3;
505 output += 4;
506 }
507 if XNN_UNLIKELY(c != 0) {
508 do {
509 int32_t vacc = *b++;
510 const int32_t vi0 = (int32_t) *i0++;
511 const int32_t vk0 = (int32_t) ((const int8_t*) w)[0];
512 vacc += vi0 * vk0;
513 const int32_t vi1 = (int32_t) *i1++;
514 const int32_t vk1 = (int32_t) ((const int8_t*) w)[1];
515 vacc += vi1 * vk1;
516 const int32_t vi2 = (int32_t) *i2++;
517 const int32_t vk2 = (int32_t) ((const int8_t*) w)[2];
518 vacc += vi2 * vk2;
519 const int32_t vi3 = (int32_t) *i3++;
520 const int32_t vk3 = (int32_t) ((const int8_t*) w)[3];
521 vacc += vi3 * vk3;
522 const int32_t vi4 = (int32_t) *i4++;
523 const int32_t vk4 = (int32_t) ((const int8_t*) w)[4];
524 vacc += vi4 * vk4;
525
526 w = (const void*) ((uintptr_t) w + 5 * sizeof(int8_t));
527
528 float vfpacc = (float) vacc * vscale;
529
530 vfpacc += vmagic_bias;
531 int32_t vout = (int32_t) float_as_uint32(vfpacc);
532 vout = math_max_s32(vout, vmagic_min);
533 vout = math_min_s32(vout, vmagic_max);
534 vout -= vmagic_bias_less_zero_point;
535
536 *output++ = (int8_t) vout;
537 } while (--c != 0);
538 }
539 }
540
541 input = (const int8_t**) ((uintptr_t) input + input_stride);
542 output = (int8_t*) ((uintptr_t) output + output_increment);
543 } while (--output_width != 0);
544 }