Debian Codegraph @

src/qs8-dwconv/gen/qs8-dwconv-8f8m9l2c1s1r-minmax-fp32-scalar-fmagic.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-dwconv/multipass-scalar.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11 #include <stddef.h>
12 #include <stdint.h>
13
14 #include "xnnpack/common.h"
15 #include "xnnpack/dwconv.h"
16 #include "xnnpack/math.h"
17 #include "xnnpack/microparams.h"
18 #include "xnnpack/unaligned.h"
19
20
21 void xnn_qs8_dwconv_minmax_fp32_ukernel_8f8m9l2c1s1r__scalar_fmagic(
22 size_t channels,
23 size_t output_width,
24 const int8_t** input,
25 const void* weights,
26 int8_t* output,
27 intptr_t input_stride,
28 size_t output_increment,
29 size_t input_offset,
30 const int8_t* zero,
31 size_t kernel_size,
32 int32_t* buffer,
33 const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)])
34 {
35 assert(channels != 0);
36 assert(output_width != 0);
37 assert(kernel_size > 8);
38
39 const float vscale = params->fp32_scalar.scale;
40 const float voutput_min_less_zero_point = (int32_t) params->fp32_scalar.output_min - (int32_t) params->fp32_scalar.output_zero_point;
41 const float voutput_max_less_zero_point = (int32_t) params->fp32_scalar.output_max - (int32_t) params->fp32_scalar.output_zero_point;
42 const float vmagic_bias = 12582912.0f;
43 const int32_t vmagic_bias_less_output_zero_point = INT32_C(0x4B400000) - (int32_t) params->fp32_scalar.output_zero_point;
44 do {
45 const void* w = weights;
46
47 // First pass to process 8 inputs.
48 {
49 int32_t* b = buffer;
50 const int8_t* i0 = input[0];
51 assert(i0 != NULL);
52 if XNN_UNPREDICTABLE(i0 != zero) {
53 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
54 }
55 const int8_t* i1 = input[1];
56 assert(i1 != NULL);
57 if XNN_UNPREDICTABLE(i1 != zero) {
58 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
59 }
60 const int8_t* i2 = input[2];
61 assert(i2 != NULL);
62 if XNN_UNPREDICTABLE(i2 != zero) {
63 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
64 }
65 const int8_t* i3 = input[3];
66 assert(i3 != NULL);
67 if XNN_UNPREDICTABLE(i3 != zero) {
68 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
69 }
70 const int8_t* i4 = input[4];
71 assert(i4 != NULL);
72 if XNN_UNPREDICTABLE(i4 != zero) {
73 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
74 }
75 const int8_t* i5 = input[5];
76 assert(i5 != NULL);
77 if XNN_UNPREDICTABLE(i5 != zero) {
78 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
79 }
80 const int8_t* i6 = input[6];
81 assert(i6 != NULL);
82 if XNN_UNPREDICTABLE(i6 != zero) {
83 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
84 }
85 const int8_t* i7 = input[7];
86 assert(i7 != NULL);
87 if XNN_UNPREDICTABLE(i7 != zero) {
88 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
89 }
90 input += 8;
91
92 size_t c = channels;
93 for (; c >= 2; c -= 2) {
94 int32_t vacc0 = unaligned_indexed_load_s32(w, 0);
95 int32_t vacc1 = unaligned_indexed_load_s32(w, 1);
96
97 const int32_t vi0x0 = (int32_t) i0[0];
98 const int32_t vi0x1 = (int32_t) i0[1];
99 i0 += 2;
100
101 const int32_t vk0x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[0];
102 const int32_t vk0x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[1];
103
104 vacc0 += vi0x0 * vk0x0;
105 vacc1 += vi0x1 * vk0x1;
106
107 const int32_t vi1x0 = (int32_t) i1[0];
108 const int32_t vi1x1 = (int32_t) i1[1];
109 i1 += 2;
110
111 const int32_t vk1x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[2];
112 const int32_t vk1x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[3];
113
114 vacc0 += vi1x0 * vk1x0;
115 vacc1 += vi1x1 * vk1x1;
116
117 const int32_t vi2x0 = (int32_t) i2[0];
118 const int32_t vi2x1 = (int32_t) i2[1];
119 i2 += 2;
120
121 const int32_t vk2x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[4];
122 const int32_t vk2x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[5];
123
124 vacc0 += vi2x0 * vk2x0;
125 vacc1 += vi2x1 * vk2x1;
126
127 const int32_t vi3x0 = (int32_t) i3[0];
128 const int32_t vi3x1 = (int32_t) i3[1];
129 i3 += 2;
130
131 const int32_t vk3x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[6];
132 const int32_t vk3x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[7];
133
134 vacc0 += vi3x0 * vk3x0;
135 vacc1 += vi3x1 * vk3x1;
136
137 const int32_t vi4x0 = (int32_t) i4[0];
138 const int32_t vi4x1 = (int32_t) i4[1];
139 i4 += 2;
140
141 const int32_t vk4x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[8];
142 const int32_t vk4x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[9];
143
144 vacc0 += vi4x0 * vk4x0;
145 vacc1 += vi4x1 * vk4x1;
146
147 const int32_t vi5x0 = (int32_t) i5[0];
148 const int32_t vi5x1 = (int32_t) i5[1];
149 i5 += 2;
150
151 const int32_t vk5x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[10];
152 const int32_t vk5x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[11];
153
154 vacc0 += vi5x0 * vk5x0;
155 vacc1 += vi5x1 * vk5x1;
156
157 const int32_t vi6x0 = (int32_t) i6[0];
158 const int32_t vi6x1 = (int32_t) i6[1];
159 i6 += 2;
160
161 const int32_t vk6x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[12];
162 const int32_t vk6x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[13];
163
164 vacc0 += vi6x0 * vk6x0;
165 vacc1 += vi6x1 * vk6x1;
166
167 const int32_t vi7x0 = (int32_t) i7[0];
168 const int32_t vi7x1 = (int32_t) i7[1];
169 i7 += 2;
170
171 const int32_t vk7x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[14];
172 const int32_t vk7x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[15];
173
174 vacc0 += vi7x0 * vk7x0;
175 vacc1 += vi7x1 * vk7x1;
176
177 w = (const void*) ((uintptr_t) w + 2 * sizeof(int32_t) + 16 * sizeof(int8_t));
178 b[0] = vacc0;
179 b[1] = vacc1;
180 b += 2;
181 }
182 if XNN_UNLIKELY(c != 0) {
183 int32_t vacc = unaligned_load_s32(w);
184
185 const int32_t vi0 = (int32_t) *i0;
186 const int32_t vk0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 1 * sizeof(int32_t)))[0];
187 vacc += vi0 * vk0;
188 const int32_t vi1 = (int32_t) *i1;
189 const int32_t vk1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 1 * sizeof(int32_t)))[1];
190 vacc += vi1 * vk1;
191 const int32_t vi2 = (int32_t) *i2;
192 const int32_t vk2 = (int32_t) ((const int8_t*) ((uintptr_t) w + 1 * sizeof(int32_t)))[2];
193 vacc += vi2 * vk2;
194 const int32_t vi3 = (int32_t) *i3;
195 const int32_t vk3 = (int32_t) ((const int8_t*) ((uintptr_t) w + 1 * sizeof(int32_t)))[3];
196 vacc += vi3 * vk3;
197 const int32_t vi4 = (int32_t) *i4;
198 const int32_t vk4 = (int32_t) ((const int8_t*) ((uintptr_t) w + 1 * sizeof(int32_t)))[4];
199 vacc += vi4 * vk4;
200 const int32_t vi5 = (int32_t) *i5;
201 const int32_t vk5 = (int32_t) ((const int8_t*) ((uintptr_t) w + 1 * sizeof(int32_t)))[5];
202 vacc += vi5 * vk5;
203 const int32_t vi6 = (int32_t) *i6;
204 const int32_t vk6 = (int32_t) ((const int8_t*) ((uintptr_t) w + 1 * sizeof(int32_t)))[6];
205 vacc += vi6 * vk6;
206 const int32_t vi7 = (int32_t) *i7;
207 const int32_t vk7 = (int32_t) ((const int8_t*) ((uintptr_t) w + 1 * sizeof(int32_t)))[7];
208 vacc += vi7 * vk7;
209
210 w = (const void*) ((uintptr_t) w + 1 * sizeof(int32_t) + 8 * sizeof(int8_t));
211 *b++ = vacc;
212 }
213 }
214
215 // Middle pass to process 8 inputs in each iteration.
216 for (size_t ks = kernel_size - 8; ks > 9; ks -= 8) {
217 int32_t* b = buffer;
218 const int8_t* i0 = input[0];
219 assert(i0 != NULL);
220 if XNN_UNPREDICTABLE(i0 != zero) {
221 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
222 }
223 const int8_t* i1 = input[1];
224 assert(i1 != NULL);
225 if XNN_UNPREDICTABLE(i1 != zero) {
226 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
227 }
228 const int8_t* i2 = input[2];
229 assert(i2 != NULL);
230 if XNN_UNPREDICTABLE(i2 != zero) {
231 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
232 }
233 const int8_t* i3 = input[3];
234 assert(i3 != NULL);
235 if XNN_UNPREDICTABLE(i3 != zero) {
236 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
237 }
238 const int8_t* i4 = input[4];
239 assert(i4 != NULL);
240 if XNN_UNPREDICTABLE(i4 != zero) {
241 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
242 }
243 const int8_t* i5 = input[5];
244 assert(i5 != NULL);
245 if XNN_UNPREDICTABLE(i5 != zero) {
246 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
247 }
248 const int8_t* i6 = input[6];
249 assert(i6 != NULL);
250 if XNN_UNPREDICTABLE(i6 != zero) {
251 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
252 }
253 const int8_t* i7 = input[7];
254 assert(i7 != NULL);
255 if XNN_UNPREDICTABLE(i7 != zero) {
256 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
257 }
258 input += 8;
259
260 size_t c = channels;
261 for (; c >= 2; c -= 2) {
262 int32_t vacc0 = b[0];
263 int32_t vacc1 = b[1];
264
265 const int32_t vi0x0 = (int32_t) i0[0];
266 const int32_t vi0x1 = (int32_t) i0[1];
267 i0 += 2;
268
269 const int32_t vk0x0 = (int32_t) ((const int8_t*) w)[0];
270 const int32_t vk0x1 = (int32_t) ((const int8_t*) w)[1];
271
272 vacc0 += vi0x0 * vk0x0;
273 vacc1 += vi0x1 * vk0x1;
274
275 const int32_t vi1x0 = (int32_t) i1[0];
276 const int32_t vi1x1 = (int32_t) i1[1];
277 i1 += 2;
278
279 const int32_t vk1x0 = (int32_t) ((const int8_t*) w)[2];
280 const int32_t vk1x1 = (int32_t) ((const int8_t*) w)[3];
281
282 vacc0 += vi1x0 * vk1x0;
283 vacc1 += vi1x1 * vk1x1;
284
285 const int32_t vi2x0 = (int32_t) i2[0];
286 const int32_t vi2x1 = (int32_t) i2[1];
287 i2 += 2;
288
289 const int32_t vk2x0 = (int32_t) ((const int8_t*) w)[4];
290 const int32_t vk2x1 = (int32_t) ((const int8_t*) w)[5];
291
292 vacc0 += vi2x0 * vk2x0;
293 vacc1 += vi2x1 * vk2x1;
294
295 const int32_t vi3x0 = (int32_t) i3[0];
296 const int32_t vi3x1 = (int32_t) i3[1];
297 i3 += 2;
298
299 const int32_t vk3x0 = (int32_t) ((const int8_t*) w)[6];
300 const int32_t vk3x1 = (int32_t) ((const int8_t*) w)[7];
301
302 vacc0 += vi3x0 * vk3x0;
303 vacc1 += vi3x1 * vk3x1;
304
305 const int32_t vi4x0 = (int32_t) i4[0];
306 const int32_t vi4x1 = (int32_t) i4[1];
307 i4 += 2;
308
309 const int32_t vk4x0 = (int32_t) ((const int8_t*) w)[8];
310 const int32_t vk4x1 = (int32_t) ((const int8_t*) w)[9];
311
312 vacc0 += vi4x0 * vk4x0;
313 vacc1 += vi4x1 * vk4x1;
314
315 const int32_t vi5x0 = (int32_t) i5[0];
316 const int32_t vi5x1 = (int32_t) i5[1];
317 i5 += 2;
318
319 const int32_t vk5x0 = (int32_t) ((const int8_t*) w)[10];
320 const int32_t vk5x1 = (int32_t) ((const int8_t*) w)[11];
321
322 vacc0 += vi5x0 * vk5x0;
323 vacc1 += vi5x1 * vk5x1;
324
325 const int32_t vi6x0 = (int32_t) i6[0];
326 const int32_t vi6x1 = (int32_t) i6[1];
327 i6 += 2;
328
329 const int32_t vk6x0 = (int32_t) ((const int8_t*) w)[12];
330 const int32_t vk6x1 = (int32_t) ((const int8_t*) w)[13];
331
332 vacc0 += vi6x0 * vk6x0;
333 vacc1 += vi6x1 * vk6x1;
334
335 const int32_t vi7x0 = (int32_t) i7[0];
336 const int32_t vi7x1 = (int32_t) i7[1];
337 i7 += 2;
338
339 const int32_t vk7x0 = (int32_t) ((const int8_t*) w)[14];
340 const int32_t vk7x1 = (int32_t) ((const int8_t*) w)[15];
341
342 vacc0 += vi7x0 * vk7x0;
343 vacc1 += vi7x1 * vk7x1;
344
345 w = (const void*) ((uintptr_t) w + 16 * sizeof(int8_t));
346 b[0] = vacc0;
347 b[1] = vacc1;
348 b += 2;
349 }
350 if XNN_UNLIKELY(c != 0) {
351 int32_t vacc = b[0];
352
353 const int32_t vi0 = (int32_t) *i0;
354 const int32_t vk0 = (int32_t) ((const int8_t*) w)[0];
355 vacc += vi0 * vk0;
356 const int32_t vi1 = (int32_t) *i1;
357 const int32_t vk1 = (int32_t) ((const int8_t*) w)[1];
358 vacc += vi1 * vk1;
359 const int32_t vi2 = (int32_t) *i2;
360 const int32_t vk2 = (int32_t) ((const int8_t*) w)[2];
361 vacc += vi2 * vk2;
362 const int32_t vi3 = (int32_t) *i3;
363 const int32_t vk3 = (int32_t) ((const int8_t*) w)[3];
364 vacc += vi3 * vk3;
365 const int32_t vi4 = (int32_t) *i4;
366 const int32_t vk4 = (int32_t) ((const int8_t*) w)[4];
367 vacc += vi4 * vk4;
368 const int32_t vi5 = (int32_t) *i5;
369 const int32_t vk5 = (int32_t) ((const int8_t*) w)[5];
370 vacc += vi5 * vk5;
371 const int32_t vi6 = (int32_t) *i6;
372 const int32_t vk6 = (int32_t) ((const int8_t*) w)[6];
373 vacc += vi6 * vk6;
374 const int32_t vi7 = (int32_t) *i7;
375 const int32_t vk7 = (int32_t) ((const int8_t*) w)[7];
376 vacc += vi7 * vk7;
377 w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
378 *b++ = vacc;
379 }
380 }
381
382 // Last pass to process up to 9 inputs.
383 {
384 const int32_t* b = buffer;
385 const int8_t* i0 = input[0];
386 assert(i0 != NULL);
387 if XNN_UNPREDICTABLE(i0 != zero) {
388 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
389 }
390 const int8_t* i1 = input[1];
391 assert(i1 != NULL);
392 if XNN_UNPREDICTABLE(i1 != zero) {
393 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
394 }
395 const int8_t* i2 = input[2];
396 assert(i2 != NULL);
397 if XNN_UNPREDICTABLE(i2 != zero) {
398 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
399 }
400 const int8_t* i3 = input[3];
401 assert(i3 != NULL);
402 if XNN_UNPREDICTABLE(i3 != zero) {
403 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
404 }
405 const int8_t* i4 = input[4];
406 assert(i4 != NULL);
407 if XNN_UNPREDICTABLE(i4 != zero) {
408 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
409 }
410 const int8_t* i5 = input[5];
411 assert(i5 != NULL);
412 if XNN_UNPREDICTABLE(i5 != zero) {
413 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
414 }
415 const int8_t* i6 = input[6];
416 assert(i6 != NULL);
417 if XNN_UNPREDICTABLE(i6 != zero) {
418 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
419 }
420 const int8_t* i7 = input[7];
421 assert(i7 != NULL);
422 if XNN_UNPREDICTABLE(i7 != zero) {
423 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
424 }
425 const int8_t* i8 = input[8];
426 assert(i8 != NULL);
427 if XNN_UNPREDICTABLE(i8 != zero) {
428 i8 = (const int8_t*) ((uintptr_t) i8 + input_offset);
429 }
430
431 size_t c = channels;
432 for (; c >= 2; c -= 2) {
433 int32_t vacc0 = b[0];
434 int32_t vacc1 = b[1];
435 b += 2;
436
437 const int32_t vi0x0 = (int32_t) i0[0];
438 const int32_t vi0x1 = (int32_t) i0[1];
439 i0 += 2;
440
441 const int32_t vk0x0 = (int32_t) ((const int8_t*) w)[0];
442 const int32_t vk0x1 = (int32_t) ((const int8_t*) w)[1];
443
444 vacc0 += vi0x0 * vk0x0;
445 vacc1 += vi0x1 * vk0x1;
446
447 const int32_t vi1x0 = (int32_t) i1[0];
448 const int32_t vi1x1 = (int32_t) i1[1];
449 i1 += 2;
450
451 const int32_t vk1x0 = (int32_t) ((const int8_t*) w)[2];
452 const int32_t vk1x1 = (int32_t) ((const int8_t*) w)[3];
453
454 vacc0 += vi1x0 * vk1x0;
455 vacc1 += vi1x1 * vk1x1;
456
457 const int32_t vi2x0 = (int32_t) i2[0];
458 const int32_t vi2x1 = (int32_t) i2[1];
459 i2 += 2;
460
461 const int32_t vk2x0 = (int32_t) ((const int8_t*) w)[4];
462 const int32_t vk2x1 = (int32_t) ((const int8_t*) w)[5];
463
464 vacc0 += vi2x0 * vk2x0;
465 vacc1 += vi2x1 * vk2x1;
466
467 const int32_t vi3x0 = (int32_t) i3[0];
468 const int32_t vi3x1 = (int32_t) i3[1];
469 i3 += 2;
470
471 const int32_t vk3x0 = (int32_t) ((const int8_t*) w)[6];
472 const int32_t vk3x1 = (int32_t) ((const int8_t*) w)[7];
473
474 vacc0 += vi3x0 * vk3x0;
475 vacc1 += vi3x1 * vk3x1;
476
477 const int32_t vi4x0 = (int32_t) i4[0];
478 const int32_t vi4x1 = (int32_t) i4[1];
479 i4 += 2;
480
481 const int32_t vk4x0 = (int32_t) ((const int8_t*) w)[8];
482 const int32_t vk4x1 = (int32_t) ((const int8_t*) w)[9];
483
484 vacc0 += vi4x0 * vk4x0;
485 vacc1 += vi4x1 * vk4x1;
486
487 const int32_t vi5x0 = (int32_t) i5[0];
488 const int32_t vi5x1 = (int32_t) i5[1];
489 i5 += 2;
490
491 const int32_t vk5x0 = (int32_t) ((const int8_t*) w)[10];
492 const int32_t vk5x1 = (int32_t) ((const int8_t*) w)[11];
493
494 vacc0 += vi5x0 * vk5x0;
495 vacc1 += vi5x1 * vk5x1;
496
497 const int32_t vi6x0 = (int32_t) i6[0];
498 const int32_t vi6x1 = (int32_t) i6[1];
499 i6 += 2;
500
501 const int32_t vk6x0 = (int32_t) ((const int8_t*) w)[12];
502 const int32_t vk6x1 = (int32_t) ((const int8_t*) w)[13];
503
504 vacc0 += vi6x0 * vk6x0;
505 vacc1 += vi6x1 * vk6x1;
506
507 const int32_t vi7x0 = (int32_t) i7[0];
508 const int32_t vi7x1 = (int32_t) i7[1];
509 i7 += 2;
510
511 const int32_t vk7x0 = (int32_t) ((const int8_t*) w)[14];
512 const int32_t vk7x1 = (int32_t) ((const int8_t*) w)[15];
513
514 vacc0 += vi7x0 * vk7x0;
515 vacc1 += vi7x1 * vk7x1;
516
517 const int32_t vi8x0 = (int32_t) i8[0];
518 const int32_t vi8x1 = (int32_t) i8[1];
519 i8 += 2;
520
521 const int32_t vk8x0 = (int32_t) ((const int8_t*) w)[16];
522 const int32_t vk8x1 = (int32_t) ((const int8_t*) w)[17];
523
524 vacc0 += vi8x0 * vk8x0;
525 vacc1 += vi8x1 * vk8x1;
526
527 w = (const void*) ((uintptr_t) w + 18 * sizeof(int8_t));
528
529 float vfpacc0 = (float) vacc0;
530 float vfpacc1 = (float) vacc1;
531
532 vfpacc0 *= vscale;
533 vfpacc1 *= vscale;
534
535 vfpacc0 = math_max_f32(vfpacc0, voutput_min_less_zero_point);
536 vfpacc1 = math_max_f32(vfpacc1, voutput_min_less_zero_point);
537
538 vfpacc0 = math_min_f32(vfpacc0, voutput_max_less_zero_point);
539 vfpacc1 = math_min_f32(vfpacc1, voutput_max_less_zero_point);
540
541 vfpacc0 += vmagic_bias;
542 vfpacc1 += vmagic_bias;
543
544 int32_t vout0 = (int32_t) float_as_uint32(vfpacc0) - vmagic_bias_less_output_zero_point;
545 int32_t vout1 = (int32_t) float_as_uint32(vfpacc1) - vmagic_bias_less_output_zero_point;
546
547 output[0] = (int8_t) vout0;
548 output[1] = (int8_t) vout1;
549 output += 2;
550 }
551 if XNN_UNLIKELY(c != 0) {
552 int32_t vacc = b[0];
553
554 const int32_t vi0 = (int32_t) *i0;
555 const int32_t vk0 = (int32_t) ((const int8_t*) w)[0];
556 vacc += vi0 * vk0;
557 const int32_t vi1 = (int32_t) *i1;
558 const int32_t vk1 = (int32_t) ((const int8_t*) w)[1];
559 vacc += vi1 * vk1;
560 const int32_t vi2 = (int32_t) *i2;
561 const int32_t vk2 = (int32_t) ((const int8_t*) w)[2];
562 vacc += vi2 * vk2;
563 const int32_t vi3 = (int32_t) *i3;
564 const int32_t vk3 = (int32_t) ((const int8_t*) w)[3];
565 vacc += vi3 * vk3;
566 const int32_t vi4 = (int32_t) *i4;
567 const int32_t vk4 = (int32_t) ((const int8_t*) w)[4];
568 vacc += vi4 * vk4;
569 const int32_t vi5 = (int32_t) *i5;
570 const int32_t vk5 = (int32_t) ((const int8_t*) w)[5];
571 vacc += vi5 * vk5;
572 const int32_t vi6 = (int32_t) *i6;
573 const int32_t vk6 = (int32_t) ((const int8_t*) w)[6];
574 vacc += vi6 * vk6;
575 const int32_t vi7 = (int32_t) *i7;
576 const int32_t vk7 = (int32_t) ((const int8_t*) w)[7];
577 vacc += vi7 * vk7;
578 const int32_t vi8 = (int32_t) *i8;
579 const int32_t vk8 = (int32_t) ((const int8_t*) w)[8];
580 vacc += vi8 * vk8;
581
582 float vfpacc = (float) vacc * vscale;
583
584 vfpacc = math_max_f32(vfpacc, voutput_min_less_zero_point);
585 vfpacc = math_min_f32(vfpacc, voutput_max_less_zero_point);
586 vfpacc += vmagic_bias;
587 int32_t vout = (int32_t) float_as_uint32(vfpacc) - vmagic_bias_less_output_zero_point;
588
589 *output++ = (int8_t) vout;
590 }
591 }
592
593 input = (const int8_t**) ((uintptr_t) input + input_stride);
594 output = (int8_t*) ((uintptr_t) output + output_increment);
595 } while (--output_width != 0);
596 }