Debian Codegraph @

src/qs8-dwconv/gen/qs8-dwconv-8f8m9l2c1s1r-minmax-fp32-scalar-imagic.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-dwconv/multipass-scalar.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11 #include <stddef.h>
12 #include <stdint.h>
13
14 #include "xnnpack/common.h"
15 #include "xnnpack/dwconv.h"
16 #include "xnnpack/math.h"
17 #include "xnnpack/microparams.h"
18 #include "xnnpack/unaligned.h"
19
20
21 void xnn_qs8_dwconv_minmax_fp32_ukernel_8f8m9l2c1s1r__scalar_imagic(
22 size_t channels,
23 size_t output_width,
24 const int8_t** input,
25 const void* weights,
26 int8_t* output,
27 intptr_t input_stride,
28 size_t output_increment,
29 size_t input_offset,
30 const int8_t* zero,
31 size_t kernel_size,
32 int32_t* buffer,
33 const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)])
34 {
35 assert(channels != 0);
36 assert(output_width != 0);
37 assert(kernel_size > 8);
38
39 const float vscale = params->fp32_scalar.scale;
40 const float vmagic_bias = 12582912.0f;
41 const int32_t output_min_less_zero_point = (int32_t) params->fp32_scalar.output_min - (int32_t) params->fp32_scalar.output_zero_point;
42 const int32_t output_max_less_zero_point = (int32_t) params->fp32_scalar.output_max - (int32_t) params->fp32_scalar.output_zero_point;
43 const int32_t vmagic_min = (int32_t) float_as_uint32(12582912.0f + output_min_less_zero_point);
44 const int32_t vmagic_max = (int32_t) float_as_uint32(12582912.0f + output_max_less_zero_point);
45 const int32_t vmagic_bias_less_zero_point = INT32_C(0x4B400000) - (int32_t) params->fp32_scalar.output_zero_point;
46 do {
47 const void* w = weights;
48
49 // First pass to process 8 inputs.
50 {
51 int32_t* b = buffer;
52 const int8_t* i0 = input[0];
53 assert(i0 != NULL);
54 if XNN_UNPREDICTABLE(i0 != zero) {
55 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
56 }
57 const int8_t* i1 = input[1];
58 assert(i1 != NULL);
59 if XNN_UNPREDICTABLE(i1 != zero) {
60 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
61 }
62 const int8_t* i2 = input[2];
63 assert(i2 != NULL);
64 if XNN_UNPREDICTABLE(i2 != zero) {
65 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
66 }
67 const int8_t* i3 = input[3];
68 assert(i3 != NULL);
69 if XNN_UNPREDICTABLE(i3 != zero) {
70 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
71 }
72 const int8_t* i4 = input[4];
73 assert(i4 != NULL);
74 if XNN_UNPREDICTABLE(i4 != zero) {
75 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
76 }
77 const int8_t* i5 = input[5];
78 assert(i5 != NULL);
79 if XNN_UNPREDICTABLE(i5 != zero) {
80 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
81 }
82 const int8_t* i6 = input[6];
83 assert(i6 != NULL);
84 if XNN_UNPREDICTABLE(i6 != zero) {
85 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
86 }
87 const int8_t* i7 = input[7];
88 assert(i7 != NULL);
89 if XNN_UNPREDICTABLE(i7 != zero) {
90 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
91 }
92 input += 8;
93
94 size_t c = channels;
95 for (; c >= 2; c -= 2) {
96 int32_t vacc0 = unaligned_indexed_load_s32(w, 0);
97 int32_t vacc1 = unaligned_indexed_load_s32(w, 1);
98
99 const int32_t vi0x0 = (int32_t) i0[0];
100 const int32_t vi0x1 = (int32_t) i0[1];
101 i0 += 2;
102
103 const int32_t vk0x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[0];
104 const int32_t vk0x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[1];
105
106 vacc0 += vi0x0 * vk0x0;
107 vacc1 += vi0x1 * vk0x1;
108
109 const int32_t vi1x0 = (int32_t) i1[0];
110 const int32_t vi1x1 = (int32_t) i1[1];
111 i1 += 2;
112
113 const int32_t vk1x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[2];
114 const int32_t vk1x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[3];
115
116 vacc0 += vi1x0 * vk1x0;
117 vacc1 += vi1x1 * vk1x1;
118
119 const int32_t vi2x0 = (int32_t) i2[0];
120 const int32_t vi2x1 = (int32_t) i2[1];
121 i2 += 2;
122
123 const int32_t vk2x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[4];
124 const int32_t vk2x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[5];
125
126 vacc0 += vi2x0 * vk2x0;
127 vacc1 += vi2x1 * vk2x1;
128
129 const int32_t vi3x0 = (int32_t) i3[0];
130 const int32_t vi3x1 = (int32_t) i3[1];
131 i3 += 2;
132
133 const int32_t vk3x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[6];
134 const int32_t vk3x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[7];
135
136 vacc0 += vi3x0 * vk3x0;
137 vacc1 += vi3x1 * vk3x1;
138
139 const int32_t vi4x0 = (int32_t) i4[0];
140 const int32_t vi4x1 = (int32_t) i4[1];
141 i4 += 2;
142
143 const int32_t vk4x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[8];
144 const int32_t vk4x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[9];
145
146 vacc0 += vi4x0 * vk4x0;
147 vacc1 += vi4x1 * vk4x1;
148
149 const int32_t vi5x0 = (int32_t) i5[0];
150 const int32_t vi5x1 = (int32_t) i5[1];
151 i5 += 2;
152
153 const int32_t vk5x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[10];
154 const int32_t vk5x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[11];
155
156 vacc0 += vi5x0 * vk5x0;
157 vacc1 += vi5x1 * vk5x1;
158
159 const int32_t vi6x0 = (int32_t) i6[0];
160 const int32_t vi6x1 = (int32_t) i6[1];
161 i6 += 2;
162
163 const int32_t vk6x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[12];
164 const int32_t vk6x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[13];
165
166 vacc0 += vi6x0 * vk6x0;
167 vacc1 += vi6x1 * vk6x1;
168
169 const int32_t vi7x0 = (int32_t) i7[0];
170 const int32_t vi7x1 = (int32_t) i7[1];
171 i7 += 2;
172
173 const int32_t vk7x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[14];
174 const int32_t vk7x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[15];
175
176 vacc0 += vi7x0 * vk7x0;
177 vacc1 += vi7x1 * vk7x1;
178
179 w = (const void*) ((uintptr_t) w + 2 * sizeof(int32_t) + 16 * sizeof(int8_t));
180 b[0] = vacc0;
181 b[1] = vacc1;
182 b += 2;
183 }
184 if XNN_UNLIKELY(c != 0) {
185 int32_t vacc = unaligned_load_s32(w);
186
187 const int32_t vi0 = (int32_t) *i0;
188 const int32_t vk0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 1 * sizeof(int32_t)))[0];
189 vacc += vi0 * vk0;
190 const int32_t vi1 = (int32_t) *i1;
191 const int32_t vk1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 1 * sizeof(int32_t)))[1];
192 vacc += vi1 * vk1;
193 const int32_t vi2 = (int32_t) *i2;
194 const int32_t vk2 = (int32_t) ((const int8_t*) ((uintptr_t) w + 1 * sizeof(int32_t)))[2];
195 vacc += vi2 * vk2;
196 const int32_t vi3 = (int32_t) *i3;
197 const int32_t vk3 = (int32_t) ((const int8_t*) ((uintptr_t) w + 1 * sizeof(int32_t)))[3];
198 vacc += vi3 * vk3;
199 const int32_t vi4 = (int32_t) *i4;
200 const int32_t vk4 = (int32_t) ((const int8_t*) ((uintptr_t) w + 1 * sizeof(int32_t)))[4];
201 vacc += vi4 * vk4;
202 const int32_t vi5 = (int32_t) *i5;
203 const int32_t vk5 = (int32_t) ((const int8_t*) ((uintptr_t) w + 1 * sizeof(int32_t)))[5];
204 vacc += vi5 * vk5;
205 const int32_t vi6 = (int32_t) *i6;
206 const int32_t vk6 = (int32_t) ((const int8_t*) ((uintptr_t) w + 1 * sizeof(int32_t)))[6];
207 vacc += vi6 * vk6;
208 const int32_t vi7 = (int32_t) *i7;
209 const int32_t vk7 = (int32_t) ((const int8_t*) ((uintptr_t) w + 1 * sizeof(int32_t)))[7];
210 vacc += vi7 * vk7;
211
212 w = (const void*) ((uintptr_t) w + 1 * sizeof(int32_t) + 8 * sizeof(int8_t));
213 *b++ = vacc;
214 }
215 }
216
217 // Middle pass to process 8 inputs in each iteration.
218 for (size_t ks = kernel_size - 8; ks > 9; ks -= 8) {
219 int32_t* b = buffer;
220 const int8_t* i0 = input[0];
221 assert(i0 != NULL);
222 if XNN_UNPREDICTABLE(i0 != zero) {
223 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
224 }
225 const int8_t* i1 = input[1];
226 assert(i1 != NULL);
227 if XNN_UNPREDICTABLE(i1 != zero) {
228 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
229 }
230 const int8_t* i2 = input[2];
231 assert(i2 != NULL);
232 if XNN_UNPREDICTABLE(i2 != zero) {
233 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
234 }
235 const int8_t* i3 = input[3];
236 assert(i3 != NULL);
237 if XNN_UNPREDICTABLE(i3 != zero) {
238 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
239 }
240 const int8_t* i4 = input[4];
241 assert(i4 != NULL);
242 if XNN_UNPREDICTABLE(i4 != zero) {
243 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
244 }
245 const int8_t* i5 = input[5];
246 assert(i5 != NULL);
247 if XNN_UNPREDICTABLE(i5 != zero) {
248 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
249 }
250 const int8_t* i6 = input[6];
251 assert(i6 != NULL);
252 if XNN_UNPREDICTABLE(i6 != zero) {
253 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
254 }
255 const int8_t* i7 = input[7];
256 assert(i7 != NULL);
257 if XNN_UNPREDICTABLE(i7 != zero) {
258 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
259 }
260 input += 8;
261
262 size_t c = channels;
263 for (; c >= 2; c -= 2) {
264 int32_t vacc0 = b[0];
265 int32_t vacc1 = b[1];
266
267 const int32_t vi0x0 = (int32_t) i0[0];
268 const int32_t vi0x1 = (int32_t) i0[1];
269 i0 += 2;
270
271 const int32_t vk0x0 = (int32_t) ((const int8_t*) w)[0];
272 const int32_t vk0x1 = (int32_t) ((const int8_t*) w)[1];
273
274 vacc0 += vi0x0 * vk0x0;
275 vacc1 += vi0x1 * vk0x1;
276
277 const int32_t vi1x0 = (int32_t) i1[0];
278 const int32_t vi1x1 = (int32_t) i1[1];
279 i1 += 2;
280
281 const int32_t vk1x0 = (int32_t) ((const int8_t*) w)[2];
282 const int32_t vk1x1 = (int32_t) ((const int8_t*) w)[3];
283
284 vacc0 += vi1x0 * vk1x0;
285 vacc1 += vi1x1 * vk1x1;
286
287 const int32_t vi2x0 = (int32_t) i2[0];
288 const int32_t vi2x1 = (int32_t) i2[1];
289 i2 += 2;
290
291 const int32_t vk2x0 = (int32_t) ((const int8_t*) w)[4];
292 const int32_t vk2x1 = (int32_t) ((const int8_t*) w)[5];
293
294 vacc0 += vi2x0 * vk2x0;
295 vacc1 += vi2x1 * vk2x1;
296
297 const int32_t vi3x0 = (int32_t) i3[0];
298 const int32_t vi3x1 = (int32_t) i3[1];
299 i3 += 2;
300
301 const int32_t vk3x0 = (int32_t) ((const int8_t*) w)[6];
302 const int32_t vk3x1 = (int32_t) ((const int8_t*) w)[7];
303
304 vacc0 += vi3x0 * vk3x0;
305 vacc1 += vi3x1 * vk3x1;
306
307 const int32_t vi4x0 = (int32_t) i4[0];
308 const int32_t vi4x1 = (int32_t) i4[1];
309 i4 += 2;
310
311 const int32_t vk4x0 = (int32_t) ((const int8_t*) w)[8];
312 const int32_t vk4x1 = (int32_t) ((const int8_t*) w)[9];
313
314 vacc0 += vi4x0 * vk4x0;
315 vacc1 += vi4x1 * vk4x1;
316
317 const int32_t vi5x0 = (int32_t) i5[0];
318 const int32_t vi5x1 = (int32_t) i5[1];
319 i5 += 2;
320
321 const int32_t vk5x0 = (int32_t) ((const int8_t*) w)[10];
322 const int32_t vk5x1 = (int32_t) ((const int8_t*) w)[11];
323
324 vacc0 += vi5x0 * vk5x0;
325 vacc1 += vi5x1 * vk5x1;
326
327 const int32_t vi6x0 = (int32_t) i6[0];
328 const int32_t vi6x1 = (int32_t) i6[1];
329 i6 += 2;
330
331 const int32_t vk6x0 = (int32_t) ((const int8_t*) w)[12];
332 const int32_t vk6x1 = (int32_t) ((const int8_t*) w)[13];
333
334 vacc0 += vi6x0 * vk6x0;
335 vacc1 += vi6x1 * vk6x1;
336
337 const int32_t vi7x0 = (int32_t) i7[0];
338 const int32_t vi7x1 = (int32_t) i7[1];
339 i7 += 2;
340
341 const int32_t vk7x0 = (int32_t) ((const int8_t*) w)[14];
342 const int32_t vk7x1 = (int32_t) ((const int8_t*) w)[15];
343
344 vacc0 += vi7x0 * vk7x0;
345 vacc1 += vi7x1 * vk7x1;
346
347 w = (const void*) ((uintptr_t) w + 16 * sizeof(int8_t));
348 b[0] = vacc0;
349 b[1] = vacc1;
350 b += 2;
351 }
352 if XNN_UNLIKELY(c != 0) {
353 int32_t vacc = b[0];
354
355 const int32_t vi0 = (int32_t) *i0;
356 const int32_t vk0 = (int32_t) ((const int8_t*) w)[0];
357 vacc += vi0 * vk0;
358 const int32_t vi1 = (int32_t) *i1;
359 const int32_t vk1 = (int32_t) ((const int8_t*) w)[1];
360 vacc += vi1 * vk1;
361 const int32_t vi2 = (int32_t) *i2;
362 const int32_t vk2 = (int32_t) ((const int8_t*) w)[2];
363 vacc += vi2 * vk2;
364 const int32_t vi3 = (int32_t) *i3;
365 const int32_t vk3 = (int32_t) ((const int8_t*) w)[3];
366 vacc += vi3 * vk3;
367 const int32_t vi4 = (int32_t) *i4;
368 const int32_t vk4 = (int32_t) ((const int8_t*) w)[4];
369 vacc += vi4 * vk4;
370 const int32_t vi5 = (int32_t) *i5;
371 const int32_t vk5 = (int32_t) ((const int8_t*) w)[5];
372 vacc += vi5 * vk5;
373 const int32_t vi6 = (int32_t) *i6;
374 const int32_t vk6 = (int32_t) ((const int8_t*) w)[6];
375 vacc += vi6 * vk6;
376 const int32_t vi7 = (int32_t) *i7;
377 const int32_t vk7 = (int32_t) ((const int8_t*) w)[7];
378 vacc += vi7 * vk7;
379 w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
380 *b++ = vacc;
381 }
382 }
383
384 // Last pass to process up to 9 inputs.
385 {
386 const int32_t* b = buffer;
387 const int8_t* i0 = input[0];
388 assert(i0 != NULL);
389 if XNN_UNPREDICTABLE(i0 != zero) {
390 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
391 }
392 const int8_t* i1 = input[1];
393 assert(i1 != NULL);
394 if XNN_UNPREDICTABLE(i1 != zero) {
395 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
396 }
397 const int8_t* i2 = input[2];
398 assert(i2 != NULL);
399 if XNN_UNPREDICTABLE(i2 != zero) {
400 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
401 }
402 const int8_t* i3 = input[3];
403 assert(i3 != NULL);
404 if XNN_UNPREDICTABLE(i3 != zero) {
405 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
406 }
407 const int8_t* i4 = input[4];
408 assert(i4 != NULL);
409 if XNN_UNPREDICTABLE(i4 != zero) {
410 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
411 }
412 const int8_t* i5 = input[5];
413 assert(i5 != NULL);
414 if XNN_UNPREDICTABLE(i5 != zero) {
415 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
416 }
417 const int8_t* i6 = input[6];
418 assert(i6 != NULL);
419 if XNN_UNPREDICTABLE(i6 != zero) {
420 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
421 }
422 const int8_t* i7 = input[7];
423 assert(i7 != NULL);
424 if XNN_UNPREDICTABLE(i7 != zero) {
425 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
426 }
427 const int8_t* i8 = input[8];
428 assert(i8 != NULL);
429 if XNN_UNPREDICTABLE(i8 != zero) {
430 i8 = (const int8_t*) ((uintptr_t) i8 + input_offset);
431 }
432
433 size_t c = channels;
434 for (; c >= 2; c -= 2) {
435 int32_t vacc0 = b[0];
436 int32_t vacc1 = b[1];
437 b += 2;
438
439 const int32_t vi0x0 = (int32_t) i0[0];
440 const int32_t vi0x1 = (int32_t) i0[1];
441 i0 += 2;
442
443 const int32_t vk0x0 = (int32_t) ((const int8_t*) w)[0];
444 const int32_t vk0x1 = (int32_t) ((const int8_t*) w)[1];
445
446 vacc0 += vi0x0 * vk0x0;
447 vacc1 += vi0x1 * vk0x1;
448
449 const int32_t vi1x0 = (int32_t) i1[0];
450 const int32_t vi1x1 = (int32_t) i1[1];
451 i1 += 2;
452
453 const int32_t vk1x0 = (int32_t) ((const int8_t*) w)[2];
454 const int32_t vk1x1 = (int32_t) ((const int8_t*) w)[3];
455
456 vacc0 += vi1x0 * vk1x0;
457 vacc1 += vi1x1 * vk1x1;
458
459 const int32_t vi2x0 = (int32_t) i2[0];
460 const int32_t vi2x1 = (int32_t) i2[1];
461 i2 += 2;
462
463 const int32_t vk2x0 = (int32_t) ((const int8_t*) w)[4];
464 const int32_t vk2x1 = (int32_t) ((const int8_t*) w)[5];
465
466 vacc0 += vi2x0 * vk2x0;
467 vacc1 += vi2x1 * vk2x1;
468
469 const int32_t vi3x0 = (int32_t) i3[0];
470 const int32_t vi3x1 = (int32_t) i3[1];
471 i3 += 2;
472
473 const int32_t vk3x0 = (int32_t) ((const int8_t*) w)[6];
474 const int32_t vk3x1 = (int32_t) ((const int8_t*) w)[7];
475
476 vacc0 += vi3x0 * vk3x0;
477 vacc1 += vi3x1 * vk3x1;
478
479 const int32_t vi4x0 = (int32_t) i4[0];
480 const int32_t vi4x1 = (int32_t) i4[1];
481 i4 += 2;
482
483 const int32_t vk4x0 = (int32_t) ((const int8_t*) w)[8];
484 const int32_t vk4x1 = (int32_t) ((const int8_t*) w)[9];
485
486 vacc0 += vi4x0 * vk4x0;
487 vacc1 += vi4x1 * vk4x1;
488
489 const int32_t vi5x0 = (int32_t) i5[0];
490 const int32_t vi5x1 = (int32_t) i5[1];
491 i5 += 2;
492
493 const int32_t vk5x0 = (int32_t) ((const int8_t*) w)[10];
494 const int32_t vk5x1 = (int32_t) ((const int8_t*) w)[11];
495
496 vacc0 += vi5x0 * vk5x0;
497 vacc1 += vi5x1 * vk5x1;
498
499 const int32_t vi6x0 = (int32_t) i6[0];
500 const int32_t vi6x1 = (int32_t) i6[1];
501 i6 += 2;
502
503 const int32_t vk6x0 = (int32_t) ((const int8_t*) w)[12];
504 const int32_t vk6x1 = (int32_t) ((const int8_t*) w)[13];
505
506 vacc0 += vi6x0 * vk6x0;
507 vacc1 += vi6x1 * vk6x1;
508
509 const int32_t vi7x0 = (int32_t) i7[0];
510 const int32_t vi7x1 = (int32_t) i7[1];
511 i7 += 2;
512
513 const int32_t vk7x0 = (int32_t) ((const int8_t*) w)[14];
514 const int32_t vk7x1 = (int32_t) ((const int8_t*) w)[15];
515
516 vacc0 += vi7x0 * vk7x0;
517 vacc1 += vi7x1 * vk7x1;
518
519 const int32_t vi8x0 = (int32_t) i8[0];
520 const int32_t vi8x1 = (int32_t) i8[1];
521 i8 += 2;
522
523 const int32_t vk8x0 = (int32_t) ((const int8_t*) w)[16];
524 const int32_t vk8x1 = (int32_t) ((const int8_t*) w)[17];
525
526 vacc0 += vi8x0 * vk8x0;
527 vacc1 += vi8x1 * vk8x1;
528
529 w = (const void*) ((uintptr_t) w + 18 * sizeof(int8_t));
530
531 float vfpacc0 = (float) vacc0;
532 float vfpacc1 = (float) vacc1;
533
534 vfpacc0 *= vscale;
535 vfpacc1 *= vscale;
536
537 vfpacc0 += vmagic_bias;
538 vfpacc1 += vmagic_bias;
539
540 int32_t vout0 = (int32_t) float_as_uint32(vfpacc0);
541 int32_t vout1 = (int32_t) float_as_uint32(vfpacc1);
542
543 vout0 = math_max_s32(vout0, vmagic_min);
544 vout1 = math_max_s32(vout1, vmagic_min);
545
546 vout0 = math_min_s32(vout0, vmagic_max);
547 vout1 = math_min_s32(vout1, vmagic_max);
548
549 vout0 -= vmagic_bias_less_zero_point;
550 vout1 -= vmagic_bias_less_zero_point;
551
552 output[0] = (int8_t) vout0;
553 output[1] = (int8_t) vout1;
554 output += 2;
555 }
556 if XNN_UNLIKELY(c != 0) {
557 int32_t vacc = b[0];
558
559 const int32_t vi0 = (int32_t) *i0;
560 const int32_t vk0 = (int32_t) ((const int8_t*) w)[0];
561 vacc += vi0 * vk0;
562 const int32_t vi1 = (int32_t) *i1;
563 const int32_t vk1 = (int32_t) ((const int8_t*) w)[1];
564 vacc += vi1 * vk1;
565 const int32_t vi2 = (int32_t) *i2;
566 const int32_t vk2 = (int32_t) ((const int8_t*) w)[2];
567 vacc += vi2 * vk2;
568 const int32_t vi3 = (int32_t) *i3;
569 const int32_t vk3 = (int32_t) ((const int8_t*) w)[3];
570 vacc += vi3 * vk3;
571 const int32_t vi4 = (int32_t) *i4;
572 const int32_t vk4 = (int32_t) ((const int8_t*) w)[4];
573 vacc += vi4 * vk4;
574 const int32_t vi5 = (int32_t) *i5;
575 const int32_t vk5 = (int32_t) ((const int8_t*) w)[5];
576 vacc += vi5 * vk5;
577 const int32_t vi6 = (int32_t) *i6;
578 const int32_t vk6 = (int32_t) ((const int8_t*) w)[6];
579 vacc += vi6 * vk6;
580 const int32_t vi7 = (int32_t) *i7;
581 const int32_t vk7 = (int32_t) ((const int8_t*) w)[7];
582 vacc += vi7 * vk7;
583 const int32_t vi8 = (int32_t) *i8;
584 const int32_t vk8 = (int32_t) ((const int8_t*) w)[8];
585 vacc += vi8 * vk8;
586
587 float vfpacc = (float) vacc * vscale;
588
589 vfpacc += vmagic_bias;
590 int32_t vout = (int32_t) float_as_uint32(vfpacc);
591 vout = math_max_s32(vout, vmagic_min);
592 vout = math_min_s32(vout, vmagic_max);
593 vout -= vmagic_bias_less_zero_point;
594
595 *output++ = (int8_t) vout;
596 }
597 }
598
599 input = (const int8_t**) ((uintptr_t) input + input_stride);
600 output = (int8_t*) ((uintptr_t) output + output_increment);
601 } while (--output_width != 0);
602 }