Debian Codegraph @

src/qs8-dwconv/gen/qs8-dwconv-5f5m5l2c1s1r-minmax-fp32-scalar-imagic.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-dwconv/multipass-scalar.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11 #include <stddef.h>
12 #include <stdint.h>
13
14 #include "xnnpack/common.h"
15 #include "xnnpack/dwconv.h"
16 #include "xnnpack/math.h"
17 #include "xnnpack/microparams.h"
18 #include "xnnpack/unaligned.h"
19
20
21 void xnn_qs8_dwconv_minmax_fp32_ukernel_5f5m5l2c1s1r__scalar_imagic(
22 size_t channels,
23 size_t output_width,
24 const int8_t** input,
25 const void* weights,
26 int8_t* output,
27 intptr_t input_stride,
28 size_t output_increment,
29 size_t input_offset,
30 const int8_t* zero,
31 size_t kernel_size,
32 int32_t* buffer,
33 const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)])
34 {
35 assert(channels != 0);
36 assert(output_width != 0);
37 assert(kernel_size > 5);
38
39 const float vscale = params->fp32_scalar.scale;
40 const float vmagic_bias = 12582912.0f;
41 const int32_t output_min_less_zero_point = (int32_t) params->fp32_scalar.output_min - (int32_t) params->fp32_scalar.output_zero_point;
42 const int32_t output_max_less_zero_point = (int32_t) params->fp32_scalar.output_max - (int32_t) params->fp32_scalar.output_zero_point;
43 const int32_t vmagic_min = (int32_t) float_as_uint32(12582912.0f + output_min_less_zero_point);
44 const int32_t vmagic_max = (int32_t) float_as_uint32(12582912.0f + output_max_less_zero_point);
45 const int32_t vmagic_bias_less_zero_point = INT32_C(0x4B400000) - (int32_t) params->fp32_scalar.output_zero_point;
46 do {
47 const void* w = weights;
48
49 // First pass to process 5 inputs.
50 {
51 int32_t* b = buffer;
52 const int8_t* i0 = input[0];
53 assert(i0 != NULL);
54 if XNN_UNPREDICTABLE(i0 != zero) {
55 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
56 }
57 const int8_t* i1 = input[1];
58 assert(i1 != NULL);
59 if XNN_UNPREDICTABLE(i1 != zero) {
60 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
61 }
62 const int8_t* i2 = input[2];
63 assert(i2 != NULL);
64 if XNN_UNPREDICTABLE(i2 != zero) {
65 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
66 }
67 const int8_t* i3 = input[3];
68 assert(i3 != NULL);
69 if XNN_UNPREDICTABLE(i3 != zero) {
70 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
71 }
72 const int8_t* i4 = input[4];
73 assert(i4 != NULL);
74 if XNN_UNPREDICTABLE(i4 != zero) {
75 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
76 }
77 input += 5;
78
79 size_t c = channels;
80 for (; c >= 2; c -= 2) {
81 int32_t vacc0 = unaligned_indexed_load_s32(w, 0);
82 int32_t vacc1 = unaligned_indexed_load_s32(w, 1);
83
84 const int32_t vi0x0 = (int32_t) i0[0];
85 const int32_t vi0x1 = (int32_t) i0[1];
86 i0 += 2;
87
88 const int32_t vk0x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[0];
89 const int32_t vk0x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[1];
90
91 vacc0 += vi0x0 * vk0x0;
92 vacc1 += vi0x1 * vk0x1;
93
94 const int32_t vi1x0 = (int32_t) i1[0];
95 const int32_t vi1x1 = (int32_t) i1[1];
96 i1 += 2;
97
98 const int32_t vk1x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[2];
99 const int32_t vk1x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[3];
100
101 vacc0 += vi1x0 * vk1x0;
102 vacc1 += vi1x1 * vk1x1;
103
104 const int32_t vi2x0 = (int32_t) i2[0];
105 const int32_t vi2x1 = (int32_t) i2[1];
106 i2 += 2;
107
108 const int32_t vk2x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[4];
109 const int32_t vk2x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[5];
110
111 vacc0 += vi2x0 * vk2x0;
112 vacc1 += vi2x1 * vk2x1;
113
114 const int32_t vi3x0 = (int32_t) i3[0];
115 const int32_t vi3x1 = (int32_t) i3[1];
116 i3 += 2;
117
118 const int32_t vk3x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[6];
119 const int32_t vk3x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[7];
120
121 vacc0 += vi3x0 * vk3x0;
122 vacc1 += vi3x1 * vk3x1;
123
124 const int32_t vi4x0 = (int32_t) i4[0];
125 const int32_t vi4x1 = (int32_t) i4[1];
126 i4 += 2;
127
128 const int32_t vk4x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[8];
129 const int32_t vk4x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[9];
130
131 vacc0 += vi4x0 * vk4x0;
132 vacc1 += vi4x1 * vk4x1;
133
134 w = (const void*) ((uintptr_t) w + 2 * sizeof(int32_t) + 10 * sizeof(int8_t));
135 b[0] = vacc0;
136 b[1] = vacc1;
137 b += 2;
138 }
139 if XNN_UNLIKELY(c != 0) {
140 int32_t vacc = unaligned_load_s32(w);
141
142 const int32_t vi0 = (int32_t) *i0;
143 const int32_t vk0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 1 * sizeof(int32_t)))[0];
144 vacc += vi0 * vk0;
145 const int32_t vi1 = (int32_t) *i1;
146 const int32_t vk1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 1 * sizeof(int32_t)))[1];
147 vacc += vi1 * vk1;
148 const int32_t vi2 = (int32_t) *i2;
149 const int32_t vk2 = (int32_t) ((const int8_t*) ((uintptr_t) w + 1 * sizeof(int32_t)))[2];
150 vacc += vi2 * vk2;
151 const int32_t vi3 = (int32_t) *i3;
152 const int32_t vk3 = (int32_t) ((const int8_t*) ((uintptr_t) w + 1 * sizeof(int32_t)))[3];
153 vacc += vi3 * vk3;
154 const int32_t vi4 = (int32_t) *i4;
155 const int32_t vk4 = (int32_t) ((const int8_t*) ((uintptr_t) w + 1 * sizeof(int32_t)))[4];
156 vacc += vi4 * vk4;
157
158 w = (const void*) ((uintptr_t) w + 1 * sizeof(int32_t) + 5 * sizeof(int8_t));
159 *b++ = vacc;
160 }
161 }
162
163 // Middle pass to process 5 inputs in each iteration.
164 for (size_t ks = kernel_size - 5; ks > 5; ks -= 5) {
165 int32_t* b = buffer;
166 const int8_t* i0 = input[0];
167 assert(i0 != NULL);
168 if XNN_UNPREDICTABLE(i0 != zero) {
169 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
170 }
171 const int8_t* i1 = input[1];
172 assert(i1 != NULL);
173 if XNN_UNPREDICTABLE(i1 != zero) {
174 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
175 }
176 const int8_t* i2 = input[2];
177 assert(i2 != NULL);
178 if XNN_UNPREDICTABLE(i2 != zero) {
179 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
180 }
181 const int8_t* i3 = input[3];
182 assert(i3 != NULL);
183 if XNN_UNPREDICTABLE(i3 != zero) {
184 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
185 }
186 const int8_t* i4 = input[4];
187 assert(i4 != NULL);
188 if XNN_UNPREDICTABLE(i4 != zero) {
189 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
190 }
191 input += 5;
192
193 size_t c = channels;
194 for (; c >= 2; c -= 2) {
195 int32_t vacc0 = b[0];
196 int32_t vacc1 = b[1];
197
198 const int32_t vi0x0 = (int32_t) i0[0];
199 const int32_t vi0x1 = (int32_t) i0[1];
200 i0 += 2;
201
202 const int32_t vk0x0 = (int32_t) ((const int8_t*) w)[0];
203 const int32_t vk0x1 = (int32_t) ((const int8_t*) w)[1];
204
205 vacc0 += vi0x0 * vk0x0;
206 vacc1 += vi0x1 * vk0x1;
207
208 const int32_t vi1x0 = (int32_t) i1[0];
209 const int32_t vi1x1 = (int32_t) i1[1];
210 i1 += 2;
211
212 const int32_t vk1x0 = (int32_t) ((const int8_t*) w)[2];
213 const int32_t vk1x1 = (int32_t) ((const int8_t*) w)[3];
214
215 vacc0 += vi1x0 * vk1x0;
216 vacc1 += vi1x1 * vk1x1;
217
218 const int32_t vi2x0 = (int32_t) i2[0];
219 const int32_t vi2x1 = (int32_t) i2[1];
220 i2 += 2;
221
222 const int32_t vk2x0 = (int32_t) ((const int8_t*) w)[4];
223 const int32_t vk2x1 = (int32_t) ((const int8_t*) w)[5];
224
225 vacc0 += vi2x0 * vk2x0;
226 vacc1 += vi2x1 * vk2x1;
227
228 const int32_t vi3x0 = (int32_t) i3[0];
229 const int32_t vi3x1 = (int32_t) i3[1];
230 i3 += 2;
231
232 const int32_t vk3x0 = (int32_t) ((const int8_t*) w)[6];
233 const int32_t vk3x1 = (int32_t) ((const int8_t*) w)[7];
234
235 vacc0 += vi3x0 * vk3x0;
236 vacc1 += vi3x1 * vk3x1;
237
238 const int32_t vi4x0 = (int32_t) i4[0];
239 const int32_t vi4x1 = (int32_t) i4[1];
240 i4 += 2;
241
242 const int32_t vk4x0 = (int32_t) ((const int8_t*) w)[8];
243 const int32_t vk4x1 = (int32_t) ((const int8_t*) w)[9];
244
245 vacc0 += vi4x0 * vk4x0;
246 vacc1 += vi4x1 * vk4x1;
247
248 w = (const void*) ((uintptr_t) w + 10 * sizeof(int8_t));
249 b[0] = vacc0;
250 b[1] = vacc1;
251 b += 2;
252 }
253 if XNN_UNLIKELY(c != 0) {
254 int32_t vacc = b[0];
255
256 const int32_t vi0 = (int32_t) *i0;
257 const int32_t vk0 = (int32_t) ((const int8_t*) w)[0];
258 vacc += vi0 * vk0;
259 const int32_t vi1 = (int32_t) *i1;
260 const int32_t vk1 = (int32_t) ((const int8_t*) w)[1];
261 vacc += vi1 * vk1;
262 const int32_t vi2 = (int32_t) *i2;
263 const int32_t vk2 = (int32_t) ((const int8_t*) w)[2];
264 vacc += vi2 * vk2;
265 const int32_t vi3 = (int32_t) *i3;
266 const int32_t vk3 = (int32_t) ((const int8_t*) w)[3];
267 vacc += vi3 * vk3;
268 const int32_t vi4 = (int32_t) *i4;
269 const int32_t vk4 = (int32_t) ((const int8_t*) w)[4];
270 vacc += vi4 * vk4;
271 w = (const void*) ((uintptr_t) w + 5 * sizeof(int8_t));
272 *b++ = vacc;
273 }
274 }
275
276 // Last pass to process up to 5 inputs.
277 {
278 const int32_t* b = buffer;
279 const int8_t* i0 = input[0];
280 assert(i0 != NULL);
281 if XNN_UNPREDICTABLE(i0 != zero) {
282 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
283 }
284 const int8_t* i1 = input[1];
285 assert(i1 != NULL);
286 if XNN_UNPREDICTABLE(i1 != zero) {
287 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
288 }
289 const int8_t* i2 = input[2];
290 assert(i2 != NULL);
291 if XNN_UNPREDICTABLE(i2 != zero) {
292 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
293 }
294 const int8_t* i3 = input[3];
295 assert(i3 != NULL);
296 if XNN_UNPREDICTABLE(i3 != zero) {
297 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
298 }
299 const int8_t* i4 = input[4];
300 assert(i4 != NULL);
301 if XNN_UNPREDICTABLE(i4 != zero) {
302 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
303 }
304
305 size_t c = channels;
306 for (; c >= 2; c -= 2) {
307 int32_t vacc0 = b[0];
308 int32_t vacc1 = b[1];
309 b += 2;
310
311 const int32_t vi0x0 = (int32_t) i0[0];
312 const int32_t vi0x1 = (int32_t) i0[1];
313 i0 += 2;
314
315 const int32_t vk0x0 = (int32_t) ((const int8_t*) w)[0];
316 const int32_t vk0x1 = (int32_t) ((const int8_t*) w)[1];
317
318 vacc0 += vi0x0 * vk0x0;
319 vacc1 += vi0x1 * vk0x1;
320
321 const int32_t vi1x0 = (int32_t) i1[0];
322 const int32_t vi1x1 = (int32_t) i1[1];
323 i1 += 2;
324
325 const int32_t vk1x0 = (int32_t) ((const int8_t*) w)[2];
326 const int32_t vk1x1 = (int32_t) ((const int8_t*) w)[3];
327
328 vacc0 += vi1x0 * vk1x0;
329 vacc1 += vi1x1 * vk1x1;
330
331 const int32_t vi2x0 = (int32_t) i2[0];
332 const int32_t vi2x1 = (int32_t) i2[1];
333 i2 += 2;
334
335 const int32_t vk2x0 = (int32_t) ((const int8_t*) w)[4];
336 const int32_t vk2x1 = (int32_t) ((const int8_t*) w)[5];
337
338 vacc0 += vi2x0 * vk2x0;
339 vacc1 += vi2x1 * vk2x1;
340
341 const int32_t vi3x0 = (int32_t) i3[0];
342 const int32_t vi3x1 = (int32_t) i3[1];
343 i3 += 2;
344
345 const int32_t vk3x0 = (int32_t) ((const int8_t*) w)[6];
346 const int32_t vk3x1 = (int32_t) ((const int8_t*) w)[7];
347
348 vacc0 += vi3x0 * vk3x0;
349 vacc1 += vi3x1 * vk3x1;
350
351 const int32_t vi4x0 = (int32_t) i4[0];
352 const int32_t vi4x1 = (int32_t) i4[1];
353 i4 += 2;
354
355 const int32_t vk4x0 = (int32_t) ((const int8_t*) w)[8];
356 const int32_t vk4x1 = (int32_t) ((const int8_t*) w)[9];
357
358 vacc0 += vi4x0 * vk4x0;
359 vacc1 += vi4x1 * vk4x1;
360
361 w = (const void*) ((uintptr_t) w + 10 * sizeof(int8_t));
362
363 float vfpacc0 = (float) vacc0;
364 float vfpacc1 = (float) vacc1;
365
366 vfpacc0 *= vscale;
367 vfpacc1 *= vscale;
368
369 vfpacc0 += vmagic_bias;
370 vfpacc1 += vmagic_bias;
371
372 int32_t vout0 = (int32_t) float_as_uint32(vfpacc0);
373 int32_t vout1 = (int32_t) float_as_uint32(vfpacc1);
374
375 vout0 = math_max_s32(vout0, vmagic_min);
376 vout1 = math_max_s32(vout1, vmagic_min);
377
378 vout0 = math_min_s32(vout0, vmagic_max);
379 vout1 = math_min_s32(vout1, vmagic_max);
380
381 vout0 -= vmagic_bias_less_zero_point;
382 vout1 -= vmagic_bias_less_zero_point;
383
384 output[0] = (int8_t) vout0;
385 output[1] = (int8_t) vout1;
386 output += 2;
387 }
388 if XNN_UNLIKELY(c != 0) {
389 int32_t vacc = b[0];
390
391 const int32_t vi0 = (int32_t) *i0;
392 const int32_t vk0 = (int32_t) ((const int8_t*) w)[0];
393 vacc += vi0 * vk0;
394 const int32_t vi1 = (int32_t) *i1;
395 const int32_t vk1 = (int32_t) ((const int8_t*) w)[1];
396 vacc += vi1 * vk1;
397 const int32_t vi2 = (int32_t) *i2;
398 const int32_t vk2 = (int32_t) ((const int8_t*) w)[2];
399 vacc += vi2 * vk2;
400 const int32_t vi3 = (int32_t) *i3;
401 const int32_t vk3 = (int32_t) ((const int8_t*) w)[3];
402 vacc += vi3 * vk3;
403 const int32_t vi4 = (int32_t) *i4;
404 const int32_t vk4 = (int32_t) ((const int8_t*) w)[4];
405 vacc += vi4 * vk4;
406
407 float vfpacc = (float) vacc * vscale;
408
409 vfpacc += vmagic_bias;
410 int32_t vout = (int32_t) float_as_uint32(vfpacc);
411 vout = math_max_s32(vout, vmagic_min);
412 vout = math_min_s32(vout, vmagic_max);
413 vout -= vmagic_bias_less_zero_point;
414
415 *output++ = (int8_t) vout;
416 }
417 }
418
419 input = (const int8_t**) ((uintptr_t) input + input_stride);
420 output = (int8_t*) ((uintptr_t) output + output_increment);
421 } while (--output_width != 0);
422 }