Debian Codegraph @

src/qs8-dwconv/gen/qs8-dwconv-5f5m5l2c1s1r-minmax-fp32-scalar-lrintf.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-dwconv/multipass-scalar.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11 #include <math.h>
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include "xnnpack/common.h"
16 #include "xnnpack/dwconv.h"
17 #include "xnnpack/math.h"
18 #include "xnnpack/microparams.h"
19 #include "xnnpack/unaligned.h"
20
21
22 void xnn_qs8_dwconv_minmax_fp32_ukernel_5f5m5l2c1s1r__scalar_lrintf(
23 size_t channels,
24 size_t output_width,
25 const int8_t** input,
26 const void* weights,
27 int8_t* output,
28 intptr_t input_stride,
29 size_t output_increment,
30 size_t input_offset,
31 const int8_t* zero,
32 size_t kernel_size,
33 int32_t* buffer,
34 const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)])
35 {
36 assert(channels != 0);
37 assert(output_width != 0);
38 assert(kernel_size > 5);
39
40 const float vscale = params->fp32_scalar.scale;
41 const float voutput_min_less_zero_point = (int32_t) params->fp32_scalar.output_min - (int32_t) params->fp32_scalar.output_zero_point;
42 const float voutput_max_less_zero_point = (int32_t) params->fp32_scalar.output_max - (int32_t) params->fp32_scalar.output_zero_point;
43 const int32_t voutput_zero_point = params->fp32_scalar.output_zero_point;
44 do {
45 const void* w = weights;
46
47 // First pass to process 5 inputs.
48 {
49 int32_t* b = buffer;
50 const int8_t* i0 = input[0];
51 assert(i0 != NULL);
52 if XNN_UNPREDICTABLE(i0 != zero) {
53 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
54 }
55 const int8_t* i1 = input[1];
56 assert(i1 != NULL);
57 if XNN_UNPREDICTABLE(i1 != zero) {
58 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
59 }
60 const int8_t* i2 = input[2];
61 assert(i2 != NULL);
62 if XNN_UNPREDICTABLE(i2 != zero) {
63 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
64 }
65 const int8_t* i3 = input[3];
66 assert(i3 != NULL);
67 if XNN_UNPREDICTABLE(i3 != zero) {
68 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
69 }
70 const int8_t* i4 = input[4];
71 assert(i4 != NULL);
72 if XNN_UNPREDICTABLE(i4 != zero) {
73 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
74 }
75 input += 5;
76
77 size_t c = channels;
78 for (; c >= 2; c -= 2) {
79 int32_t vacc0 = unaligned_indexed_load_s32(w, 0);
80 int32_t vacc1 = unaligned_indexed_load_s32(w, 1);
81
82 const int32_t vi0x0 = (int32_t) i0[0];
83 const int32_t vi0x1 = (int32_t) i0[1];
84 i0 += 2;
85
86 const int32_t vk0x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[0];
87 const int32_t vk0x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[1];
88
89 vacc0 += vi0x0 * vk0x0;
90 vacc1 += vi0x1 * vk0x1;
91
92 const int32_t vi1x0 = (int32_t) i1[0];
93 const int32_t vi1x1 = (int32_t) i1[1];
94 i1 += 2;
95
96 const int32_t vk1x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[2];
97 const int32_t vk1x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[3];
98
99 vacc0 += vi1x0 * vk1x0;
100 vacc1 += vi1x1 * vk1x1;
101
102 const int32_t vi2x0 = (int32_t) i2[0];
103 const int32_t vi2x1 = (int32_t) i2[1];
104 i2 += 2;
105
106 const int32_t vk2x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[4];
107 const int32_t vk2x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[5];
108
109 vacc0 += vi2x0 * vk2x0;
110 vacc1 += vi2x1 * vk2x1;
111
112 const int32_t vi3x0 = (int32_t) i3[0];
113 const int32_t vi3x1 = (int32_t) i3[1];
114 i3 += 2;
115
116 const int32_t vk3x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[6];
117 const int32_t vk3x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[7];
118
119 vacc0 += vi3x0 * vk3x0;
120 vacc1 += vi3x1 * vk3x1;
121
122 const int32_t vi4x0 = (int32_t) i4[0];
123 const int32_t vi4x1 = (int32_t) i4[1];
124 i4 += 2;
125
126 const int32_t vk4x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[8];
127 const int32_t vk4x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[9];
128
129 vacc0 += vi4x0 * vk4x0;
130 vacc1 += vi4x1 * vk4x1;
131
132 w = (const void*) ((uintptr_t) w + 2 * sizeof(int32_t) + 10 * sizeof(int8_t));
133 b[0] = vacc0;
134 b[1] = vacc1;
135 b += 2;
136 }
137 if XNN_UNLIKELY(c != 0) {
138 int32_t vacc = unaligned_load_s32(w);
139
140 const int32_t vi0 = (int32_t) *i0;
141 const int32_t vk0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 1 * sizeof(int32_t)))[0];
142 vacc += vi0 * vk0;
143 const int32_t vi1 = (int32_t) *i1;
144 const int32_t vk1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 1 * sizeof(int32_t)))[1];
145 vacc += vi1 * vk1;
146 const int32_t vi2 = (int32_t) *i2;
147 const int32_t vk2 = (int32_t) ((const int8_t*) ((uintptr_t) w + 1 * sizeof(int32_t)))[2];
148 vacc += vi2 * vk2;
149 const int32_t vi3 = (int32_t) *i3;
150 const int32_t vk3 = (int32_t) ((const int8_t*) ((uintptr_t) w + 1 * sizeof(int32_t)))[3];
151 vacc += vi3 * vk3;
152 const int32_t vi4 = (int32_t) *i4;
153 const int32_t vk4 = (int32_t) ((const int8_t*) ((uintptr_t) w + 1 * sizeof(int32_t)))[4];
154 vacc += vi4 * vk4;
155
156 w = (const void*) ((uintptr_t) w + 1 * sizeof(int32_t) + 5 * sizeof(int8_t));
157 *b++ = vacc;
158 }
159 }
160
161 // Middle pass to process 5 inputs in each iteration.
162 for (size_t ks = kernel_size - 5; ks > 5; ks -= 5) {
163 int32_t* b = buffer;
164 const int8_t* i0 = input[0];
165 assert(i0 != NULL);
166 if XNN_UNPREDICTABLE(i0 != zero) {
167 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
168 }
169 const int8_t* i1 = input[1];
170 assert(i1 != NULL);
171 if XNN_UNPREDICTABLE(i1 != zero) {
172 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
173 }
174 const int8_t* i2 = input[2];
175 assert(i2 != NULL);
176 if XNN_UNPREDICTABLE(i2 != zero) {
177 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
178 }
179 const int8_t* i3 = input[3];
180 assert(i3 != NULL);
181 if XNN_UNPREDICTABLE(i3 != zero) {
182 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
183 }
184 const int8_t* i4 = input[4];
185 assert(i4 != NULL);
186 if XNN_UNPREDICTABLE(i4 != zero) {
187 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
188 }
189 input += 5;
190
191 size_t c = channels;
192 for (; c >= 2; c -= 2) {
193 int32_t vacc0 = b[0];
194 int32_t vacc1 = b[1];
195
196 const int32_t vi0x0 = (int32_t) i0[0];
197 const int32_t vi0x1 = (int32_t) i0[1];
198 i0 += 2;
199
200 const int32_t vk0x0 = (int32_t) ((const int8_t*) w)[0];
201 const int32_t vk0x1 = (int32_t) ((const int8_t*) w)[1];
202
203 vacc0 += vi0x0 * vk0x0;
204 vacc1 += vi0x1 * vk0x1;
205
206 const int32_t vi1x0 = (int32_t) i1[0];
207 const int32_t vi1x1 = (int32_t) i1[1];
208 i1 += 2;
209
210 const int32_t vk1x0 = (int32_t) ((const int8_t*) w)[2];
211 const int32_t vk1x1 = (int32_t) ((const int8_t*) w)[3];
212
213 vacc0 += vi1x0 * vk1x0;
214 vacc1 += vi1x1 * vk1x1;
215
216 const int32_t vi2x0 = (int32_t) i2[0];
217 const int32_t vi2x1 = (int32_t) i2[1];
218 i2 += 2;
219
220 const int32_t vk2x0 = (int32_t) ((const int8_t*) w)[4];
221 const int32_t vk2x1 = (int32_t) ((const int8_t*) w)[5];
222
223 vacc0 += vi2x0 * vk2x0;
224 vacc1 += vi2x1 * vk2x1;
225
226 const int32_t vi3x0 = (int32_t) i3[0];
227 const int32_t vi3x1 = (int32_t) i3[1];
228 i3 += 2;
229
230 const int32_t vk3x0 = (int32_t) ((const int8_t*) w)[6];
231 const int32_t vk3x1 = (int32_t) ((const int8_t*) w)[7];
232
233 vacc0 += vi3x0 * vk3x0;
234 vacc1 += vi3x1 * vk3x1;
235
236 const int32_t vi4x0 = (int32_t) i4[0];
237 const int32_t vi4x1 = (int32_t) i4[1];
238 i4 += 2;
239
240 const int32_t vk4x0 = (int32_t) ((const int8_t*) w)[8];
241 const int32_t vk4x1 = (int32_t) ((const int8_t*) w)[9];
242
243 vacc0 += vi4x0 * vk4x0;
244 vacc1 += vi4x1 * vk4x1;
245
246 w = (const void*) ((uintptr_t) w + 10 * sizeof(int8_t));
247 b[0] = vacc0;
248 b[1] = vacc1;
249 b += 2;
250 }
251 if XNN_UNLIKELY(c != 0) {
252 int32_t vacc = b[0];
253
254 const int32_t vi0 = (int32_t) *i0;
255 const int32_t vk0 = (int32_t) ((const int8_t*) w)[0];
256 vacc += vi0 * vk0;
257 const int32_t vi1 = (int32_t) *i1;
258 const int32_t vk1 = (int32_t) ((const int8_t*) w)[1];
259 vacc += vi1 * vk1;
260 const int32_t vi2 = (int32_t) *i2;
261 const int32_t vk2 = (int32_t) ((const int8_t*) w)[2];
262 vacc += vi2 * vk2;
263 const int32_t vi3 = (int32_t) *i3;
264 const int32_t vk3 = (int32_t) ((const int8_t*) w)[3];
265 vacc += vi3 * vk3;
266 const int32_t vi4 = (int32_t) *i4;
267 const int32_t vk4 = (int32_t) ((const int8_t*) w)[4];
268 vacc += vi4 * vk4;
269 w = (const void*) ((uintptr_t) w + 5 * sizeof(int8_t));
270 *b++ = vacc;
271 }
272 }
273
274 // Last pass to process up to 5 inputs.
275 {
276 const int32_t* b = buffer;
277 const int8_t* i0 = input[0];
278 assert(i0 != NULL);
279 if XNN_UNPREDICTABLE(i0 != zero) {
280 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
281 }
282 const int8_t* i1 = input[1];
283 assert(i1 != NULL);
284 if XNN_UNPREDICTABLE(i1 != zero) {
285 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
286 }
287 const int8_t* i2 = input[2];
288 assert(i2 != NULL);
289 if XNN_UNPREDICTABLE(i2 != zero) {
290 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
291 }
292 const int8_t* i3 = input[3];
293 assert(i3 != NULL);
294 if XNN_UNPREDICTABLE(i3 != zero) {
295 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
296 }
297 const int8_t* i4 = input[4];
298 assert(i4 != NULL);
299 if XNN_UNPREDICTABLE(i4 != zero) {
300 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
301 }
302
303 size_t c = channels;
304 for (; c >= 2; c -= 2) {
305 int32_t vacc0 = b[0];
306 int32_t vacc1 = b[1];
307 b += 2;
308
309 const int32_t vi0x0 = (int32_t) i0[0];
310 const int32_t vi0x1 = (int32_t) i0[1];
311 i0 += 2;
312
313 const int32_t vk0x0 = (int32_t) ((const int8_t*) w)[0];
314 const int32_t vk0x1 = (int32_t) ((const int8_t*) w)[1];
315
316 vacc0 += vi0x0 * vk0x0;
317 vacc1 += vi0x1 * vk0x1;
318
319 const int32_t vi1x0 = (int32_t) i1[0];
320 const int32_t vi1x1 = (int32_t) i1[1];
321 i1 += 2;
322
323 const int32_t vk1x0 = (int32_t) ((const int8_t*) w)[2];
324 const int32_t vk1x1 = (int32_t) ((const int8_t*) w)[3];
325
326 vacc0 += vi1x0 * vk1x0;
327 vacc1 += vi1x1 * vk1x1;
328
329 const int32_t vi2x0 = (int32_t) i2[0];
330 const int32_t vi2x1 = (int32_t) i2[1];
331 i2 += 2;
332
333 const int32_t vk2x0 = (int32_t) ((const int8_t*) w)[4];
334 const int32_t vk2x1 = (int32_t) ((const int8_t*) w)[5];
335
336 vacc0 += vi2x0 * vk2x0;
337 vacc1 += vi2x1 * vk2x1;
338
339 const int32_t vi3x0 = (int32_t) i3[0];
340 const int32_t vi3x1 = (int32_t) i3[1];
341 i3 += 2;
342
343 const int32_t vk3x0 = (int32_t) ((const int8_t*) w)[6];
344 const int32_t vk3x1 = (int32_t) ((const int8_t*) w)[7];
345
346 vacc0 += vi3x0 * vk3x0;
347 vacc1 += vi3x1 * vk3x1;
348
349 const int32_t vi4x0 = (int32_t) i4[0];
350 const int32_t vi4x1 = (int32_t) i4[1];
351 i4 += 2;
352
353 const int32_t vk4x0 = (int32_t) ((const int8_t*) w)[8];
354 const int32_t vk4x1 = (int32_t) ((const int8_t*) w)[9];
355
356 vacc0 += vi4x0 * vk4x0;
357 vacc1 += vi4x1 * vk4x1;
358
359 w = (const void*) ((uintptr_t) w + 10 * sizeof(int8_t));
360
361 float vfpacc0 = (float) vacc0;
362 float vfpacc1 = (float) vacc1;
363
364 vfpacc0 *= vscale;
365 vfpacc1 *= vscale;
366
367 vfpacc0 = math_max_f32(vfpacc0, voutput_min_less_zero_point);
368 vfpacc1 = math_max_f32(vfpacc1, voutput_min_less_zero_point);
369
370 vfpacc0 = math_min_f32(vfpacc0, voutput_max_less_zero_point);
371 vfpacc1 = math_min_f32(vfpacc1, voutput_max_less_zero_point);
372
373 const int32_t vrndacc0 = (int32_t) lrintf(vfpacc0);
374 const int32_t vrndacc1 = (int32_t) lrintf(vfpacc1);
375
376 int32_t vout0 = (int32_t) vrndacc0 + voutput_zero_point;
377 int32_t vout1 = (int32_t) vrndacc1 + voutput_zero_point;
378
379 output[0] = (int8_t) vout0;
380 output[1] = (int8_t) vout1;
381 output += 2;
382 }
383 if XNN_UNLIKELY(c != 0) {
384 int32_t vacc = b[0];
385
386 const int32_t vi0 = (int32_t) *i0;
387 const int32_t vk0 = (int32_t) ((const int8_t*) w)[0];
388 vacc += vi0 * vk0;
389 const int32_t vi1 = (int32_t) *i1;
390 const int32_t vk1 = (int32_t) ((const int8_t*) w)[1];
391 vacc += vi1 * vk1;
392 const int32_t vi2 = (int32_t) *i2;
393 const int32_t vk2 = (int32_t) ((const int8_t*) w)[2];
394 vacc += vi2 * vk2;
395 const int32_t vi3 = (int32_t) *i3;
396 const int32_t vk3 = (int32_t) ((const int8_t*) w)[3];
397 vacc += vi3 * vk3;
398 const int32_t vi4 = (int32_t) *i4;
399 const int32_t vk4 = (int32_t) ((const int8_t*) w)[4];
400 vacc += vi4 * vk4;
401
402 float vfpacc = (float) vacc * vscale;
403
404 vfpacc = math_max_f32(vfpacc, voutput_min_less_zero_point);
405 vfpacc = math_min_f32(vfpacc, voutput_max_less_zero_point);
406 const int32_t vrndacc = (int32_t) lrintf(vfpacc);
407 int32_t vout = vrndacc + voutput_zero_point;
408
409 *output++ = (int8_t) vout;
410 }
411 }
412
413 input = (const int8_t**) ((uintptr_t) input + input_stride);
414 output = (int8_t*) ((uintptr_t) output + output_increment);
415 } while (--output_width != 0);
416 }