Debian Codegraph @

src/qs8-dwconv/gen/qs8-dwconv-6f6m7l2c1s1r-minmax-fp32-wasm-fmagic.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-dwconv/multipass-scalar.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11 #include <stddef.h>
12 #include <stdint.h>
13
14 #include "xnnpack/common.h"
15 #include "xnnpack/dwconv.h"
16 #include "xnnpack/math.h"
17 #include "xnnpack/microparams.h"
18 #include "xnnpack/unaligned.h"
19
20
21 void xnn_qs8_dwconv_minmax_fp32_ukernel_6f6m7l2c1s1r__wasm_fmagic(
22 size_t channels,
23 size_t output_width,
24 const int8_t** input,
25 const void* weights,
26 int8_t* output,
27 intptr_t input_stride,
28 size_t output_increment,
29 size_t input_offset,
30 const int8_t* zero,
31 size_t kernel_size,
32 int32_t* buffer,
33 const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)])
34 {
35 assert(channels != 0);
36 assert(output_width != 0);
37 assert(kernel_size > 6);
38
39 const float vscale = params->fp32_scalar.scale;
40 const float voutput_min_less_zero_point = (int32_t) params->fp32_scalar.output_min - (int32_t) params->fp32_scalar.output_zero_point;
41 const float voutput_max_less_zero_point = (int32_t) params->fp32_scalar.output_max - (int32_t) params->fp32_scalar.output_zero_point;
42 const float vmagic_bias = 12582912.0f;
43 const int32_t vmagic_bias_less_output_zero_point = INT32_C(0x4B400000) - (int32_t) params->fp32_scalar.output_zero_point;
44 do {
45 const void* w = weights;
46
47 // First pass to process 6 inputs.
48 {
49 int32_t* b = buffer;
50 const int8_t* i0 = input[0];
51 assert(i0 != NULL);
52 if XNN_UNPREDICTABLE(i0 != zero) {
53 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
54 }
55 const int8_t* i1 = input[1];
56 assert(i1 != NULL);
57 if XNN_UNPREDICTABLE(i1 != zero) {
58 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
59 }
60 const int8_t* i2 = input[2];
61 assert(i2 != NULL);
62 if XNN_UNPREDICTABLE(i2 != zero) {
63 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
64 }
65 const int8_t* i3 = input[3];
66 assert(i3 != NULL);
67 if XNN_UNPREDICTABLE(i3 != zero) {
68 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
69 }
70 const int8_t* i4 = input[4];
71 assert(i4 != NULL);
72 if XNN_UNPREDICTABLE(i4 != zero) {
73 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
74 }
75 const int8_t* i5 = input[5];
76 assert(i5 != NULL);
77 if XNN_UNPREDICTABLE(i5 != zero) {
78 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
79 }
80 input += 6;
81
82 size_t c = channels;
83 for (; c >= 2; c -= 2) {
84 int32_t vacc0 = unaligned_indexed_load_s32(w, 0);
85 int32_t vacc1 = unaligned_indexed_load_s32(w, 1);
86
87 const int32_t vi0x0 = (int32_t) i0[0];
88 const int32_t vi0x1 = (int32_t) i0[1];
89 i0 += 2;
90
91 const int32_t vk0x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[0];
92 const int32_t vk0x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[1];
93
94 vacc0 += vi0x0 * vk0x0;
95 vacc1 += vi0x1 * vk0x1;
96
97 const int32_t vi1x0 = (int32_t) i1[0];
98 const int32_t vi1x1 = (int32_t) i1[1];
99 i1 += 2;
100
101 const int32_t vk1x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[2];
102 const int32_t vk1x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[3];
103
104 vacc0 += vi1x0 * vk1x0;
105 vacc1 += vi1x1 * vk1x1;
106
107 const int32_t vi2x0 = (int32_t) i2[0];
108 const int32_t vi2x1 = (int32_t) i2[1];
109 i2 += 2;
110
111 const int32_t vk2x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[4];
112 const int32_t vk2x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[5];
113
114 vacc0 += vi2x0 * vk2x0;
115 vacc1 += vi2x1 * vk2x1;
116
117 const int32_t vi3x0 = (int32_t) i3[0];
118 const int32_t vi3x1 = (int32_t) i3[1];
119 i3 += 2;
120
121 const int32_t vk3x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[6];
122 const int32_t vk3x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[7];
123
124 vacc0 += vi3x0 * vk3x0;
125 vacc1 += vi3x1 * vk3x1;
126
127 const int32_t vi4x0 = (int32_t) i4[0];
128 const int32_t vi4x1 = (int32_t) i4[1];
129 i4 += 2;
130
131 const int32_t vk4x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[8];
132 const int32_t vk4x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[9];
133
134 vacc0 += vi4x0 * vk4x0;
135 vacc1 += vi4x1 * vk4x1;
136
137 const int32_t vi5x0 = (int32_t) i5[0];
138 const int32_t vi5x1 = (int32_t) i5[1];
139 i5 += 2;
140
141 const int32_t vk5x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[10];
142 const int32_t vk5x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 2 * sizeof(int32_t)))[11];
143
144 vacc0 += vi5x0 * vk5x0;
145 vacc1 += vi5x1 * vk5x1;
146
147 w = (const void*) ((uintptr_t) w + 2 * sizeof(int32_t) + 12 * sizeof(int8_t));
148 b[0] = vacc0;
149 b[1] = vacc1;
150 b += 2;
151 }
152 if XNN_UNLIKELY(c != 0) {
153 int32_t vacc = unaligned_load_s32(w);
154
155 const int32_t vi0 = (int32_t) *i0;
156 const int32_t vk0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 1 * sizeof(int32_t)))[0];
157 vacc += vi0 * vk0;
158 const int32_t vi1 = (int32_t) *i1;
159 const int32_t vk1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 1 * sizeof(int32_t)))[1];
160 vacc += vi1 * vk1;
161 const int32_t vi2 = (int32_t) *i2;
162 const int32_t vk2 = (int32_t) ((const int8_t*) ((uintptr_t) w + 1 * sizeof(int32_t)))[2];
163 vacc += vi2 * vk2;
164 const int32_t vi3 = (int32_t) *i3;
165 const int32_t vk3 = (int32_t) ((const int8_t*) ((uintptr_t) w + 1 * sizeof(int32_t)))[3];
166 vacc += vi3 * vk3;
167 const int32_t vi4 = (int32_t) *i4;
168 const int32_t vk4 = (int32_t) ((const int8_t*) ((uintptr_t) w + 1 * sizeof(int32_t)))[4];
169 vacc += vi4 * vk4;
170 const int32_t vi5 = (int32_t) *i5;
171 const int32_t vk5 = (int32_t) ((const int8_t*) ((uintptr_t) w + 1 * sizeof(int32_t)))[5];
172 vacc += vi5 * vk5;
173
174 w = (const void*) ((uintptr_t) w + 1 * sizeof(int32_t) + 6 * sizeof(int8_t));
175 *b++ = vacc;
176 }
177 }
178
179 // Middle pass to process 6 inputs in each iteration.
180 for (size_t ks = kernel_size - 6; ks > 7; ks -= 6) {
181 int32_t* b = buffer;
182 const int8_t* i0 = input[0];
183 assert(i0 != NULL);
184 if XNN_UNPREDICTABLE(i0 != zero) {
185 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
186 }
187 const int8_t* i1 = input[1];
188 assert(i1 != NULL);
189 if XNN_UNPREDICTABLE(i1 != zero) {
190 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
191 }
192 const int8_t* i2 = input[2];
193 assert(i2 != NULL);
194 if XNN_UNPREDICTABLE(i2 != zero) {
195 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
196 }
197 const int8_t* i3 = input[3];
198 assert(i3 != NULL);
199 if XNN_UNPREDICTABLE(i3 != zero) {
200 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
201 }
202 const int8_t* i4 = input[4];
203 assert(i4 != NULL);
204 if XNN_UNPREDICTABLE(i4 != zero) {
205 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
206 }
207 const int8_t* i5 = input[5];
208 assert(i5 != NULL);
209 if XNN_UNPREDICTABLE(i5 != zero) {
210 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
211 }
212 input += 6;
213
214 size_t c = channels;
215 for (; c >= 2; c -= 2) {
216 int32_t vacc0 = b[0];
217 int32_t vacc1 = b[1];
218
219 const int32_t vi0x0 = (int32_t) i0[0];
220 const int32_t vi0x1 = (int32_t) i0[1];
221 i0 += 2;
222
223 const int32_t vk0x0 = (int32_t) ((const int8_t*) w)[0];
224 const int32_t vk0x1 = (int32_t) ((const int8_t*) w)[1];
225
226 vacc0 += vi0x0 * vk0x0;
227 vacc1 += vi0x1 * vk0x1;
228
229 const int32_t vi1x0 = (int32_t) i1[0];
230 const int32_t vi1x1 = (int32_t) i1[1];
231 i1 += 2;
232
233 const int32_t vk1x0 = (int32_t) ((const int8_t*) w)[2];
234 const int32_t vk1x1 = (int32_t) ((const int8_t*) w)[3];
235
236 vacc0 += vi1x0 * vk1x0;
237 vacc1 += vi1x1 * vk1x1;
238
239 const int32_t vi2x0 = (int32_t) i2[0];
240 const int32_t vi2x1 = (int32_t) i2[1];
241 i2 += 2;
242
243 const int32_t vk2x0 = (int32_t) ((const int8_t*) w)[4];
244 const int32_t vk2x1 = (int32_t) ((const int8_t*) w)[5];
245
246 vacc0 += vi2x0 * vk2x0;
247 vacc1 += vi2x1 * vk2x1;
248
249 const int32_t vi3x0 = (int32_t) i3[0];
250 const int32_t vi3x1 = (int32_t) i3[1];
251 i3 += 2;
252
253 const int32_t vk3x0 = (int32_t) ((const int8_t*) w)[6];
254 const int32_t vk3x1 = (int32_t) ((const int8_t*) w)[7];
255
256 vacc0 += vi3x0 * vk3x0;
257 vacc1 += vi3x1 * vk3x1;
258
259 const int32_t vi4x0 = (int32_t) i4[0];
260 const int32_t vi4x1 = (int32_t) i4[1];
261 i4 += 2;
262
263 const int32_t vk4x0 = (int32_t) ((const int8_t*) w)[8];
264 const int32_t vk4x1 = (int32_t) ((const int8_t*) w)[9];
265
266 vacc0 += vi4x0 * vk4x0;
267 vacc1 += vi4x1 * vk4x1;
268
269 const int32_t vi5x0 = (int32_t) i5[0];
270 const int32_t vi5x1 = (int32_t) i5[1];
271 i5 += 2;
272
273 const int32_t vk5x0 = (int32_t) ((const int8_t*) w)[10];
274 const int32_t vk5x1 = (int32_t) ((const int8_t*) w)[11];
275
276 vacc0 += vi5x0 * vk5x0;
277 vacc1 += vi5x1 * vk5x1;
278
279 w = (const void*) ((uintptr_t) w + 12 * sizeof(int8_t));
280 b[0] = vacc0;
281 b[1] = vacc1;
282 b += 2;
283 }
284 if XNN_UNLIKELY(c != 0) {
285 int32_t vacc = b[0];
286
287 const int32_t vi0 = (int32_t) *i0;
288 const int32_t vk0 = (int32_t) ((const int8_t*) w)[0];
289 vacc += vi0 * vk0;
290 const int32_t vi1 = (int32_t) *i1;
291 const int32_t vk1 = (int32_t) ((const int8_t*) w)[1];
292 vacc += vi1 * vk1;
293 const int32_t vi2 = (int32_t) *i2;
294 const int32_t vk2 = (int32_t) ((const int8_t*) w)[2];
295 vacc += vi2 * vk2;
296 const int32_t vi3 = (int32_t) *i3;
297 const int32_t vk3 = (int32_t) ((const int8_t*) w)[3];
298 vacc += vi3 * vk3;
299 const int32_t vi4 = (int32_t) *i4;
300 const int32_t vk4 = (int32_t) ((const int8_t*) w)[4];
301 vacc += vi4 * vk4;
302 const int32_t vi5 = (int32_t) *i5;
303 const int32_t vk5 = (int32_t) ((const int8_t*) w)[5];
304 vacc += vi5 * vk5;
305 w = (const void*) ((uintptr_t) w + 6 * sizeof(int8_t));
306 *b++ = vacc;
307 }
308 }
309
310 // Last pass to process up to 7 inputs.
311 {
312 const int32_t* b = buffer;
313 const int8_t* i0 = input[0];
314 assert(i0 != NULL);
315 if XNN_UNPREDICTABLE(i0 != zero) {
316 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
317 }
318 const int8_t* i1 = input[1];
319 assert(i1 != NULL);
320 if XNN_UNPREDICTABLE(i1 != zero) {
321 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
322 }
323 const int8_t* i2 = input[2];
324 assert(i2 != NULL);
325 if XNN_UNPREDICTABLE(i2 != zero) {
326 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
327 }
328 const int8_t* i3 = input[3];
329 assert(i3 != NULL);
330 if XNN_UNPREDICTABLE(i3 != zero) {
331 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
332 }
333 const int8_t* i4 = input[4];
334 assert(i4 != NULL);
335 if XNN_UNPREDICTABLE(i4 != zero) {
336 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
337 }
338 const int8_t* i5 = input[5];
339 assert(i5 != NULL);
340 if XNN_UNPREDICTABLE(i5 != zero) {
341 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
342 }
343 const int8_t* i6 = input[6];
344 assert(i6 != NULL);
345 if XNN_UNPREDICTABLE(i6 != zero) {
346 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
347 }
348
349 size_t c = channels;
350 for (; c >= 2; c -= 2) {
351 int32_t vacc0 = b[0];
352 int32_t vacc1 = b[1];
353 b += 2;
354
355 const int32_t vi0x0 = (int32_t) i0[0];
356 const int32_t vi0x1 = (int32_t) i0[1];
357 i0 += 2;
358
359 const int32_t vk0x0 = (int32_t) ((const int8_t*) w)[0];
360 const int32_t vk0x1 = (int32_t) ((const int8_t*) w)[1];
361
362 vacc0 += vi0x0 * vk0x0;
363 vacc1 += vi0x1 * vk0x1;
364
365 const int32_t vi1x0 = (int32_t) i1[0];
366 const int32_t vi1x1 = (int32_t) i1[1];
367 i1 += 2;
368
369 const int32_t vk1x0 = (int32_t) ((const int8_t*) w)[2];
370 const int32_t vk1x1 = (int32_t) ((const int8_t*) w)[3];
371
372 vacc0 += vi1x0 * vk1x0;
373 vacc1 += vi1x1 * vk1x1;
374
375 const int32_t vi2x0 = (int32_t) i2[0];
376 const int32_t vi2x1 = (int32_t) i2[1];
377 i2 += 2;
378
379 const int32_t vk2x0 = (int32_t) ((const int8_t*) w)[4];
380 const int32_t vk2x1 = (int32_t) ((const int8_t*) w)[5];
381
382 vacc0 += vi2x0 * vk2x0;
383 vacc1 += vi2x1 * vk2x1;
384
385 const int32_t vi3x0 = (int32_t) i3[0];
386 const int32_t vi3x1 = (int32_t) i3[1];
387 i3 += 2;
388
389 const int32_t vk3x0 = (int32_t) ((const int8_t*) w)[6];
390 const int32_t vk3x1 = (int32_t) ((const int8_t*) w)[7];
391
392 vacc0 += vi3x0 * vk3x0;
393 vacc1 += vi3x1 * vk3x1;
394
395 const int32_t vi4x0 = (int32_t) i4[0];
396 const int32_t vi4x1 = (int32_t) i4[1];
397 i4 += 2;
398
399 const int32_t vk4x0 = (int32_t) ((const int8_t*) w)[8];
400 const int32_t vk4x1 = (int32_t) ((const int8_t*) w)[9];
401
402 vacc0 += vi4x0 * vk4x0;
403 vacc1 += vi4x1 * vk4x1;
404
405 const int32_t vi5x0 = (int32_t) i5[0];
406 const int32_t vi5x1 = (int32_t) i5[1];
407 i5 += 2;
408
409 const int32_t vk5x0 = (int32_t) ((const int8_t*) w)[10];
410 const int32_t vk5x1 = (int32_t) ((const int8_t*) w)[11];
411
412 vacc0 += vi5x0 * vk5x0;
413 vacc1 += vi5x1 * vk5x1;
414
415 const int32_t vi6x0 = (int32_t) i6[0];
416 const int32_t vi6x1 = (int32_t) i6[1];
417 i6 += 2;
418
419 const int32_t vk6x0 = (int32_t) ((const int8_t*) w)[12];
420 const int32_t vk6x1 = (int32_t) ((const int8_t*) w)[13];
421
422 vacc0 += vi6x0 * vk6x0;
423 vacc1 += vi6x1 * vk6x1;
424
425 w = (const void*) ((uintptr_t) w + 14 * sizeof(int8_t));
426
427 float vfpacc0 = (float) vacc0;
428 float vfpacc1 = (float) vacc1;
429
430 vfpacc0 *= vscale;
431 vfpacc1 *= vscale;
432
433 vfpacc0 = __builtin_wasm_max_f32(vfpacc0, voutput_min_less_zero_point);
434 vfpacc1 = __builtin_wasm_max_f32(vfpacc1, voutput_min_less_zero_point);
435
436 vfpacc0 = __builtin_wasm_min_f32(vfpacc0, voutput_max_less_zero_point);
437 vfpacc1 = __builtin_wasm_min_f32(vfpacc1, voutput_max_less_zero_point);
438
439 vfpacc0 += vmagic_bias;
440 vfpacc1 += vmagic_bias;
441
442 int32_t vout0 = (int32_t) float_as_uint32(vfpacc0) - vmagic_bias_less_output_zero_point;
443 int32_t vout1 = (int32_t) float_as_uint32(vfpacc1) - vmagic_bias_less_output_zero_point;
444
445 output[0] = (int8_t) vout0;
446 output[1] = (int8_t) vout1;
447 output += 2;
448 }
449 if XNN_UNLIKELY(c != 0) {
450 int32_t vacc = b[0];
451
452 const int32_t vi0 = (int32_t) *i0;
453 const int32_t vk0 = (int32_t) ((const int8_t*) w)[0];
454 vacc += vi0 * vk0;
455 const int32_t vi1 = (int32_t) *i1;
456 const int32_t vk1 = (int32_t) ((const int8_t*) w)[1];
457 vacc += vi1 * vk1;
458 const int32_t vi2 = (int32_t) *i2;
459 const int32_t vk2 = (int32_t) ((const int8_t*) w)[2];
460 vacc += vi2 * vk2;
461 const int32_t vi3 = (int32_t) *i3;
462 const int32_t vk3 = (int32_t) ((const int8_t*) w)[3];
463 vacc += vi3 * vk3;
464 const int32_t vi4 = (int32_t) *i4;
465 const int32_t vk4 = (int32_t) ((const int8_t*) w)[4];
466 vacc += vi4 * vk4;
467 const int32_t vi5 = (int32_t) *i5;
468 const int32_t vk5 = (int32_t) ((const int8_t*) w)[5];
469 vacc += vi5 * vk5;
470 const int32_t vi6 = (int32_t) *i6;
471 const int32_t vk6 = (int32_t) ((const int8_t*) w)[6];
472 vacc += vi6 * vk6;
473
474 float vfpacc = (float) vacc * vscale;
475
476 vfpacc = __builtin_wasm_max_f32(vfpacc, voutput_min_less_zero_point);
477 vfpacc = __builtin_wasm_min_f32(vfpacc, voutput_max_less_zero_point);
478 vfpacc += vmagic_bias;
479 int32_t vout = (int32_t) float_as_uint32(vfpacc) - vmagic_bias_less_output_zero_point;
480
481 *output++ = (int8_t) vout;
482 }
483 }
484
485 input = (const int8_t**) ((uintptr_t) input + input_stride);
486 output = (int8_t*) ((uintptr_t) output + output_increment);
487 } while (--output_width != 0);
488 }