Debian Codegraph @

src/qs8-dwconv/gen/qs8-dwconv-9p4c-minmax-fp32-wasm-fmagic.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-dwconv/unipass-scalar.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2021 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11 #include <stddef.h>
12 #include <stdint.h>
13
14 #include "xnnpack/common.h"
15 #include "xnnpack/dwconv.h"
16 #include "xnnpack/math.h"
17 #include "xnnpack/microparams.h"
18
19 void xnn_qs8_dwconv_minmax_fp32_ukernel_9p4c__wasm_fmagic(
20 size_t channels,
21 size_t output_width,
22 const int8_t** input,
23 const void* weights,
24 int8_t* output,
25 intptr_t input_stride,
26 size_t output_increment,
27 size_t input_offset,
28 const int8_t* zero,
29 const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)])
30 {
31 assert(channels != 0);
32 assert(output_width != 0);
33
34 const float vscale = params->fp32_scalar.scale;
35 const float voutput_min_less_zero_point = (int32_t) params->fp32_scalar.output_min - (int32_t) params->fp32_scalar.output_zero_point;
36 const float voutput_max_less_zero_point = (int32_t) params->fp32_scalar.output_max - (int32_t) params->fp32_scalar.output_zero_point;
37 const float vmagic_bias = 12582912.0f;
38 const int32_t vmagic_bias_less_output_zero_point = INT32_C(0x4B400000) - (int32_t) params->fp32_scalar.output_zero_point;
39 do {
40 const int8_t* i0 = input[0];
41 assert(i0 != NULL);
42 if XNN_UNPREDICTABLE(i0 != zero) {
43 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
44 }
45 const int8_t* i1 = input[1];
46 assert(i1 != NULL);
47 if XNN_UNPREDICTABLE(i1 != zero) {
48 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
49 }
50 const int8_t* i2 = input[2];
51 assert(i2 != NULL);
52 if XNN_UNPREDICTABLE(i2 != zero) {
53 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
54 }
55 const int8_t* i3 = input[3];
56 assert(i3 != NULL);
57 if XNN_UNPREDICTABLE(i3 != zero) {
58 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
59 }
60 const int8_t* i4 = input[4];
61 assert(i4 != NULL);
62 if XNN_UNPREDICTABLE(i4 != zero) {
63 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
64 }
65 const int8_t* i5 = input[5];
66 assert(i5 != NULL);
67 if XNN_UNPREDICTABLE(i5 != zero) {
68 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
69 }
70 const int8_t* i6 = input[6];
71 assert(i6 != NULL);
72 if XNN_UNPREDICTABLE(i6 != zero) {
73 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
74 }
75 const int8_t* i7 = input[7];
76 assert(i7 != NULL);
77 if XNN_UNPREDICTABLE(i7 != zero) {
78 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
79 }
80 const int8_t* i8 = input[8];
81 assert(i8 != NULL);
82 if XNN_UNPREDICTABLE(i8 != zero) {
83 i8 = (const int8_t*) ((uintptr_t) i8 + input_offset);
84 }
85 input = (const int8_t**) ((uintptr_t) input + input_stride);
86
87 size_t c = channels;
88 const void* w = weights;
89 for (; c >= 4; c -= 4) {
90 int32_t vacc0 = ((const int32_t*) w)[0];
91 int32_t vacc1 = ((const int32_t*) w)[1];
92 int32_t vacc2 = ((const int32_t*) w)[2];
93 int32_t vacc3 = ((const int32_t*) w)[3];
94
95
96 const int32_t vi0x0 = (int32_t) i0[0];
97 const int32_t vi0x1 = (int32_t) i0[1];
98 const int32_t vi0x2 = (int32_t) i0[2];
99 const int32_t vi0x3 = (int32_t) i0[3];
100 i0 += 4;
101
102 const int32_t vk0x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[0];
103 const int32_t vk0x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[1];
104 const int32_t vk0x2 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[2];
105 const int32_t vk0x3 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[3];
106
107 vacc0 += vi0x0 * vk0x0;
108 vacc1 += vi0x1 * vk0x1;
109 vacc2 += vi0x2 * vk0x2;
110 vacc3 += vi0x3 * vk0x3;
111
112 const int32_t vi1x0 = (int32_t) i1[0];
113 const int32_t vi1x1 = (int32_t) i1[1];
114 const int32_t vi1x2 = (int32_t) i1[2];
115 const int32_t vi1x3 = (int32_t) i1[3];
116 i1 += 4;
117
118 const int32_t vk1x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[4];
119 const int32_t vk1x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[5];
120 const int32_t vk1x2 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[6];
121 const int32_t vk1x3 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[7];
122
123 vacc0 += vi1x0 * vk1x0;
124 vacc1 += vi1x1 * vk1x1;
125 vacc2 += vi1x2 * vk1x2;
126 vacc3 += vi1x3 * vk1x3;
127
128 const int32_t vi2x0 = (int32_t) i2[0];
129 const int32_t vi2x1 = (int32_t) i2[1];
130 const int32_t vi2x2 = (int32_t) i2[2];
131 const int32_t vi2x3 = (int32_t) i2[3];
132 i2 += 4;
133
134 const int32_t vk2x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[8];
135 const int32_t vk2x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[9];
136 const int32_t vk2x2 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[10];
137 const int32_t vk2x3 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[11];
138
139 vacc0 += vi2x0 * vk2x0;
140 vacc1 += vi2x1 * vk2x1;
141 vacc2 += vi2x2 * vk2x2;
142 vacc3 += vi2x3 * vk2x3;
143
144 const int32_t vi3x0 = (int32_t) i3[0];
145 const int32_t vi3x1 = (int32_t) i3[1];
146 const int32_t vi3x2 = (int32_t) i3[2];
147 const int32_t vi3x3 = (int32_t) i3[3];
148 i3 += 4;
149
150 const int32_t vk3x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[12];
151 const int32_t vk3x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[13];
152 const int32_t vk3x2 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[14];
153 const int32_t vk3x3 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[15];
154
155 vacc0 += vi3x0 * vk3x0;
156 vacc1 += vi3x1 * vk3x1;
157 vacc2 += vi3x2 * vk3x2;
158 vacc3 += vi3x3 * vk3x3;
159
160 const int32_t vi4x0 = (int32_t) i4[0];
161 const int32_t vi4x1 = (int32_t) i4[1];
162 const int32_t vi4x2 = (int32_t) i4[2];
163 const int32_t vi4x3 = (int32_t) i4[3];
164 i4 += 4;
165
166 const int32_t vk4x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[16];
167 const int32_t vk4x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[17];
168 const int32_t vk4x2 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[18];
169 const int32_t vk4x3 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[19];
170
171 vacc0 += vi4x0 * vk4x0;
172 vacc1 += vi4x1 * vk4x1;
173 vacc2 += vi4x2 * vk4x2;
174 vacc3 += vi4x3 * vk4x3;
175
176 const int32_t vi5x0 = (int32_t) i5[0];
177 const int32_t vi5x1 = (int32_t) i5[1];
178 const int32_t vi5x2 = (int32_t) i5[2];
179 const int32_t vi5x3 = (int32_t) i5[3];
180 i5 += 4;
181
182 const int32_t vk5x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[20];
183 const int32_t vk5x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[21];
184 const int32_t vk5x2 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[22];
185 const int32_t vk5x3 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[23];
186
187 vacc0 += vi5x0 * vk5x0;
188 vacc1 += vi5x1 * vk5x1;
189 vacc2 += vi5x2 * vk5x2;
190 vacc3 += vi5x3 * vk5x3;
191
192 const int32_t vi6x0 = (int32_t) i6[0];
193 const int32_t vi6x1 = (int32_t) i6[1];
194 const int32_t vi6x2 = (int32_t) i6[2];
195 const int32_t vi6x3 = (int32_t) i6[3];
196 i6 += 4;
197
198 const int32_t vk6x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[24];
199 const int32_t vk6x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[25];
200 const int32_t vk6x2 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[26];
201 const int32_t vk6x3 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[27];
202
203 vacc0 += vi6x0 * vk6x0;
204 vacc1 += vi6x1 * vk6x1;
205 vacc2 += vi6x2 * vk6x2;
206 vacc3 += vi6x3 * vk6x3;
207
208 const int32_t vi7x0 = (int32_t) i7[0];
209 const int32_t vi7x1 = (int32_t) i7[1];
210 const int32_t vi7x2 = (int32_t) i7[2];
211 const int32_t vi7x3 = (int32_t) i7[3];
212 i7 += 4;
213
214 const int32_t vk7x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[28];
215 const int32_t vk7x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[29];
216 const int32_t vk7x2 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[30];
217 const int32_t vk7x3 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[31];
218
219 vacc0 += vi7x0 * vk7x0;
220 vacc1 += vi7x1 * vk7x1;
221 vacc2 += vi7x2 * vk7x2;
222 vacc3 += vi7x3 * vk7x3;
223
224 const int32_t vi8x0 = (int32_t) i8[0];
225 const int32_t vi8x1 = (int32_t) i8[1];
226 const int32_t vi8x2 = (int32_t) i8[2];
227 const int32_t vi8x3 = (int32_t) i8[3];
228 i8 += 4;
229
230 const int32_t vk8x0 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[32];
231 const int32_t vk8x1 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[33];
232 const int32_t vk8x2 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[34];
233 const int32_t vk8x3 = (int32_t) ((const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t)))[35];
234
235 vacc0 += vi8x0 * vk8x0;
236 vacc1 += vi8x1 * vk8x1;
237 vacc2 += vi8x2 * vk8x2;
238 vacc3 += vi8x3 * vk8x3;
239
240 w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t) + 36 * sizeof(int8_t));
241
242 float vfpacc0 = (float) vacc0;
243 float vfpacc1 = (float) vacc1;
244 float vfpacc2 = (float) vacc2;
245 float vfpacc3 = (float) vacc3;
246
247 vfpacc0 *= vscale;
248 vfpacc1 *= vscale;
249 vfpacc2 *= vscale;
250 vfpacc3 *= vscale;
251
252 vfpacc0 = __builtin_wasm_max_f32(vfpacc0, voutput_min_less_zero_point);
253 vfpacc1 = __builtin_wasm_max_f32(vfpacc1, voutput_min_less_zero_point);
254 vfpacc2 = __builtin_wasm_max_f32(vfpacc2, voutput_min_less_zero_point);
255 vfpacc3 = __builtin_wasm_max_f32(vfpacc3, voutput_min_less_zero_point);
256
257 vfpacc0 = __builtin_wasm_min_f32(vfpacc0, voutput_max_less_zero_point);
258 vfpacc1 = __builtin_wasm_min_f32(vfpacc1, voutput_max_less_zero_point);
259 vfpacc2 = __builtin_wasm_min_f32(vfpacc2, voutput_max_less_zero_point);
260 vfpacc3 = __builtin_wasm_min_f32(vfpacc3, voutput_max_less_zero_point);
261
262 vfpacc0 += vmagic_bias;
263 vfpacc1 += vmagic_bias;
264 vfpacc2 += vmagic_bias;
265 vfpacc3 += vmagic_bias;
266
267 int32_t vout0 = (int32_t) float_as_uint32(vfpacc0) - vmagic_bias_less_output_zero_point;
268 int32_t vout1 = (int32_t) float_as_uint32(vfpacc1) - vmagic_bias_less_output_zero_point;
269 int32_t vout2 = (int32_t) float_as_uint32(vfpacc2) - vmagic_bias_less_output_zero_point;
270 int32_t vout3 = (int32_t) float_as_uint32(vfpacc3) - vmagic_bias_less_output_zero_point;
271
272 output[0] = (int8_t) vout0;
273 output[1] = (int8_t) vout1;
274 output[2] = (int8_t) vout2;
275 output[3] = (int8_t) vout3;
276 output += 4;
277 }
278 if XNN_UNLIKELY(c != 0) {
279 const int8_t* k = (const int8_t*) ((uintptr_t) w + 4 * sizeof(int32_t));
280 do {
281 int32_t vacc = *((const int32_t*) w);
282 w = (const int32_t*) w + 1;
283
284 const int32_t vi0 = (int32_t) *i0++;
285 const int32_t vk0 = (int32_t) k[0];
286 vacc += vi0 * vk0;
287 const int32_t vi1 = (int32_t) *i1++;
288 const int32_t vk1 = (int32_t) k[4];
289 vacc += vi1 * vk1;
290 const int32_t vi2 = (int32_t) *i2++;
291 const int32_t vk2 = (int32_t) k[8];
292 vacc += vi2 * vk2;
293 const int32_t vi3 = (int32_t) *i3++;
294 const int32_t vk3 = (int32_t) k[12];
295 vacc += vi3 * vk3;
296 const int32_t vi4 = (int32_t) *i4++;
297 const int32_t vk4 = (int32_t) k[16];
298 vacc += vi4 * vk4;
299 const int32_t vi5 = (int32_t) *i5++;
300 const int32_t vk5 = (int32_t) k[20];
301 vacc += vi5 * vk5;
302 const int32_t vi6 = (int32_t) *i6++;
303 const int32_t vk6 = (int32_t) k[24];
304 vacc += vi6 * vk6;
305 const int32_t vi7 = (int32_t) *i7++;
306 const int32_t vk7 = (int32_t) k[28];
307 vacc += vi7 * vk7;
308 const int32_t vi8 = (int32_t) *i8++;
309 const int32_t vk8 = (int32_t) k[32];
310 vacc += vi8 * vk8;
311 k += 1;
312
313 float vfpacc = (float) vacc * vscale;
314
315 vfpacc = __builtin_wasm_max_f32(vfpacc, voutput_min_less_zero_point);
316 vfpacc = __builtin_wasm_min_f32(vfpacc, voutput_max_less_zero_point);
317 vfpacc += vmagic_bias;
318 int32_t vout = (int32_t) float_as_uint32(vfpacc) - vmagic_bias_less_output_zero_point;
319
320 *output++ = (int8_t) vout;
321 } while (--c != 0);
322 }
323
324 output = (int8_t*) ((uintptr_t) output + output_increment);
325 } while (--output_width != 0);
326 }