Debian Codegraph @

src/qd8-f32-qc8w-gemm/gen/qd8-f32-qc8w-gemm-2x8-minmax-wasm.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-gemm/scalar.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2021 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include "xnnpack/gemm.h"
13 #include "xnnpack/math.h"
14
15
16 void xnn_qd8_f32_qc8w_gemm_minmax_ukernel_2x8__wasm(
17 size_t mr,
18 size_t nc,
19 size_t kc,
20 const int8_t* restrict a,
21 size_t a_stride,
22 const void* restrict w,
23 float* restrict c,
24 size_t cm_stride,
25 size_t cn_stride,
26 const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)],
27 const struct xnn_qd8_quantization_params quantization_params[restrict XNN_MIN_ELEMENTS(1)])
28 {
29 assert(mr != 0);
30 assert(mr <= 2);
31 assert(nc != 0);
32 assert(kc != 0);
33
34 const int8_t* a0 = a;
35 float* c0 = c;
36 const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
37 float* c1 = (float*) ((uintptr_t) c0 + cm_stride);
38 if XNN_UNPREDICTABLE(mr != 2) {
39 a1 = a0;
40 c1 = c0;
41 }
42
43
44 do {
45 const int32_t vksum0 = ((const int32_t*) w)[0];
46 const int32_t vksum1 = ((const int32_t*) w)[1];
47 const int32_t vksum2 = ((const int32_t*) w)[2];
48 const int32_t vksum3 = ((const int32_t*) w)[3];
49 const int32_t vksum4 = ((const int32_t*) w)[4];
50 const int32_t vksum5 = ((const int32_t*) w)[5];
51 const int32_t vksum6 = ((const int32_t*) w)[6];
52 const int32_t vksum7 = ((const int32_t*) w)[7];
53 const int32_t vinput_zero_point0 = quantization_params[0].zero_point;
54 int32_t vacc0x0 = vksum0 * vinput_zero_point0;
55 int32_t vacc0x1 = vksum1 * vinput_zero_point0;
56 int32_t vacc0x2 = vksum2 * vinput_zero_point0;
57 int32_t vacc0x3 = vksum3 * vinput_zero_point0;
58 int32_t vacc0x4 = vksum4 * vinput_zero_point0;
59 int32_t vacc0x5 = vksum5 * vinput_zero_point0;
60 int32_t vacc0x6 = vksum6 * vinput_zero_point0;
61 int32_t vacc0x7 = vksum7 * vinput_zero_point0;
62 const int32_t vinput_zero_point1 = quantization_params[1].zero_point;
63 int32_t vacc1x0 = vksum0 * vinput_zero_point1;
64 int32_t vacc1x1 = vksum1 * vinput_zero_point1;
65 int32_t vacc1x2 = vksum2 * vinput_zero_point1;
66 int32_t vacc1x3 = vksum3 * vinput_zero_point1;
67 int32_t vacc1x4 = vksum4 * vinput_zero_point1;
68 int32_t vacc1x5 = vksum5 * vinput_zero_point1;
69 int32_t vacc1x6 = vksum6 * vinput_zero_point1;
70 int32_t vacc1x7 = vksum7 * vinput_zero_point1;
71 w = (const int32_t*) w + 8;
72
73 size_t k = kc;
74 do {
75 const int32_t va0 = (int32_t) *a0++;
76 const int32_t va1 = (int32_t) *a1++;
77
78 const int32_t vb0 = (int32_t) ((const int8_t*) w)[0];
79 const int32_t vb1 = (int32_t) ((const int8_t*) w)[1];
80 const int32_t vb2 = (int32_t) ((const int8_t*) w)[2];
81 const int32_t vb3 = (int32_t) ((const int8_t*) w)[3];
82 const int32_t vb4 = (int32_t) ((const int8_t*) w)[4];
83 const int32_t vb5 = (int32_t) ((const int8_t*) w)[5];
84 const int32_t vb6 = (int32_t) ((const int8_t*) w)[6];
85 const int32_t vb7 = (int32_t) ((const int8_t*) w)[7];
86 w = (const int8_t*) w + 8;
87
88 vacc0x0 += va0 * vb0;
89 vacc0x1 += va0 * vb1;
90 vacc0x2 += va0 * vb2;
91 vacc0x3 += va0 * vb3;
92 vacc0x4 += va0 * vb4;
93 vacc0x5 += va0 * vb5;
94 vacc0x6 += va0 * vb6;
95 vacc0x7 += va0 * vb7;
96 vacc1x0 += va1 * vb0;
97 vacc1x1 += va1 * vb1;
98 vacc1x2 += va1 * vb2;
99 vacc1x3 += va1 * vb3;
100 vacc1x4 += va1 * vb4;
101 vacc1x5 += va1 * vb5;
102 vacc1x6 += va1 * vb6;
103 vacc1x7 += va1 * vb7;
104
105 k -= sizeof(int8_t);
106 } while (k != 0);
107
108 float vout0x0 = (float) vacc0x0;
109 float vout0x1 = (float) vacc0x1;
110 float vout0x2 = (float) vacc0x2;
111 float vout0x3 = (float) vacc0x3;
112 float vout0x4 = (float) vacc0x4;
113 float vout0x5 = (float) vacc0x5;
114 float vout0x6 = (float) vacc0x6;
115 float vout0x7 = (float) vacc0x7;
116 float vout1x0 = (float) vacc1x0;
117 float vout1x1 = (float) vacc1x1;
118 float vout1x2 = (float) vacc1x2;
119 float vout1x3 = (float) vacc1x3;
120 float vout1x4 = (float) vacc1x4;
121 float vout1x5 = (float) vacc1x5;
122 float vout1x6 = (float) vacc1x6;
123 float vout1x7 = (float) vacc1x7;
124
125 const float vinput_scale0 = quantization_params[0].inv_scale;
126 vout0x0 *= vinput_scale0;
127 vout0x1 *= vinput_scale0;
128 vout0x2 *= vinput_scale0;
129 vout0x3 *= vinput_scale0;
130 vout0x4 *= vinput_scale0;
131 vout0x5 *= vinput_scale0;
132 vout0x6 *= vinput_scale0;
133 vout0x7 *= vinput_scale0;
134 const float vinput_scale1 = quantization_params[1].inv_scale;
135 vout1x0 *= vinput_scale1;
136 vout1x1 *= vinput_scale1;
137 vout1x2 *= vinput_scale1;
138 vout1x3 *= vinput_scale1;
139 vout1x4 *= vinput_scale1;
140 vout1x5 *= vinput_scale1;
141 vout1x6 *= vinput_scale1;
142 vout1x7 *= vinput_scale1;
143
144 const float vfilter_output_scale0 = ((const float*) w)[0];
145 vout0x0 *= vfilter_output_scale0;
146 vout1x0 *= vfilter_output_scale0;
147 const float vfilter_output_scale1 = ((const float*) w)[1];
148 vout0x1 *= vfilter_output_scale1;
149 vout1x1 *= vfilter_output_scale1;
150 const float vfilter_output_scale2 = ((const float*) w)[2];
151 vout0x2 *= vfilter_output_scale2;
152 vout1x2 *= vfilter_output_scale2;
153 const float vfilter_output_scale3 = ((const float*) w)[3];
154 vout0x3 *= vfilter_output_scale3;
155 vout1x3 *= vfilter_output_scale3;
156 const float vfilter_output_scale4 = ((const float*) w)[4];
157 vout0x4 *= vfilter_output_scale4;
158 vout1x4 *= vfilter_output_scale4;
159 const float vfilter_output_scale5 = ((const float*) w)[5];
160 vout0x5 *= vfilter_output_scale5;
161 vout1x5 *= vfilter_output_scale5;
162 const float vfilter_output_scale6 = ((const float*) w)[6];
163 vout0x6 *= vfilter_output_scale6;
164 vout1x6 *= vfilter_output_scale6;
165 const float vfilter_output_scale7 = ((const float*) w)[7];
166 vout0x7 *= vfilter_output_scale7;
167 vout1x7 *= vfilter_output_scale7;
168
169 const float vbias0 = ((const float*) w)[8];
170 vout0x0 += vbias0;
171 vout1x0 += vbias0;
172 const float vbias1 = ((const float*) w)[9];
173 vout0x1 += vbias1;
174 vout1x1 += vbias1;
175 const float vbias2 = ((const float*) w)[10];
176 vout0x2 += vbias2;
177 vout1x2 += vbias2;
178 const float vbias3 = ((const float*) w)[11];
179 vout0x3 += vbias3;
180 vout1x3 += vbias3;
181 const float vbias4 = ((const float*) w)[12];
182 vout0x4 += vbias4;
183 vout1x4 += vbias4;
184 const float vbias5 = ((const float*) w)[13];
185 vout0x5 += vbias5;
186 vout1x5 += vbias5;
187 const float vbias6 = ((const float*) w)[14];
188 vout0x6 += vbias6;
189 vout1x6 += vbias6;
190 const float vbias7 = ((const float*) w)[15];
191 vout0x7 += vbias7;
192 vout1x7 += vbias7;
193
194 w = (const float*) w + 16;
195
196 const float voutput_min = params->scalar.min;
197 vout0x0 = __builtin_wasm_max_f32(vout0x0, voutput_min);
198 vout1x0 = __builtin_wasm_max_f32(vout1x0, voutput_min);
199 vout0x1 = __builtin_wasm_max_f32(vout0x1, voutput_min);
200 vout1x1 = __builtin_wasm_max_f32(vout1x1, voutput_min);
201 vout0x2 = __builtin_wasm_max_f32(vout0x2, voutput_min);
202 vout1x2 = __builtin_wasm_max_f32(vout1x2, voutput_min);
203 vout0x3 = __builtin_wasm_max_f32(vout0x3, voutput_min);
204 vout1x3 = __builtin_wasm_max_f32(vout1x3, voutput_min);
205 vout0x4 = __builtin_wasm_max_f32(vout0x4, voutput_min);
206 vout1x4 = __builtin_wasm_max_f32(vout1x4, voutput_min);
207 vout0x5 = __builtin_wasm_max_f32(vout0x5, voutput_min);
208 vout1x5 = __builtin_wasm_max_f32(vout1x5, voutput_min);
209 vout0x6 = __builtin_wasm_max_f32(vout0x6, voutput_min);
210 vout1x6 = __builtin_wasm_max_f32(vout1x6, voutput_min);
211 vout0x7 = __builtin_wasm_max_f32(vout0x7, voutput_min);
212 vout1x7 = __builtin_wasm_max_f32(vout1x7, voutput_min);
213
214 const float voutput_max = params->scalar.max;
215 vout0x0 = __builtin_wasm_min_f32(vout0x0, voutput_max);
216 vout1x0 = __builtin_wasm_min_f32(vout1x0, voutput_max);
217 vout0x1 = __builtin_wasm_min_f32(vout0x1, voutput_max);
218 vout1x1 = __builtin_wasm_min_f32(vout1x1, voutput_max);
219 vout0x2 = __builtin_wasm_min_f32(vout0x2, voutput_max);
220 vout1x2 = __builtin_wasm_min_f32(vout1x2, voutput_max);
221 vout0x3 = __builtin_wasm_min_f32(vout0x3, voutput_max);
222 vout1x3 = __builtin_wasm_min_f32(vout1x3, voutput_max);
223 vout0x4 = __builtin_wasm_min_f32(vout0x4, voutput_max);
224 vout1x4 = __builtin_wasm_min_f32(vout1x4, voutput_max);
225 vout0x5 = __builtin_wasm_min_f32(vout0x5, voutput_max);
226 vout1x5 = __builtin_wasm_min_f32(vout1x5, voutput_max);
227 vout0x6 = __builtin_wasm_min_f32(vout0x6, voutput_max);
228 vout1x6 = __builtin_wasm_min_f32(vout1x6, voutput_max);
229 vout0x7 = __builtin_wasm_min_f32(vout0x7, voutput_max);
230 vout1x7 = __builtin_wasm_min_f32(vout1x7, voutput_max);
231
232 if XNN_LIKELY(nc >= 8) {
233 c0[0] = vout0x0;
234 c0[1] = vout0x1;
235 c0[2] = vout0x2;
236 c0[3] = vout0x3;
237 c0[4] = vout0x4;
238 c0[5] = vout0x5;
239 c0[6] = vout0x6;
240 c0[7] = vout0x7;
241 c1[0] = vout1x0;
242 c1[1] = vout1x1;
243 c1[2] = vout1x2;
244 c1[3] = vout1x3;
245 c1[4] = vout1x4;
246 c1[5] = vout1x5;
247 c1[6] = vout1x6;
248 c1[7] = vout1x7;
249
250 a0 = (const int8_t*) ((uintptr_t) a0 - kc);
251 a1 = (const int8_t*) ((uintptr_t) a1 - kc);
252
253 c0 = (float*) ((uintptr_t) c0 + cn_stride);
254 c1 = (float*) ((uintptr_t) c1 + cn_stride);
255
256 nc -= 8;
257 } else {
258 if (nc & 4) {
259 c0[0] = vout0x0;
260 c0[1] = vout0x1;
261 c0[2] = vout0x2;
262 c0[3] = vout0x3;
263 vout0x0 = vout0x4;
264 vout0x1 = vout0x5;
265 vout0x2 = vout0x6;
266 c0 += 4;
267 c1[0] = vout1x0;
268 c1[1] = vout1x1;
269 c1[2] = vout1x2;
270 c1[3] = vout1x3;
271 vout1x0 = vout1x4;
272 vout1x1 = vout1x5;
273 vout1x2 = vout1x6;
274 c1 += 4;
275 }
276 if (nc & 2) {
277 c0[0] = vout0x0;
278 c0[1] = vout0x1;
279 vout0x0 = vout0x2;
280 vout0x1 = vout0x3;
281 vout0x2 = vout0x4;
282 vout0x3 = vout0x5;
283 vout0x4 = vout0x6;
284 c0 += 2;
285 c1[0] = vout1x0;
286 c1[1] = vout1x1;
287 vout1x0 = vout1x2;
288 vout1x1 = vout1x3;
289 vout1x2 = vout1x4;
290 vout1x3 = vout1x5;
291 vout1x4 = vout1x6;
292 c1 += 2;
293 }
294 if (nc & 1) {
295 c0[0] = vout0x0;
296 c1[0] = vout1x0;
297 }
298
299 nc = 0;
300 }
301 } while (nc != 0);
302 }