Debian Codegraph @

src/qd8-f32-qc4w-gemm/gen/qd8-f32-qc4w-gemm-2x8-minmax-wasm.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-gemm/scalar.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2021 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include "xnnpack/gemm.h"
13 #include "xnnpack/math.h"
14
15
16 void xnn_qd8_f32_qc4w_gemm_minmax_ukernel_2x8__wasm(
17 size_t mr,
18 size_t nc,
19 size_t kc,
20 const int8_t* restrict a,
21 size_t a_stride,
22 const void* restrict w,
23 float* restrict c,
24 size_t cm_stride,
25 size_t cn_stride,
26 const struct xnn_f32_qc4w_minmax_params params[restrict XNN_MIN_ELEMENTS(1)],
27 const struct xnn_qd8_quantization_params quantization_params[restrict XNN_MIN_ELEMENTS(1)])
28 {
29 assert(mr != 0);
30 assert(mr <= 2);
31 assert(nc != 0);
32 assert(kc != 0);
33
34 const int8_t* a0 = a;
35 float* c0 = c;
36 const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
37 float* c1 = (float*) ((uintptr_t) c0 + cm_stride);
38 if XNN_UNPREDICTABLE(mr != 2) {
39 a1 = a0;
40 c1 = c0;
41 }
42
43
44 kc = round_up_po2(kc, 2);
45 do {
46 const int32_t vksum0 = ((const int32_t*) w)[0];
47 const int32_t vksum1 = ((const int32_t*) w)[1];
48 const int32_t vksum2 = ((const int32_t*) w)[2];
49 const int32_t vksum3 = ((const int32_t*) w)[3];
50 const int32_t vksum4 = ((const int32_t*) w)[4];
51 const int32_t vksum5 = ((const int32_t*) w)[5];
52 const int32_t vksum6 = ((const int32_t*) w)[6];
53 const int32_t vksum7 = ((const int32_t*) w)[7];
54 const int32_t vinput_zero_point0 = quantization_params[0].zero_point;
55 int32_t vacc0x0 = vksum0 * vinput_zero_point0;
56 int32_t vacc0x1 = vksum1 * vinput_zero_point0;
57 int32_t vacc0x2 = vksum2 * vinput_zero_point0;
58 int32_t vacc0x3 = vksum3 * vinput_zero_point0;
59 int32_t vacc0x4 = vksum4 * vinput_zero_point0;
60 int32_t vacc0x5 = vksum5 * vinput_zero_point0;
61 int32_t vacc0x6 = vksum6 * vinput_zero_point0;
62 int32_t vacc0x7 = vksum7 * vinput_zero_point0;
63 const int32_t vinput_zero_point1 = quantization_params[1].zero_point;
64 int32_t vacc1x0 = vksum0 * vinput_zero_point1;
65 int32_t vacc1x1 = vksum1 * vinput_zero_point1;
66 int32_t vacc1x2 = vksum2 * vinput_zero_point1;
67 int32_t vacc1x3 = vksum3 * vinput_zero_point1;
68 int32_t vacc1x4 = vksum4 * vinput_zero_point1;
69 int32_t vacc1x5 = vksum5 * vinput_zero_point1;
70 int32_t vacc1x6 = vksum6 * vinput_zero_point1;
71 int32_t vacc1x7 = vksum7 * vinput_zero_point1;
72 w = (const int32_t*) w + 8;
73
74 size_t k = kc;
75 for (; k >= 2 * sizeof(uint8_t); k -= 2 * sizeof(uint8_t)) {
76 const int32_t va0c0 = (int32_t) a0[0];
77 const int32_t va0c1 = (int32_t) a0[1];
78 a0 += 2;
79 const int32_t va1c0 = (int32_t) a1[0];
80 const int32_t va1c1 = (int32_t) a1[1];
81 a1 += 2;
82
83 const uint8_t vbi0 = ((const uint8_t*) w)[0];
84 const uint8_t vbi1 = ((const uint8_t*) w)[1];
85 const uint8_t vbi2 = ((const uint8_t*) w)[2];
86 const uint8_t vbi3 = ((const uint8_t*) w)[3];
87 const uint8_t vbi4 = ((const uint8_t*) w)[4];
88 const uint8_t vbi5 = ((const uint8_t*) w)[5];
89 const uint8_t vbi6 = ((const uint8_t*) w)[6];
90 const uint8_t vbi7 = ((const uint8_t*) w)[7];
91 w = (const uint8_t*) w + 8;
92 const int32_t vb0c0 = (int32_t) (int8_t) (vbi0 << 4);
93 const int32_t vb0c1 = (int32_t) (int8_t) (vbi0 & 0xF0);
94 const int32_t vb1c0 = (int32_t) (int8_t) (vbi1 << 4);
95 const int32_t vb1c1 = (int32_t) (int8_t) (vbi1 & 0xF0);
96 const int32_t vb2c0 = (int32_t) (int8_t) (vbi2 << 4);
97 const int32_t vb2c1 = (int32_t) (int8_t) (vbi2 & 0xF0);
98 const int32_t vb3c0 = (int32_t) (int8_t) (vbi3 << 4);
99 const int32_t vb3c1 = (int32_t) (int8_t) (vbi3 & 0xF0);
100 const int32_t vb4c0 = (int32_t) (int8_t) (vbi4 << 4);
101 const int32_t vb4c1 = (int32_t) (int8_t) (vbi4 & 0xF0);
102 const int32_t vb5c0 = (int32_t) (int8_t) (vbi5 << 4);
103 const int32_t vb5c1 = (int32_t) (int8_t) (vbi5 & 0xF0);
104 const int32_t vb6c0 = (int32_t) (int8_t) (vbi6 << 4);
105 const int32_t vb6c1 = (int32_t) (int8_t) (vbi6 & 0xF0);
106 const int32_t vb7c0 = (int32_t) (int8_t) (vbi7 << 4);
107 const int32_t vb7c1 = (int32_t) (int8_t) (vbi7 & 0xF0);
108
109 vacc0x0 += va0c0 * vb0c0;
110 vacc0x1 += va0c0 * vb1c0;
111 vacc0x2 += va0c0 * vb2c0;
112 vacc0x3 += va0c0 * vb3c0;
113 vacc0x4 += va0c0 * vb4c0;
114 vacc0x5 += va0c0 * vb5c0;
115 vacc0x6 += va0c0 * vb6c0;
116 vacc0x7 += va0c0 * vb7c0;
117 vacc1x0 += va1c0 * vb0c0;
118 vacc1x1 += va1c0 * vb1c0;
119 vacc1x2 += va1c0 * vb2c0;
120 vacc1x3 += va1c0 * vb3c0;
121 vacc1x4 += va1c0 * vb4c0;
122 vacc1x5 += va1c0 * vb5c0;
123 vacc1x6 += va1c0 * vb6c0;
124 vacc1x7 += va1c0 * vb7c0;
125 vacc0x0 += va0c1 * vb0c1;
126 vacc0x1 += va0c1 * vb1c1;
127 vacc0x2 += va0c1 * vb2c1;
128 vacc0x3 += va0c1 * vb3c1;
129 vacc0x4 += va0c1 * vb4c1;
130 vacc0x5 += va0c1 * vb5c1;
131 vacc0x6 += va0c1 * vb6c1;
132 vacc0x7 += va0c1 * vb7c1;
133 vacc1x0 += va1c1 * vb0c1;
134 vacc1x1 += va1c1 * vb1c1;
135 vacc1x2 += va1c1 * vb2c1;
136 vacc1x3 += va1c1 * vb3c1;
137 vacc1x4 += va1c1 * vb4c1;
138 vacc1x5 += va1c1 * vb5c1;
139 vacc1x6 += va1c1 * vb6c1;
140 vacc1x7 += va1c1 * vb7c1;
141 }
142
143 float vout0x0 = (float) math_asr_s32(vacc0x0, 4);
144 float vout0x1 = (float) math_asr_s32(vacc0x1, 4);
145 float vout0x2 = (float) math_asr_s32(vacc0x2, 4);
146 float vout0x3 = (float) math_asr_s32(vacc0x3, 4);
147 float vout0x4 = (float) math_asr_s32(vacc0x4, 4);
148 float vout0x5 = (float) math_asr_s32(vacc0x5, 4);
149 float vout0x6 = (float) math_asr_s32(vacc0x6, 4);
150 float vout0x7 = (float) math_asr_s32(vacc0x7, 4);
151 float vout1x0 = (float) math_asr_s32(vacc1x0, 4);
152 float vout1x1 = (float) math_asr_s32(vacc1x1, 4);
153 float vout1x2 = (float) math_asr_s32(vacc1x2, 4);
154 float vout1x3 = (float) math_asr_s32(vacc1x3, 4);
155 float vout1x4 = (float) math_asr_s32(vacc1x4, 4);
156 float vout1x5 = (float) math_asr_s32(vacc1x5, 4);
157 float vout1x6 = (float) math_asr_s32(vacc1x6, 4);
158 float vout1x7 = (float) math_asr_s32(vacc1x7, 4);
159
160 const float vinput_scale0 = quantization_params[0].inv_scale;
161 vout0x0 *= vinput_scale0;
162 vout0x1 *= vinput_scale0;
163 vout0x2 *= vinput_scale0;
164 vout0x3 *= vinput_scale0;
165 vout0x4 *= vinput_scale0;
166 vout0x5 *= vinput_scale0;
167 vout0x6 *= vinput_scale0;
168 vout0x7 *= vinput_scale0;
169 const float vinput_scale1 = quantization_params[1].inv_scale;
170 vout1x0 *= vinput_scale1;
171 vout1x1 *= vinput_scale1;
172 vout1x2 *= vinput_scale1;
173 vout1x3 *= vinput_scale1;
174 vout1x4 *= vinput_scale1;
175 vout1x5 *= vinput_scale1;
176 vout1x6 *= vinput_scale1;
177 vout1x7 *= vinput_scale1;
178
179 const float vfilter_output_scale0 = ((const float*) w)[0];
180 vout0x0 *= vfilter_output_scale0;
181 vout1x0 *= vfilter_output_scale0;
182 const float vfilter_output_scale1 = ((const float*) w)[1];
183 vout0x1 *= vfilter_output_scale1;
184 vout1x1 *= vfilter_output_scale1;
185 const float vfilter_output_scale2 = ((const float*) w)[2];
186 vout0x2 *= vfilter_output_scale2;
187 vout1x2 *= vfilter_output_scale2;
188 const float vfilter_output_scale3 = ((const float*) w)[3];
189 vout0x3 *= vfilter_output_scale3;
190 vout1x3 *= vfilter_output_scale3;
191 const float vfilter_output_scale4 = ((const float*) w)[4];
192 vout0x4 *= vfilter_output_scale4;
193 vout1x4 *= vfilter_output_scale4;
194 const float vfilter_output_scale5 = ((const float*) w)[5];
195 vout0x5 *= vfilter_output_scale5;
196 vout1x5 *= vfilter_output_scale5;
197 const float vfilter_output_scale6 = ((const float*) w)[6];
198 vout0x6 *= vfilter_output_scale6;
199 vout1x6 *= vfilter_output_scale6;
200 const float vfilter_output_scale7 = ((const float*) w)[7];
201 vout0x7 *= vfilter_output_scale7;
202 vout1x7 *= vfilter_output_scale7;
203
204 const float vbias0 = ((const float*) w)[8];
205 vout0x0 += vbias0;
206 vout1x0 += vbias0;
207 const float vbias1 = ((const float*) w)[9];
208 vout0x1 += vbias1;
209 vout1x1 += vbias1;
210 const float vbias2 = ((const float*) w)[10];
211 vout0x2 += vbias2;
212 vout1x2 += vbias2;
213 const float vbias3 = ((const float*) w)[11];
214 vout0x3 += vbias3;
215 vout1x3 += vbias3;
216 const float vbias4 = ((const float*) w)[12];
217 vout0x4 += vbias4;
218 vout1x4 += vbias4;
219 const float vbias5 = ((const float*) w)[13];
220 vout0x5 += vbias5;
221 vout1x5 += vbias5;
222 const float vbias6 = ((const float*) w)[14];
223 vout0x6 += vbias6;
224 vout1x6 += vbias6;
225 const float vbias7 = ((const float*) w)[15];
226 vout0x7 += vbias7;
227 vout1x7 += vbias7;
228
229 w = (const float*) w + 16;
230
231 const float voutput_min = params->scalar.min;
232 vout0x0 = __builtin_wasm_max_f32(vout0x0, voutput_min);
233 vout1x0 = __builtin_wasm_max_f32(vout1x0, voutput_min);
234 vout0x1 = __builtin_wasm_max_f32(vout0x1, voutput_min);
235 vout1x1 = __builtin_wasm_max_f32(vout1x1, voutput_min);
236 vout0x2 = __builtin_wasm_max_f32(vout0x2, voutput_min);
237 vout1x2 = __builtin_wasm_max_f32(vout1x2, voutput_min);
238 vout0x3 = __builtin_wasm_max_f32(vout0x3, voutput_min);
239 vout1x3 = __builtin_wasm_max_f32(vout1x3, voutput_min);
240 vout0x4 = __builtin_wasm_max_f32(vout0x4, voutput_min);
241 vout1x4 = __builtin_wasm_max_f32(vout1x4, voutput_min);
242 vout0x5 = __builtin_wasm_max_f32(vout0x5, voutput_min);
243 vout1x5 = __builtin_wasm_max_f32(vout1x5, voutput_min);
244 vout0x6 = __builtin_wasm_max_f32(vout0x6, voutput_min);
245 vout1x6 = __builtin_wasm_max_f32(vout1x6, voutput_min);
246 vout0x7 = __builtin_wasm_max_f32(vout0x7, voutput_min);
247 vout1x7 = __builtin_wasm_max_f32(vout1x7, voutput_min);
248
249 const float voutput_max = params->scalar.max;
250 vout0x0 = __builtin_wasm_min_f32(vout0x0, voutput_max);
251 vout1x0 = __builtin_wasm_min_f32(vout1x0, voutput_max);
252 vout0x1 = __builtin_wasm_min_f32(vout0x1, voutput_max);
253 vout1x1 = __builtin_wasm_min_f32(vout1x1, voutput_max);
254 vout0x2 = __builtin_wasm_min_f32(vout0x2, voutput_max);
255 vout1x2 = __builtin_wasm_min_f32(vout1x2, voutput_max);
256 vout0x3 = __builtin_wasm_min_f32(vout0x3, voutput_max);
257 vout1x3 = __builtin_wasm_min_f32(vout1x3, voutput_max);
258 vout0x4 = __builtin_wasm_min_f32(vout0x4, voutput_max);
259 vout1x4 = __builtin_wasm_min_f32(vout1x4, voutput_max);
260 vout0x5 = __builtin_wasm_min_f32(vout0x5, voutput_max);
261 vout1x5 = __builtin_wasm_min_f32(vout1x5, voutput_max);
262 vout0x6 = __builtin_wasm_min_f32(vout0x6, voutput_max);
263 vout1x6 = __builtin_wasm_min_f32(vout1x6, voutput_max);
264 vout0x7 = __builtin_wasm_min_f32(vout0x7, voutput_max);
265 vout1x7 = __builtin_wasm_min_f32(vout1x7, voutput_max);
266
267 if XNN_LIKELY(nc >= 8) {
268 c0[0] = vout0x0;
269 c0[1] = vout0x1;
270 c0[2] = vout0x2;
271 c0[3] = vout0x3;
272 c0[4] = vout0x4;
273 c0[5] = vout0x5;
274 c0[6] = vout0x6;
275 c0[7] = vout0x7;
276 c1[0] = vout1x0;
277 c1[1] = vout1x1;
278 c1[2] = vout1x2;
279 c1[3] = vout1x3;
280 c1[4] = vout1x4;
281 c1[5] = vout1x5;
282 c1[6] = vout1x6;
283 c1[7] = vout1x7;
284
285 a0 = (const int8_t*) ((uintptr_t) a0 - kc);
286 a1 = (const int8_t*) ((uintptr_t) a1 - kc);
287
288 c0 = (float*) ((uintptr_t) c0 + cn_stride);
289 c1 = (float*) ((uintptr_t) c1 + cn_stride);
290
291 nc -= 8;
292 } else {
293 if (nc & 4) {
294 c0[0] = vout0x0;
295 c0[1] = vout0x1;
296 c0[2] = vout0x2;
297 c0[3] = vout0x3;
298 vout0x0 = vout0x4;
299 vout0x1 = vout0x5;
300 vout0x2 = vout0x6;
301 c0 += 4;
302 c1[0] = vout1x0;
303 c1[1] = vout1x1;
304 c1[2] = vout1x2;
305 c1[3] = vout1x3;
306 vout1x0 = vout1x4;
307 vout1x1 = vout1x5;
308 vout1x2 = vout1x6;
309 c1 += 4;
310 }
311 if (nc & 2) {
312 c0[0] = vout0x0;
313 c0[1] = vout0x1;
314 vout0x0 = vout0x2;
315 vout0x1 = vout0x3;
316 vout0x2 = vout0x4;
317 vout0x3 = vout0x5;
318 vout0x4 = vout0x6;
319 c0 += 2;
320 c1[0] = vout1x0;
321 c1[1] = vout1x1;
322 vout1x0 = vout1x2;
323 vout1x1 = vout1x3;
324 vout1x2 = vout1x4;
325 vout1x3 = vout1x5;
326 vout1x4 = vout1x6;
327 c1 += 2;
328 }
329 if (nc & 1) {
330 c0[0] = vout0x0;
331 c1[0] = vout1x0;
332 }
333
334 nc = 0;
335 }
336 } while (nc != 0);
337 }