Debian Codegraph @

src/qd8-f32-qc8w-gemm/gen/qd8-f32-qc8w-gemm-4x4-minmax-wasm.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-gemm/scalar.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2021 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include "xnnpack/gemm.h"
13 #include "xnnpack/math.h"
14
15
16 void xnn_qd8_f32_qc8w_gemm_minmax_ukernel_4x4__wasm(
17 size_t mr,
18 size_t nc,
19 size_t kc,
20 const int8_t* restrict a,
21 size_t a_stride,
22 const void* restrict w,
23 float* restrict c,
24 size_t cm_stride,
25 size_t cn_stride,
26 const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)],
27 const struct xnn_qd8_quantization_params quantization_params[restrict XNN_MIN_ELEMENTS(1)])
28 {
29 assert(mr != 0);
30 assert(mr <= 4);
31 assert(nc != 0);
32 assert(kc != 0);
33
34 const int8_t* a0 = a;
35 float* c0 = c;
36 const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
37 float* c1 = (float*) ((uintptr_t) c0 + cm_stride);
38 if XNN_UNPREDICTABLE(mr < 2) {
39 a1 = a0;
40 c1 = c0;
41 }
42 const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
43 float* c2 = (float*) ((uintptr_t) c1 + cm_stride);
44 if XNN_UNPREDICTABLE(mr <= 2) {
45 a2 = a1;
46 c2 = c1;
47 }
48 const int8_t* a3 = (const int8_t*) ((uintptr_t) a2 + a_stride);
49 float* c3 = (float*) ((uintptr_t) c2 + cm_stride);
50 if XNN_UNPREDICTABLE(mr != 4) {
51 a3 = a2;
52 c3 = c2;
53 }
54
55
56 do {
57 const int32_t vksum0 = ((const int32_t*) w)[0];
58 const int32_t vksum1 = ((const int32_t*) w)[1];
59 const int32_t vksum2 = ((const int32_t*) w)[2];
60 const int32_t vksum3 = ((const int32_t*) w)[3];
61 const int32_t vinput_zero_point0 = quantization_params[0].zero_point;
62 int32_t vacc0x0 = vksum0 * vinput_zero_point0;
63 int32_t vacc0x1 = vksum1 * vinput_zero_point0;
64 int32_t vacc0x2 = vksum2 * vinput_zero_point0;
65 int32_t vacc0x3 = vksum3 * vinput_zero_point0;
66 const int32_t vinput_zero_point1 = quantization_params[1].zero_point;
67 int32_t vacc1x0 = vksum0 * vinput_zero_point1;
68 int32_t vacc1x1 = vksum1 * vinput_zero_point1;
69 int32_t vacc1x2 = vksum2 * vinput_zero_point1;
70 int32_t vacc1x3 = vksum3 * vinput_zero_point1;
71 const int32_t vinput_zero_point2 = quantization_params[2].zero_point;
72 int32_t vacc2x0 = vksum0 * vinput_zero_point2;
73 int32_t vacc2x1 = vksum1 * vinput_zero_point2;
74 int32_t vacc2x2 = vksum2 * vinput_zero_point2;
75 int32_t vacc2x3 = vksum3 * vinput_zero_point2;
76 const int32_t vinput_zero_point3 = quantization_params[3].zero_point;
77 int32_t vacc3x0 = vksum0 * vinput_zero_point3;
78 int32_t vacc3x1 = vksum1 * vinput_zero_point3;
79 int32_t vacc3x2 = vksum2 * vinput_zero_point3;
80 int32_t vacc3x3 = vksum3 * vinput_zero_point3;
81 w = (const int32_t*) w + 4;
82
83 size_t k = kc;
84 do {
85 const int32_t va0 = (int32_t) *a0++;
86 const int32_t va1 = (int32_t) *a1++;
87 const int32_t va2 = (int32_t) *a2++;
88 const int32_t va3 = (int32_t) *a3++;
89
90 const int32_t vb0 = (int32_t) ((const int8_t*) w)[0];
91 const int32_t vb1 = (int32_t) ((const int8_t*) w)[1];
92 const int32_t vb2 = (int32_t) ((const int8_t*) w)[2];
93 const int32_t vb3 = (int32_t) ((const int8_t*) w)[3];
94 w = (const int8_t*) w + 4;
95
96 vacc0x0 += va0 * vb0;
97 vacc0x1 += va0 * vb1;
98 vacc0x2 += va0 * vb2;
99 vacc0x3 += va0 * vb3;
100 vacc1x0 += va1 * vb0;
101 vacc1x1 += va1 * vb1;
102 vacc1x2 += va1 * vb2;
103 vacc1x3 += va1 * vb3;
104 vacc2x0 += va2 * vb0;
105 vacc2x1 += va2 * vb1;
106 vacc2x2 += va2 * vb2;
107 vacc2x3 += va2 * vb3;
108 vacc3x0 += va3 * vb0;
109 vacc3x1 += va3 * vb1;
110 vacc3x2 += va3 * vb2;
111 vacc3x3 += va3 * vb3;
112
113 k -= sizeof(int8_t);
114 } while (k != 0);
115
116 float vout0x0 = (float) vacc0x0;
117 float vout0x1 = (float) vacc0x1;
118 float vout0x2 = (float) vacc0x2;
119 float vout0x3 = (float) vacc0x3;
120 float vout1x0 = (float) vacc1x0;
121 float vout1x1 = (float) vacc1x1;
122 float vout1x2 = (float) vacc1x2;
123 float vout1x3 = (float) vacc1x3;
124 float vout2x0 = (float) vacc2x0;
125 float vout2x1 = (float) vacc2x1;
126 float vout2x2 = (float) vacc2x2;
127 float vout2x3 = (float) vacc2x3;
128 float vout3x0 = (float) vacc3x0;
129 float vout3x1 = (float) vacc3x1;
130 float vout3x2 = (float) vacc3x2;
131 float vout3x3 = (float) vacc3x3;
132
133 const float vinput_scale0 = quantization_params[0].inv_scale;
134 vout0x0 *= vinput_scale0;
135 vout0x1 *= vinput_scale0;
136 vout0x2 *= vinput_scale0;
137 vout0x3 *= vinput_scale0;
138 const float vinput_scale1 = quantization_params[1].inv_scale;
139 vout1x0 *= vinput_scale1;
140 vout1x1 *= vinput_scale1;
141 vout1x2 *= vinput_scale1;
142 vout1x3 *= vinput_scale1;
143 const float vinput_scale2 = quantization_params[2].inv_scale;
144 vout2x0 *= vinput_scale2;
145 vout2x1 *= vinput_scale2;
146 vout2x2 *= vinput_scale2;
147 vout2x3 *= vinput_scale2;
148 const float vinput_scale3 = quantization_params[3].inv_scale;
149 vout3x0 *= vinput_scale3;
150 vout3x1 *= vinput_scale3;
151 vout3x2 *= vinput_scale3;
152 vout3x3 *= vinput_scale3;
153
154 const float vfilter_output_scale0 = ((const float*) w)[0];
155 vout0x0 *= vfilter_output_scale0;
156 vout1x0 *= vfilter_output_scale0;
157 vout2x0 *= vfilter_output_scale0;
158 vout3x0 *= vfilter_output_scale0;
159 const float vfilter_output_scale1 = ((const float*) w)[1];
160 vout0x1 *= vfilter_output_scale1;
161 vout1x1 *= vfilter_output_scale1;
162 vout2x1 *= vfilter_output_scale1;
163 vout3x1 *= vfilter_output_scale1;
164 const float vfilter_output_scale2 = ((const float*) w)[2];
165 vout0x2 *= vfilter_output_scale2;
166 vout1x2 *= vfilter_output_scale2;
167 vout2x2 *= vfilter_output_scale2;
168 vout3x2 *= vfilter_output_scale2;
169 const float vfilter_output_scale3 = ((const float*) w)[3];
170 vout0x3 *= vfilter_output_scale3;
171 vout1x3 *= vfilter_output_scale3;
172 vout2x3 *= vfilter_output_scale3;
173 vout3x3 *= vfilter_output_scale3;
174
175 const float vbias0 = ((const float*) w)[4];
176 vout0x0 += vbias0;
177 vout1x0 += vbias0;
178 vout2x0 += vbias0;
179 vout3x0 += vbias0;
180 const float vbias1 = ((const float*) w)[5];
181 vout0x1 += vbias1;
182 vout1x1 += vbias1;
183 vout2x1 += vbias1;
184 vout3x1 += vbias1;
185 const float vbias2 = ((const float*) w)[6];
186 vout0x2 += vbias2;
187 vout1x2 += vbias2;
188 vout2x2 += vbias2;
189 vout3x2 += vbias2;
190 const float vbias3 = ((const float*) w)[7];
191 vout0x3 += vbias3;
192 vout1x3 += vbias3;
193 vout2x3 += vbias3;
194 vout3x3 += vbias3;
195
196 w = (const float*) w + 8;
197
198 const float voutput_min = params->scalar.min;
199 vout0x0 = __builtin_wasm_max_f32(vout0x0, voutput_min);
200 vout1x0 = __builtin_wasm_max_f32(vout1x0, voutput_min);
201 vout2x0 = __builtin_wasm_max_f32(vout2x0, voutput_min);
202 vout3x0 = __builtin_wasm_max_f32(vout3x0, voutput_min);
203 vout0x1 = __builtin_wasm_max_f32(vout0x1, voutput_min);
204 vout1x1 = __builtin_wasm_max_f32(vout1x1, voutput_min);
205 vout2x1 = __builtin_wasm_max_f32(vout2x1, voutput_min);
206 vout3x1 = __builtin_wasm_max_f32(vout3x1, voutput_min);
207 vout0x2 = __builtin_wasm_max_f32(vout0x2, voutput_min);
208 vout1x2 = __builtin_wasm_max_f32(vout1x2, voutput_min);
209 vout2x2 = __builtin_wasm_max_f32(vout2x2, voutput_min);
210 vout3x2 = __builtin_wasm_max_f32(vout3x2, voutput_min);
211 vout0x3 = __builtin_wasm_max_f32(vout0x3, voutput_min);
212 vout1x3 = __builtin_wasm_max_f32(vout1x3, voutput_min);
213 vout2x3 = __builtin_wasm_max_f32(vout2x3, voutput_min);
214 vout3x3 = __builtin_wasm_max_f32(vout3x3, voutput_min);
215
216 const float voutput_max = params->scalar.max;
217 vout0x0 = __builtin_wasm_min_f32(vout0x0, voutput_max);
218 vout1x0 = __builtin_wasm_min_f32(vout1x0, voutput_max);
219 vout2x0 = __builtin_wasm_min_f32(vout2x0, voutput_max);
220 vout3x0 = __builtin_wasm_min_f32(vout3x0, voutput_max);
221 vout0x1 = __builtin_wasm_min_f32(vout0x1, voutput_max);
222 vout1x1 = __builtin_wasm_min_f32(vout1x1, voutput_max);
223 vout2x1 = __builtin_wasm_min_f32(vout2x1, voutput_max);
224 vout3x1 = __builtin_wasm_min_f32(vout3x1, voutput_max);
225 vout0x2 = __builtin_wasm_min_f32(vout0x2, voutput_max);
226 vout1x2 = __builtin_wasm_min_f32(vout1x2, voutput_max);
227 vout2x2 = __builtin_wasm_min_f32(vout2x2, voutput_max);
228 vout3x2 = __builtin_wasm_min_f32(vout3x2, voutput_max);
229 vout0x3 = __builtin_wasm_min_f32(vout0x3, voutput_max);
230 vout1x3 = __builtin_wasm_min_f32(vout1x3, voutput_max);
231 vout2x3 = __builtin_wasm_min_f32(vout2x3, voutput_max);
232 vout3x3 = __builtin_wasm_min_f32(vout3x3, voutput_max);
233
234 if XNN_LIKELY(nc >= 4) {
235 c0[0] = vout0x0;
236 c0[1] = vout0x1;
237 c0[2] = vout0x2;
238 c0[3] = vout0x3;
239 c1[0] = vout1x0;
240 c1[1] = vout1x1;
241 c1[2] = vout1x2;
242 c1[3] = vout1x3;
243 c2[0] = vout2x0;
244 c2[1] = vout2x1;
245 c2[2] = vout2x2;
246 c2[3] = vout2x3;
247 c3[0] = vout3x0;
248 c3[1] = vout3x1;
249 c3[2] = vout3x2;
250 c3[3] = vout3x3;
251
252 a0 = (const int8_t*) ((uintptr_t) a0 - kc);
253 a1 = (const int8_t*) ((uintptr_t) a1 - kc);
254 a2 = (const int8_t*) ((uintptr_t) a2 - kc);
255 a3 = (const int8_t*) ((uintptr_t) a3 - kc);
256
257 c0 = (float*) ((uintptr_t) c0 + cn_stride);
258 c1 = (float*) ((uintptr_t) c1 + cn_stride);
259 c2 = (float*) ((uintptr_t) c2 + cn_stride);
260 c3 = (float*) ((uintptr_t) c3 + cn_stride);
261
262 nc -= 4;
263 } else {
264 if (nc & 2) {
265 c0[0] = vout0x0;
266 c0[1] = vout0x1;
267 vout0x0 = vout0x2;
268 c0 += 2;
269 c1[0] = vout1x0;
270 c1[1] = vout1x1;
271 vout1x0 = vout1x2;
272 c1 += 2;
273 c2[0] = vout2x0;
274 c2[1] = vout2x1;
275 vout2x0 = vout2x2;
276 c2 += 2;
277 c3[0] = vout3x0;
278 c3[1] = vout3x1;
279 vout3x0 = vout3x2;
280 c3 += 2;
281 }
282 if (nc & 1) {
283 c0[0] = vout0x0;
284 c1[0] = vout1x0;
285 c2[0] = vout2x0;
286 c3[0] = vout3x0;
287 }
288
289 nc = 0;
290 }
291 } while (nc != 0);
292 }