Debian Codegraph @

src/qd8-f16-qb4w-gemm/gen/qd8-f16-qb4w-gemm-4x4-minmax-scalar.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-gemm/scalar.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2021 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include "xnnpack/gemm.h"
13 #include "xnnpack/math.h"
14
15
16 void xnn_qd8_f16_qb4w_gemm_minmax_ukernel_4x4__scalar(
17 size_t mr,
18 size_t nc,
19 size_t kc,
20 const int8_t* restrict a,
21 size_t a_stride,
22 const void* restrict w,
23 xnn_float16* restrict c,
24 size_t cm_stride,
25 size_t cn_stride,
26 const struct xnn_f16_qb4w_minmax_params params[restrict XNN_MIN_ELEMENTS(1)],
27 const struct xnn_qd8_quantization_params quantization_params[restrict XNN_MIN_ELEMENTS(1)])
28 {
29 assert(mr != 0);
30 assert(mr <= 4);
31 assert(nc != 0);
32 assert(kc != 0);
33 size_t bl = params->scalar.blocksize;
34 assert(bl <= round_up_po2(kc, 2));
35 assert(bl != 0);
36 assert(bl % 32 == 0);
37
38 const int8_t* a0 = a;
39 xnn_float16* c0 = c;
40 const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
41 xnn_float16* c1 = (xnn_float16*) ((uintptr_t) c0 + cm_stride);
42 if XNN_UNPREDICTABLE(mr < 2) {
43 a1 = a0;
44 c1 = c0;
45 }
46 const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
47 xnn_float16* c2 = (xnn_float16*) ((uintptr_t) c1 + cm_stride);
48 if XNN_UNPREDICTABLE(mr <= 2) {
49 a2 = a1;
50 c2 = c1;
51 }
52 const int8_t* a3 = (const int8_t*) ((uintptr_t) a2 + a_stride);
53 xnn_float16* c3 = (xnn_float16*) ((uintptr_t) c2 + cm_stride);
54 if XNN_UNPREDICTABLE(mr != 4) {
55 a3 = a2;
56 c3 = c2;
57 }
58
59
60 kc = round_up_po2(kc, 2);
61 do {
62 const float vksum0 = ((const float*) w)[0];
63 const float vksum1 = ((const float*) w)[1];
64 const float vksum2 = ((const float*) w)[2];
65 const float vksum3 = ((const float*) w)[3];
66 const float vinput_zero_point0 = (const float) quantization_params[0].zero_point;
67 float vout0x0 = vksum0 * vinput_zero_point0;
68 float vout0x1 = vksum1 * vinput_zero_point0;
69 float vout0x2 = vksum2 * vinput_zero_point0;
70 float vout0x3 = vksum3 * vinput_zero_point0;
71 const float vinput_zero_point1 = (const float) quantization_params[1].zero_point;
72 float vout1x0 = vksum0 * vinput_zero_point1;
73 float vout1x1 = vksum1 * vinput_zero_point1;
74 float vout1x2 = vksum2 * vinput_zero_point1;
75 float vout1x3 = vksum3 * vinput_zero_point1;
76 const float vinput_zero_point2 = (const float) quantization_params[2].zero_point;
77 float vout2x0 = vksum0 * vinput_zero_point2;
78 float vout2x1 = vksum1 * vinput_zero_point2;
79 float vout2x2 = vksum2 * vinput_zero_point2;
80 float vout2x3 = vksum3 * vinput_zero_point2;
81 const float vinput_zero_point3 = (const float) quantization_params[3].zero_point;
82 float vout3x0 = vksum0 * vinput_zero_point3;
83 float vout3x1 = vksum1 * vinput_zero_point3;
84 float vout3x2 = vksum2 * vinput_zero_point3;
85 float vout3x3 = vksum3 * vinput_zero_point3;
86 w = (const float*) w + 4;
87
88 for (size_t kb=0; kb < kc; kb += bl) {
89 int32_t vacc0x0 = 0;
90 int32_t vacc0x1 = 0;
91 int32_t vacc0x2 = 0;
92 int32_t vacc0x3 = 0;
93 int32_t vacc1x0 = 0;
94 int32_t vacc1x1 = 0;
95 int32_t vacc1x2 = 0;
96 int32_t vacc1x3 = 0;
97 int32_t vacc2x0 = 0;
98 int32_t vacc2x1 = 0;
99 int32_t vacc2x2 = 0;
100 int32_t vacc2x3 = 0;
101 int32_t vacc3x0 = 0;
102 int32_t vacc3x1 = 0;
103 int32_t vacc3x2 = 0;
104 int32_t vacc3x3 = 0;
105 size_t k = bl;
106 for (; k >= 2 * sizeof(uint8_t); k -= 2 * sizeof(uint8_t)) {
107 const int32_t va0c0 = (int32_t) a0[0];
108 const int32_t va0c1 = (int32_t) a0[1];
109 a0 += 2;
110 const int32_t va1c0 = (int32_t) a1[0];
111 const int32_t va1c1 = (int32_t) a1[1];
112 a1 += 2;
113 const int32_t va2c0 = (int32_t) a2[0];
114 const int32_t va2c1 = (int32_t) a2[1];
115 a2 += 2;
116 const int32_t va3c0 = (int32_t) a3[0];
117 const int32_t va3c1 = (int32_t) a3[1];
118 a3 += 2;
119
120 const uint8_t vbi0 = ((const uint8_t*) w)[0];
121 const uint8_t vbi1 = ((const uint8_t*) w)[1];
122 const uint8_t vbi2 = ((const uint8_t*) w)[2];
123 const uint8_t vbi3 = ((const uint8_t*) w)[3];
124 w = (const uint8_t*) w + 4;
125 const int32_t vb0c0 = (int32_t) (int8_t) (vbi0 << 4);
126 const int32_t vb0c1 = (int32_t) (int8_t) (vbi0 & 0xF0);
127 const int32_t vb1c0 = (int32_t) (int8_t) (vbi1 << 4);
128 const int32_t vb1c1 = (int32_t) (int8_t) (vbi1 & 0xF0);
129 const int32_t vb2c0 = (int32_t) (int8_t) (vbi2 << 4);
130 const int32_t vb2c1 = (int32_t) (int8_t) (vbi2 & 0xF0);
131 const int32_t vb3c0 = (int32_t) (int8_t) (vbi3 << 4);
132 const int32_t vb3c1 = (int32_t) (int8_t) (vbi3 & 0xF0);
133
134 vacc0x0 += va0c0 * vb0c0;
135 vacc0x1 += va0c0 * vb1c0;
136 vacc0x2 += va0c0 * vb2c0;
137 vacc0x3 += va0c0 * vb3c0;
138 vacc1x0 += va1c0 * vb0c0;
139 vacc1x1 += va1c0 * vb1c0;
140 vacc1x2 += va1c0 * vb2c0;
141 vacc1x3 += va1c0 * vb3c0;
142 vacc2x0 += va2c0 * vb0c0;
143 vacc2x1 += va2c0 * vb1c0;
144 vacc2x2 += va2c0 * vb2c0;
145 vacc2x3 += va2c0 * vb3c0;
146 vacc3x0 += va3c0 * vb0c0;
147 vacc3x1 += va3c0 * vb1c0;
148 vacc3x2 += va3c0 * vb2c0;
149 vacc3x3 += va3c0 * vb3c0;
150 vacc0x0 += va0c1 * vb0c1;
151 vacc0x1 += va0c1 * vb1c1;
152 vacc0x2 += va0c1 * vb2c1;
153 vacc0x3 += va0c1 * vb3c1;
154 vacc1x0 += va1c1 * vb0c1;
155 vacc1x1 += va1c1 * vb1c1;
156 vacc1x2 += va1c1 * vb2c1;
157 vacc1x3 += va1c1 * vb3c1;
158 vacc2x0 += va2c1 * vb0c1;
159 vacc2x1 += va2c1 * vb1c1;
160 vacc2x2 += va2c1 * vb2c1;
161 vacc2x3 += va2c1 * vb3c1;
162 vacc3x0 += va3c1 * vb0c1;
163 vacc3x1 += va3c1 * vb1c1;
164 vacc3x2 += va3c1 * vb2c1;
165 vacc3x3 += va3c1 * vb3c1;
166 }
167 // accumulate in float
168 float vf0x0 = vacc0x0;
169 const float vfilter_output_scale0 = math_cvt_fp32_bf16(((const uint16_t*) w)[0]);
170 float vf0x1 = vacc0x1;
171 const float vfilter_output_scale1 = math_cvt_fp32_bf16(((const uint16_t*) w)[1]);
172 float vf0x2 = vacc0x2;
173 const float vfilter_output_scale2 = math_cvt_fp32_bf16(((const uint16_t*) w)[2]);
174 float vf0x3 = vacc0x3;
175 const float vfilter_output_scale3 = math_cvt_fp32_bf16(((const uint16_t*) w)[3]);
176 float vf1x0 = vacc1x0;
177 float vf1x1 = vacc1x1;
178 float vf1x2 = vacc1x2;
179 float vf1x3 = vacc1x3;
180 float vf2x0 = vacc2x0;
181 float vf2x1 = vacc2x1;
182 float vf2x2 = vacc2x2;
183 float vf2x3 = vacc2x3;
184 float vf3x0 = vacc3x0;
185 float vf3x1 = vacc3x1;
186 float vf3x2 = vacc3x2;
187 float vf3x3 = vacc3x3;
188
189 vf0x0 *= vfilter_output_scale0;
190 vout0x0 += vf0x0;
191 vf0x1 *= vfilter_output_scale1;
192 vout0x1 += vf0x1;
193 vf0x2 *= vfilter_output_scale2;
194 vout0x2 += vf0x2;
195 vf0x3 *= vfilter_output_scale3;
196 vout0x3 += vf0x3;
197 vf1x0 *= vfilter_output_scale0;
198 vout1x0 += vf1x0;
199 vf1x1 *= vfilter_output_scale1;
200 vout1x1 += vf1x1;
201 vf1x2 *= vfilter_output_scale2;
202 vout1x2 += vf1x2;
203 vf1x3 *= vfilter_output_scale3;
204 vout1x3 += vf1x3;
205 vf2x0 *= vfilter_output_scale0;
206 vout2x0 += vf2x0;
207 vf2x1 *= vfilter_output_scale1;
208 vout2x1 += vf2x1;
209 vf2x2 *= vfilter_output_scale2;
210 vout2x2 += vf2x2;
211 vf2x3 *= vfilter_output_scale3;
212 vout2x3 += vf2x3;
213 vf3x0 *= vfilter_output_scale0;
214 vout3x0 += vf3x0;
215 vf3x1 *= vfilter_output_scale1;
216 vout3x1 += vf3x1;
217 vf3x2 *= vfilter_output_scale2;
218 vout3x2 += vf3x2;
219 vf3x3 *= vfilter_output_scale3;
220 vout3x3 += vf3x3;
221 w = (const uint16_t*) w + 4;
222 }
223
224
225 const float vinput_scale0 = quantization_params[0].inv_scale;
226 vout0x0 *= vinput_scale0;
227 vout0x1 *= vinput_scale0;
228 vout0x2 *= vinput_scale0;
229 vout0x3 *= vinput_scale0;
230 const float vinput_scale1 = quantization_params[1].inv_scale;
231 vout1x0 *= vinput_scale1;
232 vout1x1 *= vinput_scale1;
233 vout1x2 *= vinput_scale1;
234 vout1x3 *= vinput_scale1;
235 const float vinput_scale2 = quantization_params[2].inv_scale;
236 vout2x0 *= vinput_scale2;
237 vout2x1 *= vinput_scale2;
238 vout2x2 *= vinput_scale2;
239 vout2x3 *= vinput_scale2;
240 const float vinput_scale3 = quantization_params[3].inv_scale;
241 vout3x0 *= vinput_scale3;
242 vout3x1 *= vinput_scale3;
243 vout3x2 *= vinput_scale3;
244 vout3x3 *= vinput_scale3;
245
246
247 const float vbias0 = ((const float*) w)[0];
248 vout0x0 += vbias0;
249 vout1x0 += vbias0;
250 vout2x0 += vbias0;
251 vout3x0 += vbias0;
252 const float vbias1 = ((const float*) w)[1];
253 vout0x1 += vbias1;
254 vout1x1 += vbias1;
255 vout2x1 += vbias1;
256 vout3x1 += vbias1;
257 const float vbias2 = ((const float*) w)[2];
258 vout0x2 += vbias2;
259 vout1x2 += vbias2;
260 vout2x2 += vbias2;
261 vout3x2 += vbias2;
262 const float vbias3 = ((const float*) w)[3];
263 vout0x3 += vbias3;
264 vout1x3 += vbias3;
265 vout2x3 += vbias3;
266 vout3x3 += vbias3;
267
268 w = (const float*) w + 4;
269
270 const float voutput_min = xnn_float16_to_float(params->scalar.min);
271 vout0x0 = math_max_f32(vout0x0, voutput_min);
272 vout1x0 = math_max_f32(vout1x0, voutput_min);
273 vout2x0 = math_max_f32(vout2x0, voutput_min);
274 vout3x0 = math_max_f32(vout3x0, voutput_min);
275 vout0x1 = math_max_f32(vout0x1, voutput_min);
276 vout1x1 = math_max_f32(vout1x1, voutput_min);
277 vout2x1 = math_max_f32(vout2x1, voutput_min);
278 vout3x1 = math_max_f32(vout3x1, voutput_min);
279 vout0x2 = math_max_f32(vout0x2, voutput_min);
280 vout1x2 = math_max_f32(vout1x2, voutput_min);
281 vout2x2 = math_max_f32(vout2x2, voutput_min);
282 vout3x2 = math_max_f32(vout3x2, voutput_min);
283 vout0x3 = math_max_f32(vout0x3, voutput_min);
284 vout1x3 = math_max_f32(vout1x3, voutput_min);
285 vout2x3 = math_max_f32(vout2x3, voutput_min);
286 vout3x3 = math_max_f32(vout3x3, voutput_min);
287
288 const float voutput_max = xnn_float16_to_float(params->scalar.max);
289 vout0x0 = math_min_f32(vout0x0, voutput_max);
290 vout1x0 = math_min_f32(vout1x0, voutput_max);
291 vout2x0 = math_min_f32(vout2x0, voutput_max);
292 vout3x0 = math_min_f32(vout3x0, voutput_max);
293 vout0x1 = math_min_f32(vout0x1, voutput_max);
294 vout1x1 = math_min_f32(vout1x1, voutput_max);
295 vout2x1 = math_min_f32(vout2x1, voutput_max);
296 vout3x1 = math_min_f32(vout3x1, voutput_max);
297 vout0x2 = math_min_f32(vout0x2, voutput_max);
298 vout1x2 = math_min_f32(vout1x2, voutput_max);
299 vout2x2 = math_min_f32(vout2x2, voutput_max);
300 vout3x2 = math_min_f32(vout3x2, voutput_max);
301 vout0x3 = math_min_f32(vout0x3, voutput_max);
302 vout1x3 = math_min_f32(vout1x3, voutput_max);
303 vout2x3 = math_min_f32(vout2x3, voutput_max);
304 vout3x3 = math_min_f32(vout3x3, voutput_max);
305
306 if XNN_LIKELY(nc >= 4) {
307 c0[0] = xnn_float16_from_float(vout0x0);
308 c0[1] = xnn_float16_from_float(vout0x1);
309 c0[2] = xnn_float16_from_float(vout0x2);
310 c0[3] = xnn_float16_from_float(vout0x3);
311 c1[0] = xnn_float16_from_float(vout1x0);
312 c1[1] = xnn_float16_from_float(vout1x1);
313 c1[2] = xnn_float16_from_float(vout1x2);
314 c1[3] = xnn_float16_from_float(vout1x3);
315 c2[0] = xnn_float16_from_float(vout2x0);
316 c2[1] = xnn_float16_from_float(vout2x1);
317 c2[2] = xnn_float16_from_float(vout2x2);
318 c2[3] = xnn_float16_from_float(vout2x3);
319 c3[0] = xnn_float16_from_float(vout3x0);
320 c3[1] = xnn_float16_from_float(vout3x1);
321 c3[2] = xnn_float16_from_float(vout3x2);
322 c3[3] = xnn_float16_from_float(vout3x3);
323
324 a0 = (const int8_t*) ((uintptr_t) a0 - kc);
325 a1 = (const int8_t*) ((uintptr_t) a1 - kc);
326 a2 = (const int8_t*) ((uintptr_t) a2 - kc);
327 a3 = (const int8_t*) ((uintptr_t) a3 - kc);
328
329 c0 = (xnn_float16*) ((uintptr_t) c0 + cn_stride);
330 c1 = (xnn_float16*) ((uintptr_t) c1 + cn_stride);
331 c2 = (xnn_float16*) ((uintptr_t) c2 + cn_stride);
332 c3 = (xnn_float16*) ((uintptr_t) c3 + cn_stride);
333
334 nc -= 4;
335 } else {
336 if (nc & 2) {
337 c0[0] = xnn_float16_from_float(vout0x0);
338 c0[1] = xnn_float16_from_float(vout0x1);
339 vout0x0 = vout0x2;
340 c0 += 2;
341 c1[0] = xnn_float16_from_float(vout1x0);
342 c1[1] = xnn_float16_from_float(vout1x1);
343 vout1x0 = vout1x2;
344 c1 += 2;
345 c2[0] = xnn_float16_from_float(vout2x0);
346 c2[1] = xnn_float16_from_float(vout2x1);
347 vout2x0 = vout2x2;
348 c2 += 2;
349 c3[0] = xnn_float16_from_float(vout3x0);
350 c3[1] = xnn_float16_from_float(vout3x1);
351 vout3x0 = vout3x2;
352 c3 += 2;
353 }
354 if (nc & 1) {
355 c0[0] = xnn_float16_from_float(vout0x0);
356 c1[0] = xnn_float16_from_float(vout1x0);
357 c2[0] = xnn_float16_from_float(vout2x0);
358 c3[0] = xnn_float16_from_float(vout3x0);
359 }
360
361 nc = 0;
362 }
363 } while (nc != 0);
364 }