Debian Codegraph @

src/qd8-f16-qb4w-gemm/gen/qd8-f16-qb4w-gemm-2x8-minmax-scalar.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-gemm/scalar.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2021 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include "xnnpack/gemm.h"
13 #include "xnnpack/math.h"
14
15
16 void xnn_qd8_f16_qb4w_gemm_minmax_ukernel_2x8__scalar(
17 size_t mr,
18 size_t nc,
19 size_t kc,
20 const int8_t* restrict a,
21 size_t a_stride,
22 const void* restrict w,
23 xnn_float16* restrict c,
24 size_t cm_stride,
25 size_t cn_stride,
26 const struct xnn_f16_qb4w_minmax_params params[restrict XNN_MIN_ELEMENTS(1)],
27 const struct xnn_qd8_quantization_params quantization_params[restrict XNN_MIN_ELEMENTS(1)])
28 {
29 assert(mr != 0);
30 assert(mr <= 2);
31 assert(nc != 0);
32 assert(kc != 0);
33 size_t bl = params->scalar.blocksize;
34 assert(bl <= round_up_po2(kc, 2));
35 assert(bl != 0);
36 assert(bl % 32 == 0);
37
38 const int8_t* a0 = a;
39 xnn_float16* c0 = c;
40 const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
41 xnn_float16* c1 = (xnn_float16*) ((uintptr_t) c0 + cm_stride);
42 if XNN_UNPREDICTABLE(mr != 2) {
43 a1 = a0;
44 c1 = c0;
45 }
46
47
48 kc = round_up_po2(kc, 2);
49 do {
50 const float vksum0 = ((const float*) w)[0];
51 const float vksum1 = ((const float*) w)[1];
52 const float vksum2 = ((const float*) w)[2];
53 const float vksum3 = ((const float*) w)[3];
54 const float vksum4 = ((const float*) w)[4];
55 const float vksum5 = ((const float*) w)[5];
56 const float vksum6 = ((const float*) w)[6];
57 const float vksum7 = ((const float*) w)[7];
58 const float vinput_zero_point0 = (const float) quantization_params[0].zero_point;
59 float vout0x0 = vksum0 * vinput_zero_point0;
60 float vout0x1 = vksum1 * vinput_zero_point0;
61 float vout0x2 = vksum2 * vinput_zero_point0;
62 float vout0x3 = vksum3 * vinput_zero_point0;
63 float vout0x4 = vksum4 * vinput_zero_point0;
64 float vout0x5 = vksum5 * vinput_zero_point0;
65 float vout0x6 = vksum6 * vinput_zero_point0;
66 float vout0x7 = vksum7 * vinput_zero_point0;
67 const float vinput_zero_point1 = (const float) quantization_params[1].zero_point;
68 float vout1x0 = vksum0 * vinput_zero_point1;
69 float vout1x1 = vksum1 * vinput_zero_point1;
70 float vout1x2 = vksum2 * vinput_zero_point1;
71 float vout1x3 = vksum3 * vinput_zero_point1;
72 float vout1x4 = vksum4 * vinput_zero_point1;
73 float vout1x5 = vksum5 * vinput_zero_point1;
74 float vout1x6 = vksum6 * vinput_zero_point1;
75 float vout1x7 = vksum7 * vinput_zero_point1;
76 w = (const float*) w + 8;
77
78 for (size_t kb=0; kb < kc; kb += bl) {
79 int32_t vacc0x0 = 0;
80 int32_t vacc0x1 = 0;
81 int32_t vacc0x2 = 0;
82 int32_t vacc0x3 = 0;
83 int32_t vacc0x4 = 0;
84 int32_t vacc0x5 = 0;
85 int32_t vacc0x6 = 0;
86 int32_t vacc0x7 = 0;
87 int32_t vacc1x0 = 0;
88 int32_t vacc1x1 = 0;
89 int32_t vacc1x2 = 0;
90 int32_t vacc1x3 = 0;
91 int32_t vacc1x4 = 0;
92 int32_t vacc1x5 = 0;
93 int32_t vacc1x6 = 0;
94 int32_t vacc1x7 = 0;
95 size_t k = bl;
96 for (; k >= 2 * sizeof(uint8_t); k -= 2 * sizeof(uint8_t)) {
97 const int32_t va0c0 = (int32_t) a0[0];
98 const int32_t va0c1 = (int32_t) a0[1];
99 a0 += 2;
100 const int32_t va1c0 = (int32_t) a1[0];
101 const int32_t va1c1 = (int32_t) a1[1];
102 a1 += 2;
103
104 const uint8_t vbi0 = ((const uint8_t*) w)[0];
105 const uint8_t vbi1 = ((const uint8_t*) w)[1];
106 const uint8_t vbi2 = ((const uint8_t*) w)[2];
107 const uint8_t vbi3 = ((const uint8_t*) w)[3];
108 const uint8_t vbi4 = ((const uint8_t*) w)[4];
109 const uint8_t vbi5 = ((const uint8_t*) w)[5];
110 const uint8_t vbi6 = ((const uint8_t*) w)[6];
111 const uint8_t vbi7 = ((const uint8_t*) w)[7];
112 w = (const uint8_t*) w + 8;
113 const int32_t vb0c0 = (int32_t) (int8_t) (vbi0 << 4);
114 const int32_t vb0c1 = (int32_t) (int8_t) (vbi0 & 0xF0);
115 const int32_t vb1c0 = (int32_t) (int8_t) (vbi1 << 4);
116 const int32_t vb1c1 = (int32_t) (int8_t) (vbi1 & 0xF0);
117 const int32_t vb2c0 = (int32_t) (int8_t) (vbi2 << 4);
118 const int32_t vb2c1 = (int32_t) (int8_t) (vbi2 & 0xF0);
119 const int32_t vb3c0 = (int32_t) (int8_t) (vbi3 << 4);
120 const int32_t vb3c1 = (int32_t) (int8_t) (vbi3 & 0xF0);
121 const int32_t vb4c0 = (int32_t) (int8_t) (vbi4 << 4);
122 const int32_t vb4c1 = (int32_t) (int8_t) (vbi4 & 0xF0);
123 const int32_t vb5c0 = (int32_t) (int8_t) (vbi5 << 4);
124 const int32_t vb5c1 = (int32_t) (int8_t) (vbi5 & 0xF0);
125 const int32_t vb6c0 = (int32_t) (int8_t) (vbi6 << 4);
126 const int32_t vb6c1 = (int32_t) (int8_t) (vbi6 & 0xF0);
127 const int32_t vb7c0 = (int32_t) (int8_t) (vbi7 << 4);
128 const int32_t vb7c1 = (int32_t) (int8_t) (vbi7 & 0xF0);
129
130 vacc0x0 += va0c0 * vb0c0;
131 vacc0x1 += va0c0 * vb1c0;
132 vacc0x2 += va0c0 * vb2c0;
133 vacc0x3 += va0c0 * vb3c0;
134 vacc0x4 += va0c0 * vb4c0;
135 vacc0x5 += va0c0 * vb5c0;
136 vacc0x6 += va0c0 * vb6c0;
137 vacc0x7 += va0c0 * vb7c0;
138 vacc1x0 += va1c0 * vb0c0;
139 vacc1x1 += va1c0 * vb1c0;
140 vacc1x2 += va1c0 * vb2c0;
141 vacc1x3 += va1c0 * vb3c0;
142 vacc1x4 += va1c0 * vb4c0;
143 vacc1x5 += va1c0 * vb5c0;
144 vacc1x6 += va1c0 * vb6c0;
145 vacc1x7 += va1c0 * vb7c0;
146 vacc0x0 += va0c1 * vb0c1;
147 vacc0x1 += va0c1 * vb1c1;
148 vacc0x2 += va0c1 * vb2c1;
149 vacc0x3 += va0c1 * vb3c1;
150 vacc0x4 += va0c1 * vb4c1;
151 vacc0x5 += va0c1 * vb5c1;
152 vacc0x6 += va0c1 * vb6c1;
153 vacc0x7 += va0c1 * vb7c1;
154 vacc1x0 += va1c1 * vb0c1;
155 vacc1x1 += va1c1 * vb1c1;
156 vacc1x2 += va1c1 * vb2c1;
157 vacc1x3 += va1c1 * vb3c1;
158 vacc1x4 += va1c1 * vb4c1;
159 vacc1x5 += va1c1 * vb5c1;
160 vacc1x6 += va1c1 * vb6c1;
161 vacc1x7 += va1c1 * vb7c1;
162 }
163 // accumulate in float
164 float vf0x0 = vacc0x0;
165 const float vfilter_output_scale0 = math_cvt_fp32_bf16(((const uint16_t*) w)[0]);
166 float vf0x1 = vacc0x1;
167 const float vfilter_output_scale1 = math_cvt_fp32_bf16(((const uint16_t*) w)[1]);
168 float vf0x2 = vacc0x2;
169 const float vfilter_output_scale2 = math_cvt_fp32_bf16(((const uint16_t*) w)[2]);
170 float vf0x3 = vacc0x3;
171 const float vfilter_output_scale3 = math_cvt_fp32_bf16(((const uint16_t*) w)[3]);
172 float vf0x4 = vacc0x4;
173 const float vfilter_output_scale4 = math_cvt_fp32_bf16(((const uint16_t*) w)[4]);
174 float vf0x5 = vacc0x5;
175 const float vfilter_output_scale5 = math_cvt_fp32_bf16(((const uint16_t*) w)[5]);
176 float vf0x6 = vacc0x6;
177 const float vfilter_output_scale6 = math_cvt_fp32_bf16(((const uint16_t*) w)[6]);
178 float vf0x7 = vacc0x7;
179 const float vfilter_output_scale7 = math_cvt_fp32_bf16(((const uint16_t*) w)[7]);
180 float vf1x0 = vacc1x0;
181 float vf1x1 = vacc1x1;
182 float vf1x2 = vacc1x2;
183 float vf1x3 = vacc1x3;
184 float vf1x4 = vacc1x4;
185 float vf1x5 = vacc1x5;
186 float vf1x6 = vacc1x6;
187 float vf1x7 = vacc1x7;
188
189 vf0x0 *= vfilter_output_scale0;
190 vout0x0 += vf0x0;
191 vf0x1 *= vfilter_output_scale1;
192 vout0x1 += vf0x1;
193 vf0x2 *= vfilter_output_scale2;
194 vout0x2 += vf0x2;
195 vf0x3 *= vfilter_output_scale3;
196 vout0x3 += vf0x3;
197 vf0x4 *= vfilter_output_scale4;
198 vout0x4 += vf0x4;
199 vf0x5 *= vfilter_output_scale5;
200 vout0x5 += vf0x5;
201 vf0x6 *= vfilter_output_scale6;
202 vout0x6 += vf0x6;
203 vf0x7 *= vfilter_output_scale7;
204 vout0x7 += vf0x7;
205 vf1x0 *= vfilter_output_scale0;
206 vout1x0 += vf1x0;
207 vf1x1 *= vfilter_output_scale1;
208 vout1x1 += vf1x1;
209 vf1x2 *= vfilter_output_scale2;
210 vout1x2 += vf1x2;
211 vf1x3 *= vfilter_output_scale3;
212 vout1x3 += vf1x3;
213 vf1x4 *= vfilter_output_scale4;
214 vout1x4 += vf1x4;
215 vf1x5 *= vfilter_output_scale5;
216 vout1x5 += vf1x5;
217 vf1x6 *= vfilter_output_scale6;
218 vout1x6 += vf1x6;
219 vf1x7 *= vfilter_output_scale7;
220 vout1x7 += vf1x7;
221 w = (const uint16_t*) w + 8;
222 }
223
224
225 const float vinput_scale0 = quantization_params[0].inv_scale;
226 vout0x0 *= vinput_scale0;
227 vout0x1 *= vinput_scale0;
228 vout0x2 *= vinput_scale0;
229 vout0x3 *= vinput_scale0;
230 vout0x4 *= vinput_scale0;
231 vout0x5 *= vinput_scale0;
232 vout0x6 *= vinput_scale0;
233 vout0x7 *= vinput_scale0;
234 const float vinput_scale1 = quantization_params[1].inv_scale;
235 vout1x0 *= vinput_scale1;
236 vout1x1 *= vinput_scale1;
237 vout1x2 *= vinput_scale1;
238 vout1x3 *= vinput_scale1;
239 vout1x4 *= vinput_scale1;
240 vout1x5 *= vinput_scale1;
241 vout1x6 *= vinput_scale1;
242 vout1x7 *= vinput_scale1;
243
244
245 const float vbias0 = ((const float*) w)[0];
246 vout0x0 += vbias0;
247 vout1x0 += vbias0;
248 const float vbias1 = ((const float*) w)[1];
249 vout0x1 += vbias1;
250 vout1x1 += vbias1;
251 const float vbias2 = ((const float*) w)[2];
252 vout0x2 += vbias2;
253 vout1x2 += vbias2;
254 const float vbias3 = ((const float*) w)[3];
255 vout0x3 += vbias3;
256 vout1x3 += vbias3;
257 const float vbias4 = ((const float*) w)[4];
258 vout0x4 += vbias4;
259 vout1x4 += vbias4;
260 const float vbias5 = ((const float*) w)[5];
261 vout0x5 += vbias5;
262 vout1x5 += vbias5;
263 const float vbias6 = ((const float*) w)[6];
264 vout0x6 += vbias6;
265 vout1x6 += vbias6;
266 const float vbias7 = ((const float*) w)[7];
267 vout0x7 += vbias7;
268 vout1x7 += vbias7;
269
270 w = (const float*) w + 8;
271
272 const float voutput_min = xnn_float16_to_float(params->scalar.min);
273 vout0x0 = math_max_f32(vout0x0, voutput_min);
274 vout1x0 = math_max_f32(vout1x0, voutput_min);
275 vout0x1 = math_max_f32(vout0x1, voutput_min);
276 vout1x1 = math_max_f32(vout1x1, voutput_min);
277 vout0x2 = math_max_f32(vout0x2, voutput_min);
278 vout1x2 = math_max_f32(vout1x2, voutput_min);
279 vout0x3 = math_max_f32(vout0x3, voutput_min);
280 vout1x3 = math_max_f32(vout1x3, voutput_min);
281 vout0x4 = math_max_f32(vout0x4, voutput_min);
282 vout1x4 = math_max_f32(vout1x4, voutput_min);
283 vout0x5 = math_max_f32(vout0x5, voutput_min);
284 vout1x5 = math_max_f32(vout1x5, voutput_min);
285 vout0x6 = math_max_f32(vout0x6, voutput_min);
286 vout1x6 = math_max_f32(vout1x6, voutput_min);
287 vout0x7 = math_max_f32(vout0x7, voutput_min);
288 vout1x7 = math_max_f32(vout1x7, voutput_min);
289
290 const float voutput_max = xnn_float16_to_float(params->scalar.max);
291 vout0x0 = math_min_f32(vout0x0, voutput_max);
292 vout1x0 = math_min_f32(vout1x0, voutput_max);
293 vout0x1 = math_min_f32(vout0x1, voutput_max);
294 vout1x1 = math_min_f32(vout1x1, voutput_max);
295 vout0x2 = math_min_f32(vout0x2, voutput_max);
296 vout1x2 = math_min_f32(vout1x2, voutput_max);
297 vout0x3 = math_min_f32(vout0x3, voutput_max);
298 vout1x3 = math_min_f32(vout1x3, voutput_max);
299 vout0x4 = math_min_f32(vout0x4, voutput_max);
300 vout1x4 = math_min_f32(vout1x4, voutput_max);
301 vout0x5 = math_min_f32(vout0x5, voutput_max);
302 vout1x5 = math_min_f32(vout1x5, voutput_max);
303 vout0x6 = math_min_f32(vout0x6, voutput_max);
304 vout1x6 = math_min_f32(vout1x6, voutput_max);
305 vout0x7 = math_min_f32(vout0x7, voutput_max);
306 vout1x7 = math_min_f32(vout1x7, voutput_max);
307
308 if XNN_LIKELY(nc >= 8) {
309 c0[0] = xnn_float16_from_float(vout0x0);
310 c0[1] = xnn_float16_from_float(vout0x1);
311 c0[2] = xnn_float16_from_float(vout0x2);
312 c0[3] = xnn_float16_from_float(vout0x3);
313 c0[4] = xnn_float16_from_float(vout0x4);
314 c0[5] = xnn_float16_from_float(vout0x5);
315 c0[6] = xnn_float16_from_float(vout0x6);
316 c0[7] = xnn_float16_from_float(vout0x7);
317 c1[0] = xnn_float16_from_float(vout1x0);
318 c1[1] = xnn_float16_from_float(vout1x1);
319 c1[2] = xnn_float16_from_float(vout1x2);
320 c1[3] = xnn_float16_from_float(vout1x3);
321 c1[4] = xnn_float16_from_float(vout1x4);
322 c1[5] = xnn_float16_from_float(vout1x5);
323 c1[6] = xnn_float16_from_float(vout1x6);
324 c1[7] = xnn_float16_from_float(vout1x7);
325
326 a0 = (const int8_t*) ((uintptr_t) a0 - kc);
327 a1 = (const int8_t*) ((uintptr_t) a1 - kc);
328
329 c0 = (xnn_float16*) ((uintptr_t) c0 + cn_stride);
330 c1 = (xnn_float16*) ((uintptr_t) c1 + cn_stride);
331
332 nc -= 8;
333 } else {
334 if (nc & 4) {
335 c0[0] = xnn_float16_from_float(vout0x0);
336 c0[1] = xnn_float16_from_float(vout0x1);
337 c0[2] = xnn_float16_from_float(vout0x2);
338 c0[3] = xnn_float16_from_float(vout0x3);
339 vout0x0 = vout0x4;
340 vout0x1 = vout0x5;
341 vout0x2 = vout0x6;
342 c0 += 4;
343 c1[0] = xnn_float16_from_float(vout1x0);
344 c1[1] = xnn_float16_from_float(vout1x1);
345 c1[2] = xnn_float16_from_float(vout1x2);
346 c1[3] = xnn_float16_from_float(vout1x3);
347 vout1x0 = vout1x4;
348 vout1x1 = vout1x5;
349 vout1x2 = vout1x6;
350 c1 += 4;
351 }
352 if (nc & 2) {
353 c0[0] = xnn_float16_from_float(vout0x0);
354 c0[1] = xnn_float16_from_float(vout0x1);
355 vout0x0 = vout0x2;
356 vout0x1 = vout0x3;
357 vout0x2 = vout0x4;
358 vout0x3 = vout0x5;
359 vout0x4 = vout0x6;
360 c0 += 2;
361 c1[0] = xnn_float16_from_float(vout1x0);
362 c1[1] = xnn_float16_from_float(vout1x1);
363 vout1x0 = vout1x2;
364 vout1x1 = vout1x3;
365 vout1x2 = vout1x4;
366 vout1x3 = vout1x5;
367 vout1x4 = vout1x6;
368 c1 += 2;
369 }
370 if (nc & 1) {
371 c0[0] = xnn_float16_from_float(vout0x0);
372 c1[0] = xnn_float16_from_float(vout1x0);
373 }
374
375 nc = 0;
376 }
377 } while (nc != 0);
378 }