Debian Codegraph @

src/qd8-f32-qc8w-igemm/gen/qd8-f32-qc8w-igemm-2x8-minmax-scalar.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-igemm/scalar.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2021 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include "xnnpack/math.h"
13 #include "xnnpack/gemm.h"
14
15
16 void xnn_qd8_f32_qc8w_igemm_minmax_ukernel_2x8__scalar(
17 size_t mr,
18 size_t nc,
19 size_t kc,
20 size_t ks,
21 const int8_t** restrict a,
22 const void* restrict w,
23 float* restrict c,
24 size_t cm_stride,
25 size_t cn_stride,
26 size_t a_offset,
27 const int8_t* zero,
28 const int8_t* zero_data,
29 const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)],
30 const struct xnn_qd8_quantization_params quantization_params[restrict XNN_MIN_ELEMENTS(1)])
31 {
32 assert(mr != 0);
33 assert(mr <= 2);
34 assert(nc != 0);
35 assert(kc != 0);
36 assert(ks != 0);
37 assert(ks % (2 * sizeof(void*)) == 0);
38 assert(a != NULL);
39 assert(w != NULL);
40 assert(c != NULL);
41
42 float* c0 = c;
43 float* c1 = (float*) ((uintptr_t) c0 + cm_stride);
44 if XNN_UNPREDICTABLE(mr != 2) {
45 c1 = c0;
46 }
47
48
49 do {
50 const int32_t vksum0 = ((const int32_t*) w)[0];
51 const int32_t vksum1 = ((const int32_t*) w)[1];
52 const int32_t vksum2 = ((const int32_t*) w)[2];
53 const int32_t vksum3 = ((const int32_t*) w)[3];
54 const int32_t vksum4 = ((const int32_t*) w)[4];
55 const int32_t vksum5 = ((const int32_t*) w)[5];
56 const int32_t vksum6 = ((const int32_t*) w)[6];
57 const int32_t vksum7 = ((const int32_t*) w)[7];
58 const int32_t vinput_zero_point = quantization_params->zero_point;
59 int32_t vacc0x0 = vksum0 * vinput_zero_point;
60 int32_t vacc0x1 = vksum1 * vinput_zero_point;
61 int32_t vacc0x2 = vksum2 * vinput_zero_point;
62 int32_t vacc0x3 = vksum3 * vinput_zero_point;
63 int32_t vacc0x4 = vksum4 * vinput_zero_point;
64 int32_t vacc0x5 = vksum5 * vinput_zero_point;
65 int32_t vacc0x6 = vksum6 * vinput_zero_point;
66 int32_t vacc0x7 = vksum7 * vinput_zero_point;
67 int32_t vacc1x0 = vksum0 * vinput_zero_point;
68 int32_t vacc1x1 = vksum1 * vinput_zero_point;
69 int32_t vacc1x2 = vksum2 * vinput_zero_point;
70 int32_t vacc1x3 = vksum3 * vinput_zero_point;
71 int32_t vacc1x4 = vksum4 * vinput_zero_point;
72 int32_t vacc1x5 = vksum5 * vinput_zero_point;
73 int32_t vacc1x6 = vksum6 * vinput_zero_point;
74 int32_t vacc1x7 = vksum7 * vinput_zero_point;
75 w = (const void*) ((const int32_t*) w + 8);
76
77 size_t p = ks;
78 do {
79 const int8_t* restrict a0 = a[0];
80 assert(a0 != NULL);
81 if XNN_UNPREDICTABLE(a0 != zero) {
82 a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
83 } else {
84 a0 = zero_data;
85 }
86 const int8_t* restrict a1 = a[1];
87 assert(a1 != NULL);
88 if XNN_UNPREDICTABLE(a1 != zero) {
89 a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
90 } else {
91 a1 = zero_data;
92 }
93 a += 2;
94
95 size_t k = kc;
96 do {
97 const int32_t va0 = (int32_t) *a0++;
98 const int32_t va1 = (int32_t) *a1++;
99
100 const int32_t vb0 = (int32_t) ((const int8_t*) w)[0];
101 const int32_t vb1 = (int32_t) ((const int8_t*) w)[1];
102 const int32_t vb2 = (int32_t) ((const int8_t*) w)[2];
103 const int32_t vb3 = (int32_t) ((const int8_t*) w)[3];
104 const int32_t vb4 = (int32_t) ((const int8_t*) w)[4];
105 const int32_t vb5 = (int32_t) ((const int8_t*) w)[5];
106 const int32_t vb6 = (int32_t) ((const int8_t*) w)[6];
107 const int32_t vb7 = (int32_t) ((const int8_t*) w)[7];
108 w = (const void*) ((const int8_t*) w + 8);
109
110 vacc0x0 += va0 * vb0;
111 vacc0x1 += va0 * vb1;
112 vacc0x2 += va0 * vb2;
113 vacc0x3 += va0 * vb3;
114 vacc0x4 += va0 * vb4;
115 vacc0x5 += va0 * vb5;
116 vacc0x6 += va0 * vb6;
117 vacc0x7 += va0 * vb7;
118 vacc1x0 += va1 * vb0;
119 vacc1x1 += va1 * vb1;
120 vacc1x2 += va1 * vb2;
121 vacc1x3 += va1 * vb3;
122 vacc1x4 += va1 * vb4;
123 vacc1x5 += va1 * vb5;
124 vacc1x6 += va1 * vb6;
125 vacc1x7 += va1 * vb7;
126
127 k -= sizeof(int8_t);
128 } while (k != 0);
129 p -= 2 * sizeof(void*);
130 } while (p != 0);
131
132 float vout0x0 = (float) vacc0x0;
133 float vout0x1 = (float) vacc0x1;
134 float vout0x2 = (float) vacc0x2;
135 float vout0x3 = (float) vacc0x3;
136 float vout0x4 = (float) vacc0x4;
137 float vout0x5 = (float) vacc0x5;
138 float vout0x6 = (float) vacc0x6;
139 float vout0x7 = (float) vacc0x7;
140 float vout1x0 = (float) vacc1x0;
141 float vout1x1 = (float) vacc1x1;
142 float vout1x2 = (float) vacc1x2;
143 float vout1x3 = (float) vacc1x3;
144 float vout1x4 = (float) vacc1x4;
145 float vout1x5 = (float) vacc1x5;
146 float vout1x6 = (float) vacc1x6;
147 float vout1x7 = (float) vacc1x7;
148
149 const float vinput_scale = quantization_params->inv_scale;
150 vout0x0 *= vinput_scale;
151 vout0x1 *= vinput_scale;
152 vout0x2 *= vinput_scale;
153 vout0x3 *= vinput_scale;
154 vout0x4 *= vinput_scale;
155 vout0x5 *= vinput_scale;
156 vout0x6 *= vinput_scale;
157 vout0x7 *= vinput_scale;
158 vout1x0 *= vinput_scale;
159 vout1x1 *= vinput_scale;
160 vout1x2 *= vinput_scale;
161 vout1x3 *= vinput_scale;
162 vout1x4 *= vinput_scale;
163 vout1x5 *= vinput_scale;
164 vout1x6 *= vinput_scale;
165 vout1x7 *= vinput_scale;
166
167 const float vfilter_output_scale0 = ((const float*) w)[0];
168 vout0x0 *= vfilter_output_scale0;
169 vout1x0 *= vfilter_output_scale0;
170 const float vfilter_output_scale1 = ((const float*) w)[1];
171 vout0x1 *= vfilter_output_scale1;
172 vout1x1 *= vfilter_output_scale1;
173 const float vfilter_output_scale2 = ((const float*) w)[2];
174 vout0x2 *= vfilter_output_scale2;
175 vout1x2 *= vfilter_output_scale2;
176 const float vfilter_output_scale3 = ((const float*) w)[3];
177 vout0x3 *= vfilter_output_scale3;
178 vout1x3 *= vfilter_output_scale3;
179 const float vfilter_output_scale4 = ((const float*) w)[4];
180 vout0x4 *= vfilter_output_scale4;
181 vout1x4 *= vfilter_output_scale4;
182 const float vfilter_output_scale5 = ((const float*) w)[5];
183 vout0x5 *= vfilter_output_scale5;
184 vout1x5 *= vfilter_output_scale5;
185 const float vfilter_output_scale6 = ((const float*) w)[6];
186 vout0x6 *= vfilter_output_scale6;
187 vout1x6 *= vfilter_output_scale6;
188 const float vfilter_output_scale7 = ((const float*) w)[7];
189 vout0x7 *= vfilter_output_scale7;
190 vout1x7 *= vfilter_output_scale7;
191
192 const float vbias0 = ((const float*) w)[8];
193 vout0x0 += vbias0;
194 vout1x0 += vbias0;
195 const float vbias1 = ((const float*) w)[9];
196 vout0x1 += vbias1;
197 vout1x1 += vbias1;
198 const float vbias2 = ((const float*) w)[10];
199 vout0x2 += vbias2;
200 vout1x2 += vbias2;
201 const float vbias3 = ((const float*) w)[11];
202 vout0x3 += vbias3;
203 vout1x3 += vbias3;
204 const float vbias4 = ((const float*) w)[12];
205 vout0x4 += vbias4;
206 vout1x4 += vbias4;
207 const float vbias5 = ((const float*) w)[13];
208 vout0x5 += vbias5;
209 vout1x5 += vbias5;
210 const float vbias6 = ((const float*) w)[14];
211 vout0x6 += vbias6;
212 vout1x6 += vbias6;
213 const float vbias7 = ((const float*) w)[15];
214 vout0x7 += vbias7;
215 vout1x7 += vbias7;
216
217 w = (const float*) w + 16;
218
219 const float voutput_min = params->scalar.min;
220 vout0x0 = math_max_f32(vout0x0, voutput_min);
221 vout1x0 = math_max_f32(vout1x0, voutput_min);
222 vout0x1 = math_max_f32(vout0x1, voutput_min);
223 vout1x1 = math_max_f32(vout1x1, voutput_min);
224 vout0x2 = math_max_f32(vout0x2, voutput_min);
225 vout1x2 = math_max_f32(vout1x2, voutput_min);
226 vout0x3 = math_max_f32(vout0x3, voutput_min);
227 vout1x3 = math_max_f32(vout1x3, voutput_min);
228 vout0x4 = math_max_f32(vout0x4, voutput_min);
229 vout1x4 = math_max_f32(vout1x4, voutput_min);
230 vout0x5 = math_max_f32(vout0x5, voutput_min);
231 vout1x5 = math_max_f32(vout1x5, voutput_min);
232 vout0x6 = math_max_f32(vout0x6, voutput_min);
233 vout1x6 = math_max_f32(vout1x6, voutput_min);
234 vout0x7 = math_max_f32(vout0x7, voutput_min);
235 vout1x7 = math_max_f32(vout1x7, voutput_min);
236
237 const float voutput_max = params->scalar.max;
238 vout0x0 = math_min_f32(vout0x0, voutput_max);
239 vout1x0 = math_min_f32(vout1x0, voutput_max);
240 vout0x1 = math_min_f32(vout0x1, voutput_max);
241 vout1x1 = math_min_f32(vout1x1, voutput_max);
242 vout0x2 = math_min_f32(vout0x2, voutput_max);
243 vout1x2 = math_min_f32(vout1x2, voutput_max);
244 vout0x3 = math_min_f32(vout0x3, voutput_max);
245 vout1x3 = math_min_f32(vout1x3, voutput_max);
246 vout0x4 = math_min_f32(vout0x4, voutput_max);
247 vout1x4 = math_min_f32(vout1x4, voutput_max);
248 vout0x5 = math_min_f32(vout0x5, voutput_max);
249 vout1x5 = math_min_f32(vout1x5, voutput_max);
250 vout0x6 = math_min_f32(vout0x6, voutput_max);
251 vout1x6 = math_min_f32(vout1x6, voutput_max);
252 vout0x7 = math_min_f32(vout0x7, voutput_max);
253 vout1x7 = math_min_f32(vout1x7, voutput_max);
254
255 if XNN_LIKELY(nc >= 8) {
256 c1[0] = vout1x0;
257 c1[1] = vout1x1;
258 c1[2] = vout1x2;
259 c1[3] = vout1x3;
260 c1[4] = vout1x4;
261 c1[5] = vout1x5;
262 c1[6] = vout1x6;
263 c1[7] = vout1x7;
264 c0[0] = vout0x0;
265 c0[1] = vout0x1;
266 c0[2] = vout0x2;
267 c0[3] = vout0x3;
268 c0[4] = vout0x4;
269 c0[5] = vout0x5;
270 c0[6] = vout0x6;
271 c0[7] = vout0x7;
272
273 c0 = (float*) ((uintptr_t) c0 + cn_stride);
274 c1 = (float*) ((uintptr_t) c1 + cn_stride);
275
276 a = (const int8_t**restrict) ((uintptr_t) a - ks);
277 nc -= 8;
278 } else {
279 if (nc & 4) {
280 c1[0] = vout1x0;
281 c1[1] = vout1x1;
282 c1[2] = vout1x2;
283 c1[3] = vout1x3;
284 vout1x0 = vout1x4;
285 vout1x1 = vout1x5;
286 vout1x2 = vout1x6;
287 c1 += 4;
288 c0[0] = vout0x0;
289 c0[1] = vout0x1;
290 c0[2] = vout0x2;
291 c0[3] = vout0x3;
292 vout0x0 = vout0x4;
293 vout0x1 = vout0x5;
294 vout0x2 = vout0x6;
295 c0 += 4;
296 }
297 if (nc & 2) {
298 c1[0] = vout1x0;
299 c1[1] = vout1x1;
300 vout1x0 = vout1x2;
301 vout1x1 = vout1x3;
302 vout1x2 = vout1x4;
303 vout1x3 = vout1x5;
304 vout1x4 = vout1x6;
305 c1 += 2;
306 c0[0] = vout0x0;
307 c0[1] = vout0x1;
308 vout0x0 = vout0x2;
309 vout0x1 = vout0x3;
310 vout0x2 = vout0x4;
311 vout0x3 = vout0x5;
312 vout0x4 = vout0x6;
313 c0 += 2;
314 }
315 if (nc & 1) {
316 c1[0] = vout1x0;
317 c0[0] = vout0x0;
318 }
319
320 nc = 0;
321 }
322 } while (nc != 0);
323 }