Debian Codegraph @

src/qd8-f32-qc8w-igemm/gen/qd8-f32-qc8w-igemm-4x16c4-minmax-wasmusdot.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-igemm/MRx16c4-wasmdot.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2024 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include <wasm_simd128.h>
13
14 #include "xnnpack/gemm.h"
15 #include "xnnpack/math.h"
16
17
18 void xnn_qd8_f32_qc8w_igemm_minmax_ukernel_4x16c4__wasmusdot(
19 size_t mr,
20 size_t nc,
21 size_t kc,
22 size_t ks,
23 const int8_t** restrict a,
24 const void* restrict w,
25 float* restrict c,
26 size_t cm_stride,
27 size_t cn_stride,
28 size_t a_offset,
29 const int8_t* zero,
30 const int8_t* zero_data,
31 const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)],
32 const struct xnn_qd8_quantization_params quantization_params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
33 {
34 assert(mr != 0);
35 assert(mr <= 4);
36 assert(nc != 0);
37 assert(kc != 0);
38 assert(ks != 0);
39 assert(ks % (4 * sizeof(void*)) == 0);
40 assert(a_offset % sizeof(int8_t) == 0);
41 assert(a != NULL);
42 assert(w != NULL);
43 assert(c != NULL);
44
45 kc = round_up_po2(kc, 4 * sizeof(int8_t));
46 float* c0 = c;
47 float* c1 = (float*) ((uintptr_t) c0 + cm_stride);
48 if XNN_UNPREDICTABLE(mr < 2) {
49 c1 = c0;
50 }
51 float* c2 = (float*) ((uintptr_t) c1 + cm_stride);
52 if XNN_UNPREDICTABLE(mr <= 2) {
53 c2 = c1;
54 }
55 float* c3 = (float*) ((uintptr_t) c2 + cm_stride);
56 if XNN_UNPREDICTABLE(mr != 4) {
57 c3 = c2;
58 }
59
60 const v128_t vmin = wasm_v128_load32_splat(&params->scalar.min);
61 const v128_t vmax = wasm_v128_load32_splat(&params->scalar.max);
62 XNN_FORCE_REALIZATION(vmin);
63 XNN_FORCE_REALIZATION(vmax);
64
65 const v128_t vsign_mask = wasm_u8x16_const_splat(UINT8_C(0x80));
66 XNN_FORCE_REALIZATION(vsign_mask);
67 do {
68 v128_t vksum0123 = wasm_v128_load((const int32_t*) w);
69 v128_t vksum4567 = wasm_v128_load((const int32_t*) w + 4);
70 v128_t vksum89AB = wasm_v128_load((const int32_t*) w + 8);
71 v128_t vksumCDEF = wasm_v128_load((const int32_t*) w + 12);
72 const v128_t vinput_zero_point = wasm_i32x4_splat((int32_t) quantization_params->zero_point + 128);
73 v128_t vacc0x0123 = wasm_i32x4_mul(vksum0123, vinput_zero_point);
74 v128_t vacc0x4567 = wasm_i32x4_mul(vksum4567, vinput_zero_point);
75 v128_t vacc0x89AB = wasm_i32x4_mul(vksum89AB, vinput_zero_point);
76 v128_t vacc0xCDEF = wasm_i32x4_mul(vksumCDEF, vinput_zero_point);
77 v128_t vacc1x0123 = wasm_i32x4_mul(vksum0123, vinput_zero_point);
78 v128_t vacc1x4567 = wasm_i32x4_mul(vksum4567, vinput_zero_point);
79 v128_t vacc1x89AB = wasm_i32x4_mul(vksum89AB, vinput_zero_point);
80 v128_t vacc1xCDEF = wasm_i32x4_mul(vksumCDEF, vinput_zero_point);
81 v128_t vacc2x0123 = wasm_i32x4_mul(vksum0123, vinput_zero_point);
82 v128_t vacc2x4567 = wasm_i32x4_mul(vksum4567, vinput_zero_point);
83 v128_t vacc2x89AB = wasm_i32x4_mul(vksum89AB, vinput_zero_point);
84 v128_t vacc2xCDEF = wasm_i32x4_mul(vksumCDEF, vinput_zero_point);
85 v128_t vacc3x0123 = wasm_i32x4_mul(vksum0123, vinput_zero_point);
86 v128_t vacc3x4567 = wasm_i32x4_mul(vksum4567, vinput_zero_point);
87 v128_t vacc3x89AB = wasm_i32x4_mul(vksum89AB, vinput_zero_point);
88 v128_t vacc3xCDEF = wasm_i32x4_mul(vksumCDEF, vinput_zero_point);
89 w = (const int32_t*) w + 16;
90
91 size_t p = ks;
92 do {
93 const int8_t* restrict a0 = a[0];
94 if XNN_UNPREDICTABLE(a0 != zero) {
95 a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
96 } else {
97 a0 = zero_data;
98 }
99 const int8_t* restrict a1 = a[1];
100 if XNN_UNPREDICTABLE(a1 != zero) {
101 a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
102 } else {
103 a1 = zero_data;
104 }
105 const int8_t* restrict a2 = a[2];
106 if XNN_UNPREDICTABLE(a2 != zero) {
107 a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
108 } else {
109 a2 = zero_data;
110 }
111 const int8_t* restrict a3 = a[3];
112 if XNN_UNPREDICTABLE(a3 != zero) {
113 a3 = (const int8_t*) ((uintptr_t) a3 + a_offset);
114 } else {
115 a3 = zero_data;
116 }
117 a += 4;
118
119 size_t k = kc;
120
121 while (k != 0) {
122 v128_t va0x0123 = wasm_v128_load32_splat(a0);
123 a0 += 4;
124 v128_t va1x0123 = wasm_v128_load32_splat(a1);
125 a1 += 4;
126 v128_t va2x0123 = wasm_v128_load32_splat(a2);
127 a2 += 4;
128 v128_t va3x0123 = wasm_v128_load32_splat(a3);
129 a3 += 4;
130
131 va0x0123 = wasm_v128_xor(va0x0123, vsign_mask);
132 va1x0123 = wasm_v128_xor(va1x0123, vsign_mask);
133 va2x0123 = wasm_v128_xor(va2x0123, vsign_mask);
134 va3x0123 = wasm_v128_xor(va3x0123, vsign_mask);
135
136 const v128_t vb0123 = wasm_v128_load((const int8_t*) w);
137 const v128_t vb4567 = wasm_v128_load((const int8_t*) w + 16);
138 const v128_t vb89AB = wasm_v128_load((const int8_t*) w + 32);
139 const v128_t vbCDEF = wasm_v128_load((const int8_t*) w + 48);
140
141 vacc0x0123 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb0123, va0x0123, vacc0x0123);
142 vacc0x4567 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb4567, va0x0123, vacc0x4567);
143 vacc0x89AB = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb89AB, va0x0123, vacc0x89AB);
144 vacc0xCDEF = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vbCDEF, va0x0123, vacc0xCDEF);
145 vacc1x0123 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb0123, va1x0123, vacc1x0123);
146 vacc1x4567 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb4567, va1x0123, vacc1x4567);
147 vacc1x89AB = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb89AB, va1x0123, vacc1x89AB);
148 vacc1xCDEF = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vbCDEF, va1x0123, vacc1xCDEF);
149 vacc2x0123 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb0123, va2x0123, vacc2x0123);
150 vacc2x4567 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb4567, va2x0123, vacc2x4567);
151 vacc2x89AB = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb89AB, va2x0123, vacc2x89AB);
152 vacc2xCDEF = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vbCDEF, va2x0123, vacc2xCDEF);
153 vacc3x0123 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb0123, va3x0123, vacc3x0123);
154 vacc3x4567 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb4567, va3x0123, vacc3x4567);
155 vacc3x89AB = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vb89AB, va3x0123, vacc3x89AB);
156 vacc3xCDEF = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(vbCDEF, va3x0123, vacc3xCDEF);
157
158 w = (const int8_t*) w + 64;
159 k -= 4 * sizeof(int8_t);
160 }
161
162 p -= 4 * sizeof(void*);
163 } while (p != 0);
164
165 vacc0x0123 = wasm_f32x4_convert_i32x4(vacc0x0123);
166 vacc0x4567 = wasm_f32x4_convert_i32x4(vacc0x4567);
167 vacc0x89AB = wasm_f32x4_convert_i32x4(vacc0x89AB);
168 vacc0xCDEF = wasm_f32x4_convert_i32x4(vacc0xCDEF);
169 vacc1x0123 = wasm_f32x4_convert_i32x4(vacc1x0123);
170 vacc1x4567 = wasm_f32x4_convert_i32x4(vacc1x4567);
171 vacc1x89AB = wasm_f32x4_convert_i32x4(vacc1x89AB);
172 vacc1xCDEF = wasm_f32x4_convert_i32x4(vacc1xCDEF);
173 vacc2x0123 = wasm_f32x4_convert_i32x4(vacc2x0123);
174 vacc2x4567 = wasm_f32x4_convert_i32x4(vacc2x4567);
175 vacc2x89AB = wasm_f32x4_convert_i32x4(vacc2x89AB);
176 vacc2xCDEF = wasm_f32x4_convert_i32x4(vacc2xCDEF);
177 vacc3x0123 = wasm_f32x4_convert_i32x4(vacc3x0123);
178 vacc3x4567 = wasm_f32x4_convert_i32x4(vacc3x4567);
179 vacc3x89AB = wasm_f32x4_convert_i32x4(vacc3x89AB);
180 vacc3xCDEF = wasm_f32x4_convert_i32x4(vacc3xCDEF);
181
182 const v128_t vinput_scale = wasm_v128_load32_splat(&quantization_params->inv_scale);
183
184 vacc0x0123 = wasm_f32x4_mul(vacc0x0123, vinput_scale);
185 vacc0x4567 = wasm_f32x4_mul(vacc0x4567, vinput_scale);
186 vacc0x89AB = wasm_f32x4_mul(vacc0x89AB, vinput_scale);
187 vacc0xCDEF = wasm_f32x4_mul(vacc0xCDEF, vinput_scale);
188 vacc1x0123 = wasm_f32x4_mul(vacc1x0123, vinput_scale);
189 vacc1x4567 = wasm_f32x4_mul(vacc1x4567, vinput_scale);
190 vacc1x89AB = wasm_f32x4_mul(vacc1x89AB, vinput_scale);
191 vacc1xCDEF = wasm_f32x4_mul(vacc1xCDEF, vinput_scale);
192 vacc2x0123 = wasm_f32x4_mul(vacc2x0123, vinput_scale);
193 vacc2x4567 = wasm_f32x4_mul(vacc2x4567, vinput_scale);
194 vacc2x89AB = wasm_f32x4_mul(vacc2x89AB, vinput_scale);
195 vacc2xCDEF = wasm_f32x4_mul(vacc2xCDEF, vinput_scale);
196 vacc3x0123 = wasm_f32x4_mul(vacc3x0123, vinput_scale);
197 vacc3x4567 = wasm_f32x4_mul(vacc3x4567, vinput_scale);
198 vacc3x89AB = wasm_f32x4_mul(vacc3x89AB, vinput_scale);
199 vacc3xCDEF = wasm_f32x4_mul(vacc3xCDEF, vinput_scale);
200
201 const v128_t vfilter_output_scale0123 = wasm_v128_load(w);
202 w = (const float*) w + 4;
203 const v128_t vfilter_output_scale4567 = wasm_v128_load(w);
204 w = (const float*) w + 4;
205 const v128_t vfilter_output_scale89AB = wasm_v128_load(w);
206 w = (const float*) w + 4;
207 const v128_t vfilter_output_scaleCDEF = wasm_v128_load(w);
208 w = (const float*) w + 4;
209 vacc0x0123 = wasm_f32x4_mul(vacc0x0123, vfilter_output_scale0123);
210 vacc0x4567 = wasm_f32x4_mul(vacc0x4567, vfilter_output_scale4567);
211 vacc0x89AB = wasm_f32x4_mul(vacc0x89AB, vfilter_output_scale89AB);
212 vacc0xCDEF = wasm_f32x4_mul(vacc0xCDEF, vfilter_output_scaleCDEF);
213 vacc1x0123 = wasm_f32x4_mul(vacc1x0123, vfilter_output_scale0123);
214 vacc1x4567 = wasm_f32x4_mul(vacc1x4567, vfilter_output_scale4567);
215 vacc1x89AB = wasm_f32x4_mul(vacc1x89AB, vfilter_output_scale89AB);
216 vacc1xCDEF = wasm_f32x4_mul(vacc1xCDEF, vfilter_output_scaleCDEF);
217 vacc2x0123 = wasm_f32x4_mul(vacc2x0123, vfilter_output_scale0123);
218 vacc2x4567 = wasm_f32x4_mul(vacc2x4567, vfilter_output_scale4567);
219 vacc2x89AB = wasm_f32x4_mul(vacc2x89AB, vfilter_output_scale89AB);
220 vacc2xCDEF = wasm_f32x4_mul(vacc2xCDEF, vfilter_output_scaleCDEF);
221 vacc3x0123 = wasm_f32x4_mul(vacc3x0123, vfilter_output_scale0123);
222 vacc3x4567 = wasm_f32x4_mul(vacc3x4567, vfilter_output_scale4567);
223 vacc3x89AB = wasm_f32x4_mul(vacc3x89AB, vfilter_output_scale89AB);
224 vacc3xCDEF = wasm_f32x4_mul(vacc3xCDEF, vfilter_output_scaleCDEF);
225
226 const v128_t vbias0123 = wasm_v128_load(w);
227 w = (const float*) w + 4;
228 const v128_t vbias4567 = wasm_v128_load(w);
229 w = (const float*) w + 4;
230 const v128_t vbias89AB = wasm_v128_load(w);
231 w = (const float*) w + 4;
232 const v128_t vbiasCDEF = wasm_v128_load(w);
233 w = (const float*) w + 4;
234 vacc0x0123 = wasm_f32x4_add(vacc0x0123, vbias0123);
235 vacc0x4567 = wasm_f32x4_add(vacc0x4567, vbias4567);
236 vacc0x89AB = wasm_f32x4_add(vacc0x89AB, vbias89AB);
237 vacc0xCDEF = wasm_f32x4_add(vacc0xCDEF, vbiasCDEF);
238 vacc1x0123 = wasm_f32x4_add(vacc1x0123, vbias0123);
239 vacc1x4567 = wasm_f32x4_add(vacc1x4567, vbias4567);
240 vacc1x89AB = wasm_f32x4_add(vacc1x89AB, vbias89AB);
241 vacc1xCDEF = wasm_f32x4_add(vacc1xCDEF, vbiasCDEF);
242 vacc2x0123 = wasm_f32x4_add(vacc2x0123, vbias0123);
243 vacc2x4567 = wasm_f32x4_add(vacc2x4567, vbias4567);
244 vacc2x89AB = wasm_f32x4_add(vacc2x89AB, vbias89AB);
245 vacc2xCDEF = wasm_f32x4_add(vacc2xCDEF, vbiasCDEF);
246 vacc3x0123 = wasm_f32x4_add(vacc3x0123, vbias0123);
247 vacc3x4567 = wasm_f32x4_add(vacc3x4567, vbias4567);
248 vacc3x89AB = wasm_f32x4_add(vacc3x89AB, vbias89AB);
249 vacc3xCDEF = wasm_f32x4_add(vacc3xCDEF, vbiasCDEF);
250
251 vacc0x0123 = wasm_f32x4_pmax(vacc0x0123, vmin);
252 vacc0x4567 = wasm_f32x4_pmax(vacc0x4567, vmin);
253 vacc0x89AB = wasm_f32x4_pmax(vacc0x89AB, vmin);
254 vacc0xCDEF = wasm_f32x4_pmax(vacc0xCDEF, vmin);
255 vacc1x0123 = wasm_f32x4_pmax(vacc1x0123, vmin);
256 vacc1x4567 = wasm_f32x4_pmax(vacc1x4567, vmin);
257 vacc1x89AB = wasm_f32x4_pmax(vacc1x89AB, vmin);
258 vacc1xCDEF = wasm_f32x4_pmax(vacc1xCDEF, vmin);
259 vacc2x0123 = wasm_f32x4_pmax(vacc2x0123, vmin);
260 vacc2x4567 = wasm_f32x4_pmax(vacc2x4567, vmin);
261 vacc2x89AB = wasm_f32x4_pmax(vacc2x89AB, vmin);
262 vacc2xCDEF = wasm_f32x4_pmax(vacc2xCDEF, vmin);
263 vacc3x0123 = wasm_f32x4_pmax(vacc3x0123, vmin);
264 vacc3x4567 = wasm_f32x4_pmax(vacc3x4567, vmin);
265 vacc3x89AB = wasm_f32x4_pmax(vacc3x89AB, vmin);
266 vacc3xCDEF = wasm_f32x4_pmax(vacc3xCDEF, vmin);
267
268 vacc0x0123 = wasm_f32x4_pmin(vacc0x0123, vmax);
269 vacc0x4567 = wasm_f32x4_pmin(vacc0x4567, vmax);
270 vacc0x89AB = wasm_f32x4_pmin(vacc0x89AB, vmax);
271 vacc0xCDEF = wasm_f32x4_pmin(vacc0xCDEF, vmax);
272 vacc1x0123 = wasm_f32x4_pmin(vacc1x0123, vmax);
273 vacc1x4567 = wasm_f32x4_pmin(vacc1x4567, vmax);
274 vacc1x89AB = wasm_f32x4_pmin(vacc1x89AB, vmax);
275 vacc1xCDEF = wasm_f32x4_pmin(vacc1xCDEF, vmax);
276 vacc2x0123 = wasm_f32x4_pmin(vacc2x0123, vmax);
277 vacc2x4567 = wasm_f32x4_pmin(vacc2x4567, vmax);
278 vacc2x89AB = wasm_f32x4_pmin(vacc2x89AB, vmax);
279 vacc2xCDEF = wasm_f32x4_pmin(vacc2xCDEF, vmax);
280 vacc3x0123 = wasm_f32x4_pmin(vacc3x0123, vmax);
281 vacc3x4567 = wasm_f32x4_pmin(vacc3x4567, vmax);
282 vacc3x89AB = wasm_f32x4_pmin(vacc3x89AB, vmax);
283 vacc3xCDEF = wasm_f32x4_pmin(vacc3xCDEF, vmax);
284
285 if XNN_LIKELY(nc >= 16) {
286 wasm_v128_store(c3, vacc3x0123);
287 wasm_v128_store(c3 + 4, vacc3x4567);
288 wasm_v128_store(c3 + 8, vacc3x89AB);
289 wasm_v128_store(c3 + 12, vacc3xCDEF);
290 wasm_v128_store(c2, vacc2x0123);
291 wasm_v128_store(c2 + 4, vacc2x4567);
292 wasm_v128_store(c2 + 8, vacc2x89AB);
293 wasm_v128_store(c2 + 12, vacc2xCDEF);
294 wasm_v128_store(c1, vacc1x0123);
295 wasm_v128_store(c1 + 4, vacc1x4567);
296 wasm_v128_store(c1 + 8, vacc1x89AB);
297 wasm_v128_store(c1 + 12, vacc1xCDEF);
298 wasm_v128_store(c0, vacc0x0123);
299 wasm_v128_store(c0 + 4, vacc0x4567);
300 wasm_v128_store(c0 + 8, vacc0x89AB);
301 wasm_v128_store(c0 + 12, vacc0xCDEF);
302
303 c0 = (float*) ((uintptr_t) c0 + cn_stride);
304 c1 = (float*) ((uintptr_t) c1 + cn_stride);
305 c2 = (float*) ((uintptr_t) c2 + cn_stride);
306 c3 = (float*) ((uintptr_t) c3 + cn_stride);
307
308 a = (const int8_t**restrict) ((uintptr_t) a - ks);
309
310 nc -= 16;
311 } else {
312 if (nc & 8) {
313 wasm_v128_store(c3, vacc3x0123);
314 vacc3x0123 = vacc3x89AB;
315 c3 += 4;
316 wasm_v128_store(c3, vacc3x4567);
317 vacc3x4567 = vacc3xCDEF;
318 c3 += 4;
319 wasm_v128_store(c2, vacc2x0123);
320 vacc2x0123 = vacc2x89AB;
321 c2 += 4;
322 wasm_v128_store(c2, vacc2x4567);
323 vacc2x4567 = vacc2xCDEF;
324 c2 += 4;
325 wasm_v128_store(c1, vacc1x0123);
326 vacc1x0123 = vacc1x89AB;
327 c1 += 4;
328 wasm_v128_store(c1, vacc1x4567);
329 vacc1x4567 = vacc1xCDEF;
330 c1 += 4;
331 wasm_v128_store(c0, vacc0x0123);
332 vacc0x0123 = vacc0x89AB;
333 c0 += 4;
334 wasm_v128_store(c0, vacc0x4567);
335 vacc0x4567 = vacc0xCDEF;
336 c0 += 4;
337 }
338 if (nc & 4) {
339 wasm_v128_store(c3, vacc3x0123);
340 vacc3x0123 = vacc3x4567;
341 c3 += 4;
342 wasm_v128_store(c2, vacc2x0123);
343 vacc2x0123 = vacc2x4567;
344 c2 += 4;
345 wasm_v128_store(c1, vacc1x0123);
346 vacc1x0123 = vacc1x4567;
347 c1 += 4;
348 wasm_v128_store(c0, vacc0x0123);
349 vacc0x0123 = vacc0x4567;
350 c0 += 4;
351 }
352 if (nc & 2) {
353 wasm_v128_store64_lane(c3, vacc3x0123, 0);
354 vacc3x0123 = wasm_v64x2_shuffle(vacc3x0123, vacc3x0123, 1, 1);
355 c3 += 2;
356 wasm_v128_store64_lane(c2, vacc2x0123, 0);
357 vacc2x0123 = wasm_v64x2_shuffle(vacc2x0123, vacc2x0123, 1, 1);
358 c2 += 2;
359 wasm_v128_store64_lane(c1, vacc1x0123, 0);
360 vacc1x0123 = wasm_v64x2_shuffle(vacc1x0123, vacc1x0123, 1, 1);
361 c1 += 2;
362 wasm_v128_store64_lane(c0, vacc0x0123, 0);
363 vacc0x0123 = wasm_v64x2_shuffle(vacc0x0123, vacc0x0123, 1, 1);
364 c0 += 2;
365 }
366 if (nc & 1) {
367 wasm_v128_store32_lane(c3, vacc3x0123, 0);
368 wasm_v128_store32_lane(c2, vacc2x0123, 0);
369 wasm_v128_store32_lane(c1, vacc1x0123, 0);
370 wasm_v128_store32_lane(c0, vacc0x0123, 0);
371 }
372 nc = 0;
373 }
374 } while (nc != 0);
375 }