Debian Codegraph @

src/qd8-f32-qc8w-igemm/gen/qd8-f32-qc8w-igemm-16x32c4-minmax-avx512amx-prfm.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-igemm/c4-avx512amx.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2024 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include <immintrin.h>
13
14 #include "xnnpack/gemm.h"
15 #include "xnnpack/intrinsics-polyfill.h"
16 #include "xnnpack/math.h"
17 #include "xnnpack/unaligned.h"
18 #include "xnnpack/prefetch.h"
19
20
21 void xnn_qd8_f32_qc8w_igemm_minmax_ukernel_16x32c4__avx512amx_prfm(
22 size_t mr,
23 size_t nc,
24 size_t kc,
25 size_t ks,
26 const int8_t** restrict a,
27 const void* restrict w,
28 float* restrict c,
29 size_t cm_stride,
30 size_t cn_stride,
31 size_t a_offset,
32 const int8_t* zero,
33 const int8_t* zero_data,
34 const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)],
35 const struct xnn_qd8_quantization_params quantization_params[restrict XNN_MIN_ELEMENTS(1)])
36 {
37 assert(mr != 0);
38 assert(mr <= 16);
39 assert(nc != 0);
40 assert(kc != 0);
41 assert(kc % sizeof(int8_t) == 0);
42 assert(a != NULL);
43 assert(w != NULL);
44 assert(c != NULL);
45
46 // TODO: amxintrin.h only provide intrinsics for __x86_64__
47 // Update if amxintrin changes
48 #if defined(__x86_64__)
49 __attribute__((aligned(64))) int32_t vintile[16 * 16];
50 __attribute__((aligned(64))) int32_t res0[16 * 16];
51 __attribute__((aligned(64))) int32_t res1[16 * 16];
52
53 kc = round_up_po2(kc, 4 * sizeof(int8_t));
54 const size_t kremainder = (kc & 63) ? (kc & 63) : 64;
55 const __mmask16 kremainder_mask = _cvtu32_mask16((UINT32_C(1) << (kremainder >> 2)) - 1);
56
57 // Define tile config data structure
58 struct __tile_config {
59 uint8_t palette_id;
60 uint8_t start_row;
61 uint8_t reserved_0[14];
62 uint16_t colsb[8];
63 uint16_t reserved_1[8];
64 uint8_t rows[8];
65 uint8_t reserved_2[8];
66 };
67
68 // Load tile configuration
69 __attribute__((aligned(64))) struct __tile_config tile_data = {0};
70 tile_data.palette_id = 1;
71 tile_data.rows[0] = mr; // tmm0 = res 0
72 tile_data.rows[1] = mr; // tmm1 = res 1
73 tile_data.rows[2] = mr; // tmm2 = res 2
74 tile_data.rows[3] = mr; // tmm3 = res 3
75 tile_data.rows[4] = mr; // tmm4 = input
76 tile_data.rows[5] = 16; // tmm5 = weights
77 tile_data.rows[6] = mr; // tmm6 = input remainder
78 tile_data.rows[7] = kremainder >> 2; // tmm7 = weights remainder
79
80 tile_data.colsb[0] = 64; // tmm0 = res 0
81 tile_data.colsb[1] = 64; // tmm1 = res 1
82 tile_data.colsb[2] = 64; // tmm2 = res 1
83 tile_data.colsb[3] = 64; // tmm3 = res 1
84 tile_data.colsb[4] = 64; // tmm4 = input
85 tile_data.colsb[5] = 64; // tmm5 = weights
86 tile_data.colsb[6] = kremainder; // tmm6 = input remainder
87 tile_data.colsb[7] = 64; // tmm7 = weights remainder
88
89 //_tile_loadconfig(&tile_data);
90 __asm__ volatile ("ldtilecfg %0" :: "m" (tile_data));
91
92 float* c0 = c;
93 float* c1 = (float*) ((uintptr_t) c0 + cm_stride);
94 if XNN_UNPREDICTABLE(mr < 2) {
95 c1 = c0;
96 }
97 float* c2 = (float*) ((uintptr_t) c1 + cm_stride);
98 if XNN_UNPREDICTABLE(mr <= 2) {
99 c2 = c1;
100 }
101 float* c3 = (float*) ((uintptr_t) c2 + cm_stride);
102 if XNN_UNPREDICTABLE(mr < 4) {
103 c3 = c2;
104 }
105 float* c4 = (float*) ((uintptr_t) c3 + cm_stride);
106 if XNN_UNPREDICTABLE(mr <= 4) {
107 c4 = c3;
108 }
109 float* c5 = (float*) ((uintptr_t) c4 + cm_stride);
110 if XNN_UNPREDICTABLE(mr < 6) {
111 c5 = c4;
112 }
113 float* c6 = (float*) ((uintptr_t) c5 + cm_stride);
114 if XNN_UNPREDICTABLE(mr <= 6) {
115 c6 = c5;
116 }
117 float* c7 = (float*) ((uintptr_t) c6 + cm_stride);
118 if XNN_UNPREDICTABLE(mr < 8) {
119 c7 = c6;
120 }
121 float* c8 = (float*) ((uintptr_t) c7 + cm_stride);
122 if XNN_UNPREDICTABLE(mr <= 8) {
123 c8 = c7;
124 }
125 float* c9 = (float*) ((uintptr_t) c8 + cm_stride);
126 if XNN_UNPREDICTABLE(mr < 10) {
127 c9 = c8;
128 }
129 float* c10 = (float*) ((uintptr_t) c9 + cm_stride);
130 if XNN_UNPREDICTABLE(mr <= 10) {
131 c10 = c9;
132 }
133 float* c11 = (float*) ((uintptr_t) c10 + cm_stride);
134 if XNN_UNPREDICTABLE(mr < 12) {
135 c11 = c10;
136 }
137 float* c12 = (float*) ((uintptr_t) c11 + cm_stride);
138 if XNN_UNPREDICTABLE(mr <= 12) {
139 c12 = c11;
140 }
141 float* c13 = (float*) ((uintptr_t) c12 + cm_stride);
142 if XNN_UNPREDICTABLE(mr < 14) {
143 c13 = c12;
144 }
145 float* c14 = (float*) ((uintptr_t) c13 + cm_stride);
146 if XNN_UNPREDICTABLE(mr <= 14) {
147 c14 = c13;
148 }
149 float* c15 = (float*) ((uintptr_t) c14 + cm_stride);
150 if XNN_UNPREDICTABLE(mr != 16) {
151 c15 = c14;
152 }
153
154 const __m512 voutput_min = _mm512_set1_ps(params->scalar.min);
155 const __m512 voutput_max = _mm512_set1_ps(params->scalar.max);
156 // XNN_FORCE_REALIZATION(voutput_min);
157 // XNN_FORCE_REALIZATION(voutput_max);
158
159 do {
160 const __m512i vksum0123456789ABCDEF = _mm512_loadu_epi32((const int32_t*) w + 0);
161 const __m512i vksumGHIJKLMNOPQRSTUV = _mm512_loadu_epi32((const int32_t*) w + 16);
162 w = (const int32_t*) w + 32;
163
164 // Zero tile accumulator
165 __asm__ volatile (
166 "tilezero %%tmm0\n"
167 "tilezero %%tmm1\n"
168 ::);
169
170 size_t p = ks;
171 do {
172 const int8_t* restrict a0 = a[0];
173 if XNN_UNPREDICTABLE(a0 != zero) {
174 a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
175 } else {
176 a0 = zero_data;
177 }
178 const int8_t* restrict a1 = a[1];
179 if XNN_UNPREDICTABLE(a1 != zero) {
180 a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
181 } else {
182 a1 = zero_data;
183 }
184 const int8_t* restrict a2 = a[2];
185 if XNN_UNPREDICTABLE(a2 != zero) {
186 a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
187 } else {
188 a2 = zero_data;
189 }
190 const int8_t* restrict a3 = a[3];
191 if XNN_UNPREDICTABLE(a3 != zero) {
192 a3 = (const int8_t*) ((uintptr_t) a3 + a_offset);
193 } else {
194 a3 = zero_data;
195 }
196 const int8_t* restrict a4 = a[4];
197 if XNN_UNPREDICTABLE(a4 != zero) {
198 a4 = (const int8_t*) ((uintptr_t) a4 + a_offset);
199 } else {
200 a4 = zero_data;
201 }
202 const int8_t* restrict a5 = a[5];
203 if XNN_UNPREDICTABLE(a5 != zero) {
204 a5 = (const int8_t*) ((uintptr_t) a5 + a_offset);
205 } else {
206 a5 = zero_data;
207 }
208 const int8_t* restrict a6 = a[6];
209 if XNN_UNPREDICTABLE(a6 != zero) {
210 a6 = (const int8_t*) ((uintptr_t) a6 + a_offset);
211 } else {
212 a6 = zero_data;
213 }
214 const int8_t* restrict a7 = a[7];
215 if XNN_UNPREDICTABLE(a7 != zero) {
216 a7 = (const int8_t*) ((uintptr_t) a7 + a_offset);
217 } else {
218 a7 = zero_data;
219 }
220 const int8_t* restrict a8 = a[8];
221 if XNN_UNPREDICTABLE(a8 != zero) {
222 a8 = (const int8_t*) ((uintptr_t) a8 + a_offset);
223 } else {
224 a8 = zero_data;
225 }
226 const int8_t* restrict a9 = a[9];
227 if XNN_UNPREDICTABLE(a9 != zero) {
228 a9 = (const int8_t*) ((uintptr_t) a9 + a_offset);
229 } else {
230 a9 = zero_data;
231 }
232 const int8_t* restrict a10 = a[10];
233 if XNN_UNPREDICTABLE(a10 != zero) {
234 a10 = (const int8_t*) ((uintptr_t) a10 + a_offset);
235 } else {
236 a10 = zero_data;
237 }
238 const int8_t* restrict a11 = a[11];
239 if XNN_UNPREDICTABLE(a11 != zero) {
240 a11 = (const int8_t*) ((uintptr_t) a11 + a_offset);
241 } else {
242 a11 = zero_data;
243 }
244 const int8_t* restrict a12 = a[12];
245 if XNN_UNPREDICTABLE(a12 != zero) {
246 a12 = (const int8_t*) ((uintptr_t) a12 + a_offset);
247 } else {
248 a12 = zero_data;
249 }
250 const int8_t* restrict a13 = a[13];
251 if XNN_UNPREDICTABLE(a13 != zero) {
252 a13 = (const int8_t*) ((uintptr_t) a13 + a_offset);
253 } else {
254 a13 = zero_data;
255 }
256 const int8_t* restrict a14 = a[14];
257 if XNN_UNPREDICTABLE(a14 != zero) {
258 a14 = (const int8_t*) ((uintptr_t) a14 + a_offset);
259 } else {
260 a14 = zero_data;
261 }
262 const int8_t* restrict a15 = a[15];
263 if XNN_UNPREDICTABLE(a15 != zero) {
264 a15 = (const int8_t*) ((uintptr_t) a15 + a_offset);
265 } else {
266 a15 = zero_data;
267 }
268 a += 16;
269
270 size_t k = kc;
271 if (mr == 1)
272 {
273 while (k >= 64 * sizeof(int8_t)) {
274 _tile_loadd(4, a0, 64); // Directly load input for mr=1
275 a15 += 64;
276 _tile_loadd(5, (const int8_t*) w + 0, 128);
277 _tile_dpbssd(0, 4, 5);
278 _tile_loadd(5, (const int8_t*) w + 64, 128);
279 _tile_dpbssd(1, 4, 5);
280 xnn_prefetch_to_l1((const int8_t*) w + 4096);
281 xnn_prefetch_to_l1((const int8_t*) w + 4160);
282 xnn_prefetch_to_l1((const int8_t*) w + 4224);
283 xnn_prefetch_to_l1((const int8_t*) w + 4288);
284 xnn_prefetch_to_l1((const int8_t*) w + 4352);
285 xnn_prefetch_to_l1((const int8_t*) w + 4416);
286 xnn_prefetch_to_l1((const int8_t*) w + 4480);
287 xnn_prefetch_to_l1((const int8_t*) w + 4544);
288 xnn_prefetch_to_l1((const int8_t*) w + 4608);
289 xnn_prefetch_to_l1((const int8_t*) w + 4672);
290 xnn_prefetch_to_l1((const int8_t*) w + 4736);
291 xnn_prefetch_to_l1((const int8_t*) w + 4800);
292 xnn_prefetch_to_l1((const int8_t*) w + 4864);
293 xnn_prefetch_to_l1((const int8_t*) w + 4928);
294 xnn_prefetch_to_l1((const int8_t*) w + 4992);
295 xnn_prefetch_to_l1((const int8_t*) w + 5056);
296 xnn_prefetch_to_l1((const int8_t*) w + 5120);
297 xnn_prefetch_to_l1((const int8_t*) w + 5184);
298 xnn_prefetch_to_l1((const int8_t*) w + 5248);
299 xnn_prefetch_to_l1((const int8_t*) w + 5312);
300 xnn_prefetch_to_l1((const int8_t*) w + 5376);
301 xnn_prefetch_to_l1((const int8_t*) w + 5440);
302 xnn_prefetch_to_l1((const int8_t*) w + 5504);
303 xnn_prefetch_to_l1((const int8_t*) w + 5568);
304 xnn_prefetch_to_l1((const int8_t*) w + 5632);
305 xnn_prefetch_to_l1((const int8_t*) w + 5696);
306 xnn_prefetch_to_l1((const int8_t*) w + 5760);
307 xnn_prefetch_to_l1((const int8_t*) w + 5824);
308 xnn_prefetch_to_l1((const int8_t*) w + 5888);
309 xnn_prefetch_to_l1((const int8_t*) w + 5952);
310 xnn_prefetch_to_l1((const int8_t*) w + 6016);
311 xnn_prefetch_to_l1((const int8_t*) w + 6080);
312 xnn_prefetch_to_l1((const int8_t*) w + 6144);
313 xnn_prefetch_to_l1((const int8_t*) w + 6208);
314 xnn_prefetch_to_l1((const int8_t*) w + 6272);
315 xnn_prefetch_to_l1((const int8_t*) w + 6336);
316 xnn_prefetch_to_l1((const int8_t*) w + 6400);
317 xnn_prefetch_to_l1((const int8_t*) w + 6464);
318 xnn_prefetch_to_l1((const int8_t*) w + 6528);
319 xnn_prefetch_to_l1((const int8_t*) w + 6592);
320 xnn_prefetch_to_l1((const int8_t*) w + 6656);
321 xnn_prefetch_to_l1((const int8_t*) w + 6720);
322 xnn_prefetch_to_l1((const int8_t*) w + 6784);
323 xnn_prefetch_to_l1((const int8_t*) w + 6848);
324 xnn_prefetch_to_l1((const int8_t*) w + 6912);
325 xnn_prefetch_to_l1((const int8_t*) w + 6976);
326 xnn_prefetch_to_l1((const int8_t*) w + 7040);
327 xnn_prefetch_to_l1((const int8_t*) w + 7104);
328
329 w = (const int8_t*) w + 2048;
330 k -= 64 * sizeof(int8_t);
331 }
332 }
333 else {
334 while (k >= 64 * sizeof(int8_t)) {
335 const __m512i vin0 = _mm512_loadu_epi32(a0);
336 a0 += 64;
337 _mm512_store_epi32(vintile + 0, vin0);
338 const __m512i vin1 = _mm512_loadu_epi32(a1);
339 a1 += 64;
340 _mm512_store_epi32(vintile + 16, vin1);
341 const __m512i vin2 = _mm512_loadu_epi32(a2);
342 a2 += 64;
343 _mm512_store_epi32(vintile + 32, vin2);
344 const __m512i vin3 = _mm512_loadu_epi32(a3);
345 a3 += 64;
346 _mm512_store_epi32(vintile + 48, vin3);
347 const __m512i vin4 = _mm512_loadu_epi32(a4);
348 a4 += 64;
349 _mm512_store_epi32(vintile + 64, vin4);
350 const __m512i vin5 = _mm512_loadu_epi32(a5);
351 a5 += 64;
352 _mm512_store_epi32(vintile + 80, vin5);
353 const __m512i vin6 = _mm512_loadu_epi32(a6);
354 a6 += 64;
355 _mm512_store_epi32(vintile + 96, vin6);
356 const __m512i vin7 = _mm512_loadu_epi32(a7);
357 a7 += 64;
358 _mm512_store_epi32(vintile + 112, vin7);
359 const __m512i vin8 = _mm512_loadu_epi32(a8);
360 a8 += 64;
361 _mm512_store_epi32(vintile + 128, vin8);
362 const __m512i vin9 = _mm512_loadu_epi32(a9);
363 a9 += 64;
364 _mm512_store_epi32(vintile + 144, vin9);
365 const __m512i vin10 = _mm512_loadu_epi32(a10);
366 a10 += 64;
367 _mm512_store_epi32(vintile + 160, vin10);
368 const __m512i vin11 = _mm512_loadu_epi32(a11);
369 a11 += 64;
370 _mm512_store_epi32(vintile + 176, vin11);
371 const __m512i vin12 = _mm512_loadu_epi32(a12);
372 a12 += 64;
373 _mm512_store_epi32(vintile + 192, vin12);
374 const __m512i vin13 = _mm512_loadu_epi32(a13);
375 a13 += 64;
376 _mm512_store_epi32(vintile + 208, vin13);
377 const __m512i vin14 = _mm512_loadu_epi32(a14);
378 a14 += 64;
379 _mm512_store_epi32(vintile + 224, vin14);
380 const __m512i vin15 = _mm512_loadu_epi32(a15);
381 a15 += 64;
382 _mm512_store_epi32(vintile + 240, vin15);
383 _tile_loadd(4, vintile, 64);
384 _tile_loadd(5, (const int8_t*) w + 0, 128);
385 _tile_dpbssd(0, 4, 5);
386 _tile_loadd(5, (const int8_t*) w + 64, 128);
387 _tile_dpbssd(1, 4, 5);
388 xnn_prefetch_to_l1((const int8_t*) w + 4096);
389 xnn_prefetch_to_l1((const int8_t*) w + 4160);
390 xnn_prefetch_to_l1((const int8_t*) w + 4224);
391 xnn_prefetch_to_l1((const int8_t*) w + 4288);
392 xnn_prefetch_to_l1((const int8_t*) w + 4352);
393 xnn_prefetch_to_l1((const int8_t*) w + 4416);
394 xnn_prefetch_to_l1((const int8_t*) w + 4480);
395 xnn_prefetch_to_l1((const int8_t*) w + 4544);
396 xnn_prefetch_to_l1((const int8_t*) w + 4608);
397 xnn_prefetch_to_l1((const int8_t*) w + 4672);
398 xnn_prefetch_to_l1((const int8_t*) w + 4736);
399 xnn_prefetch_to_l1((const int8_t*) w + 4800);
400 xnn_prefetch_to_l1((const int8_t*) w + 4864);
401 xnn_prefetch_to_l1((const int8_t*) w + 4928);
402 xnn_prefetch_to_l1((const int8_t*) w + 4992);
403 xnn_prefetch_to_l1((const int8_t*) w + 5056);
404 xnn_prefetch_to_l1((const int8_t*) w + 5120);
405 xnn_prefetch_to_l1((const int8_t*) w + 5184);
406 xnn_prefetch_to_l1((const int8_t*) w + 5248);
407 xnn_prefetch_to_l1((const int8_t*) w + 5312);
408 xnn_prefetch_to_l1((const int8_t*) w + 5376);
409 xnn_prefetch_to_l1((const int8_t*) w + 5440);
410 xnn_prefetch_to_l1((const int8_t*) w + 5504);
411 xnn_prefetch_to_l1((const int8_t*) w + 5568);
412 xnn_prefetch_to_l1((const int8_t*) w + 5632);
413 xnn_prefetch_to_l1((const int8_t*) w + 5696);
414 xnn_prefetch_to_l1((const int8_t*) w + 5760);
415 xnn_prefetch_to_l1((const int8_t*) w + 5824);
416 xnn_prefetch_to_l1((const int8_t*) w + 5888);
417 xnn_prefetch_to_l1((const int8_t*) w + 5952);
418 xnn_prefetch_to_l1((const int8_t*) w + 6016);
419 xnn_prefetch_to_l1((const int8_t*) w + 6080);
420 xnn_prefetch_to_l1((const int8_t*) w + 6144);
421 xnn_prefetch_to_l1((const int8_t*) w + 6208);
422 xnn_prefetch_to_l1((const int8_t*) w + 6272);
423 xnn_prefetch_to_l1((const int8_t*) w + 6336);
424 xnn_prefetch_to_l1((const int8_t*) w + 6400);
425 xnn_prefetch_to_l1((const int8_t*) w + 6464);
426 xnn_prefetch_to_l1((const int8_t*) w + 6528);
427 xnn_prefetch_to_l1((const int8_t*) w + 6592);
428 xnn_prefetch_to_l1((const int8_t*) w + 6656);
429 xnn_prefetch_to_l1((const int8_t*) w + 6720);
430 xnn_prefetch_to_l1((const int8_t*) w + 6784);
431 xnn_prefetch_to_l1((const int8_t*) w + 6848);
432 xnn_prefetch_to_l1((const int8_t*) w + 6912);
433 xnn_prefetch_to_l1((const int8_t*) w + 6976);
434 xnn_prefetch_to_l1((const int8_t*) w + 7040);
435 xnn_prefetch_to_l1((const int8_t*) w + 7104);
436
437 w = (const int8_t*) w + 2048;
438 k -= 64 * sizeof(int8_t);
439 }
440 }
441
442 if XNN_UNLIKELY(k != 0) {
443 const __m512i vin0 = _mm512_maskz_loadu_epi32(kremainder_mask, a0);
444 a0 += kremainder;
445 _mm512_store_epi32(vintile + 0, vin0);
446 const __m512i vin1 = _mm512_maskz_loadu_epi32(kremainder_mask, a1);
447 a1 += kremainder;
448 _mm512_store_epi32(vintile + 16, vin1);
449 const __m512i vin2 = _mm512_maskz_loadu_epi32(kremainder_mask, a2);
450 a2 += kremainder;
451 _mm512_store_epi32(vintile + 32, vin2);
452 const __m512i vin3 = _mm512_maskz_loadu_epi32(kremainder_mask, a3);
453 a3 += kremainder;
454 _mm512_store_epi32(vintile + 48, vin3);
455 const __m512i vin4 = _mm512_maskz_loadu_epi32(kremainder_mask, a4);
456 a4 += kremainder;
457 _mm512_store_epi32(vintile + 64, vin4);
458 const __m512i vin5 = _mm512_maskz_loadu_epi32(kremainder_mask, a5);
459 a5 += kremainder;
460 _mm512_store_epi32(vintile + 80, vin5);
461 const __m512i vin6 = _mm512_maskz_loadu_epi32(kremainder_mask, a6);
462 a6 += kremainder;
463 _mm512_store_epi32(vintile + 96, vin6);
464 const __m512i vin7 = _mm512_maskz_loadu_epi32(kremainder_mask, a7);
465 a7 += kremainder;
466 _mm512_store_epi32(vintile + 112, vin7);
467 const __m512i vin8 = _mm512_maskz_loadu_epi32(kremainder_mask, a8);
468 a8 += kremainder;
469 _mm512_store_epi32(vintile + 128, vin8);
470 const __m512i vin9 = _mm512_maskz_loadu_epi32(kremainder_mask, a9);
471 a9 += kremainder;
472 _mm512_store_epi32(vintile + 144, vin9);
473 const __m512i vin10 = _mm512_maskz_loadu_epi32(kremainder_mask, a10);
474 a10 += kremainder;
475 _mm512_store_epi32(vintile + 160, vin10);
476 const __m512i vin11 = _mm512_maskz_loadu_epi32(kremainder_mask, a11);
477 a11 += kremainder;
478 _mm512_store_epi32(vintile + 176, vin11);
479 const __m512i vin12 = _mm512_maskz_loadu_epi32(kremainder_mask, a12);
480 a12 += kremainder;
481 _mm512_store_epi32(vintile + 192, vin12);
482 const __m512i vin13 = _mm512_maskz_loadu_epi32(kremainder_mask, a13);
483 a13 += kremainder;
484 _mm512_store_epi32(vintile + 208, vin13);
485 const __m512i vin14 = _mm512_maskz_loadu_epi32(kremainder_mask, a14);
486 a14 += kremainder;
487 _mm512_store_epi32(vintile + 224, vin14);
488 const __m512i vin15 = _mm512_maskz_loadu_epi32(kremainder_mask, a15);
489 a15 += kremainder;
490 _mm512_store_epi32(vintile + 240, vin15);
491 _tile_loadd(6, vintile, 64);
492 _tile_loadd(7, (const int8_t*) w + 0, 128);
493 _tile_dpbssd(0, 6, 7);
494 _tile_loadd(7, (const int8_t*) w + 64, 128);
495 _tile_dpbssd(1, 6, 7);
496
497 w = (const int8_t*) w + kremainder * 32;
498 k -= kremainder * sizeof(int8_t);
499 }
500
501 p -= 16 * sizeof(void*);
502 } while (p != 0);
503
504 // Add tile to bias
505 _tile_stored(0, res0, 64);
506 _tile_stored(1, res1, 64);
507
508 __m512i vacc0x0123456789ABCDEF = _mm512_mullo_epi32(vksum0123456789ABCDEF, _mm512_set1_epi32((int) quantization_params->zero_point));
509 __m512i vacc0xGHIJKLMNOPQRSTUV = _mm512_mullo_epi32(vksumGHIJKLMNOPQRSTUV, _mm512_set1_epi32((int) quantization_params->zero_point));
510 __m512i vacc1x0123456789ABCDEF = _mm512_mullo_epi32(vksum0123456789ABCDEF, _mm512_set1_epi32((int) quantization_params->zero_point));
511 __m512i vacc1xGHIJKLMNOPQRSTUV = _mm512_mullo_epi32(vksumGHIJKLMNOPQRSTUV, _mm512_set1_epi32((int) quantization_params->zero_point));
512 __m512i vacc2x0123456789ABCDEF = _mm512_mullo_epi32(vksum0123456789ABCDEF, _mm512_set1_epi32((int) quantization_params->zero_point));
513 __m512i vacc2xGHIJKLMNOPQRSTUV = _mm512_mullo_epi32(vksumGHIJKLMNOPQRSTUV, _mm512_set1_epi32((int) quantization_params->zero_point));
514 __m512i vacc3x0123456789ABCDEF = _mm512_mullo_epi32(vksum0123456789ABCDEF, _mm512_set1_epi32((int) quantization_params->zero_point));
515 __m512i vacc3xGHIJKLMNOPQRSTUV = _mm512_mullo_epi32(vksumGHIJKLMNOPQRSTUV, _mm512_set1_epi32((int) quantization_params->zero_point));
516 __m512i vacc4x0123456789ABCDEF = _mm512_mullo_epi32(vksum0123456789ABCDEF, _mm512_set1_epi32((int) quantization_params->zero_point));
517 __m512i vacc4xGHIJKLMNOPQRSTUV = _mm512_mullo_epi32(vksumGHIJKLMNOPQRSTUV, _mm512_set1_epi32((int) quantization_params->zero_point));
518 __m512i vacc5x0123456789ABCDEF = _mm512_mullo_epi32(vksum0123456789ABCDEF, _mm512_set1_epi32((int) quantization_params->zero_point));
519 __m512i vacc5xGHIJKLMNOPQRSTUV = _mm512_mullo_epi32(vksumGHIJKLMNOPQRSTUV, _mm512_set1_epi32((int) quantization_params->zero_point));
520 __m512i vacc6x0123456789ABCDEF = _mm512_mullo_epi32(vksum0123456789ABCDEF, _mm512_set1_epi32((int) quantization_params->zero_point));
521 __m512i vacc6xGHIJKLMNOPQRSTUV = _mm512_mullo_epi32(vksumGHIJKLMNOPQRSTUV, _mm512_set1_epi32((int) quantization_params->zero_point));
522 __m512i vacc7x0123456789ABCDEF = _mm512_mullo_epi32(vksum0123456789ABCDEF, _mm512_set1_epi32((int) quantization_params->zero_point));
523 __m512i vacc7xGHIJKLMNOPQRSTUV = _mm512_mullo_epi32(vksumGHIJKLMNOPQRSTUV, _mm512_set1_epi32((int) quantization_params->zero_point));
524 __m512i vacc8x0123456789ABCDEF = _mm512_mullo_epi32(vksum0123456789ABCDEF, _mm512_set1_epi32((int) quantization_params->zero_point));
525 __m512i vacc8xGHIJKLMNOPQRSTUV = _mm512_mullo_epi32(vksumGHIJKLMNOPQRSTUV, _mm512_set1_epi32((int) quantization_params->zero_point));
526 __m512i vacc9x0123456789ABCDEF = _mm512_mullo_epi32(vksum0123456789ABCDEF, _mm512_set1_epi32((int) quantization_params->zero_point));
527 __m512i vacc9xGHIJKLMNOPQRSTUV = _mm512_mullo_epi32(vksumGHIJKLMNOPQRSTUV, _mm512_set1_epi32((int) quantization_params->zero_point));
528 __m512i vacc10x0123456789ABCDEF = _mm512_mullo_epi32(vksum0123456789ABCDEF, _mm512_set1_epi32((int) quantization_params->zero_point));
529 __m512i vacc10xGHIJKLMNOPQRSTUV = _mm512_mullo_epi32(vksumGHIJKLMNOPQRSTUV, _mm512_set1_epi32((int) quantization_params->zero_point));
530 __m512i vacc11x0123456789ABCDEF = _mm512_mullo_epi32(vksum0123456789ABCDEF, _mm512_set1_epi32((int) quantization_params->zero_point));
531 __m512i vacc11xGHIJKLMNOPQRSTUV = _mm512_mullo_epi32(vksumGHIJKLMNOPQRSTUV, _mm512_set1_epi32((int) quantization_params->zero_point));
532 __m512i vacc12x0123456789ABCDEF = _mm512_mullo_epi32(vksum0123456789ABCDEF, _mm512_set1_epi32((int) quantization_params->zero_point));
533 __m512i vacc12xGHIJKLMNOPQRSTUV = _mm512_mullo_epi32(vksumGHIJKLMNOPQRSTUV, _mm512_set1_epi32((int) quantization_params->zero_point));
534 __m512i vacc13x0123456789ABCDEF = _mm512_mullo_epi32(vksum0123456789ABCDEF, _mm512_set1_epi32((int) quantization_params->zero_point));
535 __m512i vacc13xGHIJKLMNOPQRSTUV = _mm512_mullo_epi32(vksumGHIJKLMNOPQRSTUV, _mm512_set1_epi32((int) quantization_params->zero_point));
536 __m512i vacc14x0123456789ABCDEF = _mm512_mullo_epi32(vksum0123456789ABCDEF, _mm512_set1_epi32((int) quantization_params->zero_point));
537 __m512i vacc14xGHIJKLMNOPQRSTUV = _mm512_mullo_epi32(vksumGHIJKLMNOPQRSTUV, _mm512_set1_epi32((int) quantization_params->zero_point));
538 __m512i vacc15x0123456789ABCDEF = _mm512_mullo_epi32(vksum0123456789ABCDEF, _mm512_set1_epi32((int) quantization_params->zero_point));
539 __m512i vacc15xGHIJKLMNOPQRSTUV = _mm512_mullo_epi32(vksumGHIJKLMNOPQRSTUV, _mm512_set1_epi32((int) quantization_params->zero_point));
540 vacc0x0123456789ABCDEF = _mm512_add_epi32(vacc0x0123456789ABCDEF, _mm512_load_epi32(res0 + 0));
541 vacc0xGHIJKLMNOPQRSTUV = _mm512_add_epi32(vacc0xGHIJKLMNOPQRSTUV, _mm512_load_epi32(res1 + 0));
542 vacc1x0123456789ABCDEF = _mm512_add_epi32(vacc1x0123456789ABCDEF, _mm512_load_epi32(res0 + 16));
543 vacc1xGHIJKLMNOPQRSTUV = _mm512_add_epi32(vacc1xGHIJKLMNOPQRSTUV, _mm512_load_epi32(res1 + 16));
544 vacc2x0123456789ABCDEF = _mm512_add_epi32(vacc2x0123456789ABCDEF, _mm512_load_epi32(res0 + 32));
545 vacc2xGHIJKLMNOPQRSTUV = _mm512_add_epi32(vacc2xGHIJKLMNOPQRSTUV, _mm512_load_epi32(res1 + 32));
546 vacc3x0123456789ABCDEF = _mm512_add_epi32(vacc3x0123456789ABCDEF, _mm512_load_epi32(res0 + 48));
547 vacc3xGHIJKLMNOPQRSTUV = _mm512_add_epi32(vacc3xGHIJKLMNOPQRSTUV, _mm512_load_epi32(res1 + 48));
548 vacc4x0123456789ABCDEF = _mm512_add_epi32(vacc4x0123456789ABCDEF, _mm512_load_epi32(res0 + 64));
549 vacc4xGHIJKLMNOPQRSTUV = _mm512_add_epi32(vacc4xGHIJKLMNOPQRSTUV, _mm512_load_epi32(res1 + 64));
550 vacc5x0123456789ABCDEF = _mm512_add_epi32(vacc5x0123456789ABCDEF, _mm512_load_epi32(res0 + 80));
551 vacc5xGHIJKLMNOPQRSTUV = _mm512_add_epi32(vacc5xGHIJKLMNOPQRSTUV, _mm512_load_epi32(res1 + 80));
552 vacc6x0123456789ABCDEF = _mm512_add_epi32(vacc6x0123456789ABCDEF, _mm512_load_epi32(res0 + 96));
553 vacc6xGHIJKLMNOPQRSTUV = _mm512_add_epi32(vacc6xGHIJKLMNOPQRSTUV, _mm512_load_epi32(res1 + 96));
554 vacc7x0123456789ABCDEF = _mm512_add_epi32(vacc7x0123456789ABCDEF, _mm512_load_epi32(res0 + 112));
555 vacc7xGHIJKLMNOPQRSTUV = _mm512_add_epi32(vacc7xGHIJKLMNOPQRSTUV, _mm512_load_epi32(res1 + 112));
556 vacc8x0123456789ABCDEF = _mm512_add_epi32(vacc8x0123456789ABCDEF, _mm512_load_epi32(res0 + 128));
557 vacc8xGHIJKLMNOPQRSTUV = _mm512_add_epi32(vacc8xGHIJKLMNOPQRSTUV, _mm512_load_epi32(res1 + 128));
558 vacc9x0123456789ABCDEF = _mm512_add_epi32(vacc9x0123456789ABCDEF, _mm512_load_epi32(res0 + 144));
559 vacc9xGHIJKLMNOPQRSTUV = _mm512_add_epi32(vacc9xGHIJKLMNOPQRSTUV, _mm512_load_epi32(res1 + 144));
560 vacc10x0123456789ABCDEF = _mm512_add_epi32(vacc10x0123456789ABCDEF, _mm512_load_epi32(res0 + 160));
561 vacc10xGHIJKLMNOPQRSTUV = _mm512_add_epi32(vacc10xGHIJKLMNOPQRSTUV, _mm512_load_epi32(res1 + 160));
562 vacc11x0123456789ABCDEF = _mm512_add_epi32(vacc11x0123456789ABCDEF, _mm512_load_epi32(res0 + 176));
563 vacc11xGHIJKLMNOPQRSTUV = _mm512_add_epi32(vacc11xGHIJKLMNOPQRSTUV, _mm512_load_epi32(res1 + 176));
564 vacc12x0123456789ABCDEF = _mm512_add_epi32(vacc12x0123456789ABCDEF, _mm512_load_epi32(res0 + 192));
565 vacc12xGHIJKLMNOPQRSTUV = _mm512_add_epi32(vacc12xGHIJKLMNOPQRSTUV, _mm512_load_epi32(res1 + 192));
566 vacc13x0123456789ABCDEF = _mm512_add_epi32(vacc13x0123456789ABCDEF, _mm512_load_epi32(res0 + 208));
567 vacc13xGHIJKLMNOPQRSTUV = _mm512_add_epi32(vacc13xGHIJKLMNOPQRSTUV, _mm512_load_epi32(res1 + 208));
568 vacc14x0123456789ABCDEF = _mm512_add_epi32(vacc14x0123456789ABCDEF, _mm512_load_epi32(res0 + 224));
569 vacc14xGHIJKLMNOPQRSTUV = _mm512_add_epi32(vacc14xGHIJKLMNOPQRSTUV, _mm512_load_epi32(res1 + 224));
570 vacc15x0123456789ABCDEF = _mm512_add_epi32(vacc15x0123456789ABCDEF, _mm512_load_epi32(res0 + 240));
571 vacc15xGHIJKLMNOPQRSTUV = _mm512_add_epi32(vacc15xGHIJKLMNOPQRSTUV, _mm512_load_epi32(res1 + 240));
572
573 __m512 vscaled0x0123456789ABCDEF = _mm512_cvtepi32_ps(vacc0x0123456789ABCDEF);
574 __m512 vscaled0xGHIJKLMNOPQRSTUV = _mm512_cvtepi32_ps(vacc0xGHIJKLMNOPQRSTUV);
575 __m512 vscaled1x0123456789ABCDEF = _mm512_cvtepi32_ps(vacc1x0123456789ABCDEF);
576 __m512 vscaled1xGHIJKLMNOPQRSTUV = _mm512_cvtepi32_ps(vacc1xGHIJKLMNOPQRSTUV);
577 __m512 vscaled2x0123456789ABCDEF = _mm512_cvtepi32_ps(vacc2x0123456789ABCDEF);
578 __m512 vscaled2xGHIJKLMNOPQRSTUV = _mm512_cvtepi32_ps(vacc2xGHIJKLMNOPQRSTUV);
579 __m512 vscaled3x0123456789ABCDEF = _mm512_cvtepi32_ps(vacc3x0123456789ABCDEF);
580 __m512 vscaled3xGHIJKLMNOPQRSTUV = _mm512_cvtepi32_ps(vacc3xGHIJKLMNOPQRSTUV);
581 __m512 vscaled4x0123456789ABCDEF = _mm512_cvtepi32_ps(vacc4x0123456789ABCDEF);
582 __m512 vscaled4xGHIJKLMNOPQRSTUV = _mm512_cvtepi32_ps(vacc4xGHIJKLMNOPQRSTUV);
583 __m512 vscaled5x0123456789ABCDEF = _mm512_cvtepi32_ps(vacc5x0123456789ABCDEF);
584 __m512 vscaled5xGHIJKLMNOPQRSTUV = _mm512_cvtepi32_ps(vacc5xGHIJKLMNOPQRSTUV);
585 __m512 vscaled6x0123456789ABCDEF = _mm512_cvtepi32_ps(vacc6x0123456789ABCDEF);
586 __m512 vscaled6xGHIJKLMNOPQRSTUV = _mm512_cvtepi32_ps(vacc6xGHIJKLMNOPQRSTUV);
587 __m512 vscaled7x0123456789ABCDEF = _mm512_cvtepi32_ps(vacc7x0123456789ABCDEF);
588 __m512 vscaled7xGHIJKLMNOPQRSTUV = _mm512_cvtepi32_ps(vacc7xGHIJKLMNOPQRSTUV);
589 __m512 vscaled8x0123456789ABCDEF = _mm512_cvtepi32_ps(vacc8x0123456789ABCDEF);
590 __m512 vscaled8xGHIJKLMNOPQRSTUV = _mm512_cvtepi32_ps(vacc8xGHIJKLMNOPQRSTUV);
591 __m512 vscaled9x0123456789ABCDEF = _mm512_cvtepi32_ps(vacc9x0123456789ABCDEF);
592 __m512 vscaled9xGHIJKLMNOPQRSTUV = _mm512_cvtepi32_ps(vacc9xGHIJKLMNOPQRSTUV);
593 __m512 vscaled10x0123456789ABCDEF = _mm512_cvtepi32_ps(vacc10x0123456789ABCDEF);
594 __m512 vscaled10xGHIJKLMNOPQRSTUV = _mm512_cvtepi32_ps(vacc10xGHIJKLMNOPQRSTUV);
595 __m512 vscaled11x0123456789ABCDEF = _mm512_cvtepi32_ps(vacc11x0123456789ABCDEF);
596 __m512 vscaled11xGHIJKLMNOPQRSTUV = _mm512_cvtepi32_ps(vacc11xGHIJKLMNOPQRSTUV);
597 __m512 vscaled12x0123456789ABCDEF = _mm512_cvtepi32_ps(vacc12x0123456789ABCDEF);
598 __m512 vscaled12xGHIJKLMNOPQRSTUV = _mm512_cvtepi32_ps(vacc12xGHIJKLMNOPQRSTUV);
599 __m512 vscaled13x0123456789ABCDEF = _mm512_cvtepi32_ps(vacc13x0123456789ABCDEF);
600 __m512 vscaled13xGHIJKLMNOPQRSTUV = _mm512_cvtepi32_ps(vacc13xGHIJKLMNOPQRSTUV);
601 __m512 vscaled14x0123456789ABCDEF = _mm512_cvtepi32_ps(vacc14x0123456789ABCDEF);
602 __m512 vscaled14xGHIJKLMNOPQRSTUV = _mm512_cvtepi32_ps(vacc14xGHIJKLMNOPQRSTUV);
603 __m512 vscaled15x0123456789ABCDEF = _mm512_cvtepi32_ps(vacc15x0123456789ABCDEF);
604 __m512 vscaled15xGHIJKLMNOPQRSTUV = _mm512_cvtepi32_ps(vacc15xGHIJKLMNOPQRSTUV);
605
606 vscaled0x0123456789ABCDEF = _mm512_mul_ps(vscaled0x0123456789ABCDEF, _mm512_set1_ps(quantization_params->inv_scale));
607 vscaled0xGHIJKLMNOPQRSTUV = _mm512_mul_ps(vscaled0xGHIJKLMNOPQRSTUV, _mm512_set1_ps(quantization_params->inv_scale));
608 vscaled1x0123456789ABCDEF = _mm512_mul_ps(vscaled1x0123456789ABCDEF, _mm512_set1_ps(quantization_params->inv_scale));
609 vscaled1xGHIJKLMNOPQRSTUV = _mm512_mul_ps(vscaled1xGHIJKLMNOPQRSTUV, _mm512_set1_ps(quantization_params->inv_scale));
610 vscaled2x0123456789ABCDEF = _mm512_mul_ps(vscaled2x0123456789ABCDEF, _mm512_set1_ps(quantization_params->inv_scale));
611 vscaled2xGHIJKLMNOPQRSTUV = _mm512_mul_ps(vscaled2xGHIJKLMNOPQRSTUV, _mm512_set1_ps(quantization_params->inv_scale));
612 vscaled3x0123456789ABCDEF = _mm512_mul_ps(vscaled3x0123456789ABCDEF, _mm512_set1_ps(quantization_params->inv_scale));
613 vscaled3xGHIJKLMNOPQRSTUV = _mm512_mul_ps(vscaled3xGHIJKLMNOPQRSTUV, _mm512_set1_ps(quantization_params->inv_scale));
614 vscaled4x0123456789ABCDEF = _mm512_mul_ps(vscaled4x0123456789ABCDEF, _mm512_set1_ps(quantization_params->inv_scale));
615 vscaled4xGHIJKLMNOPQRSTUV = _mm512_mul_ps(vscaled4xGHIJKLMNOPQRSTUV, _mm512_set1_ps(quantization_params->inv_scale));
616 vscaled5x0123456789ABCDEF = _mm512_mul_ps(vscaled5x0123456789ABCDEF, _mm512_set1_ps(quantization_params->inv_scale));
617 vscaled5xGHIJKLMNOPQRSTUV = _mm512_mul_ps(vscaled5xGHIJKLMNOPQRSTUV, _mm512_set1_ps(quantization_params->inv_scale));
618 vscaled6x0123456789ABCDEF = _mm512_mul_ps(vscaled6x0123456789ABCDEF, _mm512_set1_ps(quantization_params->inv_scale));
619 vscaled6xGHIJKLMNOPQRSTUV = _mm512_mul_ps(vscaled6xGHIJKLMNOPQRSTUV, _mm512_set1_ps(quantization_params->inv_scale));
620 vscaled7x0123456789ABCDEF = _mm512_mul_ps(vscaled7x0123456789ABCDEF, _mm512_set1_ps(quantization_params->inv_scale));
621 vscaled7xGHIJKLMNOPQRSTUV = _mm512_mul_ps(vscaled7xGHIJKLMNOPQRSTUV, _mm512_set1_ps(quantization_params->inv_scale));
622 vscaled8x0123456789ABCDEF = _mm512_mul_ps(vscaled8x0123456789ABCDEF, _mm512_set1_ps(quantization_params->inv_scale));
623 vscaled8xGHIJKLMNOPQRSTUV = _mm512_mul_ps(vscaled8xGHIJKLMNOPQRSTUV, _mm512_set1_ps(quantization_params->inv_scale));
624 vscaled9x0123456789ABCDEF = _mm512_mul_ps(vscaled9x0123456789ABCDEF, _mm512_set1_ps(quantization_params->inv_scale));
625 vscaled9xGHIJKLMNOPQRSTUV = _mm512_mul_ps(vscaled9xGHIJKLMNOPQRSTUV, _mm512_set1_ps(quantization_params->inv_scale));
626 vscaled10x0123456789ABCDEF = _mm512_mul_ps(vscaled10x0123456789ABCDEF, _mm512_set1_ps(quantization_params->inv_scale));
627 vscaled10xGHIJKLMNOPQRSTUV = _mm512_mul_ps(vscaled10xGHIJKLMNOPQRSTUV, _mm512_set1_ps(quantization_params->inv_scale));
628 vscaled11x0123456789ABCDEF = _mm512_mul_ps(vscaled11x0123456789ABCDEF, _mm512_set1_ps(quantization_params->inv_scale));
629 vscaled11xGHIJKLMNOPQRSTUV = _mm512_mul_ps(vscaled11xGHIJKLMNOPQRSTUV, _mm512_set1_ps(quantization_params->inv_scale));
630 vscaled12x0123456789ABCDEF = _mm512_mul_ps(vscaled12x0123456789ABCDEF, _mm512_set1_ps(quantization_params->inv_scale));
631 vscaled12xGHIJKLMNOPQRSTUV = _mm512_mul_ps(vscaled12xGHIJKLMNOPQRSTUV, _mm512_set1_ps(quantization_params->inv_scale));
632 vscaled13x0123456789ABCDEF = _mm512_mul_ps(vscaled13x0123456789ABCDEF, _mm512_set1_ps(quantization_params->inv_scale));
633 vscaled13xGHIJKLMNOPQRSTUV = _mm512_mul_ps(vscaled13xGHIJKLMNOPQRSTUV, _mm512_set1_ps(quantization_params->inv_scale));
634 vscaled14x0123456789ABCDEF = _mm512_mul_ps(vscaled14x0123456789ABCDEF, _mm512_set1_ps(quantization_params->inv_scale));
635 vscaled14xGHIJKLMNOPQRSTUV = _mm512_mul_ps(vscaled14xGHIJKLMNOPQRSTUV, _mm512_set1_ps(quantization_params->inv_scale));
636 vscaled15x0123456789ABCDEF = _mm512_mul_ps(vscaled15x0123456789ABCDEF, _mm512_set1_ps(quantization_params->inv_scale));
637 vscaled15xGHIJKLMNOPQRSTUV = _mm512_mul_ps(vscaled15xGHIJKLMNOPQRSTUV, _mm512_set1_ps(quantization_params->inv_scale));
638
639 const __m512 vfilter_output_scale0123456789ABCDEF = _mm512_loadu_ps((const float*) w + 0);
640 const __m512 vfilter_output_scaleGHIJKLMNOPQRSTUV = _mm512_loadu_ps((const float*) w + 16);
641 w = (const int32_t*) w + 32;
642 const __m512 vbias0123456789ABCDEF = _mm512_loadu_ps((const float*) w + 0);
643 const __m512 vbiasGHIJKLMNOPQRSTUV = _mm512_loadu_ps((const float*) w + 16);
644 w = (const int32_t*) w + 32;
645
646 vscaled0x0123456789ABCDEF = _mm512_fmadd_ps(vscaled0x0123456789ABCDEF, vfilter_output_scale0123456789ABCDEF, vbias0123456789ABCDEF);
647 vscaled0xGHIJKLMNOPQRSTUV = _mm512_fmadd_ps(vscaled0xGHIJKLMNOPQRSTUV, vfilter_output_scaleGHIJKLMNOPQRSTUV, vbiasGHIJKLMNOPQRSTUV);
648 vscaled1x0123456789ABCDEF = _mm512_fmadd_ps(vscaled1x0123456789ABCDEF, vfilter_output_scale0123456789ABCDEF, vbias0123456789ABCDEF);
649 vscaled1xGHIJKLMNOPQRSTUV = _mm512_fmadd_ps(vscaled1xGHIJKLMNOPQRSTUV, vfilter_output_scaleGHIJKLMNOPQRSTUV, vbiasGHIJKLMNOPQRSTUV);
650 vscaled2x0123456789ABCDEF = _mm512_fmadd_ps(vscaled2x0123456789ABCDEF, vfilter_output_scale0123456789ABCDEF, vbias0123456789ABCDEF);
651 vscaled2xGHIJKLMNOPQRSTUV = _mm512_fmadd_ps(vscaled2xGHIJKLMNOPQRSTUV, vfilter_output_scaleGHIJKLMNOPQRSTUV, vbiasGHIJKLMNOPQRSTUV);
652 vscaled3x0123456789ABCDEF = _mm512_fmadd_ps(vscaled3x0123456789ABCDEF, vfilter_output_scale0123456789ABCDEF, vbias0123456789ABCDEF);
653 vscaled3xGHIJKLMNOPQRSTUV = _mm512_fmadd_ps(vscaled3xGHIJKLMNOPQRSTUV, vfilter_output_scaleGHIJKLMNOPQRSTUV, vbiasGHIJKLMNOPQRSTUV);
654 vscaled4x0123456789ABCDEF = _mm512_fmadd_ps(vscaled4x0123456789ABCDEF, vfilter_output_scale0123456789ABCDEF, vbias0123456789ABCDEF);
655 vscaled4xGHIJKLMNOPQRSTUV = _mm512_fmadd_ps(vscaled4xGHIJKLMNOPQRSTUV, vfilter_output_scaleGHIJKLMNOPQRSTUV, vbiasGHIJKLMNOPQRSTUV);
656 vscaled5x0123456789ABCDEF = _mm512_fmadd_ps(vscaled5x0123456789ABCDEF, vfilter_output_scale0123456789ABCDEF, vbias0123456789ABCDEF);
657 vscaled5xGHIJKLMNOPQRSTUV = _mm512_fmadd_ps(vscaled5xGHIJKLMNOPQRSTUV, vfilter_output_scaleGHIJKLMNOPQRSTUV, vbiasGHIJKLMNOPQRSTUV);
658 vscaled6x0123456789ABCDEF = _mm512_fmadd_ps(vscaled6x0123456789ABCDEF, vfilter_output_scale0123456789ABCDEF, vbias0123456789ABCDEF);
659 vscaled6xGHIJKLMNOPQRSTUV = _mm512_fmadd_ps(vscaled6xGHIJKLMNOPQRSTUV, vfilter_output_scaleGHIJKLMNOPQRSTUV, vbiasGHIJKLMNOPQRSTUV);
660 vscaled7x0123456789ABCDEF = _mm512_fmadd_ps(vscaled7x0123456789ABCDEF, vfilter_output_scale0123456789ABCDEF, vbias0123456789ABCDEF);
661 vscaled7xGHIJKLMNOPQRSTUV = _mm512_fmadd_ps(vscaled7xGHIJKLMNOPQRSTUV, vfilter_output_scaleGHIJKLMNOPQRSTUV, vbiasGHIJKLMNOPQRSTUV);
662 vscaled8x0123456789ABCDEF = _mm512_fmadd_ps(vscaled8x0123456789ABCDEF, vfilter_output_scale0123456789ABCDEF, vbias0123456789ABCDEF);
663 vscaled8xGHIJKLMNOPQRSTUV = _mm512_fmadd_ps(vscaled8xGHIJKLMNOPQRSTUV, vfilter_output_scaleGHIJKLMNOPQRSTUV, vbiasGHIJKLMNOPQRSTUV);
664 vscaled9x0123456789ABCDEF = _mm512_fmadd_ps(vscaled9x0123456789ABCDEF, vfilter_output_scale0123456789ABCDEF, vbias0123456789ABCDEF);
665 vscaled9xGHIJKLMNOPQRSTUV = _mm512_fmadd_ps(vscaled9xGHIJKLMNOPQRSTUV, vfilter_output_scaleGHIJKLMNOPQRSTUV, vbiasGHIJKLMNOPQRSTUV);
666 vscaled10x0123456789ABCDEF = _mm512_fmadd_ps(vscaled10x0123456789ABCDEF, vfilter_output_scale0123456789ABCDEF, vbias0123456789ABCDEF);
667 vscaled10xGHIJKLMNOPQRSTUV = _mm512_fmadd_ps(vscaled10xGHIJKLMNOPQRSTUV, vfilter_output_scaleGHIJKLMNOPQRSTUV, vbiasGHIJKLMNOPQRSTUV);
668 vscaled11x0123456789ABCDEF = _mm512_fmadd_ps(vscaled11x0123456789ABCDEF, vfilter_output_scale0123456789ABCDEF, vbias0123456789ABCDEF);
669 vscaled11xGHIJKLMNOPQRSTUV = _mm512_fmadd_ps(vscaled11xGHIJKLMNOPQRSTUV, vfilter_output_scaleGHIJKLMNOPQRSTUV, vbiasGHIJKLMNOPQRSTUV);
670 vscaled12x0123456789ABCDEF = _mm512_fmadd_ps(vscaled12x0123456789ABCDEF, vfilter_output_scale0123456789ABCDEF, vbias0123456789ABCDEF);
671 vscaled12xGHIJKLMNOPQRSTUV = _mm512_fmadd_ps(vscaled12xGHIJKLMNOPQRSTUV, vfilter_output_scaleGHIJKLMNOPQRSTUV, vbiasGHIJKLMNOPQRSTUV);
672 vscaled13x0123456789ABCDEF = _mm512_fmadd_ps(vscaled13x0123456789ABCDEF, vfilter_output_scale0123456789ABCDEF, vbias0123456789ABCDEF);
673 vscaled13xGHIJKLMNOPQRSTUV = _mm512_fmadd_ps(vscaled13xGHIJKLMNOPQRSTUV, vfilter_output_scaleGHIJKLMNOPQRSTUV, vbiasGHIJKLMNOPQRSTUV);
674 vscaled14x0123456789ABCDEF = _mm512_fmadd_ps(vscaled14x0123456789ABCDEF, vfilter_output_scale0123456789ABCDEF, vbias0123456789ABCDEF);
675 vscaled14xGHIJKLMNOPQRSTUV = _mm512_fmadd_ps(vscaled14xGHIJKLMNOPQRSTUV, vfilter_output_scaleGHIJKLMNOPQRSTUV, vbiasGHIJKLMNOPQRSTUV);
676 vscaled15x0123456789ABCDEF = _mm512_fmadd_ps(vscaled15x0123456789ABCDEF, vfilter_output_scale0123456789ABCDEF, vbias0123456789ABCDEF);
677 vscaled15xGHIJKLMNOPQRSTUV = _mm512_fmadd_ps(vscaled15xGHIJKLMNOPQRSTUV, vfilter_output_scaleGHIJKLMNOPQRSTUV, vbiasGHIJKLMNOPQRSTUV);
678
679 vscaled0x0123456789ABCDEF = _mm512_max_ps(vscaled0x0123456789ABCDEF, voutput_min);
680 vscaled0xGHIJKLMNOPQRSTUV = _mm512_max_ps(vscaled0xGHIJKLMNOPQRSTUV, voutput_min);
681 vscaled1x0123456789ABCDEF = _mm512_max_ps(vscaled1x0123456789ABCDEF, voutput_min);
682 vscaled1xGHIJKLMNOPQRSTUV = _mm512_max_ps(vscaled1xGHIJKLMNOPQRSTUV, voutput_min);
683 vscaled2x0123456789ABCDEF = _mm512_max_ps(vscaled2x0123456789ABCDEF, voutput_min);
684 vscaled2xGHIJKLMNOPQRSTUV = _mm512_max_ps(vscaled2xGHIJKLMNOPQRSTUV, voutput_min);
685 vscaled3x0123456789ABCDEF = _mm512_max_ps(vscaled3x0123456789ABCDEF, voutput_min);
686 vscaled3xGHIJKLMNOPQRSTUV = _mm512_max_ps(vscaled3xGHIJKLMNOPQRSTUV, voutput_min);
687 vscaled4x0123456789ABCDEF = _mm512_max_ps(vscaled4x0123456789ABCDEF, voutput_min);
688 vscaled4xGHIJKLMNOPQRSTUV = _mm512_max_ps(vscaled4xGHIJKLMNOPQRSTUV, voutput_min);
689 vscaled5x0123456789ABCDEF = _mm512_max_ps(vscaled5x0123456789ABCDEF, voutput_min);
690 vscaled5xGHIJKLMNOPQRSTUV = _mm512_max_ps(vscaled5xGHIJKLMNOPQRSTUV, voutput_min);
691 vscaled6x0123456789ABCDEF = _mm512_max_ps(vscaled6x0123456789ABCDEF, voutput_min);
692 vscaled6xGHIJKLMNOPQRSTUV = _mm512_max_ps(vscaled6xGHIJKLMNOPQRSTUV, voutput_min);
693 vscaled7x0123456789ABCDEF = _mm512_max_ps(vscaled7x0123456789ABCDEF, voutput_min);
694 vscaled7xGHIJKLMNOPQRSTUV = _mm512_max_ps(vscaled7xGHIJKLMNOPQRSTUV, voutput_min);
695 vscaled8x0123456789ABCDEF = _mm512_max_ps(vscaled8x0123456789ABCDEF, voutput_min);
696 vscaled8xGHIJKLMNOPQRSTUV = _mm512_max_ps(vscaled8xGHIJKLMNOPQRSTUV, voutput_min);
697 vscaled9x0123456789ABCDEF = _mm512_max_ps(vscaled9x0123456789ABCDEF, voutput_min);
698 vscaled9xGHIJKLMNOPQRSTUV = _mm512_max_ps(vscaled9xGHIJKLMNOPQRSTUV, voutput_min);
699 vscaled10x0123456789ABCDEF = _mm512_max_ps(vscaled10x0123456789ABCDEF, voutput_min);
700 vscaled10xGHIJKLMNOPQRSTUV = _mm512_max_ps(vscaled10xGHIJKLMNOPQRSTUV, voutput_min);
701 vscaled11x0123456789ABCDEF = _mm512_max_ps(vscaled11x0123456789ABCDEF, voutput_min);
702 vscaled11xGHIJKLMNOPQRSTUV = _mm512_max_ps(vscaled11xGHIJKLMNOPQRSTUV, voutput_min);
703 vscaled12x0123456789ABCDEF = _mm512_max_ps(vscaled12x0123456789ABCDEF, voutput_min);
704 vscaled12xGHIJKLMNOPQRSTUV = _mm512_max_ps(vscaled12xGHIJKLMNOPQRSTUV, voutput_min);
705 vscaled13x0123456789ABCDEF = _mm512_max_ps(vscaled13x0123456789ABCDEF, voutput_min);
706 vscaled13xGHIJKLMNOPQRSTUV = _mm512_max_ps(vscaled13xGHIJKLMNOPQRSTUV, voutput_min);
707 vscaled14x0123456789ABCDEF = _mm512_max_ps(vscaled14x0123456789ABCDEF, voutput_min);
708 vscaled14xGHIJKLMNOPQRSTUV = _mm512_max_ps(vscaled14xGHIJKLMNOPQRSTUV, voutput_min);
709 vscaled15x0123456789ABCDEF = _mm512_max_ps(vscaled15x0123456789ABCDEF, voutput_min);
710 vscaled15xGHIJKLMNOPQRSTUV = _mm512_max_ps(vscaled15xGHIJKLMNOPQRSTUV, voutput_min);
711
712 vscaled0x0123456789ABCDEF = _mm512_min_ps(vscaled0x0123456789ABCDEF, voutput_max);
713 vscaled0xGHIJKLMNOPQRSTUV = _mm512_min_ps(vscaled0xGHIJKLMNOPQRSTUV, voutput_max);
714 vscaled1x0123456789ABCDEF = _mm512_min_ps(vscaled1x0123456789ABCDEF, voutput_max);
715 vscaled1xGHIJKLMNOPQRSTUV = _mm512_min_ps(vscaled1xGHIJKLMNOPQRSTUV, voutput_max);
716 vscaled2x0123456789ABCDEF = _mm512_min_ps(vscaled2x0123456789ABCDEF, voutput_max);
717 vscaled2xGHIJKLMNOPQRSTUV = _mm512_min_ps(vscaled2xGHIJKLMNOPQRSTUV, voutput_max);
718 vscaled3x0123456789ABCDEF = _mm512_min_ps(vscaled3x0123456789ABCDEF, voutput_max);
719 vscaled3xGHIJKLMNOPQRSTUV = _mm512_min_ps(vscaled3xGHIJKLMNOPQRSTUV, voutput_max);
720 vscaled4x0123456789ABCDEF = _mm512_min_ps(vscaled4x0123456789ABCDEF, voutput_max);
721 vscaled4xGHIJKLMNOPQRSTUV = _mm512_min_ps(vscaled4xGHIJKLMNOPQRSTUV, voutput_max);
722 vscaled5x0123456789ABCDEF = _mm512_min_ps(vscaled5x0123456789ABCDEF, voutput_max);
723 vscaled5xGHIJKLMNOPQRSTUV = _mm512_min_ps(vscaled5xGHIJKLMNOPQRSTUV, voutput_max);
724 vscaled6x0123456789ABCDEF = _mm512_min_ps(vscaled6x0123456789ABCDEF, voutput_max);
725 vscaled6xGHIJKLMNOPQRSTUV = _mm512_min_ps(vscaled6xGHIJKLMNOPQRSTUV, voutput_max);
726 vscaled7x0123456789ABCDEF = _mm512_min_ps(vscaled7x0123456789ABCDEF, voutput_max);
727 vscaled7xGHIJKLMNOPQRSTUV = _mm512_min_ps(vscaled7xGHIJKLMNOPQRSTUV, voutput_max);
728 vscaled8x0123456789ABCDEF = _mm512_min_ps(vscaled8x0123456789ABCDEF, voutput_max);
729 vscaled8xGHIJKLMNOPQRSTUV = _mm512_min_ps(vscaled8xGHIJKLMNOPQRSTUV, voutput_max);
730 vscaled9x0123456789ABCDEF = _mm512_min_ps(vscaled9x0123456789ABCDEF, voutput_max);
731 vscaled9xGHIJKLMNOPQRSTUV = _mm512_min_ps(vscaled9xGHIJKLMNOPQRSTUV, voutput_max);
732 vscaled10x0123456789ABCDEF = _mm512_min_ps(vscaled10x0123456789ABCDEF, voutput_max);
733 vscaled10xGHIJKLMNOPQRSTUV = _mm512_min_ps(vscaled10xGHIJKLMNOPQRSTUV, voutput_max);
734 vscaled11x0123456789ABCDEF = _mm512_min_ps(vscaled11x0123456789ABCDEF, voutput_max);
735 vscaled11xGHIJKLMNOPQRSTUV = _mm512_min_ps(vscaled11xGHIJKLMNOPQRSTUV, voutput_max);
736 vscaled12x0123456789ABCDEF = _mm512_min_ps(vscaled12x0123456789ABCDEF, voutput_max);
737 vscaled12xGHIJKLMNOPQRSTUV = _mm512_min_ps(vscaled12xGHIJKLMNOPQRSTUV, voutput_max);
738 vscaled13x0123456789ABCDEF = _mm512_min_ps(vscaled13x0123456789ABCDEF, voutput_max);
739 vscaled13xGHIJKLMNOPQRSTUV = _mm512_min_ps(vscaled13xGHIJKLMNOPQRSTUV, voutput_max);
740 vscaled14x0123456789ABCDEF = _mm512_min_ps(vscaled14x0123456789ABCDEF, voutput_max);
741 vscaled14xGHIJKLMNOPQRSTUV = _mm512_min_ps(vscaled14xGHIJKLMNOPQRSTUV, voutput_max);
742 vscaled15x0123456789ABCDEF = _mm512_min_ps(vscaled15x0123456789ABCDEF, voutput_max);
743 vscaled15xGHIJKLMNOPQRSTUV = _mm512_min_ps(vscaled15xGHIJKLMNOPQRSTUV, voutput_max);
744
745 if XNN_LIKELY(nc >= 32) {
746 _mm512_storeu_ps(c15 + 0, vscaled15x0123456789ABCDEF);
747 _mm512_storeu_ps(c15 + 16, vscaled15xGHIJKLMNOPQRSTUV);
748 c15 = (float*) ((uintptr_t) c15 + cn_stride);
749 _mm512_storeu_ps(c14 + 0, vscaled14x0123456789ABCDEF);
750 _mm512_storeu_ps(c14 + 16, vscaled14xGHIJKLMNOPQRSTUV);
751 c14 = (float*) ((uintptr_t) c14 + cn_stride);
752 _mm512_storeu_ps(c13 + 0, vscaled13x0123456789ABCDEF);
753 _mm512_storeu_ps(c13 + 16, vscaled13xGHIJKLMNOPQRSTUV);
754 c13 = (float*) ((uintptr_t) c13 + cn_stride);
755 _mm512_storeu_ps(c12 + 0, vscaled12x0123456789ABCDEF);
756 _mm512_storeu_ps(c12 + 16, vscaled12xGHIJKLMNOPQRSTUV);
757 c12 = (float*) ((uintptr_t) c12 + cn_stride);
758 _mm512_storeu_ps(c11 + 0, vscaled11x0123456789ABCDEF);
759 _mm512_storeu_ps(c11 + 16, vscaled11xGHIJKLMNOPQRSTUV);
760 c11 = (float*) ((uintptr_t) c11 + cn_stride);
761 _mm512_storeu_ps(c10 + 0, vscaled10x0123456789ABCDEF);
762 _mm512_storeu_ps(c10 + 16, vscaled10xGHIJKLMNOPQRSTUV);
763 c10 = (float*) ((uintptr_t) c10 + cn_stride);
764 _mm512_storeu_ps(c9 + 0, vscaled9x0123456789ABCDEF);
765 _mm512_storeu_ps(c9 + 16, vscaled9xGHIJKLMNOPQRSTUV);
766 c9 = (float*) ((uintptr_t) c9 + cn_stride);
767 _mm512_storeu_ps(c8 + 0, vscaled8x0123456789ABCDEF);
768 _mm512_storeu_ps(c8 + 16, vscaled8xGHIJKLMNOPQRSTUV);
769 c8 = (float*) ((uintptr_t) c8 + cn_stride);
770 _mm512_storeu_ps(c7 + 0, vscaled7x0123456789ABCDEF);
771 _mm512_storeu_ps(c7 + 16, vscaled7xGHIJKLMNOPQRSTUV);
772 c7 = (float*) ((uintptr_t) c7 + cn_stride);
773 _mm512_storeu_ps(c6 + 0, vscaled6x0123456789ABCDEF);
774 _mm512_storeu_ps(c6 + 16, vscaled6xGHIJKLMNOPQRSTUV);
775 c6 = (float*) ((uintptr_t) c6 + cn_stride);
776 _mm512_storeu_ps(c5 + 0, vscaled5x0123456789ABCDEF);
777 _mm512_storeu_ps(c5 + 16, vscaled5xGHIJKLMNOPQRSTUV);
778 c5 = (float*) ((uintptr_t) c5 + cn_stride);
779 _mm512_storeu_ps(c4 + 0, vscaled4x0123456789ABCDEF);
780 _mm512_storeu_ps(c4 + 16, vscaled4xGHIJKLMNOPQRSTUV);
781 c4 = (float*) ((uintptr_t) c4 + cn_stride);
782 _mm512_storeu_ps(c3 + 0, vscaled3x0123456789ABCDEF);
783 _mm512_storeu_ps(c3 + 16, vscaled3xGHIJKLMNOPQRSTUV);
784 c3 = (float*) ((uintptr_t) c3 + cn_stride);
785 _mm512_storeu_ps(c2 + 0, vscaled2x0123456789ABCDEF);
786 _mm512_storeu_ps(c2 + 16, vscaled2xGHIJKLMNOPQRSTUV);
787 c2 = (float*) ((uintptr_t) c2 + cn_stride);
788 _mm512_storeu_ps(c1 + 0, vscaled1x0123456789ABCDEF);
789 _mm512_storeu_ps(c1 + 16, vscaled1xGHIJKLMNOPQRSTUV);
790 c1 = (float*) ((uintptr_t) c1 + cn_stride);
791 _mm512_storeu_ps(c0 + 0, vscaled0x0123456789ABCDEF);
792 _mm512_storeu_ps(c0 + 16, vscaled0xGHIJKLMNOPQRSTUV);
793 c0 = (float*) ((uintptr_t) c0 + cn_stride);
794
795 a = (const int8_t**restrict) ((uintptr_t) a - ks);
796 nc -= 32;
797 } else {
798 // Prepare mask for valid 32-bit elements (depends on nc).
799 const __mmask16 vmask0 = _cvtu32_mask16((uint32_t) ((((UINT64_C(1) << nc) - 1) >> 0) & 0xFFFF));
800 const __mmask16 vmask1 = _cvtu32_mask16((uint32_t) ((((UINT64_C(1) << nc) - 1) >> 16) & 0xFFFF));
801 _mm512_mask_storeu_ps(c15 + 0, vmask0, vscaled15x0123456789ABCDEF);
802 _mm512_mask_storeu_ps(c15 + 16, vmask1, vscaled15xGHIJKLMNOPQRSTUV);
803 _mm512_mask_storeu_ps(c14 + 0, vmask0, vscaled14x0123456789ABCDEF);
804 _mm512_mask_storeu_ps(c14 + 16, vmask1, vscaled14xGHIJKLMNOPQRSTUV);
805 _mm512_mask_storeu_ps(c13 + 0, vmask0, vscaled13x0123456789ABCDEF);
806 _mm512_mask_storeu_ps(c13 + 16, vmask1, vscaled13xGHIJKLMNOPQRSTUV);
807 _mm512_mask_storeu_ps(c12 + 0, vmask0, vscaled12x0123456789ABCDEF);
808 _mm512_mask_storeu_ps(c12 + 16, vmask1, vscaled12xGHIJKLMNOPQRSTUV);
809 _mm512_mask_storeu_ps(c11 + 0, vmask0, vscaled11x0123456789ABCDEF);
810 _mm512_mask_storeu_ps(c11 + 16, vmask1, vscaled11xGHIJKLMNOPQRSTUV);
811 _mm512_mask_storeu_ps(c10 + 0, vmask0, vscaled10x0123456789ABCDEF);
812 _mm512_mask_storeu_ps(c10 + 16, vmask1, vscaled10xGHIJKLMNOPQRSTUV);
813 _mm512_mask_storeu_ps(c9 + 0, vmask0, vscaled9x0123456789ABCDEF);
814 _mm512_mask_storeu_ps(c9 + 16, vmask1, vscaled9xGHIJKLMNOPQRSTUV);
815 _mm512_mask_storeu_ps(c8 + 0, vmask0, vscaled8x0123456789ABCDEF);
816 _mm512_mask_storeu_ps(c8 + 16, vmask1, vscaled8xGHIJKLMNOPQRSTUV);
817 _mm512_mask_storeu_ps(c7 + 0, vmask0, vscaled7x0123456789ABCDEF);
818 _mm512_mask_storeu_ps(c7 + 16, vmask1, vscaled7xGHIJKLMNOPQRSTUV);
819 _mm512_mask_storeu_ps(c6 + 0, vmask0, vscaled6x0123456789ABCDEF);
820 _mm512_mask_storeu_ps(c6 + 16, vmask1, vscaled6xGHIJKLMNOPQRSTUV);
821 _mm512_mask_storeu_ps(c5 + 0, vmask0, vscaled5x0123456789ABCDEF);
822 _mm512_mask_storeu_ps(c5 + 16, vmask1, vscaled5xGHIJKLMNOPQRSTUV);
823 _mm512_mask_storeu_ps(c4 + 0, vmask0, vscaled4x0123456789ABCDEF);
824 _mm512_mask_storeu_ps(c4 + 16, vmask1, vscaled4xGHIJKLMNOPQRSTUV);
825 _mm512_mask_storeu_ps(c3 + 0, vmask0, vscaled3x0123456789ABCDEF);
826 _mm512_mask_storeu_ps(c3 + 16, vmask1, vscaled3xGHIJKLMNOPQRSTUV);
827 _mm512_mask_storeu_ps(c2 + 0, vmask0, vscaled2x0123456789ABCDEF);
828 _mm512_mask_storeu_ps(c2 + 16, vmask1, vscaled2xGHIJKLMNOPQRSTUV);
829 _mm512_mask_storeu_ps(c1 + 0, vmask0, vscaled1x0123456789ABCDEF);
830 _mm512_mask_storeu_ps(c1 + 16, vmask1, vscaled1xGHIJKLMNOPQRSTUV);
831 _mm512_mask_storeu_ps(c0 + 0, vmask0, vscaled0x0123456789ABCDEF);
832 _mm512_mask_storeu_ps(c0 + 16, vmask1, vscaled0xGHIJKLMNOPQRSTUV);
833 nc = 0;
834 }
835 } while (nc != 0);
836
837 // Release tile config
838 // _tile_release();
839 __asm__ volatile ("tilerelease" ::);
840 #endif // defined(__x86_64__)
841 }