Debian Codegraph @

src/qd8-f16-qc8w-igemm/gen/qd8-f16-qc8w-igemm-16x64c4-minmax-avx512amx.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-igemm/c4-avx512amx.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2024 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include <immintrin.h>
13
14 #include "xnnpack/gemm.h"
15 #include "xnnpack/intrinsics-polyfill.h"
16 #include "xnnpack/math.h"
17 #include "xnnpack/unaligned.h"
18
19
20 void xnn_qd8_f16_qc8w_igemm_minmax_ukernel_16x64c4__avx512amx(
21 size_t mr,
22 size_t nc,
23 size_t kc,
24 size_t ks,
25 const int8_t** restrict a,
26 const void* restrict w,
27 xnn_float16* restrict c,
28 size_t cm_stride,
29 size_t cn_stride,
30 size_t a_offset,
31 const int8_t* zero,
32 const int8_t* zero_data,
33 const union xnn_f16_minmax_params params[restrict XNN_MIN_ELEMENTS(1)],
34 const struct xnn_qd8_quantization_params quantization_params[restrict XNN_MIN_ELEMENTS(1)])
35 {
36 assert(mr != 0);
37 assert(mr <= 16);
38 assert(nc != 0);
39 assert(kc != 0);
40 assert(kc % sizeof(int8_t) == 0);
41 assert(a != NULL);
42 assert(w != NULL);
43 assert(c != NULL);
44
45 // TODO: amxintrin.h only provide intrinsics for __x86_64__
46 // Update if amxintrin changes
47 #if defined(__x86_64__)
48 __attribute__((aligned(64))) int32_t vintile[16 * 16];
49 __attribute__((aligned(64))) int32_t res0[16 * 16];
50 __attribute__((aligned(64))) int32_t res1[16 * 16];
51 __attribute__((aligned(64))) int32_t res2[16 * 16];
52 __attribute__((aligned(64))) int32_t res3[16 * 16];
53
54 kc = round_up_po2(kc, 4 * sizeof(int8_t));
55 const size_t kremainder = (kc & 63) ? (kc & 63) : 64;
56 const __mmask16 kremainder_mask = _cvtu32_mask16((UINT32_C(1) << (kremainder >> 2)) - 1);
57
58 // Define tile config data structure
59 struct __tile_config {
60 uint8_t palette_id;
61 uint8_t start_row;
62 uint8_t reserved_0[14];
63 uint16_t colsb[8];
64 uint16_t reserved_1[8];
65 uint8_t rows[8];
66 uint8_t reserved_2[8];
67 };
68
69 // Load tile configuration
70 __attribute__((aligned(64))) struct __tile_config tile_data = {0};
71 tile_data.palette_id = 1;
72 tile_data.rows[0] = mr; // tmm0 = res 0
73 tile_data.rows[1] = mr; // tmm1 = res 1
74 tile_data.rows[2] = mr; // tmm2 = res 2
75 tile_data.rows[3] = mr; // tmm3 = res 3
76 tile_data.rows[4] = mr; // tmm4 = input
77 tile_data.rows[5] = 16; // tmm5 = weights
78 tile_data.rows[6] = mr; // tmm6 = input remainder
79 tile_data.rows[7] = kremainder >> 2; // tmm7 = weights remainder
80
81 tile_data.colsb[0] = 64; // tmm0 = res 0
82 tile_data.colsb[1] = 64; // tmm1 = res 1
83 tile_data.colsb[2] = 64; // tmm2 = res 1
84 tile_data.colsb[3] = 64; // tmm3 = res 1
85 tile_data.colsb[4] = 64; // tmm4 = input
86 tile_data.colsb[5] = 64; // tmm5 = weights
87 tile_data.colsb[6] = kremainder; // tmm6 = input remainder
88 tile_data.colsb[7] = 64; // tmm7 = weights remainder
89
90 //_tile_loadconfig(&tile_data);
91 __asm__ volatile ("ldtilecfg %0" :: "m" (tile_data));
92
93 xnn_float16* c0 = c;
94 xnn_float16* c1 = (xnn_float16*) ((uintptr_t) c0 + cm_stride);
95 if XNN_UNPREDICTABLE(mr < 2) {
96 c1 = c0;
97 }
98 xnn_float16* c2 = (xnn_float16*) ((uintptr_t) c1 + cm_stride);
99 if XNN_UNPREDICTABLE(mr <= 2) {
100 c2 = c1;
101 }
102 xnn_float16* c3 = (xnn_float16*) ((uintptr_t) c2 + cm_stride);
103 if XNN_UNPREDICTABLE(mr < 4) {
104 c3 = c2;
105 }
106 xnn_float16* c4 = (xnn_float16*) ((uintptr_t) c3 + cm_stride);
107 if XNN_UNPREDICTABLE(mr <= 4) {
108 c4 = c3;
109 }
110 xnn_float16* c5 = (xnn_float16*) ((uintptr_t) c4 + cm_stride);
111 if XNN_UNPREDICTABLE(mr < 6) {
112 c5 = c4;
113 }
114 xnn_float16* c6 = (xnn_float16*) ((uintptr_t) c5 + cm_stride);
115 if XNN_UNPREDICTABLE(mr <= 6) {
116 c6 = c5;
117 }
118 xnn_float16* c7 = (xnn_float16*) ((uintptr_t) c6 + cm_stride);
119 if XNN_UNPREDICTABLE(mr < 8) {
120 c7 = c6;
121 }
122 xnn_float16* c8 = (xnn_float16*) ((uintptr_t) c7 + cm_stride);
123 if XNN_UNPREDICTABLE(mr <= 8) {
124 c8 = c7;
125 }
126 xnn_float16* c9 = (xnn_float16*) ((uintptr_t) c8 + cm_stride);
127 if XNN_UNPREDICTABLE(mr < 10) {
128 c9 = c8;
129 }
130 xnn_float16* c10 = (xnn_float16*) ((uintptr_t) c9 + cm_stride);
131 if XNN_UNPREDICTABLE(mr <= 10) {
132 c10 = c9;
133 }
134 xnn_float16* c11 = (xnn_float16*) ((uintptr_t) c10 + cm_stride);
135 if XNN_UNPREDICTABLE(mr < 12) {
136 c11 = c10;
137 }
138 xnn_float16* c12 = (xnn_float16*) ((uintptr_t) c11 + cm_stride);
139 if XNN_UNPREDICTABLE(mr <= 12) {
140 c12 = c11;
141 }
142 xnn_float16* c13 = (xnn_float16*) ((uintptr_t) c12 + cm_stride);
143 if XNN_UNPREDICTABLE(mr < 14) {
144 c13 = c12;
145 }
146 xnn_float16* c14 = (xnn_float16*) ((uintptr_t) c13 + cm_stride);
147 if XNN_UNPREDICTABLE(mr <= 14) {
148 c14 = c13;
149 }
150 xnn_float16* c15 = (xnn_float16*) ((uintptr_t) c14 + cm_stride);
151 if XNN_UNPREDICTABLE(mr != 16) {
152 c15 = c14;
153 }
154
155 const __m512 voutput_min = _mm512_cvtph_ps(_mm256_set1_epi16(*(const uint16_t*) &params->scalar.min));
156 const __m512 voutput_max = _mm512_cvtph_ps(_mm256_set1_epi16(*(const uint16_t*) &params->scalar.max));
157 // XNN_FORCE_REALIZATION(voutput_min);
158 // XNN_FORCE_REALIZATION(voutput_max);
159
160 do {
161 const __m512i vksum0123456789ABCDEF = _mm512_loadu_epi32((const int32_t*) w + 0);
162 const __m512i vksumGHIJKLMNOPQRSTUV = _mm512_loadu_epi32((const int32_t*) w + 16);
163 const __m512i vksumWXYZabcdefghijkl = _mm512_loadu_epi32((const int32_t*) w + 32);
164 const __m512i vksummnopqrstuvwxyz01 = _mm512_loadu_epi32((const int32_t*) w + 48);
165 w = (const int32_t*) w + 64;
166
167 // Zero tile accumulator
168 __asm__ volatile (
169 "tilezero %%tmm0\n"
170 "tilezero %%tmm1\n"
171 "tilezero %%tmm2\n"
172 "tilezero %%tmm3\n"
173 ::);
174
175 size_t p = ks;
176 do {
177 const int8_t* restrict a0 = a[0];
178 if XNN_UNPREDICTABLE(a0 != zero) {
179 a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
180 } else {
181 a0 = zero_data;
182 }
183 const int8_t* restrict a1 = a[1];
184 if XNN_UNPREDICTABLE(a1 != zero) {
185 a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
186 } else {
187 a1 = zero_data;
188 }
189 const int8_t* restrict a2 = a[2];
190 if XNN_UNPREDICTABLE(a2 != zero) {
191 a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
192 } else {
193 a2 = zero_data;
194 }
195 const int8_t* restrict a3 = a[3];
196 if XNN_UNPREDICTABLE(a3 != zero) {
197 a3 = (const int8_t*) ((uintptr_t) a3 + a_offset);
198 } else {
199 a3 = zero_data;
200 }
201 const int8_t* restrict a4 = a[4];
202 if XNN_UNPREDICTABLE(a4 != zero) {
203 a4 = (const int8_t*) ((uintptr_t) a4 + a_offset);
204 } else {
205 a4 = zero_data;
206 }
207 const int8_t* restrict a5 = a[5];
208 if XNN_UNPREDICTABLE(a5 != zero) {
209 a5 = (const int8_t*) ((uintptr_t) a5 + a_offset);
210 } else {
211 a5 = zero_data;
212 }
213 const int8_t* restrict a6 = a[6];
214 if XNN_UNPREDICTABLE(a6 != zero) {
215 a6 = (const int8_t*) ((uintptr_t) a6 + a_offset);
216 } else {
217 a6 = zero_data;
218 }
219 const int8_t* restrict a7 = a[7];
220 if XNN_UNPREDICTABLE(a7 != zero) {
221 a7 = (const int8_t*) ((uintptr_t) a7 + a_offset);
222 } else {
223 a7 = zero_data;
224 }
225 const int8_t* restrict a8 = a[8];
226 if XNN_UNPREDICTABLE(a8 != zero) {
227 a8 = (const int8_t*) ((uintptr_t) a8 + a_offset);
228 } else {
229 a8 = zero_data;
230 }
231 const int8_t* restrict a9 = a[9];
232 if XNN_UNPREDICTABLE(a9 != zero) {
233 a9 = (const int8_t*) ((uintptr_t) a9 + a_offset);
234 } else {
235 a9 = zero_data;
236 }
237 const int8_t* restrict a10 = a[10];
238 if XNN_UNPREDICTABLE(a10 != zero) {
239 a10 = (const int8_t*) ((uintptr_t) a10 + a_offset);
240 } else {
241 a10 = zero_data;
242 }
243 const int8_t* restrict a11 = a[11];
244 if XNN_UNPREDICTABLE(a11 != zero) {
245 a11 = (const int8_t*) ((uintptr_t) a11 + a_offset);
246 } else {
247 a11 = zero_data;
248 }
249 const int8_t* restrict a12 = a[12];
250 if XNN_UNPREDICTABLE(a12 != zero) {
251 a12 = (const int8_t*) ((uintptr_t) a12 + a_offset);
252 } else {
253 a12 = zero_data;
254 }
255 const int8_t* restrict a13 = a[13];
256 if XNN_UNPREDICTABLE(a13 != zero) {
257 a13 = (const int8_t*) ((uintptr_t) a13 + a_offset);
258 } else {
259 a13 = zero_data;
260 }
261 const int8_t* restrict a14 = a[14];
262 if XNN_UNPREDICTABLE(a14 != zero) {
263 a14 = (const int8_t*) ((uintptr_t) a14 + a_offset);
264 } else {
265 a14 = zero_data;
266 }
267 const int8_t* restrict a15 = a[15];
268 if XNN_UNPREDICTABLE(a15 != zero) {
269 a15 = (const int8_t*) ((uintptr_t) a15 + a_offset);
270 } else {
271 a15 = zero_data;
272 }
273 a += 16;
274
275 size_t k = kc;
276 if (mr == 1)
277 {
278 while (k >= 64 * sizeof(int8_t)) {
279 _tile_loadd(4, a0, 64); // Directly load input for mr=1
280 a15 += 64;
281 _tile_loadd(5, (const int8_t*) w + 0, 256);
282 _tile_dpbssd(0, 4, 5);
283 _tile_loadd(5, (const int8_t*) w + 64, 256);
284 _tile_dpbssd(1, 4, 5);
285 _tile_loadd(5, (const int8_t*) w + 128, 256);
286 _tile_dpbssd(2, 4, 5);
287 _tile_loadd(5, (const int8_t*) w + 192, 256);
288 _tile_dpbssd(3, 4, 5);
289
290 w = (const int8_t*) w + 4096;
291 k -= 64 * sizeof(int8_t);
292 }
293 }
294 else {
295 while (k >= 64 * sizeof(int8_t)) {
296 const __m512i vin0 = _mm512_loadu_epi32(a0);
297 a0 += 64;
298 _mm512_store_epi32(vintile + 0, vin0);
299 const __m512i vin1 = _mm512_loadu_epi32(a1);
300 a1 += 64;
301 _mm512_store_epi32(vintile + 16, vin1);
302 const __m512i vin2 = _mm512_loadu_epi32(a2);
303 a2 += 64;
304 _mm512_store_epi32(vintile + 32, vin2);
305 const __m512i vin3 = _mm512_loadu_epi32(a3);
306 a3 += 64;
307 _mm512_store_epi32(vintile + 48, vin3);
308 const __m512i vin4 = _mm512_loadu_epi32(a4);
309 a4 += 64;
310 _mm512_store_epi32(vintile + 64, vin4);
311 const __m512i vin5 = _mm512_loadu_epi32(a5);
312 a5 += 64;
313 _mm512_store_epi32(vintile + 80, vin5);
314 const __m512i vin6 = _mm512_loadu_epi32(a6);
315 a6 += 64;
316 _mm512_store_epi32(vintile + 96, vin6);
317 const __m512i vin7 = _mm512_loadu_epi32(a7);
318 a7 += 64;
319 _mm512_store_epi32(vintile + 112, vin7);
320 const __m512i vin8 = _mm512_loadu_epi32(a8);
321 a8 += 64;
322 _mm512_store_epi32(vintile + 128, vin8);
323 const __m512i vin9 = _mm512_loadu_epi32(a9);
324 a9 += 64;
325 _mm512_store_epi32(vintile + 144, vin9);
326 const __m512i vin10 = _mm512_loadu_epi32(a10);
327 a10 += 64;
328 _mm512_store_epi32(vintile + 160, vin10);
329 const __m512i vin11 = _mm512_loadu_epi32(a11);
330 a11 += 64;
331 _mm512_store_epi32(vintile + 176, vin11);
332 const __m512i vin12 = _mm512_loadu_epi32(a12);
333 a12 += 64;
334 _mm512_store_epi32(vintile + 192, vin12);
335 const __m512i vin13 = _mm512_loadu_epi32(a13);
336 a13 += 64;
337 _mm512_store_epi32(vintile + 208, vin13);
338 const __m512i vin14 = _mm512_loadu_epi32(a14);
339 a14 += 64;
340 _mm512_store_epi32(vintile + 224, vin14);
341 const __m512i vin15 = _mm512_loadu_epi32(a15);
342 a15 += 64;
343 _mm512_store_epi32(vintile + 240, vin15);
344 _tile_loadd(4, vintile, 64);
345 _tile_loadd(5, (const int8_t*) w + 0, 256);
346 _tile_dpbssd(0, 4, 5);
347 _tile_loadd(5, (const int8_t*) w + 64, 256);
348 _tile_dpbssd(1, 4, 5);
349 _tile_loadd(5, (const int8_t*) w + 128, 256);
350 _tile_dpbssd(2, 4, 5);
351 _tile_loadd(5, (const int8_t*) w + 192, 256);
352 _tile_dpbssd(3, 4, 5);
353
354 w = (const int8_t*) w + 4096;
355 k -= 64 * sizeof(int8_t);
356 }
357 }
358
359 if XNN_UNLIKELY(k != 0) {
360 const __m512i vin0 = _mm512_maskz_loadu_epi32(kremainder_mask, a0);
361 a0 += kremainder;
362 _mm512_store_epi32(vintile + 0, vin0);
363 const __m512i vin1 = _mm512_maskz_loadu_epi32(kremainder_mask, a1);
364 a1 += kremainder;
365 _mm512_store_epi32(vintile + 16, vin1);
366 const __m512i vin2 = _mm512_maskz_loadu_epi32(kremainder_mask, a2);
367 a2 += kremainder;
368 _mm512_store_epi32(vintile + 32, vin2);
369 const __m512i vin3 = _mm512_maskz_loadu_epi32(kremainder_mask, a3);
370 a3 += kremainder;
371 _mm512_store_epi32(vintile + 48, vin3);
372 const __m512i vin4 = _mm512_maskz_loadu_epi32(kremainder_mask, a4);
373 a4 += kremainder;
374 _mm512_store_epi32(vintile + 64, vin4);
375 const __m512i vin5 = _mm512_maskz_loadu_epi32(kremainder_mask, a5);
376 a5 += kremainder;
377 _mm512_store_epi32(vintile + 80, vin5);
378 const __m512i vin6 = _mm512_maskz_loadu_epi32(kremainder_mask, a6);
379 a6 += kremainder;
380 _mm512_store_epi32(vintile + 96, vin6);
381 const __m512i vin7 = _mm512_maskz_loadu_epi32(kremainder_mask, a7);
382 a7 += kremainder;
383 _mm512_store_epi32(vintile + 112, vin7);
384 const __m512i vin8 = _mm512_maskz_loadu_epi32(kremainder_mask, a8);
385 a8 += kremainder;
386 _mm512_store_epi32(vintile + 128, vin8);
387 const __m512i vin9 = _mm512_maskz_loadu_epi32(kremainder_mask, a9);
388 a9 += kremainder;
389 _mm512_store_epi32(vintile + 144, vin9);
390 const __m512i vin10 = _mm512_maskz_loadu_epi32(kremainder_mask, a10);
391 a10 += kremainder;
392 _mm512_store_epi32(vintile + 160, vin10);
393 const __m512i vin11 = _mm512_maskz_loadu_epi32(kremainder_mask, a11);
394 a11 += kremainder;
395 _mm512_store_epi32(vintile + 176, vin11);
396 const __m512i vin12 = _mm512_maskz_loadu_epi32(kremainder_mask, a12);
397 a12 += kremainder;
398 _mm512_store_epi32(vintile + 192, vin12);
399 const __m512i vin13 = _mm512_maskz_loadu_epi32(kremainder_mask, a13);
400 a13 += kremainder;
401 _mm512_store_epi32(vintile + 208, vin13);
402 const __m512i vin14 = _mm512_maskz_loadu_epi32(kremainder_mask, a14);
403 a14 += kremainder;
404 _mm512_store_epi32(vintile + 224, vin14);
405 const __m512i vin15 = _mm512_maskz_loadu_epi32(kremainder_mask, a15);
406 a15 += kremainder;
407 _mm512_store_epi32(vintile + 240, vin15);
408 _tile_loadd(6, vintile, 64);
409 _tile_loadd(7, (const int8_t*) w + 0, 256);
410 _tile_dpbssd(0, 6, 7);
411 _tile_loadd(7, (const int8_t*) w + 64, 256);
412 _tile_dpbssd(1, 6, 7);
413 _tile_loadd(7, (const int8_t*) w + 128, 256);
414 _tile_dpbssd(2, 6, 7);
415 _tile_loadd(7, (const int8_t*) w + 192, 256);
416 _tile_dpbssd(3, 6, 7);
417
418 w = (const int8_t*) w + kremainder * 64;
419 k -= kremainder * sizeof(int8_t);
420 }
421
422 p -= 16 * sizeof(void*);
423 } while (p != 0);
424
425 // Add tile to bias
426 _tile_stored(0, res0, 64);
427 _tile_stored(1, res1, 64);
428 _tile_stored(2, res2, 64);
429 _tile_stored(3, res3, 64);
430
431 __m512i vacc0x0123456789ABCDEF = _mm512_mullo_epi32(vksum0123456789ABCDEF, _mm512_set1_epi32((int) quantization_params->zero_point));
432 __m512i vacc0xGHIJKLMNOPQRSTUV = _mm512_mullo_epi32(vksumGHIJKLMNOPQRSTUV, _mm512_set1_epi32((int) quantization_params->zero_point));
433 __m512i vacc0xWXYZabcdefghijkl = _mm512_mullo_epi32(vksumWXYZabcdefghijkl, _mm512_set1_epi32((int) quantization_params->zero_point));
434 __m512i vacc0xmnopqrstuvwxyz01 = _mm512_mullo_epi32(vksummnopqrstuvwxyz01, _mm512_set1_epi32((int) quantization_params->zero_point));
435 __m512i vacc1x0123456789ABCDEF = _mm512_mullo_epi32(vksum0123456789ABCDEF, _mm512_set1_epi32((int) quantization_params->zero_point));
436 __m512i vacc1xGHIJKLMNOPQRSTUV = _mm512_mullo_epi32(vksumGHIJKLMNOPQRSTUV, _mm512_set1_epi32((int) quantization_params->zero_point));
437 __m512i vacc1xWXYZabcdefghijkl = _mm512_mullo_epi32(vksumWXYZabcdefghijkl, _mm512_set1_epi32((int) quantization_params->zero_point));
438 __m512i vacc1xmnopqrstuvwxyz01 = _mm512_mullo_epi32(vksummnopqrstuvwxyz01, _mm512_set1_epi32((int) quantization_params->zero_point));
439 __m512i vacc2x0123456789ABCDEF = _mm512_mullo_epi32(vksum0123456789ABCDEF, _mm512_set1_epi32((int) quantization_params->zero_point));
440 __m512i vacc2xGHIJKLMNOPQRSTUV = _mm512_mullo_epi32(vksumGHIJKLMNOPQRSTUV, _mm512_set1_epi32((int) quantization_params->zero_point));
441 __m512i vacc2xWXYZabcdefghijkl = _mm512_mullo_epi32(vksumWXYZabcdefghijkl, _mm512_set1_epi32((int) quantization_params->zero_point));
442 __m512i vacc2xmnopqrstuvwxyz01 = _mm512_mullo_epi32(vksummnopqrstuvwxyz01, _mm512_set1_epi32((int) quantization_params->zero_point));
443 __m512i vacc3x0123456789ABCDEF = _mm512_mullo_epi32(vksum0123456789ABCDEF, _mm512_set1_epi32((int) quantization_params->zero_point));
444 __m512i vacc3xGHIJKLMNOPQRSTUV = _mm512_mullo_epi32(vksumGHIJKLMNOPQRSTUV, _mm512_set1_epi32((int) quantization_params->zero_point));
445 __m512i vacc3xWXYZabcdefghijkl = _mm512_mullo_epi32(vksumWXYZabcdefghijkl, _mm512_set1_epi32((int) quantization_params->zero_point));
446 __m512i vacc3xmnopqrstuvwxyz01 = _mm512_mullo_epi32(vksummnopqrstuvwxyz01, _mm512_set1_epi32((int) quantization_params->zero_point));
447 __m512i vacc4x0123456789ABCDEF = _mm512_mullo_epi32(vksum0123456789ABCDEF, _mm512_set1_epi32((int) quantization_params->zero_point));
448 __m512i vacc4xGHIJKLMNOPQRSTUV = _mm512_mullo_epi32(vksumGHIJKLMNOPQRSTUV, _mm512_set1_epi32((int) quantization_params->zero_point));
449 __m512i vacc4xWXYZabcdefghijkl = _mm512_mullo_epi32(vksumWXYZabcdefghijkl, _mm512_set1_epi32((int) quantization_params->zero_point));
450 __m512i vacc4xmnopqrstuvwxyz01 = _mm512_mullo_epi32(vksummnopqrstuvwxyz01, _mm512_set1_epi32((int) quantization_params->zero_point));
451 __m512i vacc5x0123456789ABCDEF = _mm512_mullo_epi32(vksum0123456789ABCDEF, _mm512_set1_epi32((int) quantization_params->zero_point));
452 __m512i vacc5xGHIJKLMNOPQRSTUV = _mm512_mullo_epi32(vksumGHIJKLMNOPQRSTUV, _mm512_set1_epi32((int) quantization_params->zero_point));
453 __m512i vacc5xWXYZabcdefghijkl = _mm512_mullo_epi32(vksumWXYZabcdefghijkl, _mm512_set1_epi32((int) quantization_params->zero_point));
454 __m512i vacc5xmnopqrstuvwxyz01 = _mm512_mullo_epi32(vksummnopqrstuvwxyz01, _mm512_set1_epi32((int) quantization_params->zero_point));
455 __m512i vacc6x0123456789ABCDEF = _mm512_mullo_epi32(vksum0123456789ABCDEF, _mm512_set1_epi32((int) quantization_params->zero_point));
456 __m512i vacc6xGHIJKLMNOPQRSTUV = _mm512_mullo_epi32(vksumGHIJKLMNOPQRSTUV, _mm512_set1_epi32((int) quantization_params->zero_point));
457 __m512i vacc6xWXYZabcdefghijkl = _mm512_mullo_epi32(vksumWXYZabcdefghijkl, _mm512_set1_epi32((int) quantization_params->zero_point));
458 __m512i vacc6xmnopqrstuvwxyz01 = _mm512_mullo_epi32(vksummnopqrstuvwxyz01, _mm512_set1_epi32((int) quantization_params->zero_point));
459 __m512i vacc7x0123456789ABCDEF = _mm512_mullo_epi32(vksum0123456789ABCDEF, _mm512_set1_epi32((int) quantization_params->zero_point));
460 __m512i vacc7xGHIJKLMNOPQRSTUV = _mm512_mullo_epi32(vksumGHIJKLMNOPQRSTUV, _mm512_set1_epi32((int) quantization_params->zero_point));
461 __m512i vacc7xWXYZabcdefghijkl = _mm512_mullo_epi32(vksumWXYZabcdefghijkl, _mm512_set1_epi32((int) quantization_params->zero_point));
462 __m512i vacc7xmnopqrstuvwxyz01 = _mm512_mullo_epi32(vksummnopqrstuvwxyz01, _mm512_set1_epi32((int) quantization_params->zero_point));
463 __m512i vacc8x0123456789ABCDEF = _mm512_mullo_epi32(vksum0123456789ABCDEF, _mm512_set1_epi32((int) quantization_params->zero_point));
464 __m512i vacc8xGHIJKLMNOPQRSTUV = _mm512_mullo_epi32(vksumGHIJKLMNOPQRSTUV, _mm512_set1_epi32((int) quantization_params->zero_point));
465 __m512i vacc8xWXYZabcdefghijkl = _mm512_mullo_epi32(vksumWXYZabcdefghijkl, _mm512_set1_epi32((int) quantization_params->zero_point));
466 __m512i vacc8xmnopqrstuvwxyz01 = _mm512_mullo_epi32(vksummnopqrstuvwxyz01, _mm512_set1_epi32((int) quantization_params->zero_point));
467 __m512i vacc9x0123456789ABCDEF = _mm512_mullo_epi32(vksum0123456789ABCDEF, _mm512_set1_epi32((int) quantization_params->zero_point));
468 __m512i vacc9xGHIJKLMNOPQRSTUV = _mm512_mullo_epi32(vksumGHIJKLMNOPQRSTUV, _mm512_set1_epi32((int) quantization_params->zero_point));
469 __m512i vacc9xWXYZabcdefghijkl = _mm512_mullo_epi32(vksumWXYZabcdefghijkl, _mm512_set1_epi32((int) quantization_params->zero_point));
470 __m512i vacc9xmnopqrstuvwxyz01 = _mm512_mullo_epi32(vksummnopqrstuvwxyz01, _mm512_set1_epi32((int) quantization_params->zero_point));
471 __m512i vacc10x0123456789ABCDEF = _mm512_mullo_epi32(vksum0123456789ABCDEF, _mm512_set1_epi32((int) quantization_params->zero_point));
472 __m512i vacc10xGHIJKLMNOPQRSTUV = _mm512_mullo_epi32(vksumGHIJKLMNOPQRSTUV, _mm512_set1_epi32((int) quantization_params->zero_point));
473 __m512i vacc10xWXYZabcdefghijkl = _mm512_mullo_epi32(vksumWXYZabcdefghijkl, _mm512_set1_epi32((int) quantization_params->zero_point));
474 __m512i vacc10xmnopqrstuvwxyz01 = _mm512_mullo_epi32(vksummnopqrstuvwxyz01, _mm512_set1_epi32((int) quantization_params->zero_point));
475 __m512i vacc11x0123456789ABCDEF = _mm512_mullo_epi32(vksum0123456789ABCDEF, _mm512_set1_epi32((int) quantization_params->zero_point));
476 __m512i vacc11xGHIJKLMNOPQRSTUV = _mm512_mullo_epi32(vksumGHIJKLMNOPQRSTUV, _mm512_set1_epi32((int) quantization_params->zero_point));
477 __m512i vacc11xWXYZabcdefghijkl = _mm512_mullo_epi32(vksumWXYZabcdefghijkl, _mm512_set1_epi32((int) quantization_params->zero_point));
478 __m512i vacc11xmnopqrstuvwxyz01 = _mm512_mullo_epi32(vksummnopqrstuvwxyz01, _mm512_set1_epi32((int) quantization_params->zero_point));
479 __m512i vacc12x0123456789ABCDEF = _mm512_mullo_epi32(vksum0123456789ABCDEF, _mm512_set1_epi32((int) quantization_params->zero_point));
480 __m512i vacc12xGHIJKLMNOPQRSTUV = _mm512_mullo_epi32(vksumGHIJKLMNOPQRSTUV, _mm512_set1_epi32((int) quantization_params->zero_point));
481 __m512i vacc12xWXYZabcdefghijkl = _mm512_mullo_epi32(vksumWXYZabcdefghijkl, _mm512_set1_epi32((int) quantization_params->zero_point));
482 __m512i vacc12xmnopqrstuvwxyz01 = _mm512_mullo_epi32(vksummnopqrstuvwxyz01, _mm512_set1_epi32((int) quantization_params->zero_point));
483 __m512i vacc13x0123456789ABCDEF = _mm512_mullo_epi32(vksum0123456789ABCDEF, _mm512_set1_epi32((int) quantization_params->zero_point));
484 __m512i vacc13xGHIJKLMNOPQRSTUV = _mm512_mullo_epi32(vksumGHIJKLMNOPQRSTUV, _mm512_set1_epi32((int) quantization_params->zero_point));
485 __m512i vacc13xWXYZabcdefghijkl = _mm512_mullo_epi32(vksumWXYZabcdefghijkl, _mm512_set1_epi32((int) quantization_params->zero_point));
486 __m512i vacc13xmnopqrstuvwxyz01 = _mm512_mullo_epi32(vksummnopqrstuvwxyz01, _mm512_set1_epi32((int) quantization_params->zero_point));
487 __m512i vacc14x0123456789ABCDEF = _mm512_mullo_epi32(vksum0123456789ABCDEF, _mm512_set1_epi32((int) quantization_params->zero_point));
488 __m512i vacc14xGHIJKLMNOPQRSTUV = _mm512_mullo_epi32(vksumGHIJKLMNOPQRSTUV, _mm512_set1_epi32((int) quantization_params->zero_point));
489 __m512i vacc14xWXYZabcdefghijkl = _mm512_mullo_epi32(vksumWXYZabcdefghijkl, _mm512_set1_epi32((int) quantization_params->zero_point));
490 __m512i vacc14xmnopqrstuvwxyz01 = _mm512_mullo_epi32(vksummnopqrstuvwxyz01, _mm512_set1_epi32((int) quantization_params->zero_point));
491 __m512i vacc15x0123456789ABCDEF = _mm512_mullo_epi32(vksum0123456789ABCDEF, _mm512_set1_epi32((int) quantization_params->zero_point));
492 __m512i vacc15xGHIJKLMNOPQRSTUV = _mm512_mullo_epi32(vksumGHIJKLMNOPQRSTUV, _mm512_set1_epi32((int) quantization_params->zero_point));
493 __m512i vacc15xWXYZabcdefghijkl = _mm512_mullo_epi32(vksumWXYZabcdefghijkl, _mm512_set1_epi32((int) quantization_params->zero_point));
494 __m512i vacc15xmnopqrstuvwxyz01 = _mm512_mullo_epi32(vksummnopqrstuvwxyz01, _mm512_set1_epi32((int) quantization_params->zero_point));
495 vacc0x0123456789ABCDEF = _mm512_add_epi32(vacc0x0123456789ABCDEF, _mm512_load_epi32(res0 + 0));
496 vacc0xGHIJKLMNOPQRSTUV = _mm512_add_epi32(vacc0xGHIJKLMNOPQRSTUV, _mm512_load_epi32(res1 + 0));
497 vacc0xWXYZabcdefghijkl = _mm512_add_epi32(vacc0xWXYZabcdefghijkl, _mm512_load_epi32(res2 + 0));
498 vacc0xmnopqrstuvwxyz01 = _mm512_add_epi32(vacc0xmnopqrstuvwxyz01, _mm512_load_epi32(res3 + 0));
499 vacc1x0123456789ABCDEF = _mm512_add_epi32(vacc1x0123456789ABCDEF, _mm512_load_epi32(res0 + 16));
500 vacc1xGHIJKLMNOPQRSTUV = _mm512_add_epi32(vacc1xGHIJKLMNOPQRSTUV, _mm512_load_epi32(res1 + 16));
501 vacc1xWXYZabcdefghijkl = _mm512_add_epi32(vacc1xWXYZabcdefghijkl, _mm512_load_epi32(res2 + 16));
502 vacc1xmnopqrstuvwxyz01 = _mm512_add_epi32(vacc1xmnopqrstuvwxyz01, _mm512_load_epi32(res3 + 16));
503 vacc2x0123456789ABCDEF = _mm512_add_epi32(vacc2x0123456789ABCDEF, _mm512_load_epi32(res0 + 32));
504 vacc2xGHIJKLMNOPQRSTUV = _mm512_add_epi32(vacc2xGHIJKLMNOPQRSTUV, _mm512_load_epi32(res1 + 32));
505 vacc2xWXYZabcdefghijkl = _mm512_add_epi32(vacc2xWXYZabcdefghijkl, _mm512_load_epi32(res2 + 32));
506 vacc2xmnopqrstuvwxyz01 = _mm512_add_epi32(vacc2xmnopqrstuvwxyz01, _mm512_load_epi32(res3 + 32));
507 vacc3x0123456789ABCDEF = _mm512_add_epi32(vacc3x0123456789ABCDEF, _mm512_load_epi32(res0 + 48));
508 vacc3xGHIJKLMNOPQRSTUV = _mm512_add_epi32(vacc3xGHIJKLMNOPQRSTUV, _mm512_load_epi32(res1 + 48));
509 vacc3xWXYZabcdefghijkl = _mm512_add_epi32(vacc3xWXYZabcdefghijkl, _mm512_load_epi32(res2 + 48));
510 vacc3xmnopqrstuvwxyz01 = _mm512_add_epi32(vacc3xmnopqrstuvwxyz01, _mm512_load_epi32(res3 + 48));
511 vacc4x0123456789ABCDEF = _mm512_add_epi32(vacc4x0123456789ABCDEF, _mm512_load_epi32(res0 + 64));
512 vacc4xGHIJKLMNOPQRSTUV = _mm512_add_epi32(vacc4xGHIJKLMNOPQRSTUV, _mm512_load_epi32(res1 + 64));
513 vacc4xWXYZabcdefghijkl = _mm512_add_epi32(vacc4xWXYZabcdefghijkl, _mm512_load_epi32(res2 + 64));
514 vacc4xmnopqrstuvwxyz01 = _mm512_add_epi32(vacc4xmnopqrstuvwxyz01, _mm512_load_epi32(res3 + 64));
515 vacc5x0123456789ABCDEF = _mm512_add_epi32(vacc5x0123456789ABCDEF, _mm512_load_epi32(res0 + 80));
516 vacc5xGHIJKLMNOPQRSTUV = _mm512_add_epi32(vacc5xGHIJKLMNOPQRSTUV, _mm512_load_epi32(res1 + 80));
517 vacc5xWXYZabcdefghijkl = _mm512_add_epi32(vacc5xWXYZabcdefghijkl, _mm512_load_epi32(res2 + 80));
518 vacc5xmnopqrstuvwxyz01 = _mm512_add_epi32(vacc5xmnopqrstuvwxyz01, _mm512_load_epi32(res3 + 80));
519 vacc6x0123456789ABCDEF = _mm512_add_epi32(vacc6x0123456789ABCDEF, _mm512_load_epi32(res0 + 96));
520 vacc6xGHIJKLMNOPQRSTUV = _mm512_add_epi32(vacc6xGHIJKLMNOPQRSTUV, _mm512_load_epi32(res1 + 96));
521 vacc6xWXYZabcdefghijkl = _mm512_add_epi32(vacc6xWXYZabcdefghijkl, _mm512_load_epi32(res2 + 96));
522 vacc6xmnopqrstuvwxyz01 = _mm512_add_epi32(vacc6xmnopqrstuvwxyz01, _mm512_load_epi32(res3 + 96));
523 vacc7x0123456789ABCDEF = _mm512_add_epi32(vacc7x0123456789ABCDEF, _mm512_load_epi32(res0 + 112));
524 vacc7xGHIJKLMNOPQRSTUV = _mm512_add_epi32(vacc7xGHIJKLMNOPQRSTUV, _mm512_load_epi32(res1 + 112));
525 vacc7xWXYZabcdefghijkl = _mm512_add_epi32(vacc7xWXYZabcdefghijkl, _mm512_load_epi32(res2 + 112));
526 vacc7xmnopqrstuvwxyz01 = _mm512_add_epi32(vacc7xmnopqrstuvwxyz01, _mm512_load_epi32(res3 + 112));
527 vacc8x0123456789ABCDEF = _mm512_add_epi32(vacc8x0123456789ABCDEF, _mm512_load_epi32(res0 + 128));
528 vacc8xGHIJKLMNOPQRSTUV = _mm512_add_epi32(vacc8xGHIJKLMNOPQRSTUV, _mm512_load_epi32(res1 + 128));
529 vacc8xWXYZabcdefghijkl = _mm512_add_epi32(vacc8xWXYZabcdefghijkl, _mm512_load_epi32(res2 + 128));
530 vacc8xmnopqrstuvwxyz01 = _mm512_add_epi32(vacc8xmnopqrstuvwxyz01, _mm512_load_epi32(res3 + 128));
531 vacc9x0123456789ABCDEF = _mm512_add_epi32(vacc9x0123456789ABCDEF, _mm512_load_epi32(res0 + 144));
532 vacc9xGHIJKLMNOPQRSTUV = _mm512_add_epi32(vacc9xGHIJKLMNOPQRSTUV, _mm512_load_epi32(res1 + 144));
533 vacc9xWXYZabcdefghijkl = _mm512_add_epi32(vacc9xWXYZabcdefghijkl, _mm512_load_epi32(res2 + 144));
534 vacc9xmnopqrstuvwxyz01 = _mm512_add_epi32(vacc9xmnopqrstuvwxyz01, _mm512_load_epi32(res3 + 144));
535 vacc10x0123456789ABCDEF = _mm512_add_epi32(vacc10x0123456789ABCDEF, _mm512_load_epi32(res0 + 160));
536 vacc10xGHIJKLMNOPQRSTUV = _mm512_add_epi32(vacc10xGHIJKLMNOPQRSTUV, _mm512_load_epi32(res1 + 160));
537 vacc10xWXYZabcdefghijkl = _mm512_add_epi32(vacc10xWXYZabcdefghijkl, _mm512_load_epi32(res2 + 160));
538 vacc10xmnopqrstuvwxyz01 = _mm512_add_epi32(vacc10xmnopqrstuvwxyz01, _mm512_load_epi32(res3 + 160));
539 vacc11x0123456789ABCDEF = _mm512_add_epi32(vacc11x0123456789ABCDEF, _mm512_load_epi32(res0 + 176));
540 vacc11xGHIJKLMNOPQRSTUV = _mm512_add_epi32(vacc11xGHIJKLMNOPQRSTUV, _mm512_load_epi32(res1 + 176));
541 vacc11xWXYZabcdefghijkl = _mm512_add_epi32(vacc11xWXYZabcdefghijkl, _mm512_load_epi32(res2 + 176));
542 vacc11xmnopqrstuvwxyz01 = _mm512_add_epi32(vacc11xmnopqrstuvwxyz01, _mm512_load_epi32(res3 + 176));
543 vacc12x0123456789ABCDEF = _mm512_add_epi32(vacc12x0123456789ABCDEF, _mm512_load_epi32(res0 + 192));
544 vacc12xGHIJKLMNOPQRSTUV = _mm512_add_epi32(vacc12xGHIJKLMNOPQRSTUV, _mm512_load_epi32(res1 + 192));
545 vacc12xWXYZabcdefghijkl = _mm512_add_epi32(vacc12xWXYZabcdefghijkl, _mm512_load_epi32(res2 + 192));
546 vacc12xmnopqrstuvwxyz01 = _mm512_add_epi32(vacc12xmnopqrstuvwxyz01, _mm512_load_epi32(res3 + 192));
547 vacc13x0123456789ABCDEF = _mm512_add_epi32(vacc13x0123456789ABCDEF, _mm512_load_epi32(res0 + 208));
548 vacc13xGHIJKLMNOPQRSTUV = _mm512_add_epi32(vacc13xGHIJKLMNOPQRSTUV, _mm512_load_epi32(res1 + 208));
549 vacc13xWXYZabcdefghijkl = _mm512_add_epi32(vacc13xWXYZabcdefghijkl, _mm512_load_epi32(res2 + 208));
550 vacc13xmnopqrstuvwxyz01 = _mm512_add_epi32(vacc13xmnopqrstuvwxyz01, _mm512_load_epi32(res3 + 208));
551 vacc14x0123456789ABCDEF = _mm512_add_epi32(vacc14x0123456789ABCDEF, _mm512_load_epi32(res0 + 224));
552 vacc14xGHIJKLMNOPQRSTUV = _mm512_add_epi32(vacc14xGHIJKLMNOPQRSTUV, _mm512_load_epi32(res1 + 224));
553 vacc14xWXYZabcdefghijkl = _mm512_add_epi32(vacc14xWXYZabcdefghijkl, _mm512_load_epi32(res2 + 224));
554 vacc14xmnopqrstuvwxyz01 = _mm512_add_epi32(vacc14xmnopqrstuvwxyz01, _mm512_load_epi32(res3 + 224));
555 vacc15x0123456789ABCDEF = _mm512_add_epi32(vacc15x0123456789ABCDEF, _mm512_load_epi32(res0 + 240));
556 vacc15xGHIJKLMNOPQRSTUV = _mm512_add_epi32(vacc15xGHIJKLMNOPQRSTUV, _mm512_load_epi32(res1 + 240));
557 vacc15xWXYZabcdefghijkl = _mm512_add_epi32(vacc15xWXYZabcdefghijkl, _mm512_load_epi32(res2 + 240));
558 vacc15xmnopqrstuvwxyz01 = _mm512_add_epi32(vacc15xmnopqrstuvwxyz01, _mm512_load_epi32(res3 + 240));
559
560 __m512 vscaled0x0123456789ABCDEF = _mm512_cvtepi32_ps(vacc0x0123456789ABCDEF);
561 __m512 vscaled0xGHIJKLMNOPQRSTUV = _mm512_cvtepi32_ps(vacc0xGHIJKLMNOPQRSTUV);
562 __m512 vscaled0xWXYZabcdefghijkl = _mm512_cvtepi32_ps(vacc0xWXYZabcdefghijkl);
563 __m512 vscaled0xmnopqrstuvwxyz01 = _mm512_cvtepi32_ps(vacc0xmnopqrstuvwxyz01);
564 __m512 vscaled1x0123456789ABCDEF = _mm512_cvtepi32_ps(vacc1x0123456789ABCDEF);
565 __m512 vscaled1xGHIJKLMNOPQRSTUV = _mm512_cvtepi32_ps(vacc1xGHIJKLMNOPQRSTUV);
566 __m512 vscaled1xWXYZabcdefghijkl = _mm512_cvtepi32_ps(vacc1xWXYZabcdefghijkl);
567 __m512 vscaled1xmnopqrstuvwxyz01 = _mm512_cvtepi32_ps(vacc1xmnopqrstuvwxyz01);
568 __m512 vscaled2x0123456789ABCDEF = _mm512_cvtepi32_ps(vacc2x0123456789ABCDEF);
569 __m512 vscaled2xGHIJKLMNOPQRSTUV = _mm512_cvtepi32_ps(vacc2xGHIJKLMNOPQRSTUV);
570 __m512 vscaled2xWXYZabcdefghijkl = _mm512_cvtepi32_ps(vacc2xWXYZabcdefghijkl);
571 __m512 vscaled2xmnopqrstuvwxyz01 = _mm512_cvtepi32_ps(vacc2xmnopqrstuvwxyz01);
572 __m512 vscaled3x0123456789ABCDEF = _mm512_cvtepi32_ps(vacc3x0123456789ABCDEF);
573 __m512 vscaled3xGHIJKLMNOPQRSTUV = _mm512_cvtepi32_ps(vacc3xGHIJKLMNOPQRSTUV);
574 __m512 vscaled3xWXYZabcdefghijkl = _mm512_cvtepi32_ps(vacc3xWXYZabcdefghijkl);
575 __m512 vscaled3xmnopqrstuvwxyz01 = _mm512_cvtepi32_ps(vacc3xmnopqrstuvwxyz01);
576 __m512 vscaled4x0123456789ABCDEF = _mm512_cvtepi32_ps(vacc4x0123456789ABCDEF);
577 __m512 vscaled4xGHIJKLMNOPQRSTUV = _mm512_cvtepi32_ps(vacc4xGHIJKLMNOPQRSTUV);
578 __m512 vscaled4xWXYZabcdefghijkl = _mm512_cvtepi32_ps(vacc4xWXYZabcdefghijkl);
579 __m512 vscaled4xmnopqrstuvwxyz01 = _mm512_cvtepi32_ps(vacc4xmnopqrstuvwxyz01);
580 __m512 vscaled5x0123456789ABCDEF = _mm512_cvtepi32_ps(vacc5x0123456789ABCDEF);
581 __m512 vscaled5xGHIJKLMNOPQRSTUV = _mm512_cvtepi32_ps(vacc5xGHIJKLMNOPQRSTUV);
582 __m512 vscaled5xWXYZabcdefghijkl = _mm512_cvtepi32_ps(vacc5xWXYZabcdefghijkl);
583 __m512 vscaled5xmnopqrstuvwxyz01 = _mm512_cvtepi32_ps(vacc5xmnopqrstuvwxyz01);
584 __m512 vscaled6x0123456789ABCDEF = _mm512_cvtepi32_ps(vacc6x0123456789ABCDEF);
585 __m512 vscaled6xGHIJKLMNOPQRSTUV = _mm512_cvtepi32_ps(vacc6xGHIJKLMNOPQRSTUV);
586 __m512 vscaled6xWXYZabcdefghijkl = _mm512_cvtepi32_ps(vacc6xWXYZabcdefghijkl);
587 __m512 vscaled6xmnopqrstuvwxyz01 = _mm512_cvtepi32_ps(vacc6xmnopqrstuvwxyz01);
588 __m512 vscaled7x0123456789ABCDEF = _mm512_cvtepi32_ps(vacc7x0123456789ABCDEF);
589 __m512 vscaled7xGHIJKLMNOPQRSTUV = _mm512_cvtepi32_ps(vacc7xGHIJKLMNOPQRSTUV);
590 __m512 vscaled7xWXYZabcdefghijkl = _mm512_cvtepi32_ps(vacc7xWXYZabcdefghijkl);
591 __m512 vscaled7xmnopqrstuvwxyz01 = _mm512_cvtepi32_ps(vacc7xmnopqrstuvwxyz01);
592 __m512 vscaled8x0123456789ABCDEF = _mm512_cvtepi32_ps(vacc8x0123456789ABCDEF);
593 __m512 vscaled8xGHIJKLMNOPQRSTUV = _mm512_cvtepi32_ps(vacc8xGHIJKLMNOPQRSTUV);
594 __m512 vscaled8xWXYZabcdefghijkl = _mm512_cvtepi32_ps(vacc8xWXYZabcdefghijkl);
595 __m512 vscaled8xmnopqrstuvwxyz01 = _mm512_cvtepi32_ps(vacc8xmnopqrstuvwxyz01);
596 __m512 vscaled9x0123456789ABCDEF = _mm512_cvtepi32_ps(vacc9x0123456789ABCDEF);
597 __m512 vscaled9xGHIJKLMNOPQRSTUV = _mm512_cvtepi32_ps(vacc9xGHIJKLMNOPQRSTUV);
598 __m512 vscaled9xWXYZabcdefghijkl = _mm512_cvtepi32_ps(vacc9xWXYZabcdefghijkl);
599 __m512 vscaled9xmnopqrstuvwxyz01 = _mm512_cvtepi32_ps(vacc9xmnopqrstuvwxyz01);
600 __m512 vscaled10x0123456789ABCDEF = _mm512_cvtepi32_ps(vacc10x0123456789ABCDEF);
601 __m512 vscaled10xGHIJKLMNOPQRSTUV = _mm512_cvtepi32_ps(vacc10xGHIJKLMNOPQRSTUV);
602 __m512 vscaled10xWXYZabcdefghijkl = _mm512_cvtepi32_ps(vacc10xWXYZabcdefghijkl);
603 __m512 vscaled10xmnopqrstuvwxyz01 = _mm512_cvtepi32_ps(vacc10xmnopqrstuvwxyz01);
604 __m512 vscaled11x0123456789ABCDEF = _mm512_cvtepi32_ps(vacc11x0123456789ABCDEF);
605 __m512 vscaled11xGHIJKLMNOPQRSTUV = _mm512_cvtepi32_ps(vacc11xGHIJKLMNOPQRSTUV);
606 __m512 vscaled11xWXYZabcdefghijkl = _mm512_cvtepi32_ps(vacc11xWXYZabcdefghijkl);
607 __m512 vscaled11xmnopqrstuvwxyz01 = _mm512_cvtepi32_ps(vacc11xmnopqrstuvwxyz01);
608 __m512 vscaled12x0123456789ABCDEF = _mm512_cvtepi32_ps(vacc12x0123456789ABCDEF);
609 __m512 vscaled12xGHIJKLMNOPQRSTUV = _mm512_cvtepi32_ps(vacc12xGHIJKLMNOPQRSTUV);
610 __m512 vscaled12xWXYZabcdefghijkl = _mm512_cvtepi32_ps(vacc12xWXYZabcdefghijkl);
611 __m512 vscaled12xmnopqrstuvwxyz01 = _mm512_cvtepi32_ps(vacc12xmnopqrstuvwxyz01);
612 __m512 vscaled13x0123456789ABCDEF = _mm512_cvtepi32_ps(vacc13x0123456789ABCDEF);
613 __m512 vscaled13xGHIJKLMNOPQRSTUV = _mm512_cvtepi32_ps(vacc13xGHIJKLMNOPQRSTUV);
614 __m512 vscaled13xWXYZabcdefghijkl = _mm512_cvtepi32_ps(vacc13xWXYZabcdefghijkl);
615 __m512 vscaled13xmnopqrstuvwxyz01 = _mm512_cvtepi32_ps(vacc13xmnopqrstuvwxyz01);
616 __m512 vscaled14x0123456789ABCDEF = _mm512_cvtepi32_ps(vacc14x0123456789ABCDEF);
617 __m512 vscaled14xGHIJKLMNOPQRSTUV = _mm512_cvtepi32_ps(vacc14xGHIJKLMNOPQRSTUV);
618 __m512 vscaled14xWXYZabcdefghijkl = _mm512_cvtepi32_ps(vacc14xWXYZabcdefghijkl);
619 __m512 vscaled14xmnopqrstuvwxyz01 = _mm512_cvtepi32_ps(vacc14xmnopqrstuvwxyz01);
620 __m512 vscaled15x0123456789ABCDEF = _mm512_cvtepi32_ps(vacc15x0123456789ABCDEF);
621 __m512 vscaled15xGHIJKLMNOPQRSTUV = _mm512_cvtepi32_ps(vacc15xGHIJKLMNOPQRSTUV);
622 __m512 vscaled15xWXYZabcdefghijkl = _mm512_cvtepi32_ps(vacc15xWXYZabcdefghijkl);
623 __m512 vscaled15xmnopqrstuvwxyz01 = _mm512_cvtepi32_ps(vacc15xmnopqrstuvwxyz01);
624
625 vscaled0x0123456789ABCDEF = _mm512_mul_ps(vscaled0x0123456789ABCDEF, _mm512_set1_ps(quantization_params->inv_scale));
626 vscaled0xGHIJKLMNOPQRSTUV = _mm512_mul_ps(vscaled0xGHIJKLMNOPQRSTUV, _mm512_set1_ps(quantization_params->inv_scale));
627 vscaled0xWXYZabcdefghijkl = _mm512_mul_ps(vscaled0xWXYZabcdefghijkl, _mm512_set1_ps(quantization_params->inv_scale));
628 vscaled0xmnopqrstuvwxyz01 = _mm512_mul_ps(vscaled0xmnopqrstuvwxyz01, _mm512_set1_ps(quantization_params->inv_scale));
629 vscaled1x0123456789ABCDEF = _mm512_mul_ps(vscaled1x0123456789ABCDEF, _mm512_set1_ps(quantization_params->inv_scale));
630 vscaled1xGHIJKLMNOPQRSTUV = _mm512_mul_ps(vscaled1xGHIJKLMNOPQRSTUV, _mm512_set1_ps(quantization_params->inv_scale));
631 vscaled1xWXYZabcdefghijkl = _mm512_mul_ps(vscaled1xWXYZabcdefghijkl, _mm512_set1_ps(quantization_params->inv_scale));
632 vscaled1xmnopqrstuvwxyz01 = _mm512_mul_ps(vscaled1xmnopqrstuvwxyz01, _mm512_set1_ps(quantization_params->inv_scale));
633 vscaled2x0123456789ABCDEF = _mm512_mul_ps(vscaled2x0123456789ABCDEF, _mm512_set1_ps(quantization_params->inv_scale));
634 vscaled2xGHIJKLMNOPQRSTUV = _mm512_mul_ps(vscaled2xGHIJKLMNOPQRSTUV, _mm512_set1_ps(quantization_params->inv_scale));
635 vscaled2xWXYZabcdefghijkl = _mm512_mul_ps(vscaled2xWXYZabcdefghijkl, _mm512_set1_ps(quantization_params->inv_scale));
636 vscaled2xmnopqrstuvwxyz01 = _mm512_mul_ps(vscaled2xmnopqrstuvwxyz01, _mm512_set1_ps(quantization_params->inv_scale));
637 vscaled3x0123456789ABCDEF = _mm512_mul_ps(vscaled3x0123456789ABCDEF, _mm512_set1_ps(quantization_params->inv_scale));
638 vscaled3xGHIJKLMNOPQRSTUV = _mm512_mul_ps(vscaled3xGHIJKLMNOPQRSTUV, _mm512_set1_ps(quantization_params->inv_scale));
639 vscaled3xWXYZabcdefghijkl = _mm512_mul_ps(vscaled3xWXYZabcdefghijkl, _mm512_set1_ps(quantization_params->inv_scale));
640 vscaled3xmnopqrstuvwxyz01 = _mm512_mul_ps(vscaled3xmnopqrstuvwxyz01, _mm512_set1_ps(quantization_params->inv_scale));
641 vscaled4x0123456789ABCDEF = _mm512_mul_ps(vscaled4x0123456789ABCDEF, _mm512_set1_ps(quantization_params->inv_scale));
642 vscaled4xGHIJKLMNOPQRSTUV = _mm512_mul_ps(vscaled4xGHIJKLMNOPQRSTUV, _mm512_set1_ps(quantization_params->inv_scale));
643 vscaled4xWXYZabcdefghijkl = _mm512_mul_ps(vscaled4xWXYZabcdefghijkl, _mm512_set1_ps(quantization_params->inv_scale));
644 vscaled4xmnopqrstuvwxyz01 = _mm512_mul_ps(vscaled4xmnopqrstuvwxyz01, _mm512_set1_ps(quantization_params->inv_scale));
645 vscaled5x0123456789ABCDEF = _mm512_mul_ps(vscaled5x0123456789ABCDEF, _mm512_set1_ps(quantization_params->inv_scale));
646 vscaled5xGHIJKLMNOPQRSTUV = _mm512_mul_ps(vscaled5xGHIJKLMNOPQRSTUV, _mm512_set1_ps(quantization_params->inv_scale));
647 vscaled5xWXYZabcdefghijkl = _mm512_mul_ps(vscaled5xWXYZabcdefghijkl, _mm512_set1_ps(quantization_params->inv_scale));
648 vscaled5xmnopqrstuvwxyz01 = _mm512_mul_ps(vscaled5xmnopqrstuvwxyz01, _mm512_set1_ps(quantization_params->inv_scale));
649 vscaled6x0123456789ABCDEF = _mm512_mul_ps(vscaled6x0123456789ABCDEF, _mm512_set1_ps(quantization_params->inv_scale));
650 vscaled6xGHIJKLMNOPQRSTUV = _mm512_mul_ps(vscaled6xGHIJKLMNOPQRSTUV, _mm512_set1_ps(quantization_params->inv_scale));
651 vscaled6xWXYZabcdefghijkl = _mm512_mul_ps(vscaled6xWXYZabcdefghijkl, _mm512_set1_ps(quantization_params->inv_scale));
652 vscaled6xmnopqrstuvwxyz01 = _mm512_mul_ps(vscaled6xmnopqrstuvwxyz01, _mm512_set1_ps(quantization_params->inv_scale));
653 vscaled7x0123456789ABCDEF = _mm512_mul_ps(vscaled7x0123456789ABCDEF, _mm512_set1_ps(quantization_params->inv_scale));
654 vscaled7xGHIJKLMNOPQRSTUV = _mm512_mul_ps(vscaled7xGHIJKLMNOPQRSTUV, _mm512_set1_ps(quantization_params->inv_scale));
655 vscaled7xWXYZabcdefghijkl = _mm512_mul_ps(vscaled7xWXYZabcdefghijkl, _mm512_set1_ps(quantization_params->inv_scale));
656 vscaled7xmnopqrstuvwxyz01 = _mm512_mul_ps(vscaled7xmnopqrstuvwxyz01, _mm512_set1_ps(quantization_params->inv_scale));
657 vscaled8x0123456789ABCDEF = _mm512_mul_ps(vscaled8x0123456789ABCDEF, _mm512_set1_ps(quantization_params->inv_scale));
658 vscaled8xGHIJKLMNOPQRSTUV = _mm512_mul_ps(vscaled8xGHIJKLMNOPQRSTUV, _mm512_set1_ps(quantization_params->inv_scale));
659 vscaled8xWXYZabcdefghijkl = _mm512_mul_ps(vscaled8xWXYZabcdefghijkl, _mm512_set1_ps(quantization_params->inv_scale));
660 vscaled8xmnopqrstuvwxyz01 = _mm512_mul_ps(vscaled8xmnopqrstuvwxyz01, _mm512_set1_ps(quantization_params->inv_scale));
661 vscaled9x0123456789ABCDEF = _mm512_mul_ps(vscaled9x0123456789ABCDEF, _mm512_set1_ps(quantization_params->inv_scale));
662 vscaled9xGHIJKLMNOPQRSTUV = _mm512_mul_ps(vscaled9xGHIJKLMNOPQRSTUV, _mm512_set1_ps(quantization_params->inv_scale));
663 vscaled9xWXYZabcdefghijkl = _mm512_mul_ps(vscaled9xWXYZabcdefghijkl, _mm512_set1_ps(quantization_params->inv_scale));
664 vscaled9xmnopqrstuvwxyz01 = _mm512_mul_ps(vscaled9xmnopqrstuvwxyz01, _mm512_set1_ps(quantization_params->inv_scale));
665 vscaled10x0123456789ABCDEF = _mm512_mul_ps(vscaled10x0123456789ABCDEF, _mm512_set1_ps(quantization_params->inv_scale));
666 vscaled10xGHIJKLMNOPQRSTUV = _mm512_mul_ps(vscaled10xGHIJKLMNOPQRSTUV, _mm512_set1_ps(quantization_params->inv_scale));
667 vscaled10xWXYZabcdefghijkl = _mm512_mul_ps(vscaled10xWXYZabcdefghijkl, _mm512_set1_ps(quantization_params->inv_scale));
668 vscaled10xmnopqrstuvwxyz01 = _mm512_mul_ps(vscaled10xmnopqrstuvwxyz01, _mm512_set1_ps(quantization_params->inv_scale));
669 vscaled11x0123456789ABCDEF = _mm512_mul_ps(vscaled11x0123456789ABCDEF, _mm512_set1_ps(quantization_params->inv_scale));
670 vscaled11xGHIJKLMNOPQRSTUV = _mm512_mul_ps(vscaled11xGHIJKLMNOPQRSTUV, _mm512_set1_ps(quantization_params->inv_scale));
671 vscaled11xWXYZabcdefghijkl = _mm512_mul_ps(vscaled11xWXYZabcdefghijkl, _mm512_set1_ps(quantization_params->inv_scale));
672 vscaled11xmnopqrstuvwxyz01 = _mm512_mul_ps(vscaled11xmnopqrstuvwxyz01, _mm512_set1_ps(quantization_params->inv_scale));
673 vscaled12x0123456789ABCDEF = _mm512_mul_ps(vscaled12x0123456789ABCDEF, _mm512_set1_ps(quantization_params->inv_scale));
674 vscaled12xGHIJKLMNOPQRSTUV = _mm512_mul_ps(vscaled12xGHIJKLMNOPQRSTUV, _mm512_set1_ps(quantization_params->inv_scale));
675 vscaled12xWXYZabcdefghijkl = _mm512_mul_ps(vscaled12xWXYZabcdefghijkl, _mm512_set1_ps(quantization_params->inv_scale));
676 vscaled12xmnopqrstuvwxyz01 = _mm512_mul_ps(vscaled12xmnopqrstuvwxyz01, _mm512_set1_ps(quantization_params->inv_scale));
677 vscaled13x0123456789ABCDEF = _mm512_mul_ps(vscaled13x0123456789ABCDEF, _mm512_set1_ps(quantization_params->inv_scale));
678 vscaled13xGHIJKLMNOPQRSTUV = _mm512_mul_ps(vscaled13xGHIJKLMNOPQRSTUV, _mm512_set1_ps(quantization_params->inv_scale));
679 vscaled13xWXYZabcdefghijkl = _mm512_mul_ps(vscaled13xWXYZabcdefghijkl, _mm512_set1_ps(quantization_params->inv_scale));
680 vscaled13xmnopqrstuvwxyz01 = _mm512_mul_ps(vscaled13xmnopqrstuvwxyz01, _mm512_set1_ps(quantization_params->inv_scale));
681 vscaled14x0123456789ABCDEF = _mm512_mul_ps(vscaled14x0123456789ABCDEF, _mm512_set1_ps(quantization_params->inv_scale));
682 vscaled14xGHIJKLMNOPQRSTUV = _mm512_mul_ps(vscaled14xGHIJKLMNOPQRSTUV, _mm512_set1_ps(quantization_params->inv_scale));
683 vscaled14xWXYZabcdefghijkl = _mm512_mul_ps(vscaled14xWXYZabcdefghijkl, _mm512_set1_ps(quantization_params->inv_scale));
684 vscaled14xmnopqrstuvwxyz01 = _mm512_mul_ps(vscaled14xmnopqrstuvwxyz01, _mm512_set1_ps(quantization_params->inv_scale));
685 vscaled15x0123456789ABCDEF = _mm512_mul_ps(vscaled15x0123456789ABCDEF, _mm512_set1_ps(quantization_params->inv_scale));
686 vscaled15xGHIJKLMNOPQRSTUV = _mm512_mul_ps(vscaled15xGHIJKLMNOPQRSTUV, _mm512_set1_ps(quantization_params->inv_scale));
687 vscaled15xWXYZabcdefghijkl = _mm512_mul_ps(vscaled15xWXYZabcdefghijkl, _mm512_set1_ps(quantization_params->inv_scale));
688 vscaled15xmnopqrstuvwxyz01 = _mm512_mul_ps(vscaled15xmnopqrstuvwxyz01, _mm512_set1_ps(quantization_params->inv_scale));
689
690 const __m512 vfilter_output_scale0123456789ABCDEF = _mm512_loadu_ps((const float*) w + 0);
691 const __m512 vfilter_output_scaleGHIJKLMNOPQRSTUV = _mm512_loadu_ps((const float*) w + 16);
692 const __m512 vfilter_output_scaleWXYZabcdefghijkl = _mm512_loadu_ps((const float*) w + 32);
693 const __m512 vfilter_output_scalemnopqrstuvwxyz01 = _mm512_loadu_ps((const float*) w + 48);
694 w = (const int32_t*) w + 64;
695 const __m512 vbias0123456789ABCDEF = _mm512_loadu_ps((const float*) w + 0);
696 const __m512 vbiasGHIJKLMNOPQRSTUV = _mm512_loadu_ps((const float*) w + 16);
697 const __m512 vbiasWXYZabcdefghijkl = _mm512_loadu_ps((const float*) w + 32);
698 const __m512 vbiasmnopqrstuvwxyz01 = _mm512_loadu_ps((const float*) w + 48);
699 w = (const int32_t*) w + 64;
700
701 vscaled0x0123456789ABCDEF = _mm512_fmadd_ps(vscaled0x0123456789ABCDEF, vfilter_output_scale0123456789ABCDEF, vbias0123456789ABCDEF);
702 vscaled0xGHIJKLMNOPQRSTUV = _mm512_fmadd_ps(vscaled0xGHIJKLMNOPQRSTUV, vfilter_output_scaleGHIJKLMNOPQRSTUV, vbiasGHIJKLMNOPQRSTUV);
703 vscaled0xWXYZabcdefghijkl = _mm512_fmadd_ps(vscaled0xWXYZabcdefghijkl, vfilter_output_scaleWXYZabcdefghijkl, vbiasWXYZabcdefghijkl);
704 vscaled0xmnopqrstuvwxyz01 = _mm512_fmadd_ps(vscaled0xmnopqrstuvwxyz01, vfilter_output_scalemnopqrstuvwxyz01, vbiasmnopqrstuvwxyz01);
705 vscaled1x0123456789ABCDEF = _mm512_fmadd_ps(vscaled1x0123456789ABCDEF, vfilter_output_scale0123456789ABCDEF, vbias0123456789ABCDEF);
706 vscaled1xGHIJKLMNOPQRSTUV = _mm512_fmadd_ps(vscaled1xGHIJKLMNOPQRSTUV, vfilter_output_scaleGHIJKLMNOPQRSTUV, vbiasGHIJKLMNOPQRSTUV);
707 vscaled1xWXYZabcdefghijkl = _mm512_fmadd_ps(vscaled1xWXYZabcdefghijkl, vfilter_output_scaleWXYZabcdefghijkl, vbiasWXYZabcdefghijkl);
708 vscaled1xmnopqrstuvwxyz01 = _mm512_fmadd_ps(vscaled1xmnopqrstuvwxyz01, vfilter_output_scalemnopqrstuvwxyz01, vbiasmnopqrstuvwxyz01);
709 vscaled2x0123456789ABCDEF = _mm512_fmadd_ps(vscaled2x0123456789ABCDEF, vfilter_output_scale0123456789ABCDEF, vbias0123456789ABCDEF);
710 vscaled2xGHIJKLMNOPQRSTUV = _mm512_fmadd_ps(vscaled2xGHIJKLMNOPQRSTUV, vfilter_output_scaleGHIJKLMNOPQRSTUV, vbiasGHIJKLMNOPQRSTUV);
711 vscaled2xWXYZabcdefghijkl = _mm512_fmadd_ps(vscaled2xWXYZabcdefghijkl, vfilter_output_scaleWXYZabcdefghijkl, vbiasWXYZabcdefghijkl);
712 vscaled2xmnopqrstuvwxyz01 = _mm512_fmadd_ps(vscaled2xmnopqrstuvwxyz01, vfilter_output_scalemnopqrstuvwxyz01, vbiasmnopqrstuvwxyz01);
713 vscaled3x0123456789ABCDEF = _mm512_fmadd_ps(vscaled3x0123456789ABCDEF, vfilter_output_scale0123456789ABCDEF, vbias0123456789ABCDEF);
714 vscaled3xGHIJKLMNOPQRSTUV = _mm512_fmadd_ps(vscaled3xGHIJKLMNOPQRSTUV, vfilter_output_scaleGHIJKLMNOPQRSTUV, vbiasGHIJKLMNOPQRSTUV);
715 vscaled3xWXYZabcdefghijkl = _mm512_fmadd_ps(vscaled3xWXYZabcdefghijkl, vfilter_output_scaleWXYZabcdefghijkl, vbiasWXYZabcdefghijkl);
716 vscaled3xmnopqrstuvwxyz01 = _mm512_fmadd_ps(vscaled3xmnopqrstuvwxyz01, vfilter_output_scalemnopqrstuvwxyz01, vbiasmnopqrstuvwxyz01);
717 vscaled4x0123456789ABCDEF = _mm512_fmadd_ps(vscaled4x0123456789ABCDEF, vfilter_output_scale0123456789ABCDEF, vbias0123456789ABCDEF);
718 vscaled4xGHIJKLMNOPQRSTUV = _mm512_fmadd_ps(vscaled4xGHIJKLMNOPQRSTUV, vfilter_output_scaleGHIJKLMNOPQRSTUV, vbiasGHIJKLMNOPQRSTUV);
719 vscaled4xWXYZabcdefghijkl = _mm512_fmadd_ps(vscaled4xWXYZabcdefghijkl, vfilter_output_scaleWXYZabcdefghijkl, vbiasWXYZabcdefghijkl);
720 vscaled4xmnopqrstuvwxyz01 = _mm512_fmadd_ps(vscaled4xmnopqrstuvwxyz01, vfilter_output_scalemnopqrstuvwxyz01, vbiasmnopqrstuvwxyz01);
721 vscaled5x0123456789ABCDEF = _mm512_fmadd_ps(vscaled5x0123456789ABCDEF, vfilter_output_scale0123456789ABCDEF, vbias0123456789ABCDEF);
722 vscaled5xGHIJKLMNOPQRSTUV = _mm512_fmadd_ps(vscaled5xGHIJKLMNOPQRSTUV, vfilter_output_scaleGHIJKLMNOPQRSTUV, vbiasGHIJKLMNOPQRSTUV);
723 vscaled5xWXYZabcdefghijkl = _mm512_fmadd_ps(vscaled5xWXYZabcdefghijkl, vfilter_output_scaleWXYZabcdefghijkl, vbiasWXYZabcdefghijkl);
724 vscaled5xmnopqrstuvwxyz01 = _mm512_fmadd_ps(vscaled5xmnopqrstuvwxyz01, vfilter_output_scalemnopqrstuvwxyz01, vbiasmnopqrstuvwxyz01);
725 vscaled6x0123456789ABCDEF = _mm512_fmadd_ps(vscaled6x0123456789ABCDEF, vfilter_output_scale0123456789ABCDEF, vbias0123456789ABCDEF);
726 vscaled6xGHIJKLMNOPQRSTUV = _mm512_fmadd_ps(vscaled6xGHIJKLMNOPQRSTUV, vfilter_output_scaleGHIJKLMNOPQRSTUV, vbiasGHIJKLMNOPQRSTUV);
727 vscaled6xWXYZabcdefghijkl = _mm512_fmadd_ps(vscaled6xWXYZabcdefghijkl, vfilter_output_scaleWXYZabcdefghijkl, vbiasWXYZabcdefghijkl);
728 vscaled6xmnopqrstuvwxyz01 = _mm512_fmadd_ps(vscaled6xmnopqrstuvwxyz01, vfilter_output_scalemnopqrstuvwxyz01, vbiasmnopqrstuvwxyz01);
729 vscaled7x0123456789ABCDEF = _mm512_fmadd_ps(vscaled7x0123456789ABCDEF, vfilter_output_scale0123456789ABCDEF, vbias0123456789ABCDEF);
730 vscaled7xGHIJKLMNOPQRSTUV = _mm512_fmadd_ps(vscaled7xGHIJKLMNOPQRSTUV, vfilter_output_scaleGHIJKLMNOPQRSTUV, vbiasGHIJKLMNOPQRSTUV);
731 vscaled7xWXYZabcdefghijkl = _mm512_fmadd_ps(vscaled7xWXYZabcdefghijkl, vfilter_output_scaleWXYZabcdefghijkl, vbiasWXYZabcdefghijkl);
732 vscaled7xmnopqrstuvwxyz01 = _mm512_fmadd_ps(vscaled7xmnopqrstuvwxyz01, vfilter_output_scalemnopqrstuvwxyz01, vbiasmnopqrstuvwxyz01);
733 vscaled8x0123456789ABCDEF = _mm512_fmadd_ps(vscaled8x0123456789ABCDEF, vfilter_output_scale0123456789ABCDEF, vbias0123456789ABCDEF);
734 vscaled8xGHIJKLMNOPQRSTUV = _mm512_fmadd_ps(vscaled8xGHIJKLMNOPQRSTUV, vfilter_output_scaleGHIJKLMNOPQRSTUV, vbiasGHIJKLMNOPQRSTUV);
735 vscaled8xWXYZabcdefghijkl = _mm512_fmadd_ps(vscaled8xWXYZabcdefghijkl, vfilter_output_scaleWXYZabcdefghijkl, vbiasWXYZabcdefghijkl);
736 vscaled8xmnopqrstuvwxyz01 = _mm512_fmadd_ps(vscaled8xmnopqrstuvwxyz01, vfilter_output_scalemnopqrstuvwxyz01, vbiasmnopqrstuvwxyz01);
737 vscaled9x0123456789ABCDEF = _mm512_fmadd_ps(vscaled9x0123456789ABCDEF, vfilter_output_scale0123456789ABCDEF, vbias0123456789ABCDEF);
738 vscaled9xGHIJKLMNOPQRSTUV = _mm512_fmadd_ps(vscaled9xGHIJKLMNOPQRSTUV, vfilter_output_scaleGHIJKLMNOPQRSTUV, vbiasGHIJKLMNOPQRSTUV);
739 vscaled9xWXYZabcdefghijkl = _mm512_fmadd_ps(vscaled9xWXYZabcdefghijkl, vfilter_output_scaleWXYZabcdefghijkl, vbiasWXYZabcdefghijkl);
740 vscaled9xmnopqrstuvwxyz01 = _mm512_fmadd_ps(vscaled9xmnopqrstuvwxyz01, vfilter_output_scalemnopqrstuvwxyz01, vbiasmnopqrstuvwxyz01);
741 vscaled10x0123456789ABCDEF = _mm512_fmadd_ps(vscaled10x0123456789ABCDEF, vfilter_output_scale0123456789ABCDEF, vbias0123456789ABCDEF);
742 vscaled10xGHIJKLMNOPQRSTUV = _mm512_fmadd_ps(vscaled10xGHIJKLMNOPQRSTUV, vfilter_output_scaleGHIJKLMNOPQRSTUV, vbiasGHIJKLMNOPQRSTUV);
743 vscaled10xWXYZabcdefghijkl = _mm512_fmadd_ps(vscaled10xWXYZabcdefghijkl, vfilter_output_scaleWXYZabcdefghijkl, vbiasWXYZabcdefghijkl);
744 vscaled10xmnopqrstuvwxyz01 = _mm512_fmadd_ps(vscaled10xmnopqrstuvwxyz01, vfilter_output_scalemnopqrstuvwxyz01, vbiasmnopqrstuvwxyz01);
745 vscaled11x0123456789ABCDEF = _mm512_fmadd_ps(vscaled11x0123456789ABCDEF, vfilter_output_scale0123456789ABCDEF, vbias0123456789ABCDEF);
746 vscaled11xGHIJKLMNOPQRSTUV = _mm512_fmadd_ps(vscaled11xGHIJKLMNOPQRSTUV, vfilter_output_scaleGHIJKLMNOPQRSTUV, vbiasGHIJKLMNOPQRSTUV);
747 vscaled11xWXYZabcdefghijkl = _mm512_fmadd_ps(vscaled11xWXYZabcdefghijkl, vfilter_output_scaleWXYZabcdefghijkl, vbiasWXYZabcdefghijkl);
748 vscaled11xmnopqrstuvwxyz01 = _mm512_fmadd_ps(vscaled11xmnopqrstuvwxyz01, vfilter_output_scalemnopqrstuvwxyz01, vbiasmnopqrstuvwxyz01);
749 vscaled12x0123456789ABCDEF = _mm512_fmadd_ps(vscaled12x0123456789ABCDEF, vfilter_output_scale0123456789ABCDEF, vbias0123456789ABCDEF);
750 vscaled12xGHIJKLMNOPQRSTUV = _mm512_fmadd_ps(vscaled12xGHIJKLMNOPQRSTUV, vfilter_output_scaleGHIJKLMNOPQRSTUV, vbiasGHIJKLMNOPQRSTUV);
751 vscaled12xWXYZabcdefghijkl = _mm512_fmadd_ps(vscaled12xWXYZabcdefghijkl, vfilter_output_scaleWXYZabcdefghijkl, vbiasWXYZabcdefghijkl);
752 vscaled12xmnopqrstuvwxyz01 = _mm512_fmadd_ps(vscaled12xmnopqrstuvwxyz01, vfilter_output_scalemnopqrstuvwxyz01, vbiasmnopqrstuvwxyz01);
753 vscaled13x0123456789ABCDEF = _mm512_fmadd_ps(vscaled13x0123456789ABCDEF, vfilter_output_scale0123456789ABCDEF, vbias0123456789ABCDEF);
754 vscaled13xGHIJKLMNOPQRSTUV = _mm512_fmadd_ps(vscaled13xGHIJKLMNOPQRSTUV, vfilter_output_scaleGHIJKLMNOPQRSTUV, vbiasGHIJKLMNOPQRSTUV);
755 vscaled13xWXYZabcdefghijkl = _mm512_fmadd_ps(vscaled13xWXYZabcdefghijkl, vfilter_output_scaleWXYZabcdefghijkl, vbiasWXYZabcdefghijkl);
756 vscaled13xmnopqrstuvwxyz01 = _mm512_fmadd_ps(vscaled13xmnopqrstuvwxyz01, vfilter_output_scalemnopqrstuvwxyz01, vbiasmnopqrstuvwxyz01);
757 vscaled14x0123456789ABCDEF = _mm512_fmadd_ps(vscaled14x0123456789ABCDEF, vfilter_output_scale0123456789ABCDEF, vbias0123456789ABCDEF);
758 vscaled14xGHIJKLMNOPQRSTUV = _mm512_fmadd_ps(vscaled14xGHIJKLMNOPQRSTUV, vfilter_output_scaleGHIJKLMNOPQRSTUV, vbiasGHIJKLMNOPQRSTUV);
759 vscaled14xWXYZabcdefghijkl = _mm512_fmadd_ps(vscaled14xWXYZabcdefghijkl, vfilter_output_scaleWXYZabcdefghijkl, vbiasWXYZabcdefghijkl);
760 vscaled14xmnopqrstuvwxyz01 = _mm512_fmadd_ps(vscaled14xmnopqrstuvwxyz01, vfilter_output_scalemnopqrstuvwxyz01, vbiasmnopqrstuvwxyz01);
761 vscaled15x0123456789ABCDEF = _mm512_fmadd_ps(vscaled15x0123456789ABCDEF, vfilter_output_scale0123456789ABCDEF, vbias0123456789ABCDEF);
762 vscaled15xGHIJKLMNOPQRSTUV = _mm512_fmadd_ps(vscaled15xGHIJKLMNOPQRSTUV, vfilter_output_scaleGHIJKLMNOPQRSTUV, vbiasGHIJKLMNOPQRSTUV);
763 vscaled15xWXYZabcdefghijkl = _mm512_fmadd_ps(vscaled15xWXYZabcdefghijkl, vfilter_output_scaleWXYZabcdefghijkl, vbiasWXYZabcdefghijkl);
764 vscaled15xmnopqrstuvwxyz01 = _mm512_fmadd_ps(vscaled15xmnopqrstuvwxyz01, vfilter_output_scalemnopqrstuvwxyz01, vbiasmnopqrstuvwxyz01);
765
766 vscaled0x0123456789ABCDEF = _mm512_max_ps(vscaled0x0123456789ABCDEF, voutput_min);
767 vscaled0xGHIJKLMNOPQRSTUV = _mm512_max_ps(vscaled0xGHIJKLMNOPQRSTUV, voutput_min);
768 vscaled0xWXYZabcdefghijkl = _mm512_max_ps(vscaled0xWXYZabcdefghijkl, voutput_min);
769 vscaled0xmnopqrstuvwxyz01 = _mm512_max_ps(vscaled0xmnopqrstuvwxyz01, voutput_min);
770 vscaled1x0123456789ABCDEF = _mm512_max_ps(vscaled1x0123456789ABCDEF, voutput_min);
771 vscaled1xGHIJKLMNOPQRSTUV = _mm512_max_ps(vscaled1xGHIJKLMNOPQRSTUV, voutput_min);
772 vscaled1xWXYZabcdefghijkl = _mm512_max_ps(vscaled1xWXYZabcdefghijkl, voutput_min);
773 vscaled1xmnopqrstuvwxyz01 = _mm512_max_ps(vscaled1xmnopqrstuvwxyz01, voutput_min);
774 vscaled2x0123456789ABCDEF = _mm512_max_ps(vscaled2x0123456789ABCDEF, voutput_min);
775 vscaled2xGHIJKLMNOPQRSTUV = _mm512_max_ps(vscaled2xGHIJKLMNOPQRSTUV, voutput_min);
776 vscaled2xWXYZabcdefghijkl = _mm512_max_ps(vscaled2xWXYZabcdefghijkl, voutput_min);
777 vscaled2xmnopqrstuvwxyz01 = _mm512_max_ps(vscaled2xmnopqrstuvwxyz01, voutput_min);
778 vscaled3x0123456789ABCDEF = _mm512_max_ps(vscaled3x0123456789ABCDEF, voutput_min);
779 vscaled3xGHIJKLMNOPQRSTUV = _mm512_max_ps(vscaled3xGHIJKLMNOPQRSTUV, voutput_min);
780 vscaled3xWXYZabcdefghijkl = _mm512_max_ps(vscaled3xWXYZabcdefghijkl, voutput_min);
781 vscaled3xmnopqrstuvwxyz01 = _mm512_max_ps(vscaled3xmnopqrstuvwxyz01, voutput_min);
782 vscaled4x0123456789ABCDEF = _mm512_max_ps(vscaled4x0123456789ABCDEF, voutput_min);
783 vscaled4xGHIJKLMNOPQRSTUV = _mm512_max_ps(vscaled4xGHIJKLMNOPQRSTUV, voutput_min);
784 vscaled4xWXYZabcdefghijkl = _mm512_max_ps(vscaled4xWXYZabcdefghijkl, voutput_min);
785 vscaled4xmnopqrstuvwxyz01 = _mm512_max_ps(vscaled4xmnopqrstuvwxyz01, voutput_min);
786 vscaled5x0123456789ABCDEF = _mm512_max_ps(vscaled5x0123456789ABCDEF, voutput_min);
787 vscaled5xGHIJKLMNOPQRSTUV = _mm512_max_ps(vscaled5xGHIJKLMNOPQRSTUV, voutput_min);
788 vscaled5xWXYZabcdefghijkl = _mm512_max_ps(vscaled5xWXYZabcdefghijkl, voutput_min);
789 vscaled5xmnopqrstuvwxyz01 = _mm512_max_ps(vscaled5xmnopqrstuvwxyz01, voutput_min);
790 vscaled6x0123456789ABCDEF = _mm512_max_ps(vscaled6x0123456789ABCDEF, voutput_min);
791 vscaled6xGHIJKLMNOPQRSTUV = _mm512_max_ps(vscaled6xGHIJKLMNOPQRSTUV, voutput_min);
792 vscaled6xWXYZabcdefghijkl = _mm512_max_ps(vscaled6xWXYZabcdefghijkl, voutput_min);
793 vscaled6xmnopqrstuvwxyz01 = _mm512_max_ps(vscaled6xmnopqrstuvwxyz01, voutput_min);
794 vscaled7x0123456789ABCDEF = _mm512_max_ps(vscaled7x0123456789ABCDEF, voutput_min);
795 vscaled7xGHIJKLMNOPQRSTUV = _mm512_max_ps(vscaled7xGHIJKLMNOPQRSTUV, voutput_min);
796 vscaled7xWXYZabcdefghijkl = _mm512_max_ps(vscaled7xWXYZabcdefghijkl, voutput_min);
797 vscaled7xmnopqrstuvwxyz01 = _mm512_max_ps(vscaled7xmnopqrstuvwxyz01, voutput_min);
798 vscaled8x0123456789ABCDEF = _mm512_max_ps(vscaled8x0123456789ABCDEF, voutput_min);
799 vscaled8xGHIJKLMNOPQRSTUV = _mm512_max_ps(vscaled8xGHIJKLMNOPQRSTUV, voutput_min);
800 vscaled8xWXYZabcdefghijkl = _mm512_max_ps(vscaled8xWXYZabcdefghijkl, voutput_min);
801 vscaled8xmnopqrstuvwxyz01 = _mm512_max_ps(vscaled8xmnopqrstuvwxyz01, voutput_min);
802 vscaled9x0123456789ABCDEF = _mm512_max_ps(vscaled9x0123456789ABCDEF, voutput_min);
803 vscaled9xGHIJKLMNOPQRSTUV = _mm512_max_ps(vscaled9xGHIJKLMNOPQRSTUV, voutput_min);
804 vscaled9xWXYZabcdefghijkl = _mm512_max_ps(vscaled9xWXYZabcdefghijkl, voutput_min);
805 vscaled9xmnopqrstuvwxyz01 = _mm512_max_ps(vscaled9xmnopqrstuvwxyz01, voutput_min);
806 vscaled10x0123456789ABCDEF = _mm512_max_ps(vscaled10x0123456789ABCDEF, voutput_min);
807 vscaled10xGHIJKLMNOPQRSTUV = _mm512_max_ps(vscaled10xGHIJKLMNOPQRSTUV, voutput_min);
808 vscaled10xWXYZabcdefghijkl = _mm512_max_ps(vscaled10xWXYZabcdefghijkl, voutput_min);
809 vscaled10xmnopqrstuvwxyz01 = _mm512_max_ps(vscaled10xmnopqrstuvwxyz01, voutput_min);
810 vscaled11x0123456789ABCDEF = _mm512_max_ps(vscaled11x0123456789ABCDEF, voutput_min);
811 vscaled11xGHIJKLMNOPQRSTUV = _mm512_max_ps(vscaled11xGHIJKLMNOPQRSTUV, voutput_min);
812 vscaled11xWXYZabcdefghijkl = _mm512_max_ps(vscaled11xWXYZabcdefghijkl, voutput_min);
813 vscaled11xmnopqrstuvwxyz01 = _mm512_max_ps(vscaled11xmnopqrstuvwxyz01, voutput_min);
814 vscaled12x0123456789ABCDEF = _mm512_max_ps(vscaled12x0123456789ABCDEF, voutput_min);
815 vscaled12xGHIJKLMNOPQRSTUV = _mm512_max_ps(vscaled12xGHIJKLMNOPQRSTUV, voutput_min);
816 vscaled12xWXYZabcdefghijkl = _mm512_max_ps(vscaled12xWXYZabcdefghijkl, voutput_min);
817 vscaled12xmnopqrstuvwxyz01 = _mm512_max_ps(vscaled12xmnopqrstuvwxyz01, voutput_min);
818 vscaled13x0123456789ABCDEF = _mm512_max_ps(vscaled13x0123456789ABCDEF, voutput_min);
819 vscaled13xGHIJKLMNOPQRSTUV = _mm512_max_ps(vscaled13xGHIJKLMNOPQRSTUV, voutput_min);
820 vscaled13xWXYZabcdefghijkl = _mm512_max_ps(vscaled13xWXYZabcdefghijkl, voutput_min);
821 vscaled13xmnopqrstuvwxyz01 = _mm512_max_ps(vscaled13xmnopqrstuvwxyz01, voutput_min);
822 vscaled14x0123456789ABCDEF = _mm512_max_ps(vscaled14x0123456789ABCDEF, voutput_min);
823 vscaled14xGHIJKLMNOPQRSTUV = _mm512_max_ps(vscaled14xGHIJKLMNOPQRSTUV, voutput_min);
824 vscaled14xWXYZabcdefghijkl = _mm512_max_ps(vscaled14xWXYZabcdefghijkl, voutput_min);
825 vscaled14xmnopqrstuvwxyz01 = _mm512_max_ps(vscaled14xmnopqrstuvwxyz01, voutput_min);
826 vscaled15x0123456789ABCDEF = _mm512_max_ps(vscaled15x0123456789ABCDEF, voutput_min);
827 vscaled15xGHIJKLMNOPQRSTUV = _mm512_max_ps(vscaled15xGHIJKLMNOPQRSTUV, voutput_min);
828 vscaled15xWXYZabcdefghijkl = _mm512_max_ps(vscaled15xWXYZabcdefghijkl, voutput_min);
829 vscaled15xmnopqrstuvwxyz01 = _mm512_max_ps(vscaled15xmnopqrstuvwxyz01, voutput_min);
830
831 vscaled0x0123456789ABCDEF = _mm512_min_ps(vscaled0x0123456789ABCDEF, voutput_max);
832 vscaled0xGHIJKLMNOPQRSTUV = _mm512_min_ps(vscaled0xGHIJKLMNOPQRSTUV, voutput_max);
833 vscaled0xWXYZabcdefghijkl = _mm512_min_ps(vscaled0xWXYZabcdefghijkl, voutput_max);
834 vscaled0xmnopqrstuvwxyz01 = _mm512_min_ps(vscaled0xmnopqrstuvwxyz01, voutput_max);
835 vscaled1x0123456789ABCDEF = _mm512_min_ps(vscaled1x0123456789ABCDEF, voutput_max);
836 vscaled1xGHIJKLMNOPQRSTUV = _mm512_min_ps(vscaled1xGHIJKLMNOPQRSTUV, voutput_max);
837 vscaled1xWXYZabcdefghijkl = _mm512_min_ps(vscaled1xWXYZabcdefghijkl, voutput_max);
838 vscaled1xmnopqrstuvwxyz01 = _mm512_min_ps(vscaled1xmnopqrstuvwxyz01, voutput_max);
839 vscaled2x0123456789ABCDEF = _mm512_min_ps(vscaled2x0123456789ABCDEF, voutput_max);
840 vscaled2xGHIJKLMNOPQRSTUV = _mm512_min_ps(vscaled2xGHIJKLMNOPQRSTUV, voutput_max);
841 vscaled2xWXYZabcdefghijkl = _mm512_min_ps(vscaled2xWXYZabcdefghijkl, voutput_max);
842 vscaled2xmnopqrstuvwxyz01 = _mm512_min_ps(vscaled2xmnopqrstuvwxyz01, voutput_max);
843 vscaled3x0123456789ABCDEF = _mm512_min_ps(vscaled3x0123456789ABCDEF, voutput_max);
844 vscaled3xGHIJKLMNOPQRSTUV = _mm512_min_ps(vscaled3xGHIJKLMNOPQRSTUV, voutput_max);
845 vscaled3xWXYZabcdefghijkl = _mm512_min_ps(vscaled3xWXYZabcdefghijkl, voutput_max);
846 vscaled3xmnopqrstuvwxyz01 = _mm512_min_ps(vscaled3xmnopqrstuvwxyz01, voutput_max);
847 vscaled4x0123456789ABCDEF = _mm512_min_ps(vscaled4x0123456789ABCDEF, voutput_max);
848 vscaled4xGHIJKLMNOPQRSTUV = _mm512_min_ps(vscaled4xGHIJKLMNOPQRSTUV, voutput_max);
849 vscaled4xWXYZabcdefghijkl = _mm512_min_ps(vscaled4xWXYZabcdefghijkl, voutput_max);
850 vscaled4xmnopqrstuvwxyz01 = _mm512_min_ps(vscaled4xmnopqrstuvwxyz01, voutput_max);
851 vscaled5x0123456789ABCDEF = _mm512_min_ps(vscaled5x0123456789ABCDEF, voutput_max);
852 vscaled5xGHIJKLMNOPQRSTUV = _mm512_min_ps(vscaled5xGHIJKLMNOPQRSTUV, voutput_max);
853 vscaled5xWXYZabcdefghijkl = _mm512_min_ps(vscaled5xWXYZabcdefghijkl, voutput_max);
854 vscaled5xmnopqrstuvwxyz01 = _mm512_min_ps(vscaled5xmnopqrstuvwxyz01, voutput_max);
855 vscaled6x0123456789ABCDEF = _mm512_min_ps(vscaled6x0123456789ABCDEF, voutput_max);
856 vscaled6xGHIJKLMNOPQRSTUV = _mm512_min_ps(vscaled6xGHIJKLMNOPQRSTUV, voutput_max);
857 vscaled6xWXYZabcdefghijkl = _mm512_min_ps(vscaled6xWXYZabcdefghijkl, voutput_max);
858 vscaled6xmnopqrstuvwxyz01 = _mm512_min_ps(vscaled6xmnopqrstuvwxyz01, voutput_max);
859 vscaled7x0123456789ABCDEF = _mm512_min_ps(vscaled7x0123456789ABCDEF, voutput_max);
860 vscaled7xGHIJKLMNOPQRSTUV = _mm512_min_ps(vscaled7xGHIJKLMNOPQRSTUV, voutput_max);
861 vscaled7xWXYZabcdefghijkl = _mm512_min_ps(vscaled7xWXYZabcdefghijkl, voutput_max);
862 vscaled7xmnopqrstuvwxyz01 = _mm512_min_ps(vscaled7xmnopqrstuvwxyz01, voutput_max);
863 vscaled8x0123456789ABCDEF = _mm512_min_ps(vscaled8x0123456789ABCDEF, voutput_max);
864 vscaled8xGHIJKLMNOPQRSTUV = _mm512_min_ps(vscaled8xGHIJKLMNOPQRSTUV, voutput_max);
865 vscaled8xWXYZabcdefghijkl = _mm512_min_ps(vscaled8xWXYZabcdefghijkl, voutput_max);
866 vscaled8xmnopqrstuvwxyz01 = _mm512_min_ps(vscaled8xmnopqrstuvwxyz01, voutput_max);
867 vscaled9x0123456789ABCDEF = _mm512_min_ps(vscaled9x0123456789ABCDEF, voutput_max);
868 vscaled9xGHIJKLMNOPQRSTUV = _mm512_min_ps(vscaled9xGHIJKLMNOPQRSTUV, voutput_max);
869 vscaled9xWXYZabcdefghijkl = _mm512_min_ps(vscaled9xWXYZabcdefghijkl, voutput_max);
870 vscaled9xmnopqrstuvwxyz01 = _mm512_min_ps(vscaled9xmnopqrstuvwxyz01, voutput_max);
871 vscaled10x0123456789ABCDEF = _mm512_min_ps(vscaled10x0123456789ABCDEF, voutput_max);
872 vscaled10xGHIJKLMNOPQRSTUV = _mm512_min_ps(vscaled10xGHIJKLMNOPQRSTUV, voutput_max);
873 vscaled10xWXYZabcdefghijkl = _mm512_min_ps(vscaled10xWXYZabcdefghijkl, voutput_max);
874 vscaled10xmnopqrstuvwxyz01 = _mm512_min_ps(vscaled10xmnopqrstuvwxyz01, voutput_max);
875 vscaled11x0123456789ABCDEF = _mm512_min_ps(vscaled11x0123456789ABCDEF, voutput_max);
876 vscaled11xGHIJKLMNOPQRSTUV = _mm512_min_ps(vscaled11xGHIJKLMNOPQRSTUV, voutput_max);
877 vscaled11xWXYZabcdefghijkl = _mm512_min_ps(vscaled11xWXYZabcdefghijkl, voutput_max);
878 vscaled11xmnopqrstuvwxyz01 = _mm512_min_ps(vscaled11xmnopqrstuvwxyz01, voutput_max);
879 vscaled12x0123456789ABCDEF = _mm512_min_ps(vscaled12x0123456789ABCDEF, voutput_max);
880 vscaled12xGHIJKLMNOPQRSTUV = _mm512_min_ps(vscaled12xGHIJKLMNOPQRSTUV, voutput_max);
881 vscaled12xWXYZabcdefghijkl = _mm512_min_ps(vscaled12xWXYZabcdefghijkl, voutput_max);
882 vscaled12xmnopqrstuvwxyz01 = _mm512_min_ps(vscaled12xmnopqrstuvwxyz01, voutput_max);
883 vscaled13x0123456789ABCDEF = _mm512_min_ps(vscaled13x0123456789ABCDEF, voutput_max);
884 vscaled13xGHIJKLMNOPQRSTUV = _mm512_min_ps(vscaled13xGHIJKLMNOPQRSTUV, voutput_max);
885 vscaled13xWXYZabcdefghijkl = _mm512_min_ps(vscaled13xWXYZabcdefghijkl, voutput_max);
886 vscaled13xmnopqrstuvwxyz01 = _mm512_min_ps(vscaled13xmnopqrstuvwxyz01, voutput_max);
887 vscaled14x0123456789ABCDEF = _mm512_min_ps(vscaled14x0123456789ABCDEF, voutput_max);
888 vscaled14xGHIJKLMNOPQRSTUV = _mm512_min_ps(vscaled14xGHIJKLMNOPQRSTUV, voutput_max);
889 vscaled14xWXYZabcdefghijkl = _mm512_min_ps(vscaled14xWXYZabcdefghijkl, voutput_max);
890 vscaled14xmnopqrstuvwxyz01 = _mm512_min_ps(vscaled14xmnopqrstuvwxyz01, voutput_max);
891 vscaled15x0123456789ABCDEF = _mm512_min_ps(vscaled15x0123456789ABCDEF, voutput_max);
892 vscaled15xGHIJKLMNOPQRSTUV = _mm512_min_ps(vscaled15xGHIJKLMNOPQRSTUV, voutput_max);
893 vscaled15xWXYZabcdefghijkl = _mm512_min_ps(vscaled15xWXYZabcdefghijkl, voutput_max);
894 vscaled15xmnopqrstuvwxyz01 = _mm512_min_ps(vscaled15xmnopqrstuvwxyz01, voutput_max);
895
896 __m256i vfp16out0x0123456789ABCDEF = _mm512_cvtps_ph(vscaled0x0123456789ABCDEF, _MM_FROUND_TO_NEAREST_INT);
897 __m256i vfp16out0xGHIJKLMNOPQRSTUV = _mm512_cvtps_ph(vscaled0xGHIJKLMNOPQRSTUV, _MM_FROUND_TO_NEAREST_INT);
898 __m256i vfp16out0xWXYZabcdefghijkl = _mm512_cvtps_ph(vscaled0xWXYZabcdefghijkl, _MM_FROUND_TO_NEAREST_INT);
899 __m256i vfp16out0xmnopqrstuvwxyz01 = _mm512_cvtps_ph(vscaled0xmnopqrstuvwxyz01, _MM_FROUND_TO_NEAREST_INT);
900 __m256i vfp16out1x0123456789ABCDEF = _mm512_cvtps_ph(vscaled1x0123456789ABCDEF, _MM_FROUND_TO_NEAREST_INT);
901 __m256i vfp16out1xGHIJKLMNOPQRSTUV = _mm512_cvtps_ph(vscaled1xGHIJKLMNOPQRSTUV, _MM_FROUND_TO_NEAREST_INT);
902 __m256i vfp16out1xWXYZabcdefghijkl = _mm512_cvtps_ph(vscaled1xWXYZabcdefghijkl, _MM_FROUND_TO_NEAREST_INT);
903 __m256i vfp16out1xmnopqrstuvwxyz01 = _mm512_cvtps_ph(vscaled1xmnopqrstuvwxyz01, _MM_FROUND_TO_NEAREST_INT);
904 __m256i vfp16out2x0123456789ABCDEF = _mm512_cvtps_ph(vscaled2x0123456789ABCDEF, _MM_FROUND_TO_NEAREST_INT);
905 __m256i vfp16out2xGHIJKLMNOPQRSTUV = _mm512_cvtps_ph(vscaled2xGHIJKLMNOPQRSTUV, _MM_FROUND_TO_NEAREST_INT);
906 __m256i vfp16out2xWXYZabcdefghijkl = _mm512_cvtps_ph(vscaled2xWXYZabcdefghijkl, _MM_FROUND_TO_NEAREST_INT);
907 __m256i vfp16out2xmnopqrstuvwxyz01 = _mm512_cvtps_ph(vscaled2xmnopqrstuvwxyz01, _MM_FROUND_TO_NEAREST_INT);
908 __m256i vfp16out3x0123456789ABCDEF = _mm512_cvtps_ph(vscaled3x0123456789ABCDEF, _MM_FROUND_TO_NEAREST_INT);
909 __m256i vfp16out3xGHIJKLMNOPQRSTUV = _mm512_cvtps_ph(vscaled3xGHIJKLMNOPQRSTUV, _MM_FROUND_TO_NEAREST_INT);
910 __m256i vfp16out3xWXYZabcdefghijkl = _mm512_cvtps_ph(vscaled3xWXYZabcdefghijkl, _MM_FROUND_TO_NEAREST_INT);
911 __m256i vfp16out3xmnopqrstuvwxyz01 = _mm512_cvtps_ph(vscaled3xmnopqrstuvwxyz01, _MM_FROUND_TO_NEAREST_INT);
912 __m256i vfp16out4x0123456789ABCDEF = _mm512_cvtps_ph(vscaled4x0123456789ABCDEF, _MM_FROUND_TO_NEAREST_INT);
913 __m256i vfp16out4xGHIJKLMNOPQRSTUV = _mm512_cvtps_ph(vscaled4xGHIJKLMNOPQRSTUV, _MM_FROUND_TO_NEAREST_INT);
914 __m256i vfp16out4xWXYZabcdefghijkl = _mm512_cvtps_ph(vscaled4xWXYZabcdefghijkl, _MM_FROUND_TO_NEAREST_INT);
915 __m256i vfp16out4xmnopqrstuvwxyz01 = _mm512_cvtps_ph(vscaled4xmnopqrstuvwxyz01, _MM_FROUND_TO_NEAREST_INT);
916 __m256i vfp16out5x0123456789ABCDEF = _mm512_cvtps_ph(vscaled5x0123456789ABCDEF, _MM_FROUND_TO_NEAREST_INT);
917 __m256i vfp16out5xGHIJKLMNOPQRSTUV = _mm512_cvtps_ph(vscaled5xGHIJKLMNOPQRSTUV, _MM_FROUND_TO_NEAREST_INT);
918 __m256i vfp16out5xWXYZabcdefghijkl = _mm512_cvtps_ph(vscaled5xWXYZabcdefghijkl, _MM_FROUND_TO_NEAREST_INT);
919 __m256i vfp16out5xmnopqrstuvwxyz01 = _mm512_cvtps_ph(vscaled5xmnopqrstuvwxyz01, _MM_FROUND_TO_NEAREST_INT);
920 __m256i vfp16out6x0123456789ABCDEF = _mm512_cvtps_ph(vscaled6x0123456789ABCDEF, _MM_FROUND_TO_NEAREST_INT);
921 __m256i vfp16out6xGHIJKLMNOPQRSTUV = _mm512_cvtps_ph(vscaled6xGHIJKLMNOPQRSTUV, _MM_FROUND_TO_NEAREST_INT);
922 __m256i vfp16out6xWXYZabcdefghijkl = _mm512_cvtps_ph(vscaled6xWXYZabcdefghijkl, _MM_FROUND_TO_NEAREST_INT);
923 __m256i vfp16out6xmnopqrstuvwxyz01 = _mm512_cvtps_ph(vscaled6xmnopqrstuvwxyz01, _MM_FROUND_TO_NEAREST_INT);
924 __m256i vfp16out7x0123456789ABCDEF = _mm512_cvtps_ph(vscaled7x0123456789ABCDEF, _MM_FROUND_TO_NEAREST_INT);
925 __m256i vfp16out7xGHIJKLMNOPQRSTUV = _mm512_cvtps_ph(vscaled7xGHIJKLMNOPQRSTUV, _MM_FROUND_TO_NEAREST_INT);
926 __m256i vfp16out7xWXYZabcdefghijkl = _mm512_cvtps_ph(vscaled7xWXYZabcdefghijkl, _MM_FROUND_TO_NEAREST_INT);
927 __m256i vfp16out7xmnopqrstuvwxyz01 = _mm512_cvtps_ph(vscaled7xmnopqrstuvwxyz01, _MM_FROUND_TO_NEAREST_INT);
928 __m256i vfp16out8x0123456789ABCDEF = _mm512_cvtps_ph(vscaled8x0123456789ABCDEF, _MM_FROUND_TO_NEAREST_INT);
929 __m256i vfp16out8xGHIJKLMNOPQRSTUV = _mm512_cvtps_ph(vscaled8xGHIJKLMNOPQRSTUV, _MM_FROUND_TO_NEAREST_INT);
930 __m256i vfp16out8xWXYZabcdefghijkl = _mm512_cvtps_ph(vscaled8xWXYZabcdefghijkl, _MM_FROUND_TO_NEAREST_INT);
931 __m256i vfp16out8xmnopqrstuvwxyz01 = _mm512_cvtps_ph(vscaled8xmnopqrstuvwxyz01, _MM_FROUND_TO_NEAREST_INT);
932 __m256i vfp16out9x0123456789ABCDEF = _mm512_cvtps_ph(vscaled9x0123456789ABCDEF, _MM_FROUND_TO_NEAREST_INT);
933 __m256i vfp16out9xGHIJKLMNOPQRSTUV = _mm512_cvtps_ph(vscaled9xGHIJKLMNOPQRSTUV, _MM_FROUND_TO_NEAREST_INT);
934 __m256i vfp16out9xWXYZabcdefghijkl = _mm512_cvtps_ph(vscaled9xWXYZabcdefghijkl, _MM_FROUND_TO_NEAREST_INT);
935 __m256i vfp16out9xmnopqrstuvwxyz01 = _mm512_cvtps_ph(vscaled9xmnopqrstuvwxyz01, _MM_FROUND_TO_NEAREST_INT);
936 __m256i vfp16out10x0123456789ABCDEF = _mm512_cvtps_ph(vscaled10x0123456789ABCDEF, _MM_FROUND_TO_NEAREST_INT);
937 __m256i vfp16out10xGHIJKLMNOPQRSTUV = _mm512_cvtps_ph(vscaled10xGHIJKLMNOPQRSTUV, _MM_FROUND_TO_NEAREST_INT);
938 __m256i vfp16out10xWXYZabcdefghijkl = _mm512_cvtps_ph(vscaled10xWXYZabcdefghijkl, _MM_FROUND_TO_NEAREST_INT);
939 __m256i vfp16out10xmnopqrstuvwxyz01 = _mm512_cvtps_ph(vscaled10xmnopqrstuvwxyz01, _MM_FROUND_TO_NEAREST_INT);
940 __m256i vfp16out11x0123456789ABCDEF = _mm512_cvtps_ph(vscaled11x0123456789ABCDEF, _MM_FROUND_TO_NEAREST_INT);
941 __m256i vfp16out11xGHIJKLMNOPQRSTUV = _mm512_cvtps_ph(vscaled11xGHIJKLMNOPQRSTUV, _MM_FROUND_TO_NEAREST_INT);
942 __m256i vfp16out11xWXYZabcdefghijkl = _mm512_cvtps_ph(vscaled11xWXYZabcdefghijkl, _MM_FROUND_TO_NEAREST_INT);
943 __m256i vfp16out11xmnopqrstuvwxyz01 = _mm512_cvtps_ph(vscaled11xmnopqrstuvwxyz01, _MM_FROUND_TO_NEAREST_INT);
944 __m256i vfp16out12x0123456789ABCDEF = _mm512_cvtps_ph(vscaled12x0123456789ABCDEF, _MM_FROUND_TO_NEAREST_INT);
945 __m256i vfp16out12xGHIJKLMNOPQRSTUV = _mm512_cvtps_ph(vscaled12xGHIJKLMNOPQRSTUV, _MM_FROUND_TO_NEAREST_INT);
946 __m256i vfp16out12xWXYZabcdefghijkl = _mm512_cvtps_ph(vscaled12xWXYZabcdefghijkl, _MM_FROUND_TO_NEAREST_INT);
947 __m256i vfp16out12xmnopqrstuvwxyz01 = _mm512_cvtps_ph(vscaled12xmnopqrstuvwxyz01, _MM_FROUND_TO_NEAREST_INT);
948 __m256i vfp16out13x0123456789ABCDEF = _mm512_cvtps_ph(vscaled13x0123456789ABCDEF, _MM_FROUND_TO_NEAREST_INT);
949 __m256i vfp16out13xGHIJKLMNOPQRSTUV = _mm512_cvtps_ph(vscaled13xGHIJKLMNOPQRSTUV, _MM_FROUND_TO_NEAREST_INT);
950 __m256i vfp16out13xWXYZabcdefghijkl = _mm512_cvtps_ph(vscaled13xWXYZabcdefghijkl, _MM_FROUND_TO_NEAREST_INT);
951 __m256i vfp16out13xmnopqrstuvwxyz01 = _mm512_cvtps_ph(vscaled13xmnopqrstuvwxyz01, _MM_FROUND_TO_NEAREST_INT);
952 __m256i vfp16out14x0123456789ABCDEF = _mm512_cvtps_ph(vscaled14x0123456789ABCDEF, _MM_FROUND_TO_NEAREST_INT);
953 __m256i vfp16out14xGHIJKLMNOPQRSTUV = _mm512_cvtps_ph(vscaled14xGHIJKLMNOPQRSTUV, _MM_FROUND_TO_NEAREST_INT);
954 __m256i vfp16out14xWXYZabcdefghijkl = _mm512_cvtps_ph(vscaled14xWXYZabcdefghijkl, _MM_FROUND_TO_NEAREST_INT);
955 __m256i vfp16out14xmnopqrstuvwxyz01 = _mm512_cvtps_ph(vscaled14xmnopqrstuvwxyz01, _MM_FROUND_TO_NEAREST_INT);
956 __m256i vfp16out15x0123456789ABCDEF = _mm512_cvtps_ph(vscaled15x0123456789ABCDEF, _MM_FROUND_TO_NEAREST_INT);
957 __m256i vfp16out15xGHIJKLMNOPQRSTUV = _mm512_cvtps_ph(vscaled15xGHIJKLMNOPQRSTUV, _MM_FROUND_TO_NEAREST_INT);
958 __m256i vfp16out15xWXYZabcdefghijkl = _mm512_cvtps_ph(vscaled15xWXYZabcdefghijkl, _MM_FROUND_TO_NEAREST_INT);
959 __m256i vfp16out15xmnopqrstuvwxyz01 = _mm512_cvtps_ph(vscaled15xmnopqrstuvwxyz01, _MM_FROUND_TO_NEAREST_INT);
960 if XNN_LIKELY(nc >= 64) {
961 _mm256_storeu_si256((__m256i*) (c15 + 0), vfp16out15x0123456789ABCDEF);
962 _mm256_storeu_si256((__m256i*) (c15 + 16), vfp16out15xGHIJKLMNOPQRSTUV);
963 _mm256_storeu_si256((__m256i*) (c15 + 32), vfp16out15xWXYZabcdefghijkl);
964 _mm256_storeu_si256((__m256i*) (c15 + 48), vfp16out15xmnopqrstuvwxyz01);
965 c15 = (xnn_float16*) ((uintptr_t) c15 + cn_stride);
966 _mm256_storeu_si256((__m256i*) (c14 + 0), vfp16out14x0123456789ABCDEF);
967 _mm256_storeu_si256((__m256i*) (c14 + 16), vfp16out14xGHIJKLMNOPQRSTUV);
968 _mm256_storeu_si256((__m256i*) (c14 + 32), vfp16out14xWXYZabcdefghijkl);
969 _mm256_storeu_si256((__m256i*) (c14 + 48), vfp16out14xmnopqrstuvwxyz01);
970 c14 = (xnn_float16*) ((uintptr_t) c14 + cn_stride);
971 _mm256_storeu_si256((__m256i*) (c13 + 0), vfp16out13x0123456789ABCDEF);
972 _mm256_storeu_si256((__m256i*) (c13 + 16), vfp16out13xGHIJKLMNOPQRSTUV);
973 _mm256_storeu_si256((__m256i*) (c13 + 32), vfp16out13xWXYZabcdefghijkl);
974 _mm256_storeu_si256((__m256i*) (c13 + 48), vfp16out13xmnopqrstuvwxyz01);
975 c13 = (xnn_float16*) ((uintptr_t) c13 + cn_stride);
976 _mm256_storeu_si256((__m256i*) (c12 + 0), vfp16out12x0123456789ABCDEF);
977 _mm256_storeu_si256((__m256i*) (c12 + 16), vfp16out12xGHIJKLMNOPQRSTUV);
978 _mm256_storeu_si256((__m256i*) (c12 + 32), vfp16out12xWXYZabcdefghijkl);
979 _mm256_storeu_si256((__m256i*) (c12 + 48), vfp16out12xmnopqrstuvwxyz01);
980 c12 = (xnn_float16*) ((uintptr_t) c12 + cn_stride);
981 _mm256_storeu_si256((__m256i*) (c11 + 0), vfp16out11x0123456789ABCDEF);
982 _mm256_storeu_si256((__m256i*) (c11 + 16), vfp16out11xGHIJKLMNOPQRSTUV);
983 _mm256_storeu_si256((__m256i*) (c11 + 32), vfp16out11xWXYZabcdefghijkl);
984 _mm256_storeu_si256((__m256i*) (c11 + 48), vfp16out11xmnopqrstuvwxyz01);
985 c11 = (xnn_float16*) ((uintptr_t) c11 + cn_stride);
986 _mm256_storeu_si256((__m256i*) (c10 + 0), vfp16out10x0123456789ABCDEF);
987 _mm256_storeu_si256((__m256i*) (c10 + 16), vfp16out10xGHIJKLMNOPQRSTUV);
988 _mm256_storeu_si256((__m256i*) (c10 + 32), vfp16out10xWXYZabcdefghijkl);
989 _mm256_storeu_si256((__m256i*) (c10 + 48), vfp16out10xmnopqrstuvwxyz01);
990 c10 = (xnn_float16*) ((uintptr_t) c10 + cn_stride);
991 _mm256_storeu_si256((__m256i*) (c9 + 0), vfp16out9x0123456789ABCDEF);
992 _mm256_storeu_si256((__m256i*) (c9 + 16), vfp16out9xGHIJKLMNOPQRSTUV);
993 _mm256_storeu_si256((__m256i*) (c9 + 32), vfp16out9xWXYZabcdefghijkl);
994 _mm256_storeu_si256((__m256i*) (c9 + 48), vfp16out9xmnopqrstuvwxyz01);
995 c9 = (xnn_float16*) ((uintptr_t) c9 + cn_stride);
996 _mm256_storeu_si256((__m256i*) (c8 + 0), vfp16out8x0123456789ABCDEF);
997 _mm256_storeu_si256((__m256i*) (c8 + 16), vfp16out8xGHIJKLMNOPQRSTUV);
998 _mm256_storeu_si256((__m256i*) (c8 + 32), vfp16out8xWXYZabcdefghijkl);
999 _mm256_storeu_si256((__m256i*) (c8 + 48), vfp16out8xmnopqrstuvwxyz01);
1000 c8 = (xnn_float16*) ((uintptr_t) c8 + cn_stride);
1001 _mm256_storeu_si256((__m256i*) (c7 + 0), vfp16out7x0123456789ABCDEF);
1002 _mm256_storeu_si256((__m256i*) (c7 + 16), vfp16out7xGHIJKLMNOPQRSTUV);
1003 _mm256_storeu_si256((__m256i*) (c7 + 32), vfp16out7xWXYZabcdefghijkl);
1004 _mm256_storeu_si256((__m256i*) (c7 + 48), vfp16out7xmnopqrstuvwxyz01);
1005 c7 = (xnn_float16*) ((uintptr_t) c7 + cn_stride);
1006 _mm256_storeu_si256((__m256i*) (c6 + 0), vfp16out6x0123456789ABCDEF);
1007 _mm256_storeu_si256((__m256i*) (c6 + 16), vfp16out6xGHIJKLMNOPQRSTUV);
1008 _mm256_storeu_si256((__m256i*) (c6 + 32), vfp16out6xWXYZabcdefghijkl);
1009 _mm256_storeu_si256((__m256i*) (c6 + 48), vfp16out6xmnopqrstuvwxyz01);
1010 c6 = (xnn_float16*) ((uintptr_t) c6 + cn_stride);
1011 _mm256_storeu_si256((__m256i*) (c5 + 0), vfp16out5x0123456789ABCDEF);
1012 _mm256_storeu_si256((__m256i*) (c5 + 16), vfp16out5xGHIJKLMNOPQRSTUV);
1013 _mm256_storeu_si256((__m256i*) (c5 + 32), vfp16out5xWXYZabcdefghijkl);
1014 _mm256_storeu_si256((__m256i*) (c5 + 48), vfp16out5xmnopqrstuvwxyz01);
1015 c5 = (xnn_float16*) ((uintptr_t) c5 + cn_stride);
1016 _mm256_storeu_si256((__m256i*) (c4 + 0), vfp16out4x0123456789ABCDEF);
1017 _mm256_storeu_si256((__m256i*) (c4 + 16), vfp16out4xGHIJKLMNOPQRSTUV);
1018 _mm256_storeu_si256((__m256i*) (c4 + 32), vfp16out4xWXYZabcdefghijkl);
1019 _mm256_storeu_si256((__m256i*) (c4 + 48), vfp16out4xmnopqrstuvwxyz01);
1020 c4 = (xnn_float16*) ((uintptr_t) c4 + cn_stride);
1021 _mm256_storeu_si256((__m256i*) (c3 + 0), vfp16out3x0123456789ABCDEF);
1022 _mm256_storeu_si256((__m256i*) (c3 + 16), vfp16out3xGHIJKLMNOPQRSTUV);
1023 _mm256_storeu_si256((__m256i*) (c3 + 32), vfp16out3xWXYZabcdefghijkl);
1024 _mm256_storeu_si256((__m256i*) (c3 + 48), vfp16out3xmnopqrstuvwxyz01);
1025 c3 = (xnn_float16*) ((uintptr_t) c3 + cn_stride);
1026 _mm256_storeu_si256((__m256i*) (c2 + 0), vfp16out2x0123456789ABCDEF);
1027 _mm256_storeu_si256((__m256i*) (c2 + 16), vfp16out2xGHIJKLMNOPQRSTUV);
1028 _mm256_storeu_si256((__m256i*) (c2 + 32), vfp16out2xWXYZabcdefghijkl);
1029 _mm256_storeu_si256((__m256i*) (c2 + 48), vfp16out2xmnopqrstuvwxyz01);
1030 c2 = (xnn_float16*) ((uintptr_t) c2 + cn_stride);
1031 _mm256_storeu_si256((__m256i*) (c1 + 0), vfp16out1x0123456789ABCDEF);
1032 _mm256_storeu_si256((__m256i*) (c1 + 16), vfp16out1xGHIJKLMNOPQRSTUV);
1033 _mm256_storeu_si256((__m256i*) (c1 + 32), vfp16out1xWXYZabcdefghijkl);
1034 _mm256_storeu_si256((__m256i*) (c1 + 48), vfp16out1xmnopqrstuvwxyz01);
1035 c1 = (xnn_float16*) ((uintptr_t) c1 + cn_stride);
1036 _mm256_storeu_si256((__m256i*) (c0 + 0), vfp16out0x0123456789ABCDEF);
1037 _mm256_storeu_si256((__m256i*) (c0 + 16), vfp16out0xGHIJKLMNOPQRSTUV);
1038 _mm256_storeu_si256((__m256i*) (c0 + 32), vfp16out0xWXYZabcdefghijkl);
1039 _mm256_storeu_si256((__m256i*) (c0 + 48), vfp16out0xmnopqrstuvwxyz01);
1040 c0 = (xnn_float16*) ((uintptr_t) c0 + cn_stride);
1041
1042 a = (const int8_t**restrict) ((uintptr_t) a - ks);
1043 nc -= 64;
1044 } else {
1045 // Prepare mask for valid 32-bit elements (depends on nc).
1046 const __mmask16 vmask0 = _cvtu32_mask16((uint32_t) ((((UINT64_C(1) << nc) - 1) >> 0) & 0xFFFF));
1047 const __mmask16 vmask1 = _cvtu32_mask16((uint32_t) ((((UINT64_C(1) << nc) - 1) >> 16) & 0xFFFF));
1048 const __mmask16 vmask2 = _cvtu32_mask16((uint32_t) ((((UINT64_C(1) << nc) - 1) >> 32) & 0xFFFF));
1049 const __mmask16 vmask3 = _cvtu32_mask16((uint32_t) ((((UINT64_C(1) << nc) - 1) >> 48) & 0xFFFF));
1050 _mm256_mask_storeu_epi16(c15 + 0, vmask0, vfp16out15x0123456789ABCDEF);
1051 _mm256_mask_storeu_epi16(c15 + 16, vmask1, vfp16out15xGHIJKLMNOPQRSTUV);
1052 _mm256_mask_storeu_epi16(c15 + 32, vmask2, vfp16out15xWXYZabcdefghijkl);
1053 _mm256_mask_storeu_epi16(c15 + 48, vmask3, vfp16out15xmnopqrstuvwxyz01);
1054 _mm256_mask_storeu_epi16(c14 + 0, vmask0, vfp16out14x0123456789ABCDEF);
1055 _mm256_mask_storeu_epi16(c14 + 16, vmask1, vfp16out14xGHIJKLMNOPQRSTUV);
1056 _mm256_mask_storeu_epi16(c14 + 32, vmask2, vfp16out14xWXYZabcdefghijkl);
1057 _mm256_mask_storeu_epi16(c14 + 48, vmask3, vfp16out14xmnopqrstuvwxyz01);
1058 _mm256_mask_storeu_epi16(c13 + 0, vmask0, vfp16out13x0123456789ABCDEF);
1059 _mm256_mask_storeu_epi16(c13 + 16, vmask1, vfp16out13xGHIJKLMNOPQRSTUV);
1060 _mm256_mask_storeu_epi16(c13 + 32, vmask2, vfp16out13xWXYZabcdefghijkl);
1061 _mm256_mask_storeu_epi16(c13 + 48, vmask3, vfp16out13xmnopqrstuvwxyz01);
1062 _mm256_mask_storeu_epi16(c12 + 0, vmask0, vfp16out12x0123456789ABCDEF);
1063 _mm256_mask_storeu_epi16(c12 + 16, vmask1, vfp16out12xGHIJKLMNOPQRSTUV);
1064 _mm256_mask_storeu_epi16(c12 + 32, vmask2, vfp16out12xWXYZabcdefghijkl);
1065 _mm256_mask_storeu_epi16(c12 + 48, vmask3, vfp16out12xmnopqrstuvwxyz01);
1066 _mm256_mask_storeu_epi16(c11 + 0, vmask0, vfp16out11x0123456789ABCDEF);
1067 _mm256_mask_storeu_epi16(c11 + 16, vmask1, vfp16out11xGHIJKLMNOPQRSTUV);
1068 _mm256_mask_storeu_epi16(c11 + 32, vmask2, vfp16out11xWXYZabcdefghijkl);
1069 _mm256_mask_storeu_epi16(c11 + 48, vmask3, vfp16out11xmnopqrstuvwxyz01);
1070 _mm256_mask_storeu_epi16(c10 + 0, vmask0, vfp16out10x0123456789ABCDEF);
1071 _mm256_mask_storeu_epi16(c10 + 16, vmask1, vfp16out10xGHIJKLMNOPQRSTUV);
1072 _mm256_mask_storeu_epi16(c10 + 32, vmask2, vfp16out10xWXYZabcdefghijkl);
1073 _mm256_mask_storeu_epi16(c10 + 48, vmask3, vfp16out10xmnopqrstuvwxyz01);
1074 _mm256_mask_storeu_epi16(c9 + 0, vmask0, vfp16out9x0123456789ABCDEF);
1075 _mm256_mask_storeu_epi16(c9 + 16, vmask1, vfp16out9xGHIJKLMNOPQRSTUV);
1076 _mm256_mask_storeu_epi16(c9 + 32, vmask2, vfp16out9xWXYZabcdefghijkl);
1077 _mm256_mask_storeu_epi16(c9 + 48, vmask3, vfp16out9xmnopqrstuvwxyz01);
1078 _mm256_mask_storeu_epi16(c8 + 0, vmask0, vfp16out8x0123456789ABCDEF);
1079 _mm256_mask_storeu_epi16(c8 + 16, vmask1, vfp16out8xGHIJKLMNOPQRSTUV);
1080 _mm256_mask_storeu_epi16(c8 + 32, vmask2, vfp16out8xWXYZabcdefghijkl);
1081 _mm256_mask_storeu_epi16(c8 + 48, vmask3, vfp16out8xmnopqrstuvwxyz01);
1082 _mm256_mask_storeu_epi16(c7 + 0, vmask0, vfp16out7x0123456789ABCDEF);
1083 _mm256_mask_storeu_epi16(c7 + 16, vmask1, vfp16out7xGHIJKLMNOPQRSTUV);
1084 _mm256_mask_storeu_epi16(c7 + 32, vmask2, vfp16out7xWXYZabcdefghijkl);
1085 _mm256_mask_storeu_epi16(c7 + 48, vmask3, vfp16out7xmnopqrstuvwxyz01);
1086 _mm256_mask_storeu_epi16(c6 + 0, vmask0, vfp16out6x0123456789ABCDEF);
1087 _mm256_mask_storeu_epi16(c6 + 16, vmask1, vfp16out6xGHIJKLMNOPQRSTUV);
1088 _mm256_mask_storeu_epi16(c6 + 32, vmask2, vfp16out6xWXYZabcdefghijkl);
1089 _mm256_mask_storeu_epi16(c6 + 48, vmask3, vfp16out6xmnopqrstuvwxyz01);
1090 _mm256_mask_storeu_epi16(c5 + 0, vmask0, vfp16out5x0123456789ABCDEF);
1091 _mm256_mask_storeu_epi16(c5 + 16, vmask1, vfp16out5xGHIJKLMNOPQRSTUV);
1092 _mm256_mask_storeu_epi16(c5 + 32, vmask2, vfp16out5xWXYZabcdefghijkl);
1093 _mm256_mask_storeu_epi16(c5 + 48, vmask3, vfp16out5xmnopqrstuvwxyz01);
1094 _mm256_mask_storeu_epi16(c4 + 0, vmask0, vfp16out4x0123456789ABCDEF);
1095 _mm256_mask_storeu_epi16(c4 + 16, vmask1, vfp16out4xGHIJKLMNOPQRSTUV);
1096 _mm256_mask_storeu_epi16(c4 + 32, vmask2, vfp16out4xWXYZabcdefghijkl);
1097 _mm256_mask_storeu_epi16(c4 + 48, vmask3, vfp16out4xmnopqrstuvwxyz01);
1098 _mm256_mask_storeu_epi16(c3 + 0, vmask0, vfp16out3x0123456789ABCDEF);
1099 _mm256_mask_storeu_epi16(c3 + 16, vmask1, vfp16out3xGHIJKLMNOPQRSTUV);
1100 _mm256_mask_storeu_epi16(c3 + 32, vmask2, vfp16out3xWXYZabcdefghijkl);
1101 _mm256_mask_storeu_epi16(c3 + 48, vmask3, vfp16out3xmnopqrstuvwxyz01);
1102 _mm256_mask_storeu_epi16(c2 + 0, vmask0, vfp16out2x0123456789ABCDEF);
1103 _mm256_mask_storeu_epi16(c2 + 16, vmask1, vfp16out2xGHIJKLMNOPQRSTUV);
1104 _mm256_mask_storeu_epi16(c2 + 32, vmask2, vfp16out2xWXYZabcdefghijkl);
1105 _mm256_mask_storeu_epi16(c2 + 48, vmask3, vfp16out2xmnopqrstuvwxyz01);
1106 _mm256_mask_storeu_epi16(c1 + 0, vmask0, vfp16out1x0123456789ABCDEF);
1107 _mm256_mask_storeu_epi16(c1 + 16, vmask1, vfp16out1xGHIJKLMNOPQRSTUV);
1108 _mm256_mask_storeu_epi16(c1 + 32, vmask2, vfp16out1xWXYZabcdefghijkl);
1109 _mm256_mask_storeu_epi16(c1 + 48, vmask3, vfp16out1xmnopqrstuvwxyz01);
1110 _mm256_mask_storeu_epi16(c0 + 0, vmask0, vfp16out0x0123456789ABCDEF);
1111 _mm256_mask_storeu_epi16(c0 + 16, vmask1, vfp16out0xGHIJKLMNOPQRSTUV);
1112 _mm256_mask_storeu_epi16(c0 + 32, vmask2, vfp16out0xWXYZabcdefghijkl);
1113 _mm256_mask_storeu_epi16(c0 + 48, vmask3, vfp16out0xmnopqrstuvwxyz01);
1114 nc = 0;
1115 }
1116 } while (nc != 0);
1117
1118 // Release tile config
1119 // _tile_release();
1120 __asm__ volatile ("tilerelease" ::);
1121 #endif // defined(__x86_64__)
1122 }