Debian Codegraph @

src/qd8-f16-qc8w-igemm/gen/qd8-f16-qc8w-igemm-1x64c4-minmax-avx512amx.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-igemm/c4-avx512amx.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2024 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include <immintrin.h>
13
14 #include "xnnpack/gemm.h"
15 #include "xnnpack/intrinsics-polyfill.h"
16 #include "xnnpack/math.h"
17 #include "xnnpack/unaligned.h"
18
19
20 void xnn_qd8_f16_qc8w_igemm_minmax_ukernel_1x64c4__avx512amx(
21 size_t mr,
22 size_t nc,
23 size_t kc,
24 size_t ks,
25 const int8_t** restrict a,
26 const void* restrict w,
27 xnn_float16* restrict c,
28 size_t cm_stride,
29 size_t cn_stride,
30 size_t a_offset,
31 const int8_t* zero,
32 const int8_t* zero_data,
33 const union xnn_f16_minmax_params params[restrict XNN_MIN_ELEMENTS(1)],
34 const struct xnn_qd8_quantization_params quantization_params[restrict XNN_MIN_ELEMENTS(1)])
35 {
36 assert(mr != 0);
37 assert(mr <= 1);
38 assert(nc != 0);
39 assert(kc != 0);
40 assert(kc % sizeof(int8_t) == 0);
41 assert(a != NULL);
42 assert(w != NULL);
43 assert(c != NULL);
44
45 // TODO: amxintrin.h only provide intrinsics for __x86_64__
46 // Update if amxintrin changes
47 #if defined(__x86_64__)
48 __attribute__((aligned(64))) int32_t vintile[1 * 16];
49 __attribute__((aligned(64))) int32_t res0[1 * 16];
50 __attribute__((aligned(64))) int32_t res1[1 * 16];
51 __attribute__((aligned(64))) int32_t res2[1 * 16];
52 __attribute__((aligned(64))) int32_t res3[1 * 16];
53
54 kc = round_up_po2(kc, 4 * sizeof(int8_t));
55 const size_t kremainder = (kc & 63) ? (kc & 63) : 64;
56 const __mmask16 kremainder_mask = _cvtu32_mask16((UINT32_C(1) << (kremainder >> 2)) - 1);
57
58 // Define tile config data structure
59 struct __tile_config {
60 uint8_t palette_id;
61 uint8_t start_row;
62 uint8_t reserved_0[14];
63 uint16_t colsb[8];
64 uint16_t reserved_1[8];
65 uint8_t rows[8];
66 uint8_t reserved_2[8];
67 };
68
69 // Load tile configuration
70 __attribute__((aligned(64))) struct __tile_config tile_data = {0};
71 tile_data.palette_id = 1;
72 tile_data.rows[0] = mr; // tmm0 = res 0
73 tile_data.rows[1] = mr; // tmm1 = res 1
74 tile_data.rows[2] = mr; // tmm2 = res 2
75 tile_data.rows[3] = mr; // tmm3 = res 3
76 tile_data.rows[4] = mr; // tmm4 = input
77 tile_data.rows[5] = 16; // tmm5 = weights
78 tile_data.rows[6] = mr; // tmm6 = input remainder
79 tile_data.rows[7] = kremainder >> 2; // tmm7 = weights remainder
80
81 tile_data.colsb[0] = 64; // tmm0 = res 0
82 tile_data.colsb[1] = 64; // tmm1 = res 1
83 tile_data.colsb[2] = 64; // tmm2 = res 1
84 tile_data.colsb[3] = 64; // tmm3 = res 1
85 tile_data.colsb[4] = 64; // tmm4 = input
86 tile_data.colsb[5] = 64; // tmm5 = weights
87 tile_data.colsb[6] = kremainder; // tmm6 = input remainder
88 tile_data.colsb[7] = 64; // tmm7 = weights remainder
89
90 //_tile_loadconfig(&tile_data);
91 __asm__ volatile ("ldtilecfg %0" :: "m" (tile_data));
92
93 xnn_float16* c0 = c;
94
95 const __m512 voutput_min = _mm512_cvtph_ps(_mm256_set1_epi16(*(const uint16_t*) &params->scalar.min));
96 const __m512 voutput_max = _mm512_cvtph_ps(_mm256_set1_epi16(*(const uint16_t*) &params->scalar.max));
97 // XNN_FORCE_REALIZATION(voutput_min);
98 // XNN_FORCE_REALIZATION(voutput_max);
99
100 do {
101 const __m512i vksum0123456789ABCDEF = _mm512_loadu_epi32((const int32_t*) w + 0);
102 const __m512i vksumGHIJKLMNOPQRSTUV = _mm512_loadu_epi32((const int32_t*) w + 16);
103 const __m512i vksumWXYZabcdefghijkl = _mm512_loadu_epi32((const int32_t*) w + 32);
104 const __m512i vksummnopqrstuvwxyz01 = _mm512_loadu_epi32((const int32_t*) w + 48);
105 w = (const int32_t*) w + 64;
106
107 // Zero tile accumulator
108 __asm__ volatile (
109 "tilezero %%tmm0\n"
110 "tilezero %%tmm1\n"
111 "tilezero %%tmm2\n"
112 "tilezero %%tmm3\n"
113 ::);
114
115 size_t p = ks;
116 do {
117 const int8_t* restrict a0 = a[0];
118 if XNN_UNPREDICTABLE(a0 != zero) {
119 a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
120 } else {
121 a0 = zero_data;
122 }
123 a += 1;
124
125 size_t k = kc;
126 {
127 while (k >= 64 * sizeof(int8_t)) {
128 _tile_loadd(4, a0, 64); // Directly load input for mr=1
129 a0 += 64;
130 _tile_loadd(5, (const int8_t*) w + 0, 256);
131 _tile_dpbssd(0, 4, 5);
132 _tile_loadd(5, (const int8_t*) w + 64, 256);
133 _tile_dpbssd(1, 4, 5);
134 _tile_loadd(5, (const int8_t*) w + 128, 256);
135 _tile_dpbssd(2, 4, 5);
136 _tile_loadd(5, (const int8_t*) w + 192, 256);
137 _tile_dpbssd(3, 4, 5);
138
139 w = (const int8_t*) w + 4096;
140 k -= 64 * sizeof(int8_t);
141 }
142 }
143
144 if XNN_UNLIKELY(k != 0) {
145 const __m512i vin0 = _mm512_maskz_loadu_epi32(kremainder_mask, a0);
146 a0 += kremainder;
147 _mm512_store_epi32(vintile + 0, vin0);
148 _tile_loadd(6, vintile, 64);
149 _tile_loadd(7, (const int8_t*) w + 0, 256);
150 _tile_dpbssd(0, 6, 7);
151 _tile_loadd(7, (const int8_t*) w + 64, 256);
152 _tile_dpbssd(1, 6, 7);
153 _tile_loadd(7, (const int8_t*) w + 128, 256);
154 _tile_dpbssd(2, 6, 7);
155 _tile_loadd(7, (const int8_t*) w + 192, 256);
156 _tile_dpbssd(3, 6, 7);
157
158 w = (const int8_t*) w + kremainder * 64;
159 k -= kremainder * sizeof(int8_t);
160 }
161
162 p -= 1 * sizeof(void*);
163 } while (p != 0);
164
165 // Add tile to bias
166 _tile_stored(0, res0, 64);
167 _tile_stored(1, res1, 64);
168 _tile_stored(2, res2, 64);
169 _tile_stored(3, res3, 64);
170
171 __m512i vacc0x0123456789ABCDEF = _mm512_mullo_epi32(vksum0123456789ABCDEF, _mm512_set1_epi32((int) quantization_params->zero_point));
172 __m512i vacc0xGHIJKLMNOPQRSTUV = _mm512_mullo_epi32(vksumGHIJKLMNOPQRSTUV, _mm512_set1_epi32((int) quantization_params->zero_point));
173 __m512i vacc0xWXYZabcdefghijkl = _mm512_mullo_epi32(vksumWXYZabcdefghijkl, _mm512_set1_epi32((int) quantization_params->zero_point));
174 __m512i vacc0xmnopqrstuvwxyz01 = _mm512_mullo_epi32(vksummnopqrstuvwxyz01, _mm512_set1_epi32((int) quantization_params->zero_point));
175 vacc0x0123456789ABCDEF = _mm512_add_epi32(vacc0x0123456789ABCDEF, _mm512_load_epi32(res0 + 0));
176 vacc0xGHIJKLMNOPQRSTUV = _mm512_add_epi32(vacc0xGHIJKLMNOPQRSTUV, _mm512_load_epi32(res1 + 0));
177 vacc0xWXYZabcdefghijkl = _mm512_add_epi32(vacc0xWXYZabcdefghijkl, _mm512_load_epi32(res2 + 0));
178 vacc0xmnopqrstuvwxyz01 = _mm512_add_epi32(vacc0xmnopqrstuvwxyz01, _mm512_load_epi32(res3 + 0));
179
180 __m512 vscaled0x0123456789ABCDEF = _mm512_cvtepi32_ps(vacc0x0123456789ABCDEF);
181 __m512 vscaled0xGHIJKLMNOPQRSTUV = _mm512_cvtepi32_ps(vacc0xGHIJKLMNOPQRSTUV);
182 __m512 vscaled0xWXYZabcdefghijkl = _mm512_cvtepi32_ps(vacc0xWXYZabcdefghijkl);
183 __m512 vscaled0xmnopqrstuvwxyz01 = _mm512_cvtepi32_ps(vacc0xmnopqrstuvwxyz01);
184
185 vscaled0x0123456789ABCDEF = _mm512_mul_ps(vscaled0x0123456789ABCDEF, _mm512_set1_ps(quantization_params->inv_scale));
186 vscaled0xGHIJKLMNOPQRSTUV = _mm512_mul_ps(vscaled0xGHIJKLMNOPQRSTUV, _mm512_set1_ps(quantization_params->inv_scale));
187 vscaled0xWXYZabcdefghijkl = _mm512_mul_ps(vscaled0xWXYZabcdefghijkl, _mm512_set1_ps(quantization_params->inv_scale));
188 vscaled0xmnopqrstuvwxyz01 = _mm512_mul_ps(vscaled0xmnopqrstuvwxyz01, _mm512_set1_ps(quantization_params->inv_scale));
189
190 const __m512 vfilter_output_scale0123456789ABCDEF = _mm512_loadu_ps((const float*) w + 0);
191 const __m512 vfilter_output_scaleGHIJKLMNOPQRSTUV = _mm512_loadu_ps((const float*) w + 16);
192 const __m512 vfilter_output_scaleWXYZabcdefghijkl = _mm512_loadu_ps((const float*) w + 32);
193 const __m512 vfilter_output_scalemnopqrstuvwxyz01 = _mm512_loadu_ps((const float*) w + 48);
194 w = (const int32_t*) w + 64;
195 const __m512 vbias0123456789ABCDEF = _mm512_loadu_ps((const float*) w + 0);
196 const __m512 vbiasGHIJKLMNOPQRSTUV = _mm512_loadu_ps((const float*) w + 16);
197 const __m512 vbiasWXYZabcdefghijkl = _mm512_loadu_ps((const float*) w + 32);
198 const __m512 vbiasmnopqrstuvwxyz01 = _mm512_loadu_ps((const float*) w + 48);
199 w = (const int32_t*) w + 64;
200
201 vscaled0x0123456789ABCDEF = _mm512_fmadd_ps(vscaled0x0123456789ABCDEF, vfilter_output_scale0123456789ABCDEF, vbias0123456789ABCDEF);
202 vscaled0xGHIJKLMNOPQRSTUV = _mm512_fmadd_ps(vscaled0xGHIJKLMNOPQRSTUV, vfilter_output_scaleGHIJKLMNOPQRSTUV, vbiasGHIJKLMNOPQRSTUV);
203 vscaled0xWXYZabcdefghijkl = _mm512_fmadd_ps(vscaled0xWXYZabcdefghijkl, vfilter_output_scaleWXYZabcdefghijkl, vbiasWXYZabcdefghijkl);
204 vscaled0xmnopqrstuvwxyz01 = _mm512_fmadd_ps(vscaled0xmnopqrstuvwxyz01, vfilter_output_scalemnopqrstuvwxyz01, vbiasmnopqrstuvwxyz01);
205
206 vscaled0x0123456789ABCDEF = _mm512_max_ps(vscaled0x0123456789ABCDEF, voutput_min);
207 vscaled0xGHIJKLMNOPQRSTUV = _mm512_max_ps(vscaled0xGHIJKLMNOPQRSTUV, voutput_min);
208 vscaled0xWXYZabcdefghijkl = _mm512_max_ps(vscaled0xWXYZabcdefghijkl, voutput_min);
209 vscaled0xmnopqrstuvwxyz01 = _mm512_max_ps(vscaled0xmnopqrstuvwxyz01, voutput_min);
210
211 vscaled0x0123456789ABCDEF = _mm512_min_ps(vscaled0x0123456789ABCDEF, voutput_max);
212 vscaled0xGHIJKLMNOPQRSTUV = _mm512_min_ps(vscaled0xGHIJKLMNOPQRSTUV, voutput_max);
213 vscaled0xWXYZabcdefghijkl = _mm512_min_ps(vscaled0xWXYZabcdefghijkl, voutput_max);
214 vscaled0xmnopqrstuvwxyz01 = _mm512_min_ps(vscaled0xmnopqrstuvwxyz01, voutput_max);
215
216 __m256i vfp16out0x0123456789ABCDEF = _mm512_cvtps_ph(vscaled0x0123456789ABCDEF, _MM_FROUND_TO_NEAREST_INT);
217 __m256i vfp16out0xGHIJKLMNOPQRSTUV = _mm512_cvtps_ph(vscaled0xGHIJKLMNOPQRSTUV, _MM_FROUND_TO_NEAREST_INT);
218 __m256i vfp16out0xWXYZabcdefghijkl = _mm512_cvtps_ph(vscaled0xWXYZabcdefghijkl, _MM_FROUND_TO_NEAREST_INT);
219 __m256i vfp16out0xmnopqrstuvwxyz01 = _mm512_cvtps_ph(vscaled0xmnopqrstuvwxyz01, _MM_FROUND_TO_NEAREST_INT);
220 if XNN_LIKELY(nc >= 64) {
221 _mm256_storeu_si256((__m256i*) (c0 + 0), vfp16out0x0123456789ABCDEF);
222 _mm256_storeu_si256((__m256i*) (c0 + 16), vfp16out0xGHIJKLMNOPQRSTUV);
223 _mm256_storeu_si256((__m256i*) (c0 + 32), vfp16out0xWXYZabcdefghijkl);
224 _mm256_storeu_si256((__m256i*) (c0 + 48), vfp16out0xmnopqrstuvwxyz01);
225 c0 = (xnn_float16*) ((uintptr_t) c0 + cn_stride);
226
227 a = (const int8_t**restrict) ((uintptr_t) a - ks);
228 nc -= 64;
229 } else {
230 // Prepare mask for valid 32-bit elements (depends on nc).
231 const __mmask16 vmask0 = _cvtu32_mask16((uint32_t) ((((UINT64_C(1) << nc) - 1) >> 0) & 0xFFFF));
232 const __mmask16 vmask1 = _cvtu32_mask16((uint32_t) ((((UINT64_C(1) << nc) - 1) >> 16) & 0xFFFF));
233 const __mmask16 vmask2 = _cvtu32_mask16((uint32_t) ((((UINT64_C(1) << nc) - 1) >> 32) & 0xFFFF));
234 const __mmask16 vmask3 = _cvtu32_mask16((uint32_t) ((((UINT64_C(1) << nc) - 1) >> 48) & 0xFFFF));
235 _mm256_mask_storeu_epi16(c0 + 0, vmask0, vfp16out0x0123456789ABCDEF);
236 _mm256_mask_storeu_epi16(c0 + 16, vmask1, vfp16out0xGHIJKLMNOPQRSTUV);
237 _mm256_mask_storeu_epi16(c0 + 32, vmask2, vfp16out0xWXYZabcdefghijkl);
238 _mm256_mask_storeu_epi16(c0 + 48, vmask3, vfp16out0xmnopqrstuvwxyz01);
239 nc = 0;
240 }
241 } while (nc != 0);
242
243 // Release tile config
244 // _tile_release();
245 __asm__ volatile ("tilerelease" ::);
246 #endif // defined(__x86_64__)
247 }