Debian Codegraph @

gcc/testsuite/gcc.target/i386/amx-helper.h

raw
1 #ifndef AMX_HELPER_H_INCLUDED
2 #define AMX_HELPER_H_INCLUDED
3 #include <immintrin.h>
4 #include <xmmintrin.h>
5 #include "amx-check.h"
6
7 typedef union
8 {
9 _Float16 f16;
10 uint16_t u;
11 } union16f_uw;
12
13 typedef union
14 {
15 __bf16 bf16;
16 uint16_t u;
17 } union16bh_uw;
18
19 typedef union
20 {
21 float f;
22 uint32_t u;
23 } union32f_ud;
24
25 typedef union
26 {
27 __m512 m;
28 uint8_t u[64];
29 } union512_ub;
30
31 #if defined(AMX_FP16) || defined(AMX_COMPLEX) || defined (AMX_AVX512)
32 /* Transformation functions between fp16/float */
33 static uint16_t make_f32_fp16 (float f)
34 {
35 union16f_uw tmp;
36 __m128 b = _mm_set_ss (f);
37 __m128h a;
38 tmp.f16 = _mm_cvtsh_h (_mm_cvtss_sh (a, b));
39 return tmp.u;
40 }
41
42 static float make_fp16_f32 (uint16_t fp)
43 {
44 union16f_uw tmp;
45 tmp.u = fp;
46 __m128h b = _mm_set_sh (tmp.f16);
47 __m128 a;
48 return _mm_cvtss_f32 (_mm_cvtsh_ss (a, b));
49 }
50
51 /* Init tile buffer with fp16 pairs */
52 void init_fp16_max_tile_buffer (uint8_t* buf)
53 {
54 int i, j;
55 uint16_t* ptr = (uint16_t *) buf;
56
57 for (i = 0; i < 16; i++)
58 for (j = 0; j < 32; j++)
59 {
60 float f = 2.5f * i + 1.25f * j;
61 ptr[i * 32 + j] = make_f32_fp16 (f);
62 }
63 }
64
65 /* Init tile fp16 pair buffer with zero */
66 void init_fp16_max_tile_zero_buffer (uint8_t* buf)
67 {
68 int i, j;
69 uint16_t* ptr = (uint16_t *) buf;
70
71 for (i = 0; i < 16; i++)
72 for (j = 0; j < 32; j++)
73 ptr[i * 32 + j] = make_f32_fp16 (0.0f);
74 }
75 #endif
76
77 #if defined (AMX_AVX512) || defined (AMX_BF16)
78 /* Transformation functions between bf16/float */
79 static uint16_t make_f32_bf16 (float f)
80 {
81 union16bh_uw tmp;
82 tmp.bf16 = (__bf16) f;
83 return tmp.u;
84 }
85
86 static float make_bf16_f32 (uint16_t bf)
87 {
88 union16bh_uw tmp;
89 tmp.u = bf;
90 return _mm_cvtsbh_ss (tmp.bf16);
91 }
92
93 /* Init tile buffer with bf16 pairs */
94 void init_bf16_max_tile_buffer (uint8_t *buf)
95 {
96 int i, j;
97 uint16_t* ptr = (uint16_t *) buf;
98
99 for (i = 0; i < 16; i++)
100 for (j = 0; j < 32; j++)
101 {
102 float f = 2.5f * i + 1.25f * j;
103 ptr[i * 32 + j] = make_f32_bf16 (f);
104 }
105 }
106 #endif
107
108 /* Init tile buffer with fp32 */
109 void init_fp32_max_tile_buffer (uint8_t *buf)
110 {
111 int i, j;
112 float* ptr = (float *) buf;
113
114 for (i = 0; i < 16; i++)
115 for (j = 0; j < 16; j++)
116 ptr[i * 16 + j] = 2.5f * i + 1.25f * j;
117 }
118
119 /* Init tile fp32 buffer with zero */
120 void init_fp32_max_tile_zero_buffer (uint8_t *buf)
121 {
122 int i, j;
123 float* ptr = (float *) buf;
124
125 for (i = 0; i < 16; i++)
126 for (j = 0; j < 16; j++)
127 ptr[i * 16 + j] = 0.0f;
128 }
129
130 /* Init tile buffer with int32 */
131 void init_int32_max_tile_buffer (uint8_t *buf)
132 {
133 int i, j;
134 uint32_t *ptr = (uint32_t *)buf;
135
136 for (i = 0; i < 16; i++)
137 for (j = 0; j < 16; j++)
138 ptr[i * 16 + j] = (uint32_t) (3 * j - 16 * i);
139 }
140
141 void
142 init_fp8_max_tile_buffer (uint8_t *buf)
143 {
144 int i, j;
145
146 for (i = 0; i < 16; i++)
147 for (j = 0; j < 64; j++)
148 {
149 int idx = i * 64 + j;
150
151 /* Positive Infinity (S11111.00) */
152 if (idx % 128 == 0)
153 buf[idx] = 0x7C;
154
155 /* Negative Infinity (S11111.00 with sign bit set) */
156 else if (idx % 128 == 1)
157 buf[idx] = 0xFC;
158
159 /* Positive NaN (S11111.01) */
160 else if (idx % 128 == 2)
161 buf[idx] = 0x7D;
162
163 /* Negative NaN (S11111.01 with sign bit set) */
164 else if (idx % 128 == 3)
165 buf[idx] = 0xFD;
166
167 /* insert Positive NaN (S11111.10) */
168 else if (idx % 128 == 4)
169 buf[idx] = 0x7E;
170
171 /* Negative NaN (S11111.10 with sign bit set) */
172 else if (idx % 128 == 5)
173 buf[idx] = 0xFE;
174
175 /* Positive NaN (S11111.11) */
176 else if (idx % 128 == 6)
177 buf[idx] = 0x7F;
178
179 /* Negative NaN (S11111.11 with sign bit set) */
180 else if (idx % 128 == 7)
181 buf[idx] = 0xFF;
182
183 else
184 buf[idx] = (uint8_t) ((idx * 251) & 0xFF);
185 }
186 }
187
188 #define COMPARE_ZMM(A, B) \
189 for (int j = 0; j < 16; j++) \
190 { \
191 union32f_ud fu1, fu2; \
192 fu1.f = A[j]; \
193 fu2.f = B[j]; \
194 if (fu1.u != fu2.u) \
195 abort (); \
196 }
197
198 #define COMPARE_ZMM_BF16(A, B) \
199 for (int j = 0; j < 32; j++) \
200 { \
201 union16bh_uw fu1, fu2; \
202 fu1.bf16 = A[j]; \
203 fu2.bf16 = B[j]; \
204 if (fu1.u != fu2.u) \
205 abort(); \
206 }
207
208 #define COMPARE_ZMM_FP16(A, B) \
209 for (int j = 0; j < 32; j++) \
210 { \
211 union16f_uw fu1, fu2; \
212 fu1.f16 = A[j]; \
213 fu2.f16 = B[j]; \
214 if (fu1.u != fu2.u) \
215 abort(); \
216 }
217
218 /* Mask low 13bits to zero */
219 static float zero_lower_mantissa_bits_fp32 (float x)
220 {
221 union32f_ud tmp;
222 tmp.f = x;
223 tmp.u = tmp.u & 0xffffe000;
224 return tmp.f;
225 }
226
227 /* Handle SNAN */
228 static float silence_snan_fp32 (float x)
229 {
230 union32f_ud tmp;
231 tmp.f = x;
232 if ((((tmp.u & 0x7f800000) >> 23) == 0xff) &&
233 ((tmp.u & 0x007fffff) != 0) &&
234 ((tmp.u & 0x00400000) == 0))
235 tmp.u = tmp.u | 0x00400000;
236 return tmp.f;
237 }
238
239 void init_pair_tile_src (int tmm_num, __tilepair *src, uint8_t *_buffer, int z)
240 {
241 int rows, colsb, start, i, j, t, elements[2];
242 uint16_t *buffer = (uint16_t *) _buffer;
243 uint16_t *ptr = (uint16_t *) src->buf;
244 __tilecfg_u tmp;
245
246 _tile_storeconfig (tmp.a);
247
248 tmm_num &= ~1;
249
250 rows = tmp.s.rows[tmm_num];
251 colsb = tmp.s.colsb[tmm_num];
252 start = tmp.s.start_row;
253
254 zero_pair_tile_src (src);
255
256 for (t = 0; t < 2; t++)
257 elements[t] = tmp.s.colsb[tmm_num + t] / 4;
258
259 src->colsb = (tmp.s.colsb[tmm_num] + tmp.s.colsb[tmm_num + 1]) / 2;
260 src->rows = rows;
261
262 while (start < 2 * rows)
263 {
264 int r = start / 2;
265 int w = start % 2;
266
267 if (start < 2 * rows - z)
268 for (t = 0; t < 2; t++)
269 if (tmp.s.colsb[tmm_num + t] > 0)
270 for (i = 0; i < elements[t]; i++)
271 ptr[t * rows * colsb / 2 + r * elements[t] * 2 + 2 * i + w] =
272 buffer[start * colsb / 2 + t * elements[0] + i];
273 start++;
274 }
275 }
276
277 #endif