Debian Codegraph @

visa/iga/IGALibrary/Frontend/Floats.cpp

raw
1 /*========================== begin_copyright_notice ============================
2
3 Copyright (C) 2017-2021 Intel Corporation
4
5 SPDX-License-Identifier: MIT
6
7 ============================= end_copyright_notice ===========================*/
8
9 #include "Floats.hpp"
10 #include "../strings.hpp"
11
12 #if defined(_MSC_VER)
13 // This only affects defined(_DEBUG) and we could scope this to debug builds
14 // only, but we prefer to keep Debug and Release configs as close as possible,
15 // and formatting/disassembly isn't considered a performance critical path.
16 //
17 // There's a defect in Windows Debug runtime libraries in emitting denorms
18 // when denorm mode is FTZ by the calling library and with the Debug runtime
19 // https://developercommunity.visualstudio.com/content/problem/1187587/printf-assert-failure-unexpected-input-value-log10.html
20 //
21 // To test this place _MM_SET_DENORMALS_ZERO_MODE(_MM_DENORMALS_ZERO_ON);
22 // in before calling into this module (and compiled for Debug on MSVC++).
23 #define IGA_NEEDS_DENORM_WORKAROUND
24 #endif
25
26 // For non-NaN cases this flag enables use of a native AVX instruction
27 // for half float conversion _cvtss_sh.
28 //
29 // #define IGA_USE_FP16C
30
31 #include <cmath>
32 #include <cstdint>
33 #include <cstdlib>
34 #include <iomanip>
35 #include <iostream>
36 #include <limits>
37 #include <sstream>
38 #if defined(IGA_NEEDS_DENORM_WORKAROUND) || defined(IGA_USE_FP16C)
39 #include <immintrin.h>
40 #endif
41
42 using namespace iga;
43
44 // Big Theory Statement (BTS): on NaN's in IGA
45 //
46 // NaN is an equivalence class of floating point values. Different
47 // libraries, runtimes, and hardware use different elements of the
48 // class as the characteristic value when they want a NaN value.
49 // For instance, the sign bit can be anything and the mantissa payload
50 // bits can be anything except for the leading mantissa bit (which
51 // distinguishes a qnan from an snan).
52 //
53 // NOTE: we distinguish between 'qnan' and 'snan' instead of just using 'nan'
54 // with the whole payload since we parse as 64b and only narrow once
55 // we see the type; we considered 'nan' '(' HEXLIT ')' and just letting
56 // the user sort the meaning of the mantissa out, but this gets messy
57 // when widening literals during parse or narrowing them during
58 // formatting.
59 //
60 // Without loss of generality (this generalizes to all IEEE sizes),
61 // the 32-bit IEEE 754 (C.f. section 6.2.1 of the spec) we have
62 // s eeeeeeee mmm`mmmm`mmmm`mmmm`mmmm`mmmm
63 // (where the s and the m's can be anything such that at least one m is
64 // non-zero; all 0's would imply infinity)
65 //
66 // SNAN: "snan" (signaling NaN) is:
67 // s 11111111 0xx`xxxx`xxxx`xxxx`xxxx`xxxx
68 // ^ leading bit of mantissa is 0; at least one bit
69 // of the rest of the payload (x's must be non-zero)
70 //
71 // QNAN: "qnan" (quiet NaN) is:
72 // s 11111111 1xx`xxxx`xxxx`xxxx`xxxx`xxxx
73 // ^ leading bit of mantiss is 1 and the rest can be anything
74 // including all zeros since the top mantissa bit is set
75 // (NaN only needs one mantissa bit set and the quiet bit
76 // counts)
77 //
78 // IGA supports the following syntax (from examples)
79 // NaNLit ::= '-'? ('snan'|'qnan') '(' HEXLIT ')'
80 //
81 // Examples:
82 // * "-snan(0xA):f" parses as
83 // s eeeeeeee mmm`mmmm`mmmm`mmmm`mmmm`mmmm
84 // 1 11111111 000`0000`0000`0000`0000`1010
85 // ^ negative ^ snan ^^^^ 0xA payload
86 // sign bit is respected
87 //
88 // * "qnan(0x1B):f" parses as
89 // s eeeeeeee mmm`mmmm`mmmm`mmmm`mmmm`mmmm
90 // 1 11111111 100`0000`0000`0000`0001`1011
91 // ^ positive ^ qnan ^^^^^^ 0x1B
92 //
93 // * "snan(0x0):f" would be a parse error since the mantissa must not be 0
94 // s eeeeeeee mmm`mmmm`mmmm`mmmm`mmmm`mmmm
95 // 1 11111111 000`0000`0000`0000`0000`0000
96 // ^ positive ^ snan^^^^^^^^^^^^^^^^^^^^^^^ at least one bit must be 0
97 //
98 // * "snan(0x00800000):f" illegal: payload value too large for bitfield
99 // s eeeeeeee mmm`mmmm`mmmm`mmmm`mmmm`mmmm
100 // 1 11111111 000`0000`0000`0000`0000`0000
101 // ^ high bit of payload overflows to here
102 //
103 // * "snan(0x00800000):df" payload fits since df has 53 bits
104 // s eeeeeeeeeee
105 // m`mmmm`mmmm`mmmm`mmmm`mmmm`mmmm`mmmm`mmmm`mmmm`mmmm`mmmm`mmmm 1
106 // 11111111111
107 // 0`0000`0000`0000`0000`0000`0000`1000`0000`0000`0000`0000`0000
108 // ^ 0x800000
109 //
110 // * "snan(0x400000):f" is legal, but just means qnan(0). This is because
111 // the parser parses the value as 64b and then converts it to 32b when it
112 // sees the :f. The payload fits while parsing, but conversion cannot
113 // tell that we were dealing with qnan
114
115 template <typename F> int FloatMantissaBits();
116 template <typename F> int FloatExponentBits();
117 template <> int FloatMantissaBits<double>() { return 52; }
118 template <> int FloatExponentBits<double>() { return 11; }
119 template <> int FloatMantissaBits<float>() { return 23; }
120 template <> int FloatExponentBits<float>() { return 8; }
121 template <> int FloatMantissaBits<uint16_t>() { return 10; }
122 template <> int FloatExponentBits<uint16_t>() { return 5; }
123
124 // E.g. FloatBias<float>() == 127
125 //
126 template <typename F> static int FloatBias() {
127 return (1 << (FloatExponentBits<F>() - 1)) - 1;
128 }
129
130 // FloatBiasDiff<double,float> => 1023 - 127
131 // FloatBiasDiff<float,half> => 127 - 15 (0x70)
132 template <typename FBIG, typename FSML> static int FloatBiasDiff() {
133 return FloatBias<FBIG>() - FloatBias<FSML>();
134 }
135 template <typename FBIG, typename FSML> static int FloatMantissaBitsDiff() {
136 return FloatMantissaBits<FBIG>() - FloatMantissaBits<FSML>();
137 }
138
139 template <typename F, typename I>
140 static void FormatFloatImplNaN(std::ostream &os, I bits) {
141 const int MANT_LEN = FloatMantissaBits<F>();
142 const int EXPN_LEN = FloatExponentBits<F>();
143 const I SIGN_BIT = ((I)1 << (MANT_LEN + EXPN_LEN));
144 if (bits & SIGN_BIT) {
145 os << '-';
146 }
147 // split the payload into the quiet/signaling bit
148 // (high bit of the mantissa) and the lower bits
149 const I QNAN_BIT = (I)1 << (MANT_LEN - 1);
150 if (bits & QNAN_BIT) {
151 os << "qnan";
152 } else {
153 os << "snan";
154 }
155 os << "(";
156 I lowerPayload = bits & (QNAN_BIT - 1); // lower bits of mantissa
157 fmtHex(os, (uint64_t)lowerPayload);
158 os << ")";
159 }
160
161 // Checks if a floating point number will reparse exactly as formatted.
162 //
163 // We parse floats as 64b literals and cast down. So we use strtod for all
164 // floating point types and cast down.
165 template <typename T> static bool willReparseExactly(T x, std::string str) {
166 // we always parse as a double since the parser will do the same
167 double y = strtod(str.c_str(), nullptr);
168 return ((T)y == x);
169 }
170
171 #ifdef IGA_NEEDS_DENORM_WORKAROUND
172 // C.f. with the #define for this above for notes on this
173 struct ScopedDenormWorkaround {
174 unsigned int oldDenormMode;
175 ScopedDenormWorkaround() : oldDenormMode(_MM_GET_DENORMALS_ZERO_MODE()) {
176 _MM_SET_DENORMALS_ZERO_MODE(_MM_DENORMALS_ZERO_OFF);
177 }
178 ~ScopedDenormWorkaround() { _MM_SET_DENORMALS_ZERO_MODE(oldDenormMode); }
179 };
180 #endif
181
182 // Formats a float in decimal or scientific format if it will not lose
183 // precision during reparse
184 //
185 // Returns false if we were able to format it as something representable.
186 template <typename F, typename I>
187 static bool TryFormatFloatImplNonHex(std::ostream &os, F x) {
188 #ifdef IGA_NEEDS_DENORM_WORKAROUND
189 ScopedDenormWorkaround sdw;
190 #endif
191 auto fpc = std::fpclassify(x);
192 switch (fpc) {
193 case FP_INFINITE:
194 if (x < 0) {
195 os << '-';
196 }
197 os << "inf";
198 return true;
199 case FP_NAN:
200 FormatFloatImplNaN<F, I>(os, iga::FloatToBits(x));
201 return true;
202 ///////////////////////////////////////////////////////////////////////////
203 // case FP_ZERO:
204 // case FP_NORMAL:
205 // case FP_SUBNORMAL:
206 default:
207 // Fall through and go through the pretty-print algorithm
208 break;
209 }
210
211 // we try and render it several different ways until we find something
212 // that we can parse back bit-exact.
213 // We try:
214 // - decimal
215 // - exponential
216 // - and then fall back on hex
217 // static_assert(sizeof(F) == sizeof(I));
218
219 // try as default, this lets STL pick the format.
220 // it sometimes gives nice terse output
221 // e.g. "3" for "3.0" instead of "3.0000000000..." (when possible)
222 std::stringstream ss;
223 ss.unsetf(std::ios_base::floatfield);
224 ss << x;
225 if (willReparseExactly(x, ss.str())) {
226 auto str = ss.str();
227 os << str;
228 if (str.find('.') == std::string::npos &&
229 str.find('e') == std::string::npos &&
230 str.find('E') == std::string::npos) {
231 // floats need a ".0" suffixing them if not in scientific form
232 // STL default float sometimes drops the .
233 //
234 // e.g. given "-0.0f", if we were to format that
235 // as "-0:f" (which MSVCRT does), then it parses as
236 // "0" since this is the negation the S64 value 0 (during parse)
237 //
238 // NOTE: we also have to ensure that we aren't using an
239 // exponential (scientific) form already.
240 //
241 // e.g. 1e-007 should not convert to 1e-007.0
242 os << ".0";
243 }
244 return true;
245 }
246
247 // try as scientific
248 ss.str(std::string()); // reset
249 ss << std::scientific << x;
250 if (willReparseExactly(x, ss.str())) {
251 os << ss.str();
252 return true;
253 }
254
255 // TODO: IFDEF this given a new enough compiler
256 // (must parse too)
257 // e.g. 0x1.47ae147ae147bp-7 (need to parse first)
258 // NOTE: parsing should use >>
259 // float f;
260 // e.g. std::istringstream(""0x1P-1022") >> std::hexfloat >> f;
261
262 // fallback to hex integral
263 // FormatFloatImplHex<F,I>(os, x);
264 return false;
265 }
266
267 template <typename F> static void FormatFloatAsHex(std::ostream &os, F f) {
268 fmtHex(os, (uint64_t)iga::FloatToBits(f));
269 }
270
271 void iga::FormatFloat(std::ostream &os, float x) {
272 if (!TryFormatFloatImplNonHex<float, uint32_t>(os, x)) {
273 FormatFloatAsHex<float>(os, x);
274 }
275 }
276
277 void iga::FormatFloat(std::ostream &os, double x) {
278 if (!TryFormatFloatImplNonHex<double, uint64_t>(os, x)) {
279 FormatFloatAsHex<double>(os, x);
280 }
281 }
282
283 void iga::FormatFloat(std::ostream &os, uint16_t w16) {
284 #if 0
285 // this would turn off all non-hex floats
286 fmtHex(os, (uint64_t)w16);
287 #else
288 // trys to format a half float in a friendly format
289 // falling back to hex if all else fails
290 float f32 = ConvertHalfToFloat(w16);
291 if (IS_NAN(f32)) {
292 // So we get the correct payload size for NaNs
293 FormatFloatImplNaN<uint16_t, uint16_t>(os, w16);
294 } else if (ConvertFloatToHalf(f32) != w16 ||
295 !TryFormatFloatImplNonHex<float, uint32_t>(os, f32)) {
296 FormatFloatAsHex<uint16_t>(os, w16);
297 } // else: FormatFloatImplNonHex worked
298 #endif
299 }
300
301 void iga::FormatFloat(std::ostream &os, uint8_t x) {
302 FormatFloat(os, ConvertQuarterToFloatGEN(x));
303 }
304
305 uint32_t iga::ConvertDoubleToFloatBits(double f) {
306 uint64_t f64 = FloatToBits(f);
307
308 uint64_t m64 = f64 & F64_MANT_MASK;
309 uint64_t e64 = (f64 & F64_EXP_MASK) >> FloatMantissaBits<double>();
310 if (e64 == (F64_EXP_MASK >> FloatMantissaBits<double>()) && m64 != 0) {
311 // f64 NaN
312 uint32_t m32 = (uint32_t)m64 & F32_MANT_MASK;
313 m32 |= (uint32_t)((m64 & F64_QNAN_BIT) >>
314 (FloatMantissaBits<double>() -
315 FloatMantissaBits<float>())); // preserve snan
316 if (m32 == 0) {
317 // The payload was only in the high bits which we dropped;
318 // make it non-zero so we retain NaN'ness
319 m32 = 1;
320 }
321 uint32_t s32 = (uint32_t)(f64 >> 32) & F32_SIGN_BIT;
322 return (s32 | F32_EXP_MASK | m32);
323 } else {
324 // regular conversion can deal with all the other special cases
325 return FloatToBits((float)f);
326 }
327 }
328
329 float iga::ConvertDoubleToFloat(double f) {
330 return FloatFromBits(ConvertDoubleToFloatBits(f));
331 }
332
333 uint16_t iga::ConvertFloatToHalf(float f) {
334 static const uint32_t F32_EXP_MASK = ((1 << FloatExponentBits<float>()) - 1)
335 << FloatMantissaBits<float>();
336
337 const uint32_t w32 = FloatToBits(f);
338 const uint32_t w32_u = w32 & 0x7FFFFFFF;
339 const uint32_t sign = w32 & 0x80000000;
340 const uint16_t sign16 = (uint16_t)(sign >> 16);
341
342 if (w32_u > F32_EXP_MASK) { // NaN
343 uint16_t m16 = 0;
344 m16 |= (F32_QNAN_BIT & w32_u) >> // preserve qnan bit
345 (FloatMantissaBits<float>() - FloatMantissaBits<uint16_t>());
346 m16 |= (F16_MANT_MASK >> 1) & w32_u; // and bottom 9b
347 //
348 // s eeeeeeee qmmmmmmmmmmmmmmmmmmmmm
349 // | |||||||||
350 // | vvvvvvvvv
351 // +---------->qmmmmmmmmm
352 if (m16 == 0x0) {
353 // if the nonzero payload is in the high bits and and gets
354 // dropped and the signal bit is non-zero, then m16 is 0;
355 // to maintain it as a qnan we must set at least one bit
356 m16 = 0x1;
357 }
358 return sign16 | F16_EXP_MASK | m16;
359 } else if (w32_u == F32_EXP_MASK) { // +/-Infinity
360 return sign16 | F16_EXP_MASK;
361 } else {
362 // norm/denorm
363 #ifdef IGA_USE_FP16C
364 // should be on all 3rd generation or newer CPUs (HSW/SNB)
365 // _MM_FROUND_NO_EXC
366 const auto oldCsr = _mm_getcsr();
367 _mm_setcsr(oldCsr | _MM_FROUND_NO_EXC);
368 uint16_t w16 = (uint16_t)_mm_cvtsi128_si32(
369 _mm_cvtps_ph(_mm_set_ss(f), _MM_FROUND_TO_NEAREST_INT));
370 _mm_setcsr(oldCsr);
371 return w16;
372 #else // !IGA_USE_FP16C
373 static const uint32_t F32_BIAS = FloatBias<float>();
374 static const uint32_t F16_BIAS = FloatBias<uint16_t>();
375 static const int F32_MNT_BITS = FloatMantissaBits<float>();
376 static const int F16_MNT_BITS = FloatMantissaBits<uint16_t>();
377
378 static const uint32_t LOWEST_OVERFLOW = // 0x47800000
379 (F32_BIAS + F16_BIAS + 1) << FloatMantissaBits<float>();
380 static const uint32_t LOWEST_NORM = // 0x38800000
381 (F32_BIAS - F16_BIAS + 1) << FloatMantissaBits<float>();
382 static const uint32_t LOWEST_DENORM = // 0x33000000
383 (F32_BIAS - F16_BIAS - (uint16_t)F16_MNT_BITS)
384 << FloatMantissaBits<float>();
385 auto round = [](uint32_t v, uint32_t g, uint32_t s) {
386 return v + (g & (s | v));
387 };
388 if (w32_u >= LOWEST_OVERFLOW) { // overflows to infinity
389 return sign16 | F16_EXP_MASK;
390 } else if (w32_u >= LOWEST_NORM) { // fits as normalized half
391 uint32_t v =
392 (((w32_u >> F32_MNT_BITS) - (F32_BIAS - F16_BIAS)) << F16_MNT_BITS) |
393 ((w32_u >> (F32_MNT_BITS - F16_MNT_BITS)) & F16_MANT_MASK);
394 uint32_t g = (w32_u >> (F32_MNT_BITS - F16_MNT_BITS - 1)) & 0x1;
395 uint32_t s = (w32_u & 0x0FFF) ? 1 : 0;
396 return (uint16_t)round(sign16 | v, g, s);
397 } else if (w32_u >= LOWEST_DENORM) { // fits as normalized half
398 uint32_t i = (F32_BIAS - 1 - 1) - (w32_u >> F32_MNT_BITS);
399 uint32_t w32_u2 = (w32_u & F32_MANT_MASK) | (F32_MANT_MASK + 1);
400 uint32_t v = sign16 | (w32_u2 >> (i + 1));
401 uint32_t g = (w32_u2 >> i) & 1;
402 uint32_t s = (w32_u2 & ((1 << i) - 1)) ? 1 : 0;
403 return (uint16_t)round(v, g, s);
404 } else {
405 // underflow to +-0
406 return sign16;
407 }
408 #endif // !IGA_USE_FP16C
409 }
410 }
411
412 // GEN's 8-bit restricted float ("quarter float")
413 // s eee mmmm (bias 2^(3-1) - 1 == 3)
414 // s gfe dcba
415 // mantissa shifted to the top of the float
416 // dcb`a000`0000`0000`0000`0000
417 // exponenent is unpacked by expanding
418 // gfe to the following:
419 //
420 // dgGGGGfe where G = ~g
421 // negative and positive 0 bypass this logic
422 //
423 // => NaN and infinities are illegal in this format
424 // => Denorms are not supported
425 // QUOTE:
426 // Specifically, when the exponent field is zero and the fraction
427 // field is not zero, an implied one is still present instead of
428 // taking a denormalized form (without an implied one). This results
429 // in a simple implementation but with a smaller dynamic range -
430 // the magnitude of the smallest non-zero number is 0.1328125.
431 float iga::ConvertQuarterToFloatGEN(uint8_t u8) {
432 if (u8 == 0x00) {
433 return 0.0f;
434 } else if (u8 == 0x80) {
435 return -0.0f;
436 } else {
437 uint32_t f32;
438 f32 = ((uint32_t)u8 & 0x80) << (32 - 8); // d: sign
439 f32 |= ((0x30 & (uint32_t)u8) << (23 - 4)); // fe: low bits of exp
440 if ((0x40 & u8) == 0) { // g = 0
441 // exp=011111fe
442 f32 |= 0x1F << (23 + 2);
443 } else {
444 // exp=100000fe
445 f32 |= 1ul << (23 + 7); // f32 high exp bit
446 }
447 f32 |= ((uint32_t)u8 & 0xF) << (23 - 4); // dcba: mantissa
448 return FloatFromBits(f32);
449 }
450 }
451
452 bool iga::IsNaN(uint16_t u16) {
453 return (F16_EXP_MASK & u16) == F16_EXP_MASK && (F16_MANT_MASK & u16) != 0;
454 }
455
456 bool iga::IsInf(uint16_t u16) {
457 return (F16_EXP_MASK & u16) == F16_EXP_MASK && (F16_MANT_MASK & u16) == 0;
458 }
459
460 float iga::ConvertHalfToFloat(uint16_t u16) {
461 uint16_t u16_u = u16 & 0x7FFF;
462 uint32_t s32 = ((uint32_t)u16 & F16_SIGN_BIT) << 16;
463 uint32_t m16 = u16 & F16_MANT_MASK;
464 if (u16_u > F16_EXP_MASK) {
465 // NaN
466 uint32_t m32 =
467 (u16 & F16_QNAN_BIT)
468 << FloatMantissaBitsDiff<float, uint16_t>(); // preserve sNaN bit
469 m32 |= (F16_MANT_MASK >> 1) & m16;
470 if (m32 == 0) {
471 m32 = 1; // ensure still NaN
472 }
473 return FloatFromBits(s32 | F32_EXP_MASK | m32);
474 }
475 #ifdef IGA_USE_FP16C
476 float f = _mm_cvtss_f32(_mm_cvtph_ps(_mm_cvtsi32_si128(u16)));
477 return f;
478 #else // !IGA_USE_FP16C
479 uint32_t e16 = (u16 & F16_EXP_MASK) >> FloatMantissaBits<uint16_t>();
480 uint32_t e32, m32;
481 if (u16_u == F16_EXP_MASK) {
482 // +-infinity
483 e32 = F32_EXP_MASK >> FloatMantissaBits<float>();
484 m32 = 0;
485 } else if (e16 != 0 && e16 < 0x1F) {
486 // normal number
487 e32 = e16 + FloatBiasDiff<float, uint16_t>(); // (127 - 15); // 0x70
488 m32 = m16 << FloatMantissaBitsDiff<float, uint16_t>(); // (23 - 10);
489 } else if (e16 == 0 && m16 != 0) {
490 // denorm/subnorm number (e16 == 0) => renormalize it
491 // shift the mantissa left until the hidden one gets set
492 for (e32 = FloatBiasDiff<float, uint16_t>() + 1;
493 (m16 & (F16_MANT_MASK + 1)) == 0; m16 <<= 1, e32--)
494 ;
495 m32 = (m16 << FloatMantissaBitsDiff<float, uint16_t>()) & F32_MANT_MASK;
496 } else { // if (e16 == 0) // +/- 0.0
497 e32 = 0;
498 m32 = 0;
499 }
500 return FloatFromBits(s32 | (e32 << FloatMantissaBits<float>()) | m32);
501 #endif // !IGA_USE_FP16C
502 }
503
504 double iga::ConvertFloatToDouble(float f) {
505 if (IS_NAN(f)) {
506 uint32_t f32 = FloatToBits(f);
507
508 uint64_t m64;
509 m64 = (uint64_t)(f32 & F32_QNAN_BIT)
510 << (FloatMantissaBits<double>() -
511 FloatMantissaBits<float>()); // keep the sNaN bit
512 m64 |= (F32_MANT_MASK >> 1) & f32; // keep the non sNaN part
513 // lower part of the payload
514 uint64_t bits = (((uint64_t)f32 & F32_SIGN_BIT) << 32) | // sign
515 F64_EXP_MASK | // exp
516 m64; // new mantissa
517 return FloatFromBits(bits);
518 } else {
519 // not NaN: use default value
520 return (double)f;
521 }
522 }
523
524
525 bool iga::ParseFLTLIT(const std::string &syntax, double &value) {
526 char *end = nullptr;
527 value = std::strtod(syntax.c_str(), &end);
528 if (*end) {
529 return false;
530 }
531 return true;
532 }