| 1 |
/* origin: FreeBSD /usr/src/lib/msun/src/s_fmal.c */ |
| 2 |
/*- |
| 3 |
* Copyright (c) 2005-2011 David Schultz <[email protected]> |
| 4 |
* All rights reserved. |
| 5 |
* |
| 6 |
* Redistribution and use in source and binary forms, with or without |
| 7 |
* modification, are permitted provided that the following conditions |
| 8 |
* are met: |
| 9 |
* 1. Redistributions of source code must retain the above copyright |
| 10 |
* notice, this list of conditions and the following disclaimer. |
| 11 |
* 2. Redistributions in binary form must reproduce the above copyright |
| 12 |
* notice, this list of conditions and the following disclaimer in the |
| 13 |
* documentation and/or other materials provided with the distribution. |
| 14 |
* |
| 15 |
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND |
| 16 |
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
| 17 |
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
| 18 |
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE |
| 19 |
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL |
| 20 |
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS |
| 21 |
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) |
| 22 |
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
| 23 |
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY |
| 24 |
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF |
| 25 |
* SUCH DAMAGE. |
| 26 |
*/ |
| 27 |
|
| 28 |
|
| 29 |
#include "libm.h" |
| 30 |
#if LDBL_MANT_DIG == 53 && LDBL_MAX_EXP == 1024 |
| 31 |
long double fmal(long double x, long double y, long double z) |
| 32 |
{ |
| 33 |
return fma(x, y, z); |
| 34 |
} |
| 35 |
#elif (LDBL_MANT_DIG == 64 || LDBL_MANT_DIG == 113) && LDBL_MAX_EXP == 16384 |
| 36 |
#include <fenv.h> |
| 37 |
#if LDBL_MANT_DIG == 64 |
| 38 |
#define LASTBIT(u) (u.i.m & 1) |
| 39 |
#define SPLIT (0x1p32L + 1) |
| 40 |
#elif LDBL_MANT_DIG == 113 |
| 41 |
#define LASTBIT(u) (u.i.lo & 1) |
| 42 |
#define SPLIT (0x1p57L + 1) |
| 43 |
#endif |
| 44 |
|
| 45 |
/* |
| 46 |
* A struct dd represents a floating-point number with twice the precision |
| 47 |
* of a long double. We maintain the invariant that "hi" stores the high-order |
| 48 |
* bits of the result. |
| 49 |
*/ |
| 50 |
struct dd { |
| 51 |
long double hi; |
| 52 |
long double lo; |
| 53 |
}; |
| 54 |
|
| 55 |
/* |
| 56 |
* Compute a+b exactly, returning the exact result in a struct dd. We assume |
| 57 |
* that both a and b are finite, but make no assumptions about their relative |
| 58 |
* magnitudes. |
| 59 |
*/ |
| 60 |
static inline struct dd dd_add(long double a, long double b) |
| 61 |
{ |
| 62 |
struct dd ret; |
| 63 |
long double s; |
| 64 |
|
| 65 |
ret.hi = a + b; |
| 66 |
s = ret.hi - a; |
| 67 |
ret.lo = (a - (ret.hi - s)) + (b - s); |
| 68 |
return (ret); |
| 69 |
} |
| 70 |
|
| 71 |
/* |
| 72 |
* Compute a+b, with a small tweak: The least significant bit of the |
| 73 |
* result is adjusted into a sticky bit summarizing all the bits that |
| 74 |
* were lost to rounding. This adjustment negates the effects of double |
| 75 |
* rounding when the result is added to another number with a higher |
| 76 |
* exponent. For an explanation of round and sticky bits, see any reference |
| 77 |
* on FPU design, e.g., |
| 78 |
* |
| 79 |
* J. Coonen. An Implementation Guide to a Proposed Standard for |
| 80 |
* Floating-Point Arithmetic. Computer, vol. 13, no. 1, Jan 1980. |
| 81 |
*/ |
| 82 |
static inline long double add_adjusted(long double a, long double b) |
| 83 |
{ |
| 84 |
struct dd sum; |
| 85 |
union ldshape u; |
| 86 |
|
| 87 |
sum = dd_add(a, b); |
| 88 |
if (sum.lo != 0) { |
| 89 |
u.f = sum.hi; |
| 90 |
if (!LASTBIT(u)) |
| 91 |
sum.hi = nextafterl(sum.hi, INFINITY * sum.lo); |
| 92 |
} |
| 93 |
return (sum.hi); |
| 94 |
} |
| 95 |
|
| 96 |
/* |
| 97 |
* Compute ldexp(a+b, scale) with a single rounding error. It is assumed |
| 98 |
* that the result will be subnormal, and care is taken to ensure that |
| 99 |
* double rounding does not occur. |
| 100 |
*/ |
| 101 |
static inline long double add_and_denormalize(long double a, long double b, int scale) |
| 102 |
{ |
| 103 |
struct dd sum; |
| 104 |
int bits_lost; |
| 105 |
union ldshape u; |
| 106 |
|
| 107 |
sum = dd_add(a, b); |
| 108 |
|
| 109 |
/* |
| 110 |
* If we are losing at least two bits of accuracy to denormalization, |
| 111 |
* then the first lost bit becomes a round bit, and we adjust the |
| 112 |
* lowest bit of sum.hi to make it a sticky bit summarizing all the |
| 113 |
* bits in sum.lo. With the sticky bit adjusted, the hardware will |
| 114 |
* break any ties in the correct direction. |
| 115 |
* |
| 116 |
* If we are losing only one bit to denormalization, however, we must |
| 117 |
* break the ties manually. |
| 118 |
*/ |
| 119 |
if (sum.lo != 0) { |
| 120 |
u.f = sum.hi; |
| 121 |
bits_lost = -u.i.se - scale + 1; |
| 122 |
if ((bits_lost != 1) ^ LASTBIT(u)) |
| 123 |
sum.hi = nextafterl(sum.hi, INFINITY * sum.lo); |
| 124 |
} |
| 125 |
return scalbnl(sum.hi, scale); |
| 126 |
} |
| 127 |
|
| 128 |
/* |
| 129 |
* Compute a*b exactly, returning the exact result in a struct dd. We assume |
| 130 |
* that both a and b are normalized, so no underflow or overflow will occur. |
| 131 |
* The current rounding mode must be round-to-nearest. |
| 132 |
*/ |
| 133 |
static inline struct dd dd_mul(long double a, long double b) |
| 134 |
{ |
| 135 |
struct dd ret; |
| 136 |
long double ha, hb, la, lb, p, q; |
| 137 |
|
| 138 |
p = a * SPLIT; |
| 139 |
ha = a - p; |
| 140 |
ha += p; |
| 141 |
la = a - ha; |
| 142 |
|
| 143 |
p = b * SPLIT; |
| 144 |
hb = b - p; |
| 145 |
hb += p; |
| 146 |
lb = b - hb; |
| 147 |
|
| 148 |
p = ha * hb; |
| 149 |
q = ha * lb + la * hb; |
| 150 |
|
| 151 |
ret.hi = p + q; |
| 152 |
ret.lo = p - ret.hi + q + la * lb; |
| 153 |
return (ret); |
| 154 |
} |
| 155 |
|
| 156 |
/* |
| 157 |
* Fused multiply-add: Compute x * y + z with a single rounding error. |
| 158 |
* |
| 159 |
* We use scaling to avoid overflow/underflow, along with the |
| 160 |
* canonical precision-doubling technique adapted from: |
| 161 |
* |
| 162 |
* Dekker, T. A Floating-Point Technique for Extending the |
| 163 |
* Available Precision. Numer. Math. 18, 224-242 (1971). |
| 164 |
*/ |
| 165 |
long double fmal(long double x, long double y, long double z) |
| 166 |
{ |
| 167 |
#pragma STDC FENV_ACCESS ON |
| 168 |
long double xs, ys, zs, adj; |
| 169 |
struct dd xy, r; |
| 170 |
int oround; |
| 171 |
int ex, ey, ez; |
| 172 |
int spread; |
| 173 |
|
| 174 |
/* |
| 175 |
* Handle special cases. The order of operations and the particular |
| 176 |
* return values here are crucial in handling special cases involving |
| 177 |
* infinities, NaNs, overflows, and signed zeroes correctly. |
| 178 |
*/ |
| 179 |
if (!isfinite(x) || !isfinite(y)) |
| 180 |
return (x * y + z); |
| 181 |
if (!isfinite(z)) |
| 182 |
return (z); |
| 183 |
if (x == 0.0 || y == 0.0) |
| 184 |
return (x * y + z); |
| 185 |
if (z == 0.0) |
| 186 |
return (x * y); |
| 187 |
|
| 188 |
xs = frexpl(x, &ex); |
| 189 |
ys = frexpl(y, &ey); |
| 190 |
zs = frexpl(z, &ez); |
| 191 |
oround = fegetround(); |
| 192 |
spread = ex + ey - ez; |
| 193 |
|
| 194 |
/* |
| 195 |
* If x * y and z are many orders of magnitude apart, the scaling |
| 196 |
* will overflow, so we handle these cases specially. Rounding |
| 197 |
* modes other than FE_TONEAREST are painful. |
| 198 |
*/ |
| 199 |
if (spread < -LDBL_MANT_DIG) { |
| 200 |
#ifdef FE_INEXACT |
| 201 |
feraiseexcept(FE_INEXACT); |
| 202 |
#endif |
| 203 |
#ifdef FE_UNDERFLOW |
| 204 |
if (!isnormal(z)) |
| 205 |
feraiseexcept(FE_UNDERFLOW); |
| 206 |
#endif |
| 207 |
switch (oround) { |
| 208 |
default: /* FE_TONEAREST */ |
| 209 |
return (z); |
| 210 |
#ifdef FE_TOWARDZERO |
| 211 |
case FE_TOWARDZERO: |
| 212 |
if (x > 0.0 ^ y < 0.0 ^ z < 0.0) |
| 213 |
return (z); |
| 214 |
else |
| 215 |
return (nextafterl(z, 0)); |
| 216 |
#endif |
| 217 |
#ifdef FE_DOWNWARD |
| 218 |
case FE_DOWNWARD: |
| 219 |
if (x > 0.0 ^ y < 0.0) |
| 220 |
return (z); |
| 221 |
else |
| 222 |
return (nextafterl(z, -INFINITY)); |
| 223 |
#endif |
| 224 |
#ifdef FE_UPWARD |
| 225 |
case FE_UPWARD: |
| 226 |
if (x > 0.0 ^ y < 0.0) |
| 227 |
return (nextafterl(z, INFINITY)); |
| 228 |
else |
| 229 |
return (z); |
| 230 |
#endif |
| 231 |
} |
| 232 |
} |
| 233 |
if (spread <= LDBL_MANT_DIG * 2) |
| 234 |
zs = scalbnl(zs, -spread); |
| 235 |
else |
| 236 |
zs = copysignl(LDBL_MIN, zs); |
| 237 |
|
| 238 |
fesetround(FE_TONEAREST); |
| 239 |
|
| 240 |
/* |
| 241 |
* Basic approach for round-to-nearest: |
| 242 |
* |
| 243 |
* (xy.hi, xy.lo) = x * y (exact) |
| 244 |
* (r.hi, r.lo) = xy.hi + z (exact) |
| 245 |
* adj = xy.lo + r.lo (inexact; low bit is sticky) |
| 246 |
* result = r.hi + adj (correctly rounded) |
| 247 |
*/ |
| 248 |
xy = dd_mul(xs, ys); |
| 249 |
r = dd_add(xy.hi, zs); |
| 250 |
|
| 251 |
spread = ex + ey; |
| 252 |
|
| 253 |
if (r.hi == 0.0) { |
| 254 |
/* |
| 255 |
* When the addends cancel to 0, ensure that the result has |
| 256 |
* the correct sign. |
| 257 |
*/ |
| 258 |
fesetround(oround); |
| 259 |
#ifdef __wasilibc_unmodified_upstream // WASI doesn't need old GCC workarounds |
| 260 |
volatile long double vzs = zs; /* XXX gcc CSE bug workaround */ |
| 261 |
#else |
| 262 |
long double vzs = zs; |
| 263 |
#endif |
| 264 |
return xy.hi + vzs + scalbnl(xy.lo, spread); |
| 265 |
} |
| 266 |
|
| 267 |
if (oround != FE_TONEAREST) { |
| 268 |
/* |
| 269 |
* There is no need to worry about double rounding in directed |
| 270 |
* rounding modes. |
| 271 |
* But underflow may not be raised correctly, example in downward rounding: |
| 272 |
* fmal(0x1.0000000001p-16000L, 0x1.0000000001p-400L, -0x1p-16440L) |
| 273 |
*/ |
| 274 |
long double ret; |
| 275 |
#if defined(FE_INEXACT) && defined(FE_UNDERFLOW) |
| 276 |
int e = fetestexcept(FE_INEXACT); |
| 277 |
feclearexcept(FE_INEXACT); |
| 278 |
#endif |
| 279 |
fesetround(oround); |
| 280 |
adj = r.lo + xy.lo; |
| 281 |
ret = scalbnl(r.hi + adj, spread); |
| 282 |
#if defined(FE_INEXACT) && defined(FE_UNDERFLOW) |
| 283 |
if (ilogbl(ret) < -16382 && fetestexcept(FE_INEXACT)) |
| 284 |
feraiseexcept(FE_UNDERFLOW); |
| 285 |
else if (e) |
| 286 |
feraiseexcept(FE_INEXACT); |
| 287 |
#endif |
| 288 |
return ret; |
| 289 |
} |
| 290 |
|
| 291 |
adj = add_adjusted(r.lo, xy.lo); |
| 292 |
if (spread + ilogbl(r.hi) > -16383) |
| 293 |
return scalbnl(r.hi + adj, spread); |
| 294 |
else |
| 295 |
return add_and_denormalize(r.hi, adj, spread); |
| 296 |
} |
| 297 |
#endif |