Debian Codegraph @

.cargo/vendor/pxfm/src/double_double.rs

raw
1 /*
2 * // Copyright (c) Radzivon Bartoshyk 6/2025. All rights reserved.
3 * //
4 * // Redistribution and use in source and binary forms, with or without modification,
5 * // are permitted provided that the following conditions are met:
6 * //
7 * // 1. Redistributions of source code must retain the above copyright notice, this
8 * // list of conditions and the following disclaimer.
9 * //
10 * // 2. Redistributions in binary form must reproduce the above copyright notice,
11 * // this list of conditions and the following disclaimer in the documentation
12 * // and/or other materials provided with the distribution.
13 * //
14 * // 3. Neither the name of the copyright holder nor the names of its
15 * // contributors may be used to endorse or promote products derived from
16 * // this software without specific prior written permission.
17 * //
18 * // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19 * // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21 * // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
22 * // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
24 * // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
25 * // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
26 * // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 */
29 use crate::bits::get_exponent_f64;
30 #[allow(unused_imports)]
31 use crate::common::*;
32 use std::ops::{Mul, Neg};
33 // https://hal.science/hal-01351529v3/document
34
35 #[derive(Copy, Clone, Default, Debug)]
36 pub(crate) struct DoubleDouble {
37 pub(crate) lo: f64,
38 pub(crate) hi: f64,
39 }
40
41 impl Neg for DoubleDouble {
42 type Output = Self;
43
44 #[inline]
45 fn neg(self) -> Self::Output {
46 Self {
47 hi: -self.hi,
48 lo: -self.lo,
49 }
50 }
51 }
52
53 impl DoubleDouble {
54 #[inline]
55 pub(crate) const fn from_bit_pair(pair: (u64, u64)) -> Self {
56 Self {
57 lo: f64::from_bits(pair.0),
58 hi: f64::from_bits(pair.1),
59 }
60 }
61
62 #[inline]
63 pub(crate) const fn new(lo: f64, hi: f64) -> Self {
64 DoubleDouble { lo, hi }
65 }
66
67 // Non FMA helper
68 #[allow(dead_code)]
69 #[inline]
70 pub(crate) const fn split(a: f64) -> DoubleDouble {
71 // CN = 2^N.
72 const CN: f64 = (1 << 27) as f64;
73 const C: f64 = CN + 1.0;
74 let t1 = C * a;
75 let t2 = a - t1;
76 let r_hi = t1 + t2;
77 let r_lo = a - r_hi;
78 DoubleDouble::new(r_lo, r_hi)
79 }
80
81 // Non FMA helper
82 #[allow(dead_code)]
83 #[inline]
84 fn from_exact_mult_impl_non_fma(asz: DoubleDouble, a: f64, b: f64) -> Self {
85 let bs = DoubleDouble::split(b);
86
87 let r_hi = a * b;
88 let t1 = asz.hi * bs.hi - r_hi;
89 let t2 = asz.hi * bs.lo + t1;
90 let t3 = asz.lo * bs.hi + t2;
91 let r_lo = asz.lo * bs.lo + t3;
92 DoubleDouble::new(r_lo, r_hi)
93 }
94
95 // valid only for |a| > b
96 #[inline]
97 pub(crate) const fn from_exact_add(a: f64, b: f64) -> DoubleDouble {
98 let r_hi = a + b;
99 let t = r_hi - a;
100 let r_lo = b - t;
101 DoubleDouble::new(r_lo, r_hi)
102 }
103
104 // valid only for |a| > b
105 #[inline]
106 pub(crate) const fn from_exact_sub(a: f64, b: f64) -> DoubleDouble {
107 let r_hi = a - b;
108 let t = a - r_hi;
109 let r_lo = t - b;
110 DoubleDouble::new(r_lo, r_hi)
111 }
112
113 #[inline]
114 pub(crate) const fn from_full_exact_add(a: f64, b: f64) -> DoubleDouble {
115 let r_hi = a + b;
116 let t1 = r_hi - a;
117 let t2 = r_hi - t1;
118 let t3 = b - t1;
119 let t4 = a - t2;
120 let r_lo = t3 + t4;
121 DoubleDouble::new(r_lo, r_hi)
122 }
123
124 #[allow(unused)]
125 #[inline]
126 pub(crate) fn dd_f64_mul_add(a: f64, b: f64, c: f64) -> f64 {
127 let ddx2 = DoubleDouble::from_exact_mult(a, b);
128 let zv = DoubleDouble::full_add_f64(ddx2, c);
129 zv.to_f64()
130 }
131
132 #[inline]
133 pub(crate) const fn from_full_exact_sub(a: f64, b: f64) -> Self {
134 let r_hi = a - b;
135 let t1 = r_hi - a;
136 let t2 = r_hi - t1;
137 let t3 = -b - t1;
138 let t4 = a - t2;
139 let r_lo = t3 + t4;
140 DoubleDouble::new(r_lo, r_hi)
141 }
142
143 #[inline]
144 pub(crate) fn add(a: DoubleDouble, b: DoubleDouble) -> DoubleDouble {
145 let s = a.hi + b.hi;
146 let d = s - a.hi;
147 let l = ((b.hi - d) + (a.hi + (d - s))) + (a.lo + b.lo);
148 DoubleDouble::new(l, s)
149 }
150
151 #[inline]
152 pub(crate) fn quick_dd_add(a: DoubleDouble, b: DoubleDouble) -> DoubleDouble {
153 let DoubleDouble { hi: sh, lo: sl } = DoubleDouble::from_full_exact_add(a.hi, b.hi);
154 let v = a.lo + b.lo;
155 let w = sl + v;
156 DoubleDouble::from_exact_add(sh, w)
157 }
158
159 #[inline]
160 pub(crate) fn quick_dd_sub(a: DoubleDouble, b: DoubleDouble) -> DoubleDouble {
161 let DoubleDouble { hi: sh, lo: sl } = DoubleDouble::from_full_exact_sub(a.hi, b.hi);
162 let v = a.lo - b.lo;
163 let w = sl + v;
164 DoubleDouble::from_exact_add(sh, w)
165 }
166
167 #[inline]
168 pub(crate) fn full_dd_add(a: DoubleDouble, b: DoubleDouble) -> DoubleDouble {
169 let DoubleDouble { hi: sh, lo: sl } = DoubleDouble::from_full_exact_add(a.hi, b.hi);
170 let DoubleDouble { hi: th, lo: tl } = DoubleDouble::from_full_exact_add(a.lo, b.lo);
171 let c = sl + th;
172 let v = DoubleDouble::from_exact_add(sh, c);
173 let w = tl + v.lo;
174 DoubleDouble::from_exact_add(v.hi, w)
175 }
176
177 #[inline]
178 pub(crate) fn full_dd_sub(a: DoubleDouble, b: DoubleDouble) -> DoubleDouble {
179 DoubleDouble::full_dd_add(a, -b)
180 }
181
182 #[inline]
183 pub(crate) fn sub(a: DoubleDouble, b: DoubleDouble) -> DoubleDouble {
184 let s = a.hi - b.hi;
185 let d = s - a.hi;
186 let l = ((-b.hi - d) + (a.hi + (d - s))) + (a.lo - b.lo);
187 DoubleDouble::new(l, s)
188 }
189
190 /// DoubleDouble-style square root for a double-double number
191 #[inline]
192 pub(crate) fn sqrt(self) -> DoubleDouble {
193 let a = self.hi + self.lo;
194
195 if a == 0.0 {
196 return DoubleDouble { hi: 0.0, lo: 0.0 };
197 }
198 if a < 0.0 || a.is_nan() {
199 return DoubleDouble {
200 hi: f64::NAN,
201 lo: 0.0,
202 };
203 }
204 if a.is_infinite() {
205 return DoubleDouble {
206 hi: f64::INFINITY,
207 lo: 0.0,
208 };
209 }
210
211 let x = a.sqrt();
212
213 let x2 = DoubleDouble::from_exact_mult(x, x);
214
215 // Residual = self - x²
216 let mut r = self.hi - x2.hi;
217 r += self.lo;
218 r -= x2.lo;
219
220 let dx = r / (2.0 * x);
221 let hi = x + dx;
222 let lo = (x - hi) + dx;
223
224 DoubleDouble { hi, lo }
225 }
226
227 /// DoubleDouble-style square root for a double-double number
228 #[inline]
229 pub(crate) fn fast_sqrt(self) -> DoubleDouble {
230 let a = self.hi + self.lo;
231 let x = a.sqrt();
232
233 let x2 = DoubleDouble::from_exact_mult(x, x);
234
235 // Residual = self - x²
236 let mut r = self.hi - x2.hi;
237 r += self.lo;
238 r -= x2.lo;
239
240 let dx = r / (2.0 * x);
241 let hi = x + dx;
242 let lo = (x - hi) + dx;
243
244 DoubleDouble { hi, lo }
245 }
246
247 /// `a*b+c`
248 ///
249 /// *Accurate dot product (Ogita, Rump and Oishi 2004)*
250 #[inline]
251 pub(crate) fn mul_add_f64(a: DoubleDouble, b: DoubleDouble, c: f64) -> DoubleDouble {
252 let DoubleDouble { hi: h, lo: r } = DoubleDouble::quick_mult(a, b);
253 let DoubleDouble { hi: p, lo: q } = DoubleDouble::from_full_exact_add(c, h);
254 DoubleDouble::new(r + q, p)
255 }
256
257 /// `a*b+c`
258 ///
259 /// *Accurate dot product (Ogita, Rump and Oishi 2004)*
260 #[inline(always)]
261 #[allow(unused)]
262 pub(crate) fn mul_add_f64_fma(a: DoubleDouble, b: DoubleDouble, c: f64) -> DoubleDouble {
263 let DoubleDouble { hi: h, lo: r } = DoubleDouble::quick_mult_fma(a, b);
264 let DoubleDouble { hi: p, lo: q } = DoubleDouble::from_full_exact_add(c, h);
265 DoubleDouble::new(r + q, p)
266 }
267
268 /// `a*b+c`
269 ///
270 /// *Accurate dot product (Ogita, Rump and Oishi 2004)*
271 #[inline]
272 pub(crate) fn quick_mul_add_f64(a: DoubleDouble, b: DoubleDouble, c: f64) -> DoubleDouble {
273 let DoubleDouble { hi: h, lo: r } = DoubleDouble::quick_mult(a, b);
274 let DoubleDouble { hi: p, lo: q } = DoubleDouble::from_exact_add(c, h);
275 DoubleDouble::new(r + q, p)
276 }
277
278 /// `a*b+c`
279 ///
280 /// *Accurate dot product (Ogita, Rump and Oishi 2004)*
281 #[inline]
282 pub(crate) fn mul_f64_add_f64(a: DoubleDouble, b: f64, c: f64) -> DoubleDouble {
283 let DoubleDouble { hi: h, lo: r } = DoubleDouble::quick_mult_f64(a, b);
284 let DoubleDouble { hi: p, lo: q } = DoubleDouble::from_full_exact_add(c, h);
285 DoubleDouble::new(r + q, p)
286 }
287
288 // /// Accurate reciprocal: 1 / self
289 // #[inline]
290 // pub(crate) fn recip_raphson(self) -> DoubleDouble {
291 // let y0 = DoubleDouble::recip(self);
292 // let z = DoubleDouble::mul_add_f64(-self, y0, 1.0);
293 // DoubleDouble::mul_add(y0, z, y0)
294 // }
295
296 /// Accurate reciprocal: 1 / self
297 #[inline]
298 pub(crate) fn recip(self) -> DoubleDouble {
299 #[cfg(any(
300 all(
301 any(target_arch = "x86", target_arch = "x86_64"),
302 target_feature = "fma"
303 ),
304 target_arch = "aarch64"
305 ))]
306 {
307 let y = 1. / self.hi;
308 let e1 = f_fmla(-self.hi, y, 1.0);
309 let e2 = f_fmla(-self.lo, y, e1);
310 let e = y * e2;
311 DoubleDouble::new(e, y)
312 }
313 #[cfg(not(any(
314 all(
315 any(target_arch = "x86", target_arch = "x86_64"),
316 target_feature = "fma"
317 ),
318 target_arch = "aarch64"
319 )))]
320 {
321 let y = 1.0 / self.hi;
322
323 let DoubleDouble { hi: p1, lo: err1 } = DoubleDouble::from_exact_mult(self.hi, y);
324 let e1 = (1.0 - p1) - err1;
325 let DoubleDouble { hi: p2, lo: err2 } = DoubleDouble::from_exact_mult(self.lo, y);
326 let e2 = (e1 - p2) - err2;
327 let e = y * e2;
328
329 DoubleDouble::new(e, y)
330 }
331 }
332
333 #[inline]
334 pub(crate) fn from_recip(b: f64) -> Self {
335 #[cfg(any(
336 all(
337 any(target_arch = "x86", target_arch = "x86_64"),
338 target_feature = "fma"
339 ),
340 target_arch = "aarch64"
341 ))]
342 {
343 let x_hi = 1.0 / b;
344 let err = f_fmla(-x_hi, b, 1.0);
345 let x_lo = err / b;
346 Self::new(x_lo, x_hi)
347 }
348 #[cfg(not(any(
349 all(
350 any(target_arch = "x86", target_arch = "x86_64"),
351 target_feature = "fma"
352 ),
353 target_arch = "aarch64"
354 )))]
355 {
356 let x_hi = 1.0 / b;
357 let prod = Self::from_exact_mult(x_hi, b);
358 let err = (1.0 - prod.hi) - prod.lo;
359 let x_lo = err / b;
360 Self::new(x_lo, x_hi)
361 }
362 }
363
364 #[inline(always)]
365 #[allow(unused)]
366 pub(crate) fn from_quick_recip_fma(b: f64) -> Self {
367 let h = 1.0 / b;
368 let hl = f64::mul_add(h, -b, 1.) * h;
369 DoubleDouble::new(hl, h)
370 }
371
372 #[inline]
373 pub(crate) fn from_quick_recip(b: f64) -> Self {
374 #[cfg(any(
375 all(
376 any(target_arch = "x86", target_arch = "x86_64"),
377 target_feature = "fma"
378 ),
379 target_arch = "aarch64"
380 ))]
381 {
382 let h = 1.0 / b;
383 let hl = f_fmla(h, -b, 1.) * h;
384 DoubleDouble::new(hl, h)
385 }
386 #[cfg(not(any(
387 all(
388 any(target_arch = "x86", target_arch = "x86_64"),
389 target_feature = "fma"
390 ),
391 target_arch = "aarch64"
392 )))]
393 {
394 let h = 1.0 / b;
395 let pr = DoubleDouble::from_exact_mult(h, b);
396 let err = (1.0 - pr.hi) - pr.lo;
397 let hl = err * h;
398 DoubleDouble::new(hl, h)
399 }
400 }
401
402 #[inline]
403 pub(crate) fn from_exact_div(a: f64, b: f64) -> Self {
404 #[cfg(any(
405 all(
406 any(target_arch = "x86", target_arch = "x86_64"),
407 target_feature = "fma"
408 ),
409 target_arch = "aarch64"
410 ))]
411 {
412 let q_hi = a / b;
413 let r = f_fmla(-q_hi, b, a);
414 let q_lo = r / b;
415 Self::new(q_lo, q_hi)
416 }
417
418 #[cfg(not(any(
419 all(
420 any(target_arch = "x86", target_arch = "x86_64"),
421 target_feature = "fma"
422 ),
423 target_arch = "aarch64"
424 )))]
425 {
426 let q_hi = a / b;
427
428 let p = DoubleDouble::from_exact_mult(q_hi, b);
429 let r = DoubleDouble::from_exact_sub(a, p.hi);
430 let r = r.hi + (r.lo - p.lo);
431 let q_lo = r / b;
432
433 Self::new(q_lo, q_hi)
434 }
435 }
436
437 // Resistant to overflow without FMA
438 #[inline]
439 pub(crate) fn from_exact_div_fma(a: f64, b: f64) -> Self {
440 let q_hi = a / b;
441 let r = f64::mul_add(-q_hi, b, a);
442 let q_lo = r / b;
443 Self::new(q_lo, q_hi)
444 }
445
446 #[inline]
447 pub(crate) fn from_sqrt(x: f64) -> Self {
448 #[cfg(any(
449 all(
450 any(target_arch = "x86", target_arch = "x86_64"),
451 target_feature = "fma"
452 ),
453 target_arch = "aarch64"
454 ))]
455 {
456 let h = x.sqrt();
457 /* h = sqrt(x) * (1 + e1) with |e1| < 2^-52
458 thus h^2 = x * (1 + e2) with |e2| < 2^-50.999 */
459 let e = -f_fmla(h, h, -x); // exact
460
461 /* e = x - h^2 */
462 let l = e / (h + h);
463 DoubleDouble::new(l, h)
464 }
465 #[cfg(not(any(
466 all(
467 any(target_arch = "x86", target_arch = "x86_64"),
468 target_feature = "fma"
469 ),
470 target_arch = "aarch64"
471 )))]
472 {
473 let h = x.sqrt();
474 let prod_hh = DoubleDouble::from_exact_mult(h, h);
475 let e = (x - prod_hh.hi) - prod_hh.lo; // exact
476
477 /* e = x - h^2 */
478 let l = e / (h + h);
479 DoubleDouble::new(l, h)
480 }
481 }
482
483 /// Safe to overflow underflow division using mandatory FMA.
484 #[inline]
485 #[allow(dead_code)]
486 pub(crate) fn div_safe_dd_f64(a: DoubleDouble, b: f64) -> Self {
487 let q1 = a.hi / b;
488 let r = f64::mul_add(-q1, b, a.hi);
489 let r = r + a.lo;
490 let q2 = r / b;
491
492 DoubleDouble::new(q2, q1)
493 }
494
495 #[inline]
496 pub(crate) fn div_dd_f64(a: DoubleDouble, b: f64) -> Self {
497 #[cfg(any(
498 all(
499 any(target_arch = "x86", target_arch = "x86_64"),
500 target_feature = "fma"
501 ),
502 target_arch = "aarch64"
503 ))]
504 {
505 let q1 = a.hi / b;
506 let r = f_fmla(-q1, b, a.hi);
507 let r = r + a.lo;
508 let q2 = r / b;
509
510 DoubleDouble::new(q2, q1)
511 }
512 #[cfg(not(any(
513 all(
514 any(target_arch = "x86", target_arch = "x86_64"),
515 target_feature = "fma"
516 ),
517 target_arch = "aarch64"
518 )))]
519 {
520 let th = a.hi / b;
521 let prod = DoubleDouble::from_exact_mult(th, b);
522 let beta_h = a.hi - prod.hi;
523 let beta_l = beta_h - prod.lo;
524 let beta = beta_l + a.lo;
525 let tl = beta / b;
526 DoubleDouble::new(tl, th)
527 }
528 }
529
530 // /// Dekker division with one refinement step
531 // #[inline]
532 // pub(crate) fn div_dd_f64_newton_raphson(a: DoubleDouble, b: f64) -> Self {
533 // // Initial estimate q = a / b
534 // let q = DoubleDouble::div_dd_f64(a, b);
535 //
536 // // One Newton-Raphson refinement step:
537 // // e = a - q * b
538 // let qb = DoubleDouble::quick_mult_f64(q, b);
539 // let e = DoubleDouble::sub(a, qb);
540 // let e_div_b = DoubleDouble::div_dd_f64(e, b);
541 //
542 // DoubleDouble::add(q, e_div_b)
543 // }
544
545 // /// Dekker division with two Newton-Raphson refinement steps
546 // #[inline]
547 // pub(crate) fn div_dd_f64_newton_raphson_2(a: Dekker, b: f64) -> Self {
548 // // First estimate: q = a / b (one round of Dekker division)
549 // let q1 = Dekker::div_dd_f64(a, b);
550 //
551 // // First refinement: q2 = q1 + (a - q1 * b) / b
552 // let qb1 = Dekker::quick_mult_f64(q1, b);
553 // let e1 = Dekker::sub(a, qb1);
554 // let dq1 = Dekker::div_dd_f64(e1, b);
555 // let q2 = Dekker::add(q1, dq1);
556 //
557 // // Second refinement: q3 = q2 + (a - q2 * b) / b
558 // let qb2 = Dekker::quick_mult_f64(q2, b);
559 // let e2 = Dekker::sub(a, qb2);
560 // let dq2 = Dekker::div_dd_f64(e2, b);
561 //
562 // Dekker::add(q2, dq2)
563 // }
564
565 // #[inline]
566 // pub(crate) fn neg(self) -> Self {
567 // Self {
568 // lo: -self.lo, hi: -self.hi,
569 // }
570 // }
571
572 #[inline]
573 pub(crate) fn from_f64_div_dd(a: f64, b: DoubleDouble) -> Self {
574 let q1 = a / b.hi;
575
576 let prod = DoubleDouble::from_exact_mult(q1, b.hi);
577 let prod_lo = f_fmla(q1, b.lo, prod.lo);
578 let rem = f_fmla(-1.0, prod.hi, a) - prod_lo;
579
580 let q2 = rem / b.hi;
581
582 DoubleDouble::new(q2, q1)
583 }
584
585 #[inline(always)]
586 #[allow(unused)]
587 pub(crate) fn from_f64_div_dd_fma(a: f64, b: DoubleDouble) -> Self {
588 let q1 = a / b.hi;
589
590 let prod = DoubleDouble::from_exact_mult_fma(q1, b.hi);
591 let prod_lo = f64::mul_add(q1, b.lo, prod.lo);
592 let rem = f64::mul_add(-1.0, prod.hi, a) - prod_lo;
593
594 let q2 = rem / b.hi;
595
596 DoubleDouble::new(q2, q1)
597 }
598
599 #[inline(always)]
600 #[allow(unused)]
601 pub(crate) fn div_fma(a: DoubleDouble, b: DoubleDouble) -> DoubleDouble {
602 let q = 1.0 / b.hi;
603 let r_hi = a.hi * q;
604 let e_hi = f64::mul_add(b.hi, -r_hi, a.hi);
605 let e_lo = f64::mul_add(b.lo, -r_hi, a.lo);
606 let r_lo = q * (e_hi + e_lo);
607 DoubleDouble::new(r_lo, r_hi)
608 }
609
610 #[inline]
611 pub(crate) fn div(a: DoubleDouble, b: DoubleDouble) -> DoubleDouble {
612 let q = 1.0 / b.hi;
613 let r_hi = a.hi * q;
614 #[cfg(any(
615 all(
616 any(target_arch = "x86", target_arch = "x86_64"),
617 target_feature = "fma"
618 ),
619 target_arch = "aarch64"
620 ))]
621 {
622 let e_hi = f_fmla(b.hi, -r_hi, a.hi);
623 let e_lo = f_fmla(b.lo, -r_hi, a.lo);
624 let r_lo = q * (e_hi + e_lo);
625 DoubleDouble::new(r_lo, r_hi)
626 }
627
628 #[cfg(not(any(
629 all(
630 any(target_arch = "x86", target_arch = "x86_64"),
631 target_feature = "fma"
632 ),
633 target_arch = "aarch64"
634 )))]
635 {
636 let b_hi_r_hi = DoubleDouble::from_exact_mult(b.hi, -r_hi);
637 let b_lo_r_hi = DoubleDouble::from_exact_mult(b.lo, -r_hi);
638 let e_hi = (a.hi + b_hi_r_hi.hi) + b_hi_r_hi.lo;
639 let e_lo = (a.lo + b_lo_r_hi.hi) + b_lo_r_hi.lo;
640 let r_lo = q * (e_hi + e_lo);
641 DoubleDouble::new(r_lo, r_hi)
642 }
643 }
644
645 #[inline]
646 pub(crate) fn from_exact_mult(a: f64, b: f64) -> Self {
647 #[cfg(any(
648 all(
649 any(target_arch = "x86", target_arch = "x86_64"),
650 target_feature = "fma"
651 ),
652 target_arch = "aarch64"
653 ))]
654 {
655 let r_hi = a * b;
656 let r_lo = f_fmla(a, b, -r_hi);
657 DoubleDouble::new(r_lo, r_hi)
658 }
659 #[cfg(not(any(
660 all(
661 any(target_arch = "x86", target_arch = "x86_64"),
662 target_feature = "fma"
663 ),
664 target_arch = "aarch64"
665 )))]
666 {
667 let splat = DoubleDouble::split(a);
668 DoubleDouble::from_exact_mult_impl_non_fma(splat, a, b)
669 }
670 }
671
672 #[inline(always)]
673 #[allow(unused)]
674 pub(crate) fn from_exact_mult_fma(a: f64, b: f64) -> Self {
675 let r_hi = a * b;
676 let r_lo = f64::mul_add(a, b, -r_hi);
677 DoubleDouble::new(r_lo, r_hi)
678 }
679
680 // #[inline]
681 // pub(crate) fn add_f64(&self, other: f64) -> DoubleDouble {
682 // let r = DoubleDouble::from_exact_add(self.hi, other);
683 // Dekker::from_exact_add(r.hi, r.lo + self.lo)
684 // }
685
686 // #[inline]
687 // pub(crate) fn to_triple(self) -> TripleDouble {
688 // TripleDouble::new(0., self.lo, self.hi)
689 // }
690
691 /// Computes `a * b + c`
692 /// `b` is an `f64`, `a` and `c` are `DoubleDouble`.
693 ///
694 /// *Accurate dot product (Ogita, Rump and Oishi 2004)*
695 #[inline]
696 pub(crate) fn mul_f64_add(a: DoubleDouble, b: f64, c: DoubleDouble) -> Self {
697 let DoubleDouble { hi: h, lo: r } = DoubleDouble::quick_mult_f64(a, b);
698 let DoubleDouble { hi: p, lo: q } = DoubleDouble::full_add_f64(c, h);
699 DoubleDouble::new(r + q, p)
700 }
701
702 /// Computes `a * b + c`
703 /// `b` is an `f64`, `a` and `c` are `DoubleDouble`.
704 ///
705 /// *Accurate dot product (Ogita, Rump and Oishi 2004)*
706 ///
707 /// *Correctness*
708 /// |c.hi| > |a.hi * b.hi|
709 #[inline]
710 pub(crate) fn quick_mul_f64_add(a: DoubleDouble, b: f64, c: DoubleDouble) -> Self {
711 let DoubleDouble { hi: h, lo: r } = DoubleDouble::quick_mult_f64(a, b);
712 let DoubleDouble { hi: p, lo: q } = DoubleDouble::add_f64(c, h);
713 DoubleDouble::new(r + q, p)
714 }
715
716 /// Computes `a * b + c`
717 ///
718 /// *Accurate dot product (Ogita, Rump and Oishi 2004)*
719 ///
720 /// *Correctness*
721 /// |c.hi| > |a.hi * b.hi|
722 #[inline]
723 pub(crate) fn quick_mul_f64_add_f64(a: DoubleDouble, b: f64, c: f64) -> Self {
724 let DoubleDouble { hi: h, lo: r } = DoubleDouble::quick_mult_f64(a, b);
725 let DoubleDouble { hi: p, lo: q } = DoubleDouble::from_exact_add(c, h);
726 DoubleDouble::new(r + q, p)
727 }
728
729 // #[inline]
730 // pub(crate) fn mul_f64_add_full(a: DoubleDouble, b: f64, c: DoubleDouble) -> Self {
731 // /*
732 // double _t1, _t2, _t3, _t4, _t5, _t6, _t7, _t8; \
733 // \
734 // Mul12(&_t1,&_t2,(a),(bh)); \
735 // Add12(_t3,_t4,(ch),_t1); \
736 // _t5 = (bl) * (a); \
737 // _t6 = (cl) + _t2; \
738 // _t7 = _t5 + _t6; \
739 // _t8 = _t7 + _t4; \
740 // Add12((*(resh)),(*(resl)),_t3,_t8); \
741 // */
742 // let DoubleDouble { hi: t1, lo: t2 } = DoubleDouble::from_exact_mult(a.hi, b);
743 // let DoubleDouble { hi: t3, lo: t4 } = DoubleDouble::from_full_exact_add(c.hi, t1);
744 // let t5 = a.lo * b;
745 // let t6 = c.lo + t2;
746 // let t7 = t5 + t6;
747 // let t8 = t7 + t4;
748 // DoubleDouble::from_full_exact_add(t3, t8)
749 // }
750
751 /// Computes `a * b + c`
752 /// `b` is an `f64`, `a` and `c` are `DoubleDouble`.
753 ///
754 /// *Accurate dot product (Ogita, Rump and Oishi 2004)*
755 #[inline]
756 pub(crate) fn f64_mul_f64_add(a: f64, b: f64, c: DoubleDouble) -> Self {
757 let DoubleDouble { hi: h, lo: r } = DoubleDouble::from_exact_mult(a, b);
758 let DoubleDouble { hi: p, lo: q } = DoubleDouble::full_add_f64(c, h);
759 DoubleDouble::new(r + q, p)
760 }
761
762 // /// Computes `a * b + c`
763 // /// `b` is an `f64`, `a` and `c` are `DoubleDouble`.
764 // ///
765 // /// *Accurate dot product (Ogita, Rump and Oishi 2004)*
766 // #[inline]
767 // pub(crate) fn single_mul_add(a: f64, b: f64, c: f64) -> Self {
768 // let DoubleDouble { hi: h, lo: r } = DoubleDouble::from_exact_mult(a, b);
769 // let DoubleDouble { hi: p, lo: q } = DoubleDouble::from_full_exact_add(c, h);
770 // DoubleDouble::new(r + q, p)
771 // }
772
773 // /// Computes `a * b + c` safe to overflow without FMA
774 // /// `b` is an `f64`, `a` and `c` are `DoubleDouble`.
775 // ///
776 // /// *Accurate dot product (Ogita, Rump and Oishi 2004)*
777 // #[inline]
778 // pub(crate) fn mul_f64_safe_add(a: DoubleDouble, b: f64, c: DoubleDouble) -> Self {
779 // let DoubleDouble { hi: h, lo: r } = DoubleDouble::quick_mult_safe_f64(a, b);
780 // let DoubleDouble { hi: p, lo: q } = DoubleDouble::full_add_f64(c, h);
781 // DoubleDouble::new(r + q, p)
782 // }
783
784 /// `a*b+c`
785 ///
786 /// *Accurate dot product (Ogita, Rump and Oishi 2004)*
787 #[inline]
788 pub(crate) fn mul_add(a: DoubleDouble, b: DoubleDouble, c: DoubleDouble) -> Self {
789 let DoubleDouble { hi: h, lo: r } = DoubleDouble::quick_mult(a, b);
790 let DoubleDouble { hi: p, lo: q } = DoubleDouble::full_add_f64(c, h);
791 DoubleDouble::new(r + q, p)
792 }
793
794 /// `a*b+c`
795 ///
796 /// *Accurate dot product (Ogita, Rump and Oishi 2004)*
797 #[inline(always)]
798 #[allow(unused)]
799 pub(crate) fn mul_add_fma(a: DoubleDouble, b: DoubleDouble, c: DoubleDouble) -> Self {
800 let DoubleDouble { hi: h, lo: r } = DoubleDouble::quick_mult_fma(a, b);
801 let DoubleDouble { hi: p, lo: q } = DoubleDouble::full_add_f64(c, h);
802 DoubleDouble::new(r + q, p)
803 }
804
805 /// `a*b+c`
806 ///
807 /// *Accurate dot product (Ogita, Rump and Oishi 2004)*
808 ///
809 /// *Correctness*
810 /// |c.hi| > |a.hi * b.hi|
811 #[inline]
812 pub(crate) fn quick_mul_add(a: DoubleDouble, b: DoubleDouble, c: DoubleDouble) -> Self {
813 let DoubleDouble { hi: h, lo: r } = DoubleDouble::quick_mult(a, b);
814 let DoubleDouble { hi: p, lo: q } = DoubleDouble::add_f64(c, h);
815 DoubleDouble::new(r + q, p)
816 }
817
818 /// `a*b+c`
819 ///
820 /// *Accurate dot product (Ogita, Rump and Oishi 2004)*
821 ///
822 /// *Correctness*
823 /// |c.hi| > |a.hi * b.hi|
824 #[inline(always)]
825 #[allow(unused)]
826 pub(crate) fn quick_mul_add_fma(a: DoubleDouble, b: DoubleDouble, c: DoubleDouble) -> Self {
827 let DoubleDouble { hi: h, lo: r } = DoubleDouble::quick_mult_fma(a, b);
828 let DoubleDouble { hi: p, lo: q } = DoubleDouble::add_f64(c, h);
829 DoubleDouble::new(r + q, p)
830 }
831
832 #[inline]
833 pub(crate) fn quick_mult(a: DoubleDouble, b: DoubleDouble) -> Self {
834 #[cfg(any(
835 all(
836 any(target_arch = "x86", target_arch = "x86_64"),
837 target_feature = "fma"
838 ),
839 target_arch = "aarch64"
840 ))]
841 {
842 let mut r = DoubleDouble::from_exact_mult(a.hi, b.hi);
843 let t1 = f_fmla(a.hi, b.lo, r.lo);
844 let t2 = f_fmla(a.lo, b.hi, t1);
845 r.lo = t2;
846 r
847 }
848 #[cfg(not(any(
849 all(
850 any(target_arch = "x86", target_arch = "x86_64"),
851 target_feature = "fma"
852 ),
853 target_arch = "aarch64"
854 )))]
855 {
856 let DoubleDouble { hi: ch, lo: cl1 } = DoubleDouble::from_exact_mult(a.hi, b.hi);
857 let tl1 = a.hi * b.lo;
858 let tl2 = a.lo * b.hi;
859 let cl2 = tl1 + tl2;
860 let cl3 = cl1 + cl2;
861 DoubleDouble::new(cl3, ch)
862 }
863 }
864
865 #[inline(always)]
866 #[allow(unused)]
867 pub(crate) fn quick_mult_fma(a: DoubleDouble, b: DoubleDouble) -> Self {
868 let mut r = DoubleDouble::from_exact_mult_fma(a.hi, b.hi);
869 let t1 = f64::mul_add(a.hi, b.lo, r.lo);
870 let t2 = f64::mul_add(a.lo, b.hi, t1);
871 r.lo = t2;
872 r
873 }
874
875 #[inline]
876 pub(crate) fn mult(a: DoubleDouble, b: DoubleDouble) -> Self {
877 #[cfg(any(
878 all(
879 any(target_arch = "x86", target_arch = "x86_64"),
880 target_feature = "fma"
881 ),
882 target_arch = "aarch64"
883 ))]
884 {
885 let DoubleDouble { hi: ch, lo: cl1 } = DoubleDouble::from_exact_mult(a.hi, b.hi);
886 let tl0 = a.lo * b.lo;
887 let tl1 = f_fmla(a.hi, b.lo, tl0);
888 let cl2 = f_fmla(a.lo, b.hi, tl1);
889 let cl3 = cl1 + cl2;
890 DoubleDouble::from_exact_add(ch, cl3)
891 }
892 #[cfg(not(any(
893 all(
894 any(target_arch = "x86", target_arch = "x86_64"),
895 target_feature = "fma"
896 ),
897 target_arch = "aarch64"
898 )))]
899 {
900 let DoubleDouble { hi: ch, lo: cl1 } = DoubleDouble::from_exact_mult(a.hi, b.hi);
901 let tl1 = a.hi * b.lo;
902 let tl2 = a.lo * b.hi;
903 let cl2 = tl1 + tl2;
904 let cl3 = cl1 + cl2;
905 DoubleDouble::from_exact_add(ch, cl3)
906 }
907 }
908
909 #[inline]
910 pub(crate) fn mult_f64(a: DoubleDouble, b: f64) -> Self {
911 #[cfg(any(
912 all(
913 any(target_arch = "x86", target_arch = "x86_64"),
914 target_feature = "fma"
915 ),
916 target_arch = "aarch64"
917 ))]
918 {
919 let DoubleDouble { hi: ch, lo: cl1 } = DoubleDouble::from_exact_mult(a.hi, b);
920 let cl3 = f_fmla(a.lo, b, cl1);
921 DoubleDouble::from_exact_add(ch, cl3)
922 }
923 #[cfg(not(any(
924 all(
925 any(target_arch = "x86", target_arch = "x86_64"),
926 target_feature = "fma"
927 ),
928 target_arch = "aarch64"
929 )))]
930 {
931 let DoubleDouble { hi: ch, lo: cl1 } = DoubleDouble::from_exact_mult(a.hi, b);
932 let cl2 = a.lo * b;
933 let t = DoubleDouble::from_exact_add(ch, cl2);
934 let tl2 = t.lo + cl1;
935 DoubleDouble::from_exact_add(t.hi, tl2)
936 }
937 }
938
939 #[inline(always)]
940 pub(crate) fn quick_f64_mult(a: f64, b: DoubleDouble) -> DoubleDouble {
941 DoubleDouble::quick_mult_f64(b, a)
942 }
943
944 #[inline]
945 pub(crate) fn quick_mult_f64(a: DoubleDouble, b: f64) -> Self {
946 #[cfg(any(
947 all(
948 any(target_arch = "x86", target_arch = "x86_64"),
949 target_feature = "fma"
950 ),
951 target_arch = "aarch64"
952 ))]
953 {
954 let h = b * a.hi;
955 let l = f_fmla(b, a.lo, f_fmla(b, a.hi, -h));
956 Self { lo: l, hi: h }
957 }
958 #[cfg(not(any(
959 all(
960 any(target_arch = "x86", target_arch = "x86_64"),
961 target_feature = "fma"
962 ),
963 target_arch = "aarch64"
964 )))]
965 {
966 let DoubleDouble { hi: ch, lo: cl1 } = DoubleDouble::from_exact_mult(a.hi, b);
967 let cl2 = a.lo * b;
968 let cl3 = cl1 + cl2;
969 DoubleDouble::new(cl3, ch)
970 }
971 }
972
973 #[inline(always)]
974 #[allow(unused)]
975 pub(crate) fn quick_mult_f64_fma(a: DoubleDouble, b: f64) -> Self {
976 let h = b * a.hi;
977 let l = f64::mul_add(b, a.lo, f64::mul_add(b, a.hi, -h));
978 Self { lo: l, hi: h }
979 }
980
981 // /// Double-double multiplication safe to overflow without FMA
982 // #[inline]
983 // pub(crate) fn quick_mult_safe_f64(a: DoubleDouble, b: f64) -> Self {
984 // let h = b * a.hi;
985 // let l = f64::mul_add(b, a.lo, f64::mul_add(b, a.hi, -h));
986 // Self { lo: l, hi: h }
987 // }
988
989 /// Valid only |a.hi| > |b|
990 #[inline]
991 pub(crate) fn add_f64(a: DoubleDouble, b: f64) -> Self {
992 let t = DoubleDouble::from_exact_add(a.hi, b);
993 let l = a.lo + t.lo;
994 Self { lo: l, hi: t.hi }
995 }
996
997 #[inline]
998 pub(crate) fn full_add_f64(a: DoubleDouble, b: f64) -> Self {
999 let t = DoubleDouble::from_full_exact_add(a.hi, b);
1000 let l = a.lo + t.lo;
1001 Self { lo: l, hi: t.hi }
1002 }
1003
1004 /// Valid only |b| > |a.hi|
1005 #[inline]
1006 pub(crate) fn f64_add(b: f64, a: DoubleDouble) -> Self {
1007 let t = DoubleDouble::from_exact_add(b, a.hi);
1008 let l = a.lo + t.lo;
1009 Self { lo: l, hi: t.hi }
1010 }
1011
1012 #[inline]
1013 pub(crate) const fn to_f64(self) -> f64 {
1014 self.lo + self.hi
1015 }
1016
1017 // #[inline]
1018 // pub(crate) fn from_rsqrt(x: f64) -> DoubleDouble {
1019 // let r = DoubleDouble::div_dd_f64(DoubleDouble::from_sqrt(x), x);
1020 // let rx = DoubleDouble::quick_mult_safe_f64(r, x);
1021 // let drx = DoubleDouble::mul_f64_safe_add(r, x, -rx);
1022 // let h = DoubleDouble::mul_add(r, drx, DoubleDouble::mul_add_f64(r, rx, -1.0));
1023 // let dr = DoubleDouble::quick_mult(DoubleDouble::quick_mult_f64(r, 0.5), h);
1024 // DoubleDouble::add(r, dr)
1025 // }
1026
1027 #[inline]
1028 pub(crate) fn from_rsqrt_fast(x: f64) -> DoubleDouble {
1029 let sqrt_x = DoubleDouble::from_sqrt(x);
1030 sqrt_x.recip()
1031 }
1032 }
1033
1034 impl Mul<DoubleDouble> for DoubleDouble {
1035 type Output = Self;
1036
1037 #[inline]
1038 fn mul(self, rhs: DoubleDouble) -> Self::Output {
1039 DoubleDouble::quick_mult(self, rhs)
1040 }
1041 }
1042
1043 /// check if number is valid for Exact mult
1044 #[allow(dead_code)]
1045 #[inline]
1046 pub(crate) fn two_product_compatible(x: f64) -> bool {
1047 let exp = get_exponent_f64(x);
1048 !(exp >= 970 || exp <= -970)
1049 }
1050
1051 #[cfg(test)]
1052 mod tests {
1053 use super::*;
1054 #[test]
1055 fn test_f64_mult() {
1056 let d1 = 1.1231;
1057 let d2 = DoubleDouble::new(1e-22, 3.2341);
1058 let p = DoubleDouble::quick_f64_mult(d1, d2);
1059 assert_eq!(p.hi, 3.6322177100000004);
1060 assert_eq!(p.lo, -1.971941841373783e-16);
1061 }
1062
1063 #[test]
1064 fn test_mult_64() {
1065 let d1 = 1.1231;
1066 let d2 = DoubleDouble::new(1e-22, 3.2341);
1067 let p = DoubleDouble::mult_f64(d2, d1);
1068 assert_eq!(p.hi, 3.6322177100000004);
1069 assert_eq!(p.lo, -1.971941841373783e-16);
1070 }
1071
1072 #[test]
1073 fn recip_test() {
1074 let d1 = 1.54352432142;
1075 let recip = DoubleDouble::new(0., d1).recip();
1076 assert_eq!(recip.hi, d1.recip());
1077 assert_ne!(recip.lo, 0.);
1078 }
1079
1080 #[test]
1081 fn from_recip_test() {
1082 let d1 = 1.54352432142;
1083 let recip = DoubleDouble::from_recip(d1);
1084 assert_eq!(recip.hi, d1.recip());
1085 assert_ne!(recip.lo, 0.);
1086 }
1087
1088 #[test]
1089 fn from_quick_recip_test() {
1090 let d1 = 1.54352432142;
1091 let recip = DoubleDouble::from_quick_recip(d1);
1092 assert_eq!(recip.hi, d1.recip());
1093 assert_ne!(recip.lo, 0.);
1094 }
1095 }