/* * // Copyright (c) Radzivon Bartoshyk 8/2025. All rights reserved. * // * // Redistribution and use in source and binary forms, with or without modification, * // are permitted provided that the following conditions are met: * // * // 1. Redistributions of source code must retain the above copyright notice, this * // list of conditions and the following disclaimer. * // * // 2. Redistributions in binary form must reproduce the above copyright notice, * // this list of conditions and the following disclaimer in the documentation * // and/or other materials provided with the distribution. * // * // 3. Neither the name of the copyright holder nor the names of its * // contributors may be used to endorse or promote products derived from * // this software without specific prior written permission. * // * // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE * // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE * // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR * // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER * // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, * // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ use crate::common::f_fmla; use crate::double_double::DoubleDouble; use crate::exponents::fast_ldexp; use crate::polyeval::f_polyeval6; // // // y1 = y0 + 1/3 * y0 * (1 - a * y0 * y0 * y0) // #[inline] // fn raphson_step(x: f64, a: f64) -> f64 { // let h = f_fmla(-a * x, x * x, 1.0); // f_fmla(1. / 3. * h, x, x) // } // y1 = y0(k1 − c(k2 − k3c), c = x*y0*y0*y0 // k1 = 14/9 , k2 = 7/9 , k3 = 2/9 #[inline(always)] fn halleys_div_free(x: f64, a: f64) -> f64 { const K3: f64 = 2. / 9.; const K2: f64 = 7. / 9.; const K1: f64 = 14. / 9.; let c = a * x * x * x; let mut y = f_fmla(-K3, c, K2); y = f_fmla(-c, y, K1); y * x } #[inline(always)] #[allow(unused)] fn halleys_div_free_fma(x: f64, a: f64) -> f64 { const K3: f64 = 2. / 9.; const K2: f64 = 7. / 9.; const K1: f64 = 14. / 9.; let c = a * x * x * x; let mut y = f64::mul_add(-K3, c, K2); y = f64::mul_add(-c, y, K1); y * x } #[inline(always)] fn rcbrt_gen_impl(a: f64) -> f64 { // Decompose a = m * 2^e, with m in [0.5, 1) let xu = a.to_bits(); let exp = ((xu >> 52) & 0x7ff) as i32; let mut e = ((xu >> 52) & 0x7ff) as i32; let mut mant = xu & ((1u64 << 52) - 1); if exp == 0x7ff { if a.is_infinite() { return if a.is_sign_negative() { -0.0 } else { 0.0 }; } return a + a; } if exp == 0 && a == 0. { return if a.is_sign_negative() { f64::NEG_INFINITY } else { f64::INFINITY }; } // Normalize subnormal if exp == 0 { let norm = a * f64::from_bits(0x4350000000000000); // * 2^54 let norm_bits = norm.to_bits(); mant = norm_bits & ((1u64 << 52) - 1); e = ((norm_bits >> 52) & 0x7ff) as i32 - 54; } e -= 1023; mant |= 0x3ff << 52; let m = f64::from_bits(mant); // Polynomial for x^(-1/3) on [1.0; 2.0] // Generated by Sollya: // d = [1.0, 2.0]; // f_inv_cbrt = x^(-1/3); // Q = fpminimax(f_inv_cbrt, 5, [|D...|], d, relative, floating); // See ./notes/inv_cbrt.sollya let p = f_polyeval6( m, f64::from_bits(0x3ffc7f365bceaf71), f64::from_bits(0xbff90e741fb9c896), f64::from_bits(0x3ff3e68b9b2cd237), f64::from_bits(0xbfe321c5eb24a185), f64::from_bits(0x3fc3fa269b897f69), f64::from_bits(0xbf916d6f13849fd1), ); // split exponent e = 3*q + r with r in {0,1,2} // use div_euclid/rem_euclid to get r >= 0 let q = e.div_euclid(3); let rem_scale = e.rem_euclid(3); // 1; 2^{-1/3}; 2^{-2/3} static ESCALE: [u64; 3] = [1.0f64.to_bits(), 0x3fe965fea53d6e3d, 0x3fe428a2f98d728b]; let z = p * f64::from_bits(ESCALE[rem_scale as usize]); let mm = fast_ldexp(m, rem_scale); // bring domain into [1;8] // One Halley's method step // then refine in partial double-double precision with Newton-Raphson iteration let y0 = halleys_div_free(z, mm); let d2y = DoubleDouble::from_exact_mult(y0, y0); let d3y = DoubleDouble::quick_mult_f64(d2y, y0); let hb = DoubleDouble::quick_mult_f64(d3y, mm); let y: f64; #[cfg(any( all( any(target_arch = "x86", target_arch = "x86_64"), target_feature = "fma" ), target_arch = "aarch64" ))] { // decompose double-double in linear FMA sums // r = (1.0 - hb.hi - hb.lo) * y0 = y0 - hb.hi * y0 - hb.lo * y0 = fma(-hb.lo, y0, fma(-hb.hi, y0, y0)) let r = f_fmla(-hb.lo, y0, f_fmla(hb.hi, -y0, y0)); // // y1 = y0 + 1/3 * y0 * (1 - a * y0 * y0 * y0) = y0 + 1/3 * r y = f_fmla(1. / 3., r, y0); } #[cfg(not(any( all( any(target_arch = "x86", target_arch = "x86_64"), target_feature = "fma" ), target_arch = "aarch64" )))] { let m_hb = DoubleDouble::full_add_f64(-hb, 1.0); let r = DoubleDouble::quick_mult_f64(m_hb, y0); y = f_fmla(1. / 3., r.to_f64(), y0); } f64::copysign(fast_ldexp(y, -q), a) } #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] #[target_feature(enable = "avx", enable = "fma")] unsafe fn rcbrt_fma_impl(a: f64) -> f64 { // Decompose a = m * 2^e, with m in [0.5, 1) let xu = a.to_bits(); let exp = ((xu >> 52) & 0x7ff) as i32; let mut e = ((xu >> 52) & 0x7ff) as i32; let mut mant = xu & ((1u64 << 52) - 1); if exp == 0x7ff { if a.is_infinite() { return if a.is_sign_negative() { -0.0 } else { 0.0 }; } return a + a; } if exp == 0 && a == 0. { return if a.is_sign_negative() { f64::NEG_INFINITY } else { f64::INFINITY }; } // Normalize subnormal if exp == 0 { let norm = a * f64::from_bits(0x4350000000000000); // * 2^54 let norm_bits = norm.to_bits(); mant = norm_bits & ((1u64 << 52) - 1); e = ((norm_bits >> 52) & 0x7ff) as i32 - 54; } e -= 1023; mant |= 0x3ff << 52; let m = f64::from_bits(mant); // Polynomial for x^(-1/3) on [1.0; 2.0] // Generated by Sollya: // d = [1.0, 2.0]; // f_inv_cbrt = x^(-1/3); // Q = fpminimax(f_inv_cbrt, 5, [|D...|], d, relative, floating); // See ./notes/inv_cbrt.sollya use crate::polyeval::d_polyeval6; let p = d_polyeval6( m, f64::from_bits(0x3ffc7f365bceaf71), f64::from_bits(0xbff90e741fb9c896), f64::from_bits(0x3ff3e68b9b2cd237), f64::from_bits(0xbfe321c5eb24a185), f64::from_bits(0x3fc3fa269b897f69), f64::from_bits(0xbf916d6f13849fd1), ); // split exponent e = 3*q + r with r in {0,1,2} // use div_euclid/rem_euclid to get r >= 0 let q = e.div_euclid(3); let rem_scale = e.rem_euclid(3); // 1; 2^{-1/3}; 2^{-2/3} static ESCALE: [u64; 3] = [1.0f64.to_bits(), 0x3fe965fea53d6e3d, 0x3fe428a2f98d728b]; let z = p * f64::from_bits(ESCALE[rem_scale as usize]); let mm = fast_ldexp(m, rem_scale); // bring domain into [1;8] // One Halley's method step // then refine in partial double-double precision with Newton-Raphson iteration let y0 = halleys_div_free_fma(z, mm); let d2y = DoubleDouble::from_exact_mult_fma(y0, y0); let d3y = DoubleDouble::quick_mult_f64_fma(d2y, y0); let hb = DoubleDouble::quick_mult_f64_fma(d3y, mm); // decompose double-double in linear FMA sums // r = (1.0 - hb.hi - hb.lo) * y0 = y0 - hb.hi * y0 - hb.lo * y0 = fma(-hb.lo, y0, fma(-hb.hi, y0, y0)) let r = f64::mul_add(-hb.lo, y0, f64::mul_add(hb.hi, -y0, y0)); // // y1 = y0 + 1/3 * y0 * (1 - a * y0 * y0 * y0) = y0 + 1/3 * r let y = f64::mul_add(1. / 3., r, y0); f64::copysign(fast_ldexp(y, -q), a) } /// Computes 1/cbrt(x) /// /// ULP 0.5 pub fn f_rcbrt(a: f64) -> f64 { #[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))] { rcbrt_gen_impl(a) } #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { use std::sync::OnceLock; static EXECUTOR: OnceLock f64> = OnceLock::new(); let q = EXECUTOR.get_or_init(|| { if std::arch::is_x86_feature_detected!("avx") && std::arch::is_x86_feature_detected!("fma") { rcbrt_fma_impl } else { fn def_rcbrt(x: f64) -> f64 { rcbrt_gen_impl(x) } def_rcbrt } }); unsafe { q(a) } } } #[cfg(test)] mod tests { use super::*; #[test] fn test_rcbrt() { assert_eq!(f_rcbrt(0.9999999999999717), 1.0000000000000095); assert_eq!(f_rcbrt(-68355745214719140000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000.), -0.000000000000000000000000000000000000000002445728958868668); assert_eq!(f_rcbrt(-96105972807656840000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000.), -0.0000000000000000000000000000000000000000000000000000000002183148143573148); assert_eq!(f_rcbrt(0.00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000139491540182158), 8949883389846071000000000000000000000000000000000000000000000000000000000000000000000000000000000000000.); assert_eq!(f_rcbrt(0.00008386280387617153), 22.846001824951983); assert_eq!(f_rcbrt(-125.0), -0.2); assert_eq!(f_rcbrt(125.0), 0.2); assert_eq!(f_rcbrt(1.0), 1.0); assert_eq!(f_rcbrt(-1.0), -1.0); assert_eq!(f_rcbrt(0.0), f64::INFINITY); assert_eq!(f_rcbrt(-27.0), -1. / 3.); assert_eq!( f_rcbrt(2417851639214765300000000.), 0.000000007450580596938716 ); assert_eq!(f_rcbrt(27.0), 1. / 3.); assert_eq!(f_rcbrt(64.0), 0.25); assert_eq!(f_rcbrt(-64.0), -0.25); assert_eq!(f_rcbrt(f64::NEG_INFINITY), -0.0); assert_eq!(f_rcbrt(f64::INFINITY), 0.0); assert!(f_rcbrt(f64::NAN).is_nan()); } }