/* PR target/112824 */ /* { dg-do compile } */ /* { dg-options "-std=c++23 -O3 -march=skylake-avx512 -mprefer-vector-width=512" } */ /* { dg-final { scan-assembler-not "vmov(?:dqu|apd)\[ \\t\]+\[^\n\]*%ymm" } } */ #include #include #include #include template using Vec [[gnu::vector_size(W * sizeof(T))]] = T; // Omitted: 16 without AVX, 32 without AVX512F, // or for forward compatibility some AVX10 may also mean 32-only static constexpr ptrdiff_t VectorBytes = 64; template static constexpr ptrdiff_t VecWidth = 64 <= sizeof(T) ? 1 : 64/sizeof(T); template struct Vector{ static constexpr ptrdiff_t L = N; T data[L]; static constexpr auto size()->ptrdiff_t{return N;} }; template struct Vector{ static constexpr ptrdiff_t W = N >= VecWidth ? VecWidth : ptrdiff_t(std::bit_ceil(size_t(N))); static constexpr ptrdiff_t L = (N/W) + ((N%W)!=0); using V = Vec; V data[L]; static constexpr auto size()->ptrdiff_t{return N;} }; /// should be trivially copyable /// codegen is worse when passing by value, even though it seems like it should make /// aliasing simpler to analyze? template [[gnu::always_inline]] constexpr auto operator+(Vector x, Vector y) -> Vector { Vector z; for (ptrdiff_t n = 0; n < Vector::L; ++n) z.data[n] = x.data[n] + y.data[n]; return z; } template [[gnu::always_inline]] constexpr auto operator*(Vector x, Vector y) -> Vector { Vector z; for (ptrdiff_t n = 0; n < Vector::L; ++n) z.data[n] = x.data[n] * y.data[n]; return z; } template [[gnu::always_inline]] constexpr auto operator+(T x, Vector y) -> Vector { Vector z; for (ptrdiff_t n = 0; n < Vector::L; ++n) z.data[n] = x + y.data[n]; return z; } template [[gnu::always_inline]] constexpr auto operator*(T x, Vector y) -> Vector { Vector z; for (ptrdiff_t n = 0; n < Vector::L; ++n) z.data[n] = x * y.data[n]; return z; } template struct Dual { T value; Vector partials; }; // Here we have a specialization for non-power-of-2 `N` template requires(std::floating_point && (std::popcount(size_t(N))>1)) struct Dual { Vector data; }; template consteval auto firstoff(){ static_assert(std::same_as, "type not implemented"); if constexpr (W==2) return Vec<2,int64_t>{0,1} != 0; else if constexpr (W == 4) return Vec<4,int64_t>{0,1,2,3} != 0; else if constexpr (W == 8) return Vec<8,int64_t>{0,1,2,3,4,5,6,7} != 0; else static_assert(false, "vector width not implemented"); } template [[gnu::always_inline]] constexpr auto operator+(Dual a, Dual b) -> Dual { if constexpr (std::floating_point && (std::popcount(size_t(N))>1)){ Dual c; for (ptrdiff_t l = 0; l < Vector::L; ++l) c.data.data[l] = a.data.data[l] + b.data.data[l]; return c; } else return {a.value + b.value, a.partials + b.partials}; } template [[gnu::always_inline]] constexpr auto operator*(Dual a, Dual b) -> Dual { if constexpr (std::floating_point && (std::popcount(size_t(N))>1)){ using V = typename Vector::V; V va = V{}+a.data.data[0][0], vb = V{}+b.data.data[0][0]; V x = va * b.data.data[0]; Dual c; c.data.data[0] = firstoff::W,T>() ? x + vb*a.data.data[0] : x; for (ptrdiff_t l = 1; l < Vector::L; ++l) c.data.data[l] = va*b.data.data[l] + vb*a.data.data[l]; return c; } else return {a.value * b.value, a.value * b.partials + b.value * a.partials}; } void prod(Dual,2> &c, const Dual,2> &a, const Dual,2>&b){ c = a*b; }