| 1 |
//===----------------------------------------------------------------------===// |
| 2 |
// |
| 3 |
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 |
// See https://llvm.org/LICENSE.txt for license information. |
| 5 |
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 |
// |
| 7 |
//===----------------------------------------------------------------------===// |
| 8 |
|
| 9 |
#ifndef _LIBCPP___CXX03___ALGORITHM_SIMD_UTILS_H |
| 10 |
#define _LIBCPP___CXX03___ALGORITHM_SIMD_UTILS_H |
| 11 |
|
| 12 |
#include <__cxx03/__algorithm/min.h> |
| 13 |
#include <__cxx03/__bit/countl.h> |
| 14 |
#include <__cxx03/__bit/countr.h> |
| 15 |
#include <__cxx03/__config> |
| 16 |
#include <__cxx03/__type_traits/is_arithmetic.h> |
| 17 |
#include <__cxx03/__type_traits/is_same.h> |
| 18 |
#include <__cxx03/__utility/integer_sequence.h> |
| 19 |
#include <__cxx03/cstddef> |
| 20 |
#include <__cxx03/cstdint> |
| 21 |
|
| 22 |
#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) |
| 23 |
# pragma GCC system_header |
| 24 |
#endif |
| 25 |
|
| 26 |
_LIBCPP_PUSH_MACROS |
| 27 |
#include <__cxx03/__undef_macros> |
| 28 |
|
| 29 |
// TODO: Find out how altivec changes things and allow vectorizations there too. |
| 30 |
#define _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS 0 |
| 31 |
|
| 32 |
#if _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS && !defined(__OPTIMIZE_SIZE__) |
| 33 |
# define _LIBCPP_VECTORIZE_ALGORITHMS 1 |
| 34 |
#else |
| 35 |
# define _LIBCPP_VECTORIZE_ALGORITHMS 0 |
| 36 |
#endif |
| 37 |
|
| 38 |
#if _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS |
| 39 |
|
| 40 |
_LIBCPP_BEGIN_NAMESPACE_STD |
| 41 |
|
| 42 |
template <class _Tp> |
| 43 |
inline constexpr bool __can_map_to_integer_v = |
| 44 |
sizeof(_Tp) == alignof(_Tp) && (sizeof(_Tp) == 1 || sizeof(_Tp) == 2 || sizeof(_Tp) == 4 || sizeof(_Tp) == 8); |
| 45 |
|
| 46 |
template <size_t _TypeSize> |
| 47 |
struct __get_as_integer_type_impl; |
| 48 |
|
| 49 |
template <> |
| 50 |
struct __get_as_integer_type_impl<1> { |
| 51 |
using type = uint8_t; |
| 52 |
}; |
| 53 |
|
| 54 |
template <> |
| 55 |
struct __get_as_integer_type_impl<2> { |
| 56 |
using type = uint16_t; |
| 57 |
}; |
| 58 |
template <> |
| 59 |
struct __get_as_integer_type_impl<4> { |
| 60 |
using type = uint32_t; |
| 61 |
}; |
| 62 |
template <> |
| 63 |
struct __get_as_integer_type_impl<8> { |
| 64 |
using type = uint64_t; |
| 65 |
}; |
| 66 |
|
| 67 |
template <class _Tp> |
| 68 |
using __get_as_integer_type_t = typename __get_as_integer_type_impl<sizeof(_Tp)>::type; |
| 69 |
|
| 70 |
// This isn't specialized for 64 byte vectors on purpose. They have the potential to significantly reduce performance |
| 71 |
// in mixed simd/non-simd workloads and don't provide any performance improvement for currently vectorized algorithms |
| 72 |
// as far as benchmarks are concerned. |
| 73 |
# if defined(__AVX__) || defined(__MVS__) |
| 74 |
template <class _Tp> |
| 75 |
inline constexpr size_t __native_vector_size = 32 / sizeof(_Tp); |
| 76 |
# elif defined(__SSE__) || defined(__ARM_NEON__) |
| 77 |
template <class _Tp> |
| 78 |
inline constexpr size_t __native_vector_size = 16 / sizeof(_Tp); |
| 79 |
# elif defined(__MMX__) |
| 80 |
template <class _Tp> |
| 81 |
inline constexpr size_t __native_vector_size = 8 / sizeof(_Tp); |
| 82 |
# else |
| 83 |
template <class _Tp> |
| 84 |
inline constexpr size_t __native_vector_size = 1; |
| 85 |
# endif |
| 86 |
|
| 87 |
template <class _ArithmeticT, size_t _Np> |
| 88 |
using __simd_vector __attribute__((__ext_vector_type__(_Np))) = _ArithmeticT; |
| 89 |
|
| 90 |
template <class _VecT> |
| 91 |
inline constexpr size_t __simd_vector_size_v = []<bool _False = false>() -> size_t { |
| 92 |
static_assert(_False, "Not a vector!"); |
| 93 |
}(); |
| 94 |
|
| 95 |
template <class _Tp, size_t _Np> |
| 96 |
inline constexpr size_t __simd_vector_size_v<__simd_vector<_Tp, _Np>> = _Np; |
| 97 |
|
| 98 |
template <class _Tp, size_t _Np> |
| 99 |
_LIBCPP_HIDE_FROM_ABI _Tp __simd_vector_underlying_type_impl(__simd_vector<_Tp, _Np>) { |
| 100 |
return _Tp{}; |
| 101 |
} |
| 102 |
|
| 103 |
template <class _VecT> |
| 104 |
using __simd_vector_underlying_type_t = decltype(std::__simd_vector_underlying_type_impl(_VecT{})); |
| 105 |
|
| 106 |
// This isn't inlined without always_inline when loading chars. |
| 107 |
template <class _VecT, class _Iter> |
| 108 |
_LIBCPP_NODISCARD _LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI _VecT __load_vector(_Iter __iter) noexcept { |
| 109 |
return [=]<size_t... _Indices>(index_sequence<_Indices...>) _LIBCPP_ALWAYS_INLINE noexcept { |
| 110 |
return _VecT{__iter[_Indices]...}; |
| 111 |
}(make_index_sequence<__simd_vector_size_v<_VecT>>{}); |
| 112 |
} |
| 113 |
|
| 114 |
template <class _Tp, size_t _Np> |
| 115 |
_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI bool __all_of(__simd_vector<_Tp, _Np> __vec) noexcept { |
| 116 |
return __builtin_reduce_and(__builtin_convertvector(__vec, __simd_vector<bool, _Np>)); |
| 117 |
} |
| 118 |
|
| 119 |
template <class _Tp, size_t _Np> |
| 120 |
_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI size_t __find_first_set(__simd_vector<_Tp, _Np> __vec) noexcept { |
| 121 |
using __mask_vec = __simd_vector<bool, _Np>; |
| 122 |
|
| 123 |
// This has MSan disabled du to https://github.com/llvm/llvm-project/issues/85876 |
| 124 |
auto __impl = [&]<class _MaskT>(_MaskT) _LIBCPP_NO_SANITIZE("memory") noexcept { |
| 125 |
# if defined(_LIBCPP_BIG_ENDIAN) |
| 126 |
return std::min<size_t>( |
| 127 |
_Np, std::__countl_zero(__builtin_bit_cast(_MaskT, __builtin_convertvector(__vec, __mask_vec)))); |
| 128 |
# else |
| 129 |
return std::min<size_t>( |
| 130 |
_Np, std::__countr_zero(__builtin_bit_cast(_MaskT, __builtin_convertvector(__vec, __mask_vec)))); |
| 131 |
# endif |
| 132 |
}; |
| 133 |
|
| 134 |
if constexpr (sizeof(__mask_vec) == sizeof(uint8_t)) { |
| 135 |
return __impl(uint8_t{}); |
| 136 |
} else if constexpr (sizeof(__mask_vec) == sizeof(uint16_t)) { |
| 137 |
return __impl(uint16_t{}); |
| 138 |
} else if constexpr (sizeof(__mask_vec) == sizeof(uint32_t)) { |
| 139 |
return __impl(uint32_t{}); |
| 140 |
} else if constexpr (sizeof(__mask_vec) == sizeof(uint64_t)) { |
| 141 |
return __impl(uint64_t{}); |
| 142 |
} else { |
| 143 |
static_assert(sizeof(__mask_vec) == 0, "unexpected required size for mask integer type"); |
| 144 |
return 0; |
| 145 |
} |
| 146 |
} |
| 147 |
|
| 148 |
template <class _Tp, size_t _Np> |
| 149 |
_LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI size_t __find_first_not_set(__simd_vector<_Tp, _Np> __vec) noexcept { |
| 150 |
return std::__find_first_set(~__vec); |
| 151 |
} |
| 152 |
|
| 153 |
_LIBCPP_END_NAMESPACE_STD |
| 154 |
|
| 155 |
#endif // _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS |
| 156 |
|
| 157 |
_LIBCPP_POP_MACROS |
| 158 |
|
| 159 |
#endif // _LIBCPP___CXX03___ALGORITHM_SIMD_UTILS_H |