Debian Codegraph @

libcxx/include/__cxx03/__algorithm/simd_utils.h

raw
1 //===----------------------------------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #ifndef _LIBCPP___CXX03___ALGORITHM_SIMD_UTILS_H
10 #define _LIBCPP___CXX03___ALGORITHM_SIMD_UTILS_H
11
12 #include <__cxx03/__algorithm/min.h>
13 #include <__cxx03/__bit/countl.h>
14 #include <__cxx03/__bit/countr.h>
15 #include <__cxx03/__config>
16 #include <__cxx03/__type_traits/is_arithmetic.h>
17 #include <__cxx03/__type_traits/is_same.h>
18 #include <__cxx03/__utility/integer_sequence.h>
19 #include <__cxx03/cstddef>
20 #include <__cxx03/cstdint>
21
22 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
23 # pragma GCC system_header
24 #endif
25
26 _LIBCPP_PUSH_MACROS
27 #include <__cxx03/__undef_macros>
28
29 // TODO: Find out how altivec changes things and allow vectorizations there too.
30 #define _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS 0
31
32 #if _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS && !defined(__OPTIMIZE_SIZE__)
33 # define _LIBCPP_VECTORIZE_ALGORITHMS 1
34 #else
35 # define _LIBCPP_VECTORIZE_ALGORITHMS 0
36 #endif
37
38 #if _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS
39
40 _LIBCPP_BEGIN_NAMESPACE_STD
41
42 template <class _Tp>
43 inline constexpr bool __can_map_to_integer_v =
44 sizeof(_Tp) == alignof(_Tp) && (sizeof(_Tp) == 1 || sizeof(_Tp) == 2 || sizeof(_Tp) == 4 || sizeof(_Tp) == 8);
45
46 template <size_t _TypeSize>
47 struct __get_as_integer_type_impl;
48
49 template <>
50 struct __get_as_integer_type_impl<1> {
51 using type = uint8_t;
52 };
53
54 template <>
55 struct __get_as_integer_type_impl<2> {
56 using type = uint16_t;
57 };
58 template <>
59 struct __get_as_integer_type_impl<4> {
60 using type = uint32_t;
61 };
62 template <>
63 struct __get_as_integer_type_impl<8> {
64 using type = uint64_t;
65 };
66
67 template <class _Tp>
68 using __get_as_integer_type_t = typename __get_as_integer_type_impl<sizeof(_Tp)>::type;
69
70 // This isn't specialized for 64 byte vectors on purpose. They have the potential to significantly reduce performance
71 // in mixed simd/non-simd workloads and don't provide any performance improvement for currently vectorized algorithms
72 // as far as benchmarks are concerned.
73 # if defined(__AVX__) || defined(__MVS__)
74 template <class _Tp>
75 inline constexpr size_t __native_vector_size = 32 / sizeof(_Tp);
76 # elif defined(__SSE__) || defined(__ARM_NEON__)
77 template <class _Tp>
78 inline constexpr size_t __native_vector_size = 16 / sizeof(_Tp);
79 # elif defined(__MMX__)
80 template <class _Tp>
81 inline constexpr size_t __native_vector_size = 8 / sizeof(_Tp);
82 # else
83 template <class _Tp>
84 inline constexpr size_t __native_vector_size = 1;
85 # endif
86
87 template <class _ArithmeticT, size_t _Np>
88 using __simd_vector __attribute__((__ext_vector_type__(_Np))) = _ArithmeticT;
89
90 template <class _VecT>
91 inline constexpr size_t __simd_vector_size_v = []<bool _False = false>() -> size_t {
92 static_assert(_False, "Not a vector!");
93 }();
94
95 template <class _Tp, size_t _Np>
96 inline constexpr size_t __simd_vector_size_v<__simd_vector<_Tp, _Np>> = _Np;
97
98 template <class _Tp, size_t _Np>
99 _LIBCPP_HIDE_FROM_ABI _Tp __simd_vector_underlying_type_impl(__simd_vector<_Tp, _Np>) {
100 return _Tp{};
101 }
102
103 template <class _VecT>
104 using __simd_vector_underlying_type_t = decltype(std::__simd_vector_underlying_type_impl(_VecT{}));
105
106 // This isn't inlined without always_inline when loading chars.
107 template <class _VecT, class _Iter>
108 _LIBCPP_NODISCARD _LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI _VecT __load_vector(_Iter __iter) noexcept {
109 return [=]<size_t... _Indices>(index_sequence<_Indices...>) _LIBCPP_ALWAYS_INLINE noexcept {
110 return _VecT{__iter[_Indices]...};
111 }(make_index_sequence<__simd_vector_size_v<_VecT>>{});
112 }
113
114 template <class _Tp, size_t _Np>
115 _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI bool __all_of(__simd_vector<_Tp, _Np> __vec) noexcept {
116 return __builtin_reduce_and(__builtin_convertvector(__vec, __simd_vector<bool, _Np>));
117 }
118
119 template <class _Tp, size_t _Np>
120 _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI size_t __find_first_set(__simd_vector<_Tp, _Np> __vec) noexcept {
121 using __mask_vec = __simd_vector<bool, _Np>;
122
123 // This has MSan disabled du to https://github.com/llvm/llvm-project/issues/85876
124 auto __impl = [&]<class _MaskT>(_MaskT) _LIBCPP_NO_SANITIZE("memory") noexcept {
125 # if defined(_LIBCPP_BIG_ENDIAN)
126 return std::min<size_t>(
127 _Np, std::__countl_zero(__builtin_bit_cast(_MaskT, __builtin_convertvector(__vec, __mask_vec))));
128 # else
129 return std::min<size_t>(
130 _Np, std::__countr_zero(__builtin_bit_cast(_MaskT, __builtin_convertvector(__vec, __mask_vec))));
131 # endif
132 };
133
134 if constexpr (sizeof(__mask_vec) == sizeof(uint8_t)) {
135 return __impl(uint8_t{});
136 } else if constexpr (sizeof(__mask_vec) == sizeof(uint16_t)) {
137 return __impl(uint16_t{});
138 } else if constexpr (sizeof(__mask_vec) == sizeof(uint32_t)) {
139 return __impl(uint32_t{});
140 } else if constexpr (sizeof(__mask_vec) == sizeof(uint64_t)) {
141 return __impl(uint64_t{});
142 } else {
143 static_assert(sizeof(__mask_vec) == 0, "unexpected required size for mask integer type");
144 return 0;
145 }
146 }
147
148 template <class _Tp, size_t _Np>
149 _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI size_t __find_first_not_set(__simd_vector<_Tp, _Np> __vec) noexcept {
150 return std::__find_first_set(~__vec);
151 }
152
153 _LIBCPP_END_NAMESPACE_STD
154
155 #endif // _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS
156
157 _LIBCPP_POP_MACROS
158
159 #endif // _LIBCPP___CXX03___ALGORITHM_SIMD_UTILS_H