| 1 |
//@HEADER |
| 2 |
// ************************************************************************ |
| 3 |
// |
| 4 |
// Kokkos v. 4.0 |
| 5 |
// Copyright (2022) National Technology & Engineering |
| 6 |
// Solutions of Sandia, LLC (NTESS). |
| 7 |
// |
| 8 |
// Under the terms of Contract DE-NA0003525 with NTESS, |
| 9 |
// the U.S. Government retains certain rights in this software. |
| 10 |
// |
| 11 |
// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. |
| 12 |
// See https://kokkos.org/LICENSE for license information. |
| 13 |
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 14 |
// |
| 15 |
//@HEADER |
| 16 |
|
| 17 |
#pragma once |
| 18 |
|
| 19 |
#include <array> |
| 20 |
#include <type_traits> |
| 21 |
#include <utility> // index_sequence |
| 22 |
#include "../__p0009_bits/utility.hpp" |
| 23 |
|
| 24 |
// Suppress spurious warning with NVCC about no return statement. |
| 25 |
// This is a known issue in NVCC and NVC++ |
| 26 |
// Depending on the CUDA and GCC version we need both the builtin |
| 27 |
// and the diagnostic push. I tried really hard to find something shorter |
| 28 |
// but no luck ... |
| 29 |
#if defined __NVCC__ |
| 30 |
#ifdef __NVCC_DIAG_PRAGMA_SUPPORT__ |
| 31 |
#pragma nv_diagnostic push |
| 32 |
#pragma nv_diag_suppress = implicit_return_from_non_void_function |
| 33 |
#else |
| 34 |
#ifdef __CUDA_ARCH__ |
| 35 |
#pragma diagnostic push |
| 36 |
#pragma diag_suppress implicit_return_from_non_void_function |
| 37 |
#endif |
| 38 |
#endif |
| 39 |
#elif defined __NVCOMPILER |
| 40 |
#pragma diagnostic push |
| 41 |
#pragma diag_suppress = implicit_return_from_non_void_function |
| 42 |
#endif |
| 43 |
|
| 44 |
namespace MDSPAN_IMPL_STANDARD_NAMESPACE { |
| 45 |
//****************************************** |
| 46 |
// Return type of submdspan_mapping overloads |
| 47 |
//****************************************** |
| 48 |
template <class LayoutMapping> struct submdspan_mapping_result { |
| 49 |
_MDSPAN_NO_UNIQUE_ADDRESS LayoutMapping mapping{}; |
| 50 |
size_t offset; |
| 51 |
}; |
| 52 |
|
| 53 |
namespace detail { |
| 54 |
|
| 55 |
// We use const Slice& and not Slice&& because the various |
| 56 |
// submdspan_mapping_impl overloads use their slices arguments |
| 57 |
// multiple times. This makes perfect forwarding not useful, but we |
| 58 |
// still don't want to pass those (possibly of size 64 x 3 bits) |
| 59 |
// objects by value. |
| 60 |
template <class IndexType, class Slice> |
| 61 |
MDSPAN_INLINE_FUNCTION constexpr bool |
| 62 |
one_slice_out_of_bounds(const IndexType &ext, const Slice &slice) { |
| 63 |
using common_t = |
| 64 |
std::common_type_t<decltype(detail::first_of(slice)), IndexType>; |
| 65 |
return static_cast<common_t>(detail::first_of(slice)) == |
| 66 |
static_cast<common_t>(ext); |
| 67 |
} |
| 68 |
|
| 69 |
template <size_t... RankIndices, class IndexType, size_t... Exts, |
| 70 |
class... Slices> |
| 71 |
MDSPAN_INLINE_FUNCTION constexpr bool |
| 72 |
any_slice_out_of_bounds_helper(std::index_sequence<RankIndices...>, |
| 73 |
const extents<IndexType, Exts...> &exts, |
| 74 |
const Slices &... slices) { |
| 75 |
return _MDSPAN_FOLD_OR( |
| 76 |
(one_slice_out_of_bounds(exts.extent(RankIndices), slices))); |
| 77 |
} |
| 78 |
|
| 79 |
template <class IndexType, size_t... Exts, class... Slices> |
| 80 |
MDSPAN_INLINE_FUNCTION constexpr bool |
| 81 |
any_slice_out_of_bounds(const extents<IndexType, Exts...> &exts, |
| 82 |
const Slices &... slices) { |
| 83 |
return any_slice_out_of_bounds_helper( |
| 84 |
std::make_index_sequence<sizeof...(Slices)>(), exts, slices...); |
| 85 |
} |
| 86 |
|
| 87 |
// constructs sub strides |
| 88 |
template<class T, size_t N> |
| 89 |
struct sub_strides |
| 90 |
{ |
| 91 |
T values[N > 0 ? N : 1]; |
| 92 |
}; |
| 93 |
|
| 94 |
template <class SrcMapping, class... slice_strides, size_t... InvMapIdxs> |
| 95 |
MDSPAN_INLINE_FUNCTION constexpr auto construct_sub_strides( |
| 96 |
const SrcMapping &src_mapping, std::index_sequence<InvMapIdxs...>, |
| 97 |
const MDSPAN_IMPL_STANDARD_NAMESPACE::detail::tuple<slice_strides...> &slices_stride_factor) { |
| 98 |
using index_type = typename SrcMapping::index_type; |
| 99 |
return sub_strides<typename SrcMapping::index_type, sizeof...(InvMapIdxs)>{{ |
| 100 |
(static_cast<index_type>(src_mapping.stride(InvMapIdxs)) * |
| 101 |
static_cast<index_type>(get<InvMapIdxs>(slices_stride_factor)))...}}; |
| 102 |
} |
| 103 |
|
| 104 |
template<class SliceSpecifier, class IndexType> |
| 105 |
struct is_range_slice { |
| 106 |
constexpr static bool value = |
| 107 |
std::is_same_v<SliceSpecifier, full_extent_t> || |
| 108 |
index_pair_like<SliceSpecifier, IndexType>::value; |
| 109 |
}; |
| 110 |
|
| 111 |
template<class SliceSpecifier, class IndexType> |
| 112 |
constexpr bool is_range_slice_v = is_range_slice<SliceSpecifier, IndexType>::value; |
| 113 |
|
| 114 |
template<class SliceSpecifier, class IndexType> |
| 115 |
struct is_index_slice { |
| 116 |
constexpr static bool value = std::is_convertible_v<SliceSpecifier, IndexType>; |
| 117 |
}; |
| 118 |
|
| 119 |
template<class SliceSpecifier, class IndexType> |
| 120 |
constexpr bool is_index_slice_v = is_index_slice<SliceSpecifier, IndexType>::value; |
| 121 |
|
| 122 |
} // namespace detail |
| 123 |
|
| 124 |
//********************************** |
| 125 |
// layout_left submdspan_mapping |
| 126 |
//********************************* |
| 127 |
namespace detail { |
| 128 |
|
| 129 |
// Figure out whether to preserve layout_left |
| 130 |
template <class IndexType, size_t SubRank, class IndexSequence, |
| 131 |
class... SliceSpecifiers> |
| 132 |
struct deduce_layout_left_submapping; |
| 133 |
|
| 134 |
template <class IndexType, size_t SubRank, size_t... Idx, |
| 135 |
class... SliceSpecifiers> |
| 136 |
struct deduce_layout_left_submapping< |
| 137 |
IndexType, SubRank, std::index_sequence<Idx...>, SliceSpecifiers...> { |
| 138 |
|
| 139 |
using count_range = index_sequence_scan_impl< |
| 140 |
0u, (is_index_slice_v<SliceSpecifiers, IndexType> ? 0u : 1u)...>; |
| 141 |
|
| 142 |
constexpr static int gap_len = |
| 143 |
(((Idx > 0 && count_range::get(Idx) == 1 && |
| 144 |
is_index_slice_v<SliceSpecifiers, IndexType>) |
| 145 |
? 1 |
| 146 |
: 0) + |
| 147 |
... + 0); |
| 148 |
|
| 149 |
MDSPAN_INLINE_FUNCTION |
| 150 |
constexpr static bool layout_left_value() { |
| 151 |
// Use layout_left for rank 0 |
| 152 |
if constexpr (SubRank == 0) { |
| 153 |
return true; |
| 154 |
// Use layout_left for rank 1 result if leftmost slice specifier is range like |
| 155 |
} else if constexpr (SubRank == 1) { |
| 156 |
return ((Idx > 0 || is_range_slice_v<SliceSpecifiers, IndexType>)&&...); |
| 157 |
} else { |
| 158 |
// Preserve if leftmost SubRank-1 slices are full_extent_t and |
| 159 |
// the slice at idx Subrank - 1 is a range and |
| 160 |
// for idx > SubRank the slice is an index |
| 161 |
return ((((Idx < SubRank - 1) && std::is_same_v<SliceSpecifiers, full_extent_t>) || |
| 162 |
((Idx == SubRank - 1) && is_range_slice_v<SliceSpecifiers, IndexType>) || |
| 163 |
((Idx > SubRank - 1) && is_index_slice_v<SliceSpecifiers, IndexType>)) && ...); |
| 164 |
} |
| 165 |
#if defined(__NVCC__) && !defined(__CUDA_ARCH__) && defined(__GNUC__) |
| 166 |
__builtin_unreachable(); |
| 167 |
#endif |
| 168 |
} |
| 169 |
|
| 170 |
MDSPAN_INLINE_FUNCTION |
| 171 |
constexpr static bool layout_left_padded_value() { |
| 172 |
// Technically could also keep layout_left_padded for SubRank==0 |
| 173 |
// and SubRank==1 with leftmost slice specifier being a contiguous range |
| 174 |
// but we intercept these cases separately |
| 175 |
|
| 176 |
// In all other cases: |
| 177 |
// leftmost slice must be range |
| 178 |
// then there can be a gap with index slices |
| 179 |
// then SubRank - 2 full_extent slices |
| 180 |
// then another range slice |
| 181 |
// then more index slices |
| 182 |
// e.g. R I I I F F F R I I for obtaining a rank-5 from a rank-10 |
| 183 |
return ((((Idx == 0) && is_range_slice_v<SliceSpecifiers, IndexType>) || |
| 184 |
((Idx > 0 && Idx <= gap_len) && is_index_slice_v<SliceSpecifiers, IndexType>) || |
| 185 |
((Idx > gap_len && Idx < gap_len + SubRank - 1) && std::is_same_v<SliceSpecifiers, full_extent_t>) || |
| 186 |
((Idx == gap_len + SubRank - 1) && is_range_slice_v<SliceSpecifiers, IndexType>) || |
| 187 |
((Idx > gap_len + SubRank - 1) && is_index_slice_v<SliceSpecifiers, IndexType>)) && ... ); |
| 188 |
} |
| 189 |
}; |
| 190 |
|
| 191 |
// We are reusing the same thing for layout_left and layout_left_padded |
| 192 |
// For layout_left as source StaticStride is static_extent(0) |
| 193 |
template<class Extents, size_t NumGaps, size_t StaticStride> |
| 194 |
struct compute_s_static_layout_left { |
| 195 |
// Neither StaticStride nor any of the provided extents can be zero. |
| 196 |
// StaticStride can never be zero, the static_extents we are looking at are associated with |
| 197 |
// integral slice specifiers - which wouldn't be valid for zero extent |
| 198 |
template<size_t ... Idx> |
| 199 |
MDSPAN_INLINE_FUNCTION |
| 200 |
static constexpr size_t value(std::index_sequence<Idx...>) { |
| 201 |
size_t val = ((Idx>0 && Idx<=NumGaps ? (Extents::static_extent(Idx) == dynamic_extent?0:Extents::static_extent(Idx)) : 1) * ... * (StaticStride == dynamic_extent?0:StaticStride)); |
| 202 |
return val == 0?dynamic_extent:val; |
| 203 |
} |
| 204 |
}; |
| 205 |
|
| 206 |
} // namespace detail |
| 207 |
|
| 208 |
// Actual submdspan mapping call |
| 209 |
template <class Extents> |
| 210 |
template <class... SliceSpecifiers> |
| 211 |
MDSPAN_INLINE_FUNCTION constexpr auto |
| 212 |
layout_left::mapping<Extents>::submdspan_mapping_impl( |
| 213 |
SliceSpecifiers... slices) const { |
| 214 |
|
| 215 |
// compute sub extents |
| 216 |
using src_ext_t = Extents; |
| 217 |
auto dst_ext = submdspan_extents(extents(), slices...); |
| 218 |
using dst_ext_t = decltype(dst_ext); |
| 219 |
|
| 220 |
// figure out sub layout type |
| 221 |
using deduce_layout = detail::deduce_layout_left_submapping< |
| 222 |
typename dst_ext_t::index_type, dst_ext_t::rank(), |
| 223 |
std::make_index_sequence<src_ext_t::rank()>, |
| 224 |
SliceSpecifiers...>; |
| 225 |
|
| 226 |
// Figure out if any slice's lower bound equals the corresponding extent. |
| 227 |
// If so, bypass evaluating the layout mapping. This fixes LWG Issue 4060. |
| 228 |
const bool out_of_bounds = |
| 229 |
detail::any_slice_out_of_bounds(this->extents(), slices...); |
| 230 |
auto offset = static_cast<size_t>( |
| 231 |
out_of_bounds ? this->required_span_size() |
| 232 |
: this->operator()(detail::first_of(slices)...)); |
| 233 |
|
| 234 |
if constexpr (deduce_layout::layout_left_value()) { |
| 235 |
// layout_left case |
| 236 |
using dst_mapping_t = typename layout_left::template mapping<dst_ext_t>; |
| 237 |
return submdspan_mapping_result<dst_mapping_t>{dst_mapping_t(dst_ext), |
| 238 |
offset}; |
| 239 |
} else if constexpr (deduce_layout::layout_left_padded_value()) { |
| 240 |
constexpr size_t S_static = MDSPAN_IMPL_STANDARD_NAMESPACE::detail::compute_s_static_layout_left<Extents, deduce_layout::gap_len, Extents::static_extent(0)>::value(std::make_index_sequence<Extents::rank()>()); |
| 241 |
using dst_mapping_t = typename MDSPAN_IMPL_PROPOSED_NAMESPACE::layout_left_padded<S_static>::template mapping<dst_ext_t>; |
| 242 |
return submdspan_mapping_result<dst_mapping_t>{ |
| 243 |
dst_mapping_t(dst_ext, stride(1 + deduce_layout::gap_len)), offset}; |
| 244 |
} else { |
| 245 |
// layout_stride case |
| 246 |
using dst_mapping_t = typename layout_stride::mapping<dst_ext_t>; |
| 247 |
auto inv_map = detail::inv_map_rank(std::integral_constant<size_t, 0>(), |
| 248 |
std::index_sequence<>(), slices...); |
| 249 |
return submdspan_mapping_result<dst_mapping_t> { |
| 250 |
dst_mapping_t(mdspan_non_standard, dst_ext, |
| 251 |
detail::construct_sub_strides( |
| 252 |
*this, inv_map, |
| 253 |
// HIP needs deduction guides to have markups so we need to be explicit |
| 254 |
// NVCC 11.0 has a bug with deduction guide here, tested that 11.2 does not have |
| 255 |
// the issue but Clang-CUDA also doesn't accept the use of deduction guide so |
| 256 |
// disable it for CUDA altogether |
| 257 |
#if defined(_MDSPAN_HAS_HIP) || defined(_MDSPAN_HAS_CUDA) |
| 258 |
detail::tuple<decltype(detail::stride_of(slices))...>{ |
| 259 |
detail::stride_of(slices)...}).values), |
| 260 |
#else |
| 261 |
detail::tuple{detail::stride_of(slices)...}).values), |
| 262 |
#endif |
| 263 |
offset |
| 264 |
}; |
| 265 |
} |
| 266 |
#if defined(__NVCC__) && !defined(__CUDA_ARCH__) && defined(__GNUC__) |
| 267 |
__builtin_unreachable(); |
| 268 |
#endif |
| 269 |
} |
| 270 |
|
| 271 |
template <size_t PaddingValue> |
| 272 |
template <class Extents> |
| 273 |
template <class... SliceSpecifiers> |
| 274 |
MDSPAN_INLINE_FUNCTION constexpr auto |
| 275 |
MDSPAN_IMPL_PROPOSED_NAMESPACE::layout_left_padded<PaddingValue>::mapping<Extents>::submdspan_mapping_impl( |
| 276 |
SliceSpecifiers... slices) const { |
| 277 |
|
| 278 |
// compute sub extents |
| 279 |
using src_ext_t = Extents; |
| 280 |
auto dst_ext = submdspan_extents(extents(), slices...); |
| 281 |
using dst_ext_t = decltype(dst_ext); |
| 282 |
|
| 283 |
if constexpr (Extents::rank() == 0) { // rank-0 case |
| 284 |
using dst_mapping_t = typename MDSPAN_IMPL_PROPOSED_NAMESPACE::layout_left_padded<PaddingValue>::template mapping<Extents>; |
| 285 |
return submdspan_mapping_result<dst_mapping_t>{*this, 0}; |
| 286 |
} else { |
| 287 |
const bool out_of_bounds = |
| 288 |
MDSPAN_IMPL_STANDARD_NAMESPACE::detail::any_slice_out_of_bounds(this->extents(), slices...); |
| 289 |
auto offset = static_cast<size_t>( |
| 290 |
out_of_bounds ? this->required_span_size() |
| 291 |
: this->operator()(MDSPAN_IMPL_STANDARD_NAMESPACE::detail::first_of(slices)...)); |
| 292 |
if constexpr (dst_ext_t::rank() == 0) { // result rank-0 |
| 293 |
// The following for some reasons leads to compiler error later, while not using a typedef works: |
| 294 |
// Compilers: CUDA 11.2 with GCC 9.1 |
| 295 |
// |
| 296 |
// using dst_mapping_t = typename layout_left::template mapping<dst_ext_t>; |
| 297 |
// return submdspan_mapping_result<dst_mapping_t>{dst_mapping_t{dst_ext}, offset}; |
| 298 |
// |
| 299 |
// Error: submdspan_mapping.hpp:299:23: error: 'dst_mapping_t' does not name a type |
| 300 |
// 299 | using dst_mapping_t = typename layout_left::template mapping<dst_ext_t>; |
| 301 |
// The same error is given (about dst_mapping_t not naming type) when a different name is used in 299: |
| 302 |
// using dst_mapping_t2 = typename layout_left::template mapping<dst_ext_t>; |
| 303 |
|
| 304 |
return submdspan_mapping_result<typename layout_left::template mapping<dst_ext_t>> |
| 305 |
{typename layout_left::template mapping<dst_ext_t>{dst_ext}, offset}; |
| 306 |
} else { // general case |
| 307 |
// Figure out if any slice's lower bound equals the corresponding extent. |
| 308 |
// If so, bypass evaluating the layout mapping. This fixes LWG Issue 4060. |
| 309 |
// figure out sub layout type |
| 310 |
using deduce_layout = MDSPAN_IMPL_STANDARD_NAMESPACE::detail::deduce_layout_left_submapping< |
| 311 |
typename dst_ext_t::index_type, dst_ext_t::rank(), |
| 312 |
decltype(std::make_index_sequence<src_ext_t::rank()>()), |
| 313 |
SliceSpecifiers...>; |
| 314 |
|
| 315 |
if constexpr (deduce_layout::layout_left_value() && dst_ext_t::rank() == 1) { // getting rank-1 from leftmost |
| 316 |
using dst_mapping_t = typename layout_left::template mapping<dst_ext_t>; |
| 317 |
return submdspan_mapping_result<dst_mapping_t>{dst_mapping_t{dst_ext}, offset}; |
| 318 |
} else if constexpr (deduce_layout::layout_left_padded_value()) { // can keep layout_left_padded |
| 319 |
constexpr size_t S_static = MDSPAN_IMPL_STANDARD_NAMESPACE::detail::compute_s_static_layout_left<Extents, deduce_layout::gap_len, static_padding_stride>::value(std::make_index_sequence<Extents::rank()>()); |
| 320 |
using dst_mapping_t = typename MDSPAN_IMPL_PROPOSED_NAMESPACE::layout_left_padded<S_static>::template mapping<dst_ext_t>; |
| 321 |
return submdspan_mapping_result<dst_mapping_t>{ |
| 322 |
dst_mapping_t(dst_ext, stride(1 + deduce_layout::gap_len)), offset}; |
| 323 |
} else { // layout_stride |
| 324 |
auto inv_map = MDSPAN_IMPL_STANDARD_NAMESPACE::detail::inv_map_rank(std::integral_constant<size_t, 0>(), |
| 325 |
std::index_sequence<>(), slices...); |
| 326 |
using dst_mapping_t = typename layout_stride::template mapping<dst_ext_t>; |
| 327 |
return submdspan_mapping_result<dst_mapping_t> { |
| 328 |
dst_mapping_t(mdspan_non_standard, dst_ext, |
| 329 |
MDSPAN_IMPL_STANDARD_NAMESPACE::detail::construct_sub_strides( |
| 330 |
*this, inv_map, |
| 331 |
// HIP needs deduction guides to have markups so we need to be explicit |
| 332 |
// NVCC 11.0 has a bug with deduction guide here, tested that 11.2 does not have |
| 333 |
// the issue but Clang-CUDA also doesn't accept the use of deduction guide so |
| 334 |
// disable it for CUDA alltogether |
| 335 |
#if defined(_MDSPAN_HAS_HIP) || defined(_MDSPAN_HAS_CUDA) |
| 336 |
MDSPAN_IMPL_STANDARD_NAMESPACE::detail::tuple<decltype(MDSPAN_IMPL_STANDARD_NAMESPACE::detail::stride_of(slices))...>{ |
| 337 |
MDSPAN_IMPL_STANDARD_NAMESPACE::detail::stride_of(slices)...}).values), |
| 338 |
#else |
| 339 |
MDSPAN_IMPL_STANDARD_NAMESPACE::detail::tuple{MDSPAN_IMPL_STANDARD_NAMESPACE::detail::stride_of(slices)...}).values), |
| 340 |
#endif |
| 341 |
offset |
| 342 |
}; |
| 343 |
} |
| 344 |
} |
| 345 |
} |
| 346 |
|
| 347 |
|
| 348 |
#if defined(__NVCC__) && !defined(__CUDA_ARCH__) && defined(__GNUC__) |
| 349 |
__builtin_unreachable(); |
| 350 |
#endif |
| 351 |
} |
| 352 |
|
| 353 |
//********************************** |
| 354 |
// layout_right submdspan_mapping |
| 355 |
//********************************* |
| 356 |
namespace detail { |
| 357 |
|
| 358 |
// Figure out whether to preserve layout_right |
| 359 |
template <class IndexType, size_t SubRank, class IndexSequence, |
| 360 |
class... SliceSpecifiers> |
| 361 |
struct deduce_layout_right_submapping; |
| 362 |
|
| 363 |
template <class IndexType, size_t SubRank, size_t... Idx, |
| 364 |
class... SliceSpecifiers> |
| 365 |
struct deduce_layout_right_submapping< |
| 366 |
IndexType, SubRank, std::index_sequence<Idx...>, SliceSpecifiers...> { |
| 367 |
|
| 368 |
static constexpr size_t Rank = sizeof...(Idx); |
| 369 |
using count_range = index_sequence_scan_impl< |
| 370 |
0u, (std::is_convertible_v<SliceSpecifiers, IndexType> ? 0u : 1u)...>; |
| 371 |
//__static_partial_sums<!std::is_convertible_v<SliceSpecifiers, |
| 372 |
// IndexType>...>; |
| 373 |
constexpr static int gap_len = |
| 374 |
(((Idx < Rank - 1 && count_range::get(Idx) == SubRank - 1 && |
| 375 |
std::is_convertible_v<SliceSpecifiers, IndexType>) |
| 376 |
? 1 |
| 377 |
: 0) + |
| 378 |
... + 0); |
| 379 |
|
| 380 |
MDSPAN_INLINE_FUNCTION |
| 381 |
constexpr static bool layout_right_value() { |
| 382 |
// Use layout_right for rank 0 |
| 383 |
if constexpr (SubRank == 0) { |
| 384 |
return true; |
| 385 |
// Use layout_right for rank 1 result if rightmost slice specifier is range like |
| 386 |
} else if constexpr (SubRank == 1) { |
| 387 |
return ((Idx < Rank - 1 || is_range_slice_v<SliceSpecifiers, IndexType>)&&...); |
| 388 |
} else { |
| 389 |
// Preserve if rightmost SubRank-1 slices are full_extent_t and |
| 390 |
// the slice at idx Rank-Subrank is a range and |
| 391 |
// for idx < Rank - SubRank the slice is an index |
| 392 |
return ((((Idx >= Rank - SubRank) && std::is_same_v<SliceSpecifiers, full_extent_t>) || |
| 393 |
((Idx == Rank - SubRank) && is_range_slice_v<SliceSpecifiers, IndexType>) || |
| 394 |
((Idx < Rank - SubRank) && is_index_slice_v<SliceSpecifiers, IndexType>)) && ...); |
| 395 |
} |
| 396 |
#if defined(__NVCC__) && !defined(__CUDA_ARCH__) && defined(__GNUC__) |
| 397 |
__builtin_unreachable(); |
| 398 |
#endif |
| 399 |
} |
| 400 |
|
| 401 |
MDSPAN_INLINE_FUNCTION |
| 402 |
constexpr static bool layout_right_padded_value() { |
| 403 |
// Technically could also keep layout_right_padded for SubRank==0 |
| 404 |
// and SubRank==1 with rightmost slice specifier being a contiguous range |
| 405 |
// but we intercept these cases separately |
| 406 |
|
| 407 |
// In all other cases: |
| 408 |
// rightmost slice must be range |
| 409 |
// then there can be a gap with index slices |
| 410 |
// then SubRank - 2 full_extent slices |
| 411 |
// then another range slice |
| 412 |
// then more index slices |
| 413 |
// e.g. I I R F F F I I I R for obtaining a rank-5 from a rank-10 |
| 414 |
return ((((Idx == Rank - 1) && is_range_slice_v<SliceSpecifiers, IndexType>) || |
| 415 |
((Idx >= Rank - gap_len - 1 && Idx < Rank - 1) && is_index_slice_v<SliceSpecifiers, IndexType>) || |
| 416 |
((Idx > Rank - gap_len - SubRank && Idx < Rank - gap_len - 1) && std::is_same_v<SliceSpecifiers, full_extent_t>) || |
| 417 |
((Idx == Rank - gap_len - SubRank) && is_range_slice_v<SliceSpecifiers, IndexType>) || |
| 418 |
((Idx < Rank - gap_len - SubRank) && is_index_slice_v<SliceSpecifiers, IndexType>)) && ... ); |
| 419 |
} |
| 420 |
}; |
| 421 |
|
| 422 |
// We are reusing the same thing for layout_right and layout_right_padded |
| 423 |
// For layout_right as source StaticStride is static_extent(Rank-1) |
| 424 |
template<class Extents, size_t NumGaps, size_t StaticStride> |
| 425 |
struct compute_s_static_layout_right { |
| 426 |
// Neither StaticStride nor any of the provided extents can be zero. |
| 427 |
// StaticStride can never be zero, the static_extents we are looking at are associated with |
| 428 |
// integral slice specifiers - which wouldn't be valid for zero extent |
| 429 |
template<size_t ... Idx> |
| 430 |
MDSPAN_INLINE_FUNCTION |
| 431 |
static constexpr size_t value(std::index_sequence<Idx...>) { |
| 432 |
size_t val = ((Idx >= Extents::rank() - 1 - NumGaps && Idx < Extents::rank() - 1 ? (Extents::static_extent(Idx) == dynamic_extent?0:Extents::static_extent(Idx)) : 1) * ... * (StaticStride == dynamic_extent?0:StaticStride)); |
| 433 |
return val == 0?dynamic_extent:val; |
| 434 |
} |
| 435 |
}; |
| 436 |
|
| 437 |
} // namespace detail |
| 438 |
|
| 439 |
// Actual submdspan mapping call |
| 440 |
template <class Extents> |
| 441 |
template <class... SliceSpecifiers> |
| 442 |
MDSPAN_INLINE_FUNCTION constexpr auto |
| 443 |
layout_right::mapping<Extents>::submdspan_mapping_impl( |
| 444 |
SliceSpecifiers... slices) const { |
| 445 |
|
| 446 |
// compute sub extents |
| 447 |
using src_ext_t = Extents; |
| 448 |
auto dst_ext = submdspan_extents(extents(), slices...); |
| 449 |
using dst_ext_t = decltype(dst_ext); |
| 450 |
|
| 451 |
// figure out sub layout type |
| 452 |
using deduce_layout = detail::deduce_layout_right_submapping< |
| 453 |
typename dst_ext_t::index_type, dst_ext_t::rank(), |
| 454 |
std::make_index_sequence<src_ext_t::rank()>, |
| 455 |
SliceSpecifiers...>; |
| 456 |
|
| 457 |
// Figure out if any slice's lower bound equals the corresponding extent. |
| 458 |
// If so, bypass evaluating the layout mapping. This fixes LWG Issue 4060. |
| 459 |
const bool out_of_bounds = |
| 460 |
detail::any_slice_out_of_bounds(this->extents(), slices...); |
| 461 |
auto offset = static_cast<size_t>( |
| 462 |
out_of_bounds ? this->required_span_size() |
| 463 |
: this->operator()(detail::first_of(slices)...)); |
| 464 |
|
| 465 |
if constexpr (deduce_layout::layout_right_value()) { |
| 466 |
// layout_right case |
| 467 |
using dst_mapping_t = typename layout_right::mapping<dst_ext_t>; |
| 468 |
return submdspan_mapping_result<dst_mapping_t>{dst_mapping_t(dst_ext), |
| 469 |
offset}; |
| 470 |
} else if constexpr (deduce_layout::layout_right_padded_value()) { |
| 471 |
constexpr size_t S_static = MDSPAN_IMPL_STANDARD_NAMESPACE::detail::compute_s_static_layout_left<Extents, deduce_layout::gap_len, Extents::static_extent(Extents::rank() - 1)>::value(std::make_index_sequence<Extents::rank()>()); |
| 472 |
using dst_mapping_t = typename MDSPAN_IMPL_PROPOSED_NAMESPACE::layout_right_padded<S_static>::template mapping<dst_ext_t>; |
| 473 |
return submdspan_mapping_result<dst_mapping_t>{ |
| 474 |
dst_mapping_t(dst_ext, |
| 475 |
stride(src_ext_t::rank() - 2 - deduce_layout::gap_len)), |
| 476 |
offset}; |
| 477 |
} else { |
| 478 |
// layout_stride case |
| 479 |
using dst_mapping_t = typename layout_stride::mapping<dst_ext_t>; |
| 480 |
auto inv_map = detail::inv_map_rank(std::integral_constant<size_t, 0>(), |
| 481 |
std::index_sequence<>(), slices...); |
| 482 |
return submdspan_mapping_result<dst_mapping_t> { |
| 483 |
dst_mapping_t(mdspan_non_standard, dst_ext, |
| 484 |
detail::construct_sub_strides( |
| 485 |
*this, inv_map, |
| 486 |
// HIP needs deduction guides to have markups so we need to be explicit |
| 487 |
// NVCC 11.0 has a bug with deduction guide here, tested that 11.2 does not have |
| 488 |
// the issue but Clang-CUDA also doesn't accept the use of deduction guide so |
| 489 |
// disable it for CUDA altogether |
| 490 |
#if defined(_MDSPAN_HAS_HIP) || defined(_MDSPAN_HAS_CUDA) |
| 491 |
MDSPAN_IMPL_STANDARD_NAMESPACE::detail::tuple<decltype(detail::stride_of(slices))...>{ |
| 492 |
detail::stride_of(slices)...}).values), |
| 493 |
#else |
| 494 |
MDSPAN_IMPL_STANDARD_NAMESPACE::detail::tuple{detail::stride_of(slices)...}).values), |
| 495 |
#endif |
| 496 |
offset |
| 497 |
}; |
| 498 |
} |
| 499 |
#if defined(__NVCC__) && !defined(__CUDA_ARCH__) && defined(__GNUC__) |
| 500 |
__builtin_unreachable(); |
| 501 |
#endif |
| 502 |
} |
| 503 |
|
| 504 |
template <size_t PaddingValue> |
| 505 |
template <class Extents> |
| 506 |
template <class... SliceSpecifiers> |
| 507 |
MDSPAN_INLINE_FUNCTION constexpr auto |
| 508 |
MDSPAN_IMPL_PROPOSED_NAMESPACE::layout_right_padded<PaddingValue>::mapping<Extents>::submdspan_mapping_impl( |
| 509 |
SliceSpecifiers... slices) const { |
| 510 |
|
| 511 |
// compute sub extents |
| 512 |
using src_ext_t = Extents; |
| 513 |
auto dst_ext = submdspan_extents(extents(), slices...); |
| 514 |
using dst_ext_t = decltype(dst_ext); |
| 515 |
|
| 516 |
if constexpr (Extents::rank() == 0) { // rank-0 case |
| 517 |
using dst_mapping_t = typename MDSPAN_IMPL_PROPOSED_NAMESPACE::layout_right_padded<PaddingValue>::template mapping<Extents>; |
| 518 |
return submdspan_mapping_result<dst_mapping_t>{*this, 0}; |
| 519 |
} else { |
| 520 |
// Figure out if any slice's lower bound equals the corresponding extent. |
| 521 |
// If so, bypass evaluating the layout mapping. This fixes LWG Issue 4060. |
| 522 |
// figure out sub layout type |
| 523 |
const bool out_of_bounds = |
| 524 |
MDSPAN_IMPL_STANDARD_NAMESPACE::detail::any_slice_out_of_bounds(this->extents(), slices...); |
| 525 |
auto offset = static_cast<size_t>( |
| 526 |
out_of_bounds ? this->required_span_size() |
| 527 |
: this->operator()(MDSPAN_IMPL_STANDARD_NAMESPACE::detail::first_of(slices)...)); |
| 528 |
if constexpr (dst_ext_t::rank() == 0) { // result rank-0 |
| 529 |
// Same issue as in layout_left_padded: see comment there |
| 530 |
// using dst_mapping_t = typename layout_right::template mapping<dst_ext_t>; |
| 531 |
// return submdspan_mapping_result<dst_mapping_t>{dst_mapping_t{dst_ext}, offset}; |
| 532 |
return submdspan_mapping_result<typename layout_right::template mapping<dst_ext_t>> |
| 533 |
{typename layout_right::template mapping<dst_ext_t>{dst_ext}, offset}; |
| 534 |
} else { // general case |
| 535 |
using deduce_layout = MDSPAN_IMPL_STANDARD_NAMESPACE::detail::deduce_layout_right_submapping< |
| 536 |
typename dst_ext_t::index_type, dst_ext_t::rank(), |
| 537 |
decltype(std::make_index_sequence<src_ext_t::rank()>()), |
| 538 |
SliceSpecifiers...>; |
| 539 |
|
| 540 |
if constexpr (deduce_layout::layout_right_value() && dst_ext_t::rank() == 1) { // getting rank-1 from rightmost |
| 541 |
using dst_mapping_t = typename layout_right::template mapping<dst_ext_t>; |
| 542 |
return submdspan_mapping_result<dst_mapping_t>{dst_mapping_t{dst_ext}, offset}; |
| 543 |
} else if constexpr (deduce_layout::layout_right_padded_value()) { // can keep layout_right_padded |
| 544 |
constexpr size_t S_static = MDSPAN_IMPL_STANDARD_NAMESPACE::detail::compute_s_static_layout_right<Extents, deduce_layout::gap_len, static_padding_stride>::value(std::make_index_sequence<Extents::rank()>()); |
| 545 |
using dst_mapping_t = typename MDSPAN_IMPL_PROPOSED_NAMESPACE::layout_right_padded<S_static>::template mapping<dst_ext_t>; |
| 546 |
return submdspan_mapping_result<dst_mapping_t>{ |
| 547 |
dst_mapping_t(dst_ext, stride(Extents::rank() - 2 - deduce_layout::gap_len)), offset}; |
| 548 |
} else { // layout_stride |
| 549 |
auto inv_map = MDSPAN_IMPL_STANDARD_NAMESPACE::detail::inv_map_rank(std::integral_constant<size_t, 0>(), |
| 550 |
std::index_sequence<>(), slices...); |
| 551 |
using dst_mapping_t = typename layout_stride::template mapping<dst_ext_t>; |
| 552 |
return submdspan_mapping_result<dst_mapping_t> { |
| 553 |
dst_mapping_t(mdspan_non_standard, dst_ext, |
| 554 |
MDSPAN_IMPL_STANDARD_NAMESPACE::detail::construct_sub_strides( |
| 555 |
*this, inv_map, |
| 556 |
// HIP needs deduction guides to have markups so we need to be explicit |
| 557 |
// NVCC 11.0 has a bug with deduction guide here, tested that 11.2 does not have |
| 558 |
// the issue but Clang-CUDA also doesn't accept the use of deduction guide so |
| 559 |
// disable it for CUDA alltogether |
| 560 |
#if defined(_MDSPAN_HAS_HIP) || defined(_MDSPAN_HAS_CUDA) |
| 561 |
MDSPAN_IMPL_STANDARD_NAMESPACE::detail::tuple<decltype(MDSPAN_IMPL_STANDARD_NAMESPACE::detail::stride_of(slices))...>{ |
| 562 |
MDSPAN_IMPL_STANDARD_NAMESPACE::detail::stride_of(slices)...}).values), |
| 563 |
#else |
| 564 |
MDSPAN_IMPL_STANDARD_NAMESPACE::detail::tuple{MDSPAN_IMPL_STANDARD_NAMESPACE::detail::stride_of(slices)...}).values), |
| 565 |
#endif |
| 566 |
offset |
| 567 |
}; |
| 568 |
} |
| 569 |
} |
| 570 |
} |
| 571 |
|
| 572 |
|
| 573 |
#if defined(__NVCC__) && !defined(__CUDA_ARCH__) && defined(__GNUC__) |
| 574 |
__builtin_unreachable(); |
| 575 |
#endif |
| 576 |
} |
| 577 |
|
| 578 |
//********************************** |
| 579 |
// layout_stride submdspan_mapping |
| 580 |
//********************************* |
| 581 |
template <class Extents> |
| 582 |
template <class... SliceSpecifiers> |
| 583 |
MDSPAN_INLINE_FUNCTION constexpr auto |
| 584 |
layout_stride::mapping<Extents>::submdspan_mapping_impl( |
| 585 |
SliceSpecifiers... slices) const { |
| 586 |
auto dst_ext = submdspan_extents(extents(), slices...); |
| 587 |
using dst_ext_t = decltype(dst_ext); |
| 588 |
auto inv_map = detail::inv_map_rank(std::integral_constant<size_t, 0>(), |
| 589 |
std::index_sequence<>(), slices...); |
| 590 |
using dst_mapping_t = typename layout_stride::template mapping<dst_ext_t>; |
| 591 |
|
| 592 |
// Figure out if any slice's lower bound equals the corresponding extent. |
| 593 |
// If so, bypass evaluating the layout mapping. This fixes LWG Issue 4060. |
| 594 |
const bool out_of_bounds = |
| 595 |
detail::any_slice_out_of_bounds(this->extents(), slices...); |
| 596 |
auto offset = static_cast<size_t>( |
| 597 |
out_of_bounds ? this->required_span_size() |
| 598 |
: this->operator()(detail::first_of(slices)...)); |
| 599 |
|
| 600 |
return submdspan_mapping_result<dst_mapping_t> { |
| 601 |
dst_mapping_t(mdspan_non_standard, dst_ext, |
| 602 |
detail::construct_sub_strides( |
| 603 |
*this, inv_map, |
| 604 |
// HIP needs deduction guides to have markups so we need to be explicit |
| 605 |
// NVCC 11.0 has a bug with deduction guide here, tested that 11.2 does not have |
| 606 |
// the issue but Clang-CUDA also doesn't accept the use of deduction guide so |
| 607 |
// disable it for CUDA alltogether |
| 608 |
#if defined(_MDSPAN_HAS_HIP) || defined(_MDSPAN_HAS_CUDA) |
| 609 |
MDSPAN_IMPL_STANDARD_NAMESPACE::detail::tuple<decltype(detail::stride_of(slices))...>( |
| 610 |
detail::stride_of(slices)...)).values), |
| 611 |
#else |
| 612 |
MDSPAN_IMPL_STANDARD_NAMESPACE::detail::tuple(detail::stride_of(slices)...)).values), |
| 613 |
#endif |
| 614 |
offset |
| 615 |
}; |
| 616 |
} |
| 617 |
|
| 618 |
} // namespace MDSPAN_IMPL_STANDARD_NAMESPACE |
| 619 |
|
| 620 |
#if defined __NVCC__ |
| 621 |
#ifdef __NVCC_DIAG_PRAGMA_SUPPORT__ |
| 622 |
#pragma nv_diagnostic pop |
| 623 |
#else |
| 624 |
#ifdef __CUDA_ARCH__ |
| 625 |
#pragma diagnostic pop |
| 626 |
#endif |
| 627 |
#endif |
| 628 |
#elif defined __NVCOMPILER |
| 629 |
#pragma diagnostic pop |
| 630 |
#endif |