| 1 |
//@HEADER |
| 2 |
// ************************************************************************ |
| 3 |
// |
| 4 |
// Kokkos v. 4.0 |
| 5 |
// Copyright (2022) National Technology & Engineering |
| 6 |
// Solutions of Sandia, LLC (NTESS). |
| 7 |
// |
| 8 |
// Under the terms of Contract DE-NA0003525 with NTESS, |
| 9 |
// the U.S. Government retains certain rights in this software. |
| 10 |
// |
| 11 |
// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. |
| 12 |
// |
| 13 |
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 14 |
// |
| 15 |
//@HEADER |
| 16 |
#pragma once |
| 17 |
|
| 18 |
#include <cassert> |
| 19 |
#include "layout_padded_fwd.hpp" |
| 20 |
#include "../__p0009_bits/dynamic_extent.hpp" |
| 21 |
#include "../__p0009_bits/extents.hpp" |
| 22 |
#include "../__p0009_bits/mdspan.hpp" |
| 23 |
#include "../__p0009_bits/layout_left.hpp" |
| 24 |
#include "../__p0009_bits/layout_right.hpp" |
| 25 |
#include "../__p0009_bits/layout_stride.hpp" |
| 26 |
#include "../__p0009_bits/utility.hpp" |
| 27 |
|
| 28 |
namespace MDSPAN_IMPL_STANDARD_NAMESPACE { |
| 29 |
namespace MDSPAN_IMPL_PROPOSED_NAMESPACE { |
| 30 |
|
| 31 |
namespace detail { |
| 32 |
template<class _T> |
| 33 |
MDSPAN_INLINE_FUNCTION |
| 34 |
constexpr _T |
| 35 |
find_next_multiple(_T alignment, _T offset) |
| 36 |
{ |
| 37 |
if ( alignment == 0 ) { |
| 38 |
return _T(0); |
| 39 |
} else { |
| 40 |
return ( ( offset + alignment - 1 ) / alignment) * alignment; |
| 41 |
} |
| 42 |
} |
| 43 |
|
| 44 |
template <class _ExtentsType, size_t _PaddingValue, size_t _ExtentToPadIdx> |
| 45 |
MDSPAN_INLINE_FUNCTION constexpr size_t get_actual_static_padding_value() { |
| 46 |
constexpr auto rank = _ExtentsType::rank(); |
| 47 |
|
| 48 |
if constexpr (rank <= typename _ExtentsType::rank_type(1)) { |
| 49 |
return 0; |
| 50 |
} else if constexpr (_PaddingValue != dynamic_extent && |
| 51 |
_ExtentsType::static_extent(_ExtentToPadIdx) != |
| 52 |
dynamic_extent) { |
| 53 |
static_assert( |
| 54 |
(_PaddingValue != 0) || |
| 55 |
(_ExtentsType::static_extent(_ExtentToPadIdx) == 0), |
| 56 |
"padding stride can be 0 only if " |
| 57 |
"extents_type::static_extent(extent-to-pad) is 0 or dynamic_extent"); |
| 58 |
return find_next_multiple(_PaddingValue, |
| 59 |
_ExtentsType::static_extent(_ExtentToPadIdx)); |
| 60 |
} else { |
| 61 |
return dynamic_extent; |
| 62 |
} |
| 63 |
// Missing return statement warning from NVCC and ICC |
| 64 |
#if (defined(__NVCC__) || defined(__INTEL_COMPILER)) && !defined(__NVCOMPILER) |
| 65 |
return 0; |
| 66 |
#endif |
| 67 |
} |
| 68 |
|
| 69 |
template <size_t _PaddingValue, typename _Extents, size_t _ExtentToPadIdx, size_t _Rank, typename Enabled = void> |
| 70 |
struct static_array_type_for_padded_extent |
| 71 |
{ |
| 72 |
static constexpr size_t padding_value = _PaddingValue; |
| 73 |
using index_type = typename _Extents::index_type; |
| 74 |
using extents_type = _Extents; |
| 75 |
using type = ::MDSPAN_IMPL_STANDARD_NAMESPACE::detail::maybe_static_array< |
| 76 |
index_type, size_t, dynamic_extent, |
| 77 |
::MDSPAN_IMPL_STANDARD_NAMESPACE::MDSPAN_IMPL_PROPOSED_NAMESPACE::detail::get_actual_static_padding_value<extents_type, _PaddingValue, |
| 78 |
_ExtentToPadIdx>()>; |
| 79 |
}; |
| 80 |
|
| 81 |
template <size_t _PaddingValue, typename _Extents, size_t _ExtentToPadIdx, size_t Rank> |
| 82 |
struct static_array_type_for_padded_extent<_PaddingValue, _Extents, |
| 83 |
_ExtentToPadIdx, Rank, std::enable_if_t<Rank <= 1>> { |
| 84 |
using index_type = typename _Extents::index_type; |
| 85 |
using extents_type = _Extents; |
| 86 |
using type = |
| 87 |
::MDSPAN_IMPL_STANDARD_NAMESPACE::detail::maybe_static_array< |
| 88 |
index_type, size_t, dynamic_extent, 0>; |
| 89 |
}; |
| 90 |
|
| 91 |
template <size_t _PaddingValue, typename _Extents, size_t _ExtentToPadIdx> |
| 92 |
struct padded_extent { |
| 93 |
static constexpr size_t padding_value = _PaddingValue; |
| 94 |
using index_type = typename _Extents::index_type; |
| 95 |
using extents_type = _Extents; |
| 96 |
using static_array_type = typename static_array_type_for_padded_extent< |
| 97 |
padding_value, _Extents, _ExtentToPadIdx, _Extents::rank()>::type; |
| 98 |
|
| 99 |
MDSPAN_INLINE_FUNCTION |
| 100 |
static constexpr auto static_value() { return static_array_type::static_value(0); } |
| 101 |
|
| 102 |
MDSPAN_INLINE_FUNCTION |
| 103 |
static constexpr static_array_type |
| 104 |
init_padding(const _Extents &exts) { |
| 105 |
if constexpr ((_Extents::rank() > 1) && (padding_value == dynamic_extent)) { |
| 106 |
return {exts.extent(_ExtentToPadIdx)}; |
| 107 |
} else { |
| 108 |
return init_padding(exts, padding_value); |
| 109 |
} |
| 110 |
// Missing return statement warning from NVCC and ICC |
| 111 |
#if (defined(__NVCC__) || defined(__INTEL_COMPILER)) && !defined(__NVCOMPILER) |
| 112 |
return {}; |
| 113 |
#endif |
| 114 |
} |
| 115 |
|
| 116 |
MDSPAN_INLINE_FUNCTION static constexpr static_array_type |
| 117 |
init_padding([[maybe_unused]] const _Extents &exts, |
| 118 |
[[maybe_unused]] index_type pv) { |
| 119 |
if constexpr (_Extents::rank() > 1) { |
| 120 |
return {find_next_multiple(pv, |
| 121 |
exts.extent(_ExtentToPadIdx))}; |
| 122 |
} else { |
| 123 |
return {}; |
| 124 |
} |
| 125 |
// Missing return statement warning from NVCC and ICC |
| 126 |
#if (defined(__NVCC__) || defined(__INTEL_COMPILER)) && !defined(__NVCOMPILER) |
| 127 |
return {}; |
| 128 |
#endif |
| 129 |
} |
| 130 |
|
| 131 |
template <typename _Mapping, size_t _PaddingStrideIdx> |
| 132 |
MDSPAN_INLINE_FUNCTION static constexpr static_array_type |
| 133 |
init_padding([[maybe_unused]] const _Mapping &other_mapping, |
| 134 |
std::integral_constant<size_t, _PaddingStrideIdx>) { |
| 135 |
if constexpr (_Extents::rank() > 1) { |
| 136 |
return {other_mapping.stride(_PaddingStrideIdx)}; |
| 137 |
} else { |
| 138 |
return {}; |
| 139 |
} |
| 140 |
// Missing return statement warning from NVCC and ICC |
| 141 |
#if (defined(__NVCC__) || defined(__INTEL_COMPILER)) && !defined(__NVCOMPILER) |
| 142 |
return {}; |
| 143 |
#endif |
| 144 |
} |
| 145 |
}; |
| 146 |
} // namespace detail |
| 147 |
|
| 148 |
template <size_t PaddingValue> |
| 149 |
template <class Extents> |
| 150 |
class layout_left_padded<PaddingValue>::mapping { |
| 151 |
public: |
| 152 |
static constexpr size_t padding_value = PaddingValue; |
| 153 |
|
| 154 |
using extents_type = Extents; |
| 155 |
using index_type = typename extents_type::index_type; |
| 156 |
using size_type = typename extents_type::size_type; |
| 157 |
using rank_type = typename extents_type::rank_type; |
| 158 |
using layout_type = layout_left_padded<padding_value>; |
| 159 |
|
| 160 |
#ifndef MDSPAN_INTERNAL_TEST |
| 161 |
private: |
| 162 |
#endif // MDSPAN_INTERNAL_TEST |
| 163 |
|
| 164 |
static constexpr rank_type padded_stride_idx = detail::layout_padded_constants<layout_type, extents_type>::padded_stride_idx; |
| 165 |
static constexpr rank_type extent_to_pad_idx = detail::layout_padded_constants<layout_type, extents_type>::extent_to_pad_idx; |
| 166 |
|
| 167 |
static_assert((padding_value != 0) |
| 168 |
|| (extents_type::static_extent(extent_to_pad_idx) == 0) |
| 169 |
|| (extents_type::static_extent(extent_to_pad_idx) == dynamic_extent), |
| 170 |
"out of bounds access for rank 0"); |
| 171 |
|
| 172 |
using padded_stride_type = detail::padded_extent< padding_value, extents_type, extent_to_pad_idx >; |
| 173 |
|
| 174 |
static constexpr size_t static_padding_stride = padded_stride_type::static_value(); |
| 175 |
|
| 176 |
typename padded_stride_type::static_array_type padded_stride = {}; |
| 177 |
extents_type exts = {}; |
| 178 |
|
| 179 |
MDSPAN_INLINE_FUNCTION constexpr index_type |
| 180 |
compute_offset(std::index_sequence<>) const { |
| 181 |
return 0; |
| 182 |
} |
| 183 |
|
| 184 |
template <size_t Rank, class IndexOffset> |
| 185 |
MDSPAN_INLINE_FUNCTION constexpr index_type |
| 186 |
compute_offset(std::index_sequence<Rank>, IndexOffset index_offset) const { |
| 187 |
return index_offset; |
| 188 |
} |
| 189 |
|
| 190 |
template <size_t... Ranks, class... IndexOffsets> |
| 191 |
MDSPAN_INLINE_FUNCTION constexpr index_type |
| 192 |
compute_offset(std::index_sequence<Ranks...>, |
| 193 |
IndexOffsets... index_offsets) const { |
| 194 |
index_type indices[] = {static_cast<index_type>(index_offsets)...}; |
| 195 |
// self-recursive fold trick from |
| 196 |
// https://github.com/llvm/llvm-project/blob/96e1914aa2e6d8966acbfbe2f4d184201f1aa318/libcxx/include/mdspan/layout_left.h#L144 |
| 197 |
index_type res = 0; |
| 198 |
((res = indices[extents_type::rank() - 1 - Ranks] + |
| 199 |
((extents_type::rank() - 1 - Ranks) == extent_to_pad_idx |
| 200 |
? padded_stride.value(0) |
| 201 |
: exts.extent(extents_type::rank() - 1 - Ranks)) * |
| 202 |
res), |
| 203 |
...); |
| 204 |
return res; |
| 205 |
} |
| 206 |
|
| 207 |
public: |
| 208 |
#if !MDSPAN_HAS_CXX_20 || defined(__NVCC__) |
| 209 |
MDSPAN_INLINE_FUNCTION |
| 210 |
constexpr mapping() |
| 211 |
: mapping(extents_type{}) |
| 212 |
{} |
| 213 |
#else |
| 214 |
MDSPAN_INLINE_FUNCTION_DEFAULTED |
| 215 |
constexpr mapping() |
| 216 |
requires(static_padding_stride != dynamic_extent) = default; |
| 217 |
|
| 218 |
MDSPAN_INLINE_FUNCTION |
| 219 |
constexpr mapping() |
| 220 |
requires(static_padding_stride == dynamic_extent) |
| 221 |
: mapping(extents_type{}) |
| 222 |
{} |
| 223 |
#endif |
| 224 |
|
| 225 |
MDSPAN_INLINE_FUNCTION_DEFAULTED constexpr mapping(const mapping&) noexcept = default; |
| 226 |
MDSPAN_INLINE_FUNCTION_DEFAULTED constexpr mapping& operator=(const mapping&) noexcept = default; |
| 227 |
|
| 228 |
/** |
| 229 |
* Initializes the mapping with the given extents. |
| 230 |
* |
| 231 |
* \param ext the given extents |
| 232 |
*/ |
| 233 |
MDSPAN_INLINE_FUNCTION |
| 234 |
constexpr mapping(const extents_type& ext) |
| 235 |
: padded_stride(padded_stride_type::init_padding(ext)), exts(ext) |
| 236 |
{} |
| 237 |
|
| 238 |
/** |
| 239 |
* Initializes the mapping with the given extents and the specified padding value. |
| 240 |
* |
| 241 |
* This overload participates in overload resolution only if `is_convertible_v<Size, index_type>` |
| 242 |
* is `true` and `is_nothrow_constructible_v<index_type, Size>` is `true` |
| 243 |
* |
| 244 |
* \param ext the given extents |
| 245 |
* \param padding_value the padding value |
| 246 |
*/ |
| 247 |
MDSPAN_TEMPLATE_REQUIRES( |
| 248 |
class _Size, |
| 249 |
/* requires */ ( |
| 250 |
std::is_convertible_v<_Size, index_type> |
| 251 |
&& std::is_nothrow_constructible_v<index_type, _Size> |
| 252 |
) |
| 253 |
) |
| 254 |
MDSPAN_INLINE_FUNCTION |
| 255 |
constexpr mapping(const extents_type &ext, _Size dynamic_padding_value) |
| 256 |
: padded_stride(padded_stride_type::init_padding(ext, dynamic_padding_value)), exts(ext) |
| 257 |
{ |
| 258 |
assert((padding_value == dynamic_extent) || (static_cast<index_type>(padding_value) == static_cast<index_type>(dynamic_padding_value))); |
| 259 |
} |
| 260 |
|
| 261 |
/** |
| 262 |
* Converting constructor from `layout_left::mapping`. |
| 263 |
* |
| 264 |
* This overload participates in overload resolution only if |
| 265 |
* `is_constructible_v<extents_type, OtherExtents>` is true. If |
| 266 |
* `OtherExtents::rank() > 1` then one of `padding_value`, `static_extent(0)`, |
| 267 |
* or `OtherExtents::static_extent(0)` must be `dynamic_extent`; otherwise, |
| 268 |
* `OtherExtents::static_extent(0)` must be equal to the least multiple of |
| 269 |
* `padding_value` greater than or equal to `extents_type::static_extent(0)` |
| 270 |
*/ |
| 271 |
MDSPAN_TEMPLATE_REQUIRES( |
| 272 |
class _OtherExtents, |
| 273 |
/* requires */ (std::is_constructible_v<extents_type, _OtherExtents>)) |
| 274 |
MDSPAN_CONDITIONAL_EXPLICIT( |
| 275 |
(!std::is_convertible_v<_OtherExtents, extents_type>)) |
| 276 |
MDSPAN_INLINE_FUNCTION |
| 277 |
constexpr mapping(const layout_left::mapping<_OtherExtents> &other_mapping) |
| 278 |
: padded_stride(padded_stride_type::init_padding( |
| 279 |
other_mapping, |
| 280 |
std::integral_constant<size_t, padded_stride_idx>{})), |
| 281 |
exts(other_mapping.extents()) { |
| 282 |
static_assert( |
| 283 |
(_OtherExtents::rank() > 1) || |
| 284 |
(static_padding_stride != dynamic_extent) || |
| 285 |
(_OtherExtents::static_extent(extent_to_pad_idx) != dynamic_extent) || |
| 286 |
(static_padding_stride == |
| 287 |
_OtherExtents::static_extent(extent_to_pad_idx))); |
| 288 |
} |
| 289 |
|
| 290 |
/** |
| 291 |
* Converting constructor from `layout_stride::mapping`. |
| 292 |
* |
| 293 |
* This overload participates in overload resolution only if |
| 294 |
* `is_constructible_v<extents_type, OtherExtents>` is true |
| 295 |
*/ |
| 296 |
MDSPAN_TEMPLATE_REQUIRES( |
| 297 |
class _OtherExtents, |
| 298 |
/* requires */ (std::is_constructible_v<extents_type, _OtherExtents>)) |
| 299 |
MDSPAN_CONDITIONAL_EXPLICIT((extents_type::rank() > 0)) |
| 300 |
MDSPAN_INLINE_FUNCTION |
| 301 |
constexpr mapping(const layout_stride::mapping<_OtherExtents> &other_mapping) |
| 302 |
: padded_stride(padded_stride_type::init_padding( |
| 303 |
other_mapping, |
| 304 |
std::integral_constant<size_t, padded_stride_idx>{})), |
| 305 |
exts(other_mapping.extents()) {} |
| 306 |
|
| 307 |
/** |
| 308 |
* Converting constructor from `layout_left_padded::mapping`. |
| 309 |
* |
| 310 |
* This overload participates in overload resolution only if |
| 311 |
* `is_constructible_v<extents_type, OtherExtents>` is true. Either |
| 312 |
* `padding_value` or `OtherPaddingStride` must be `std::dynamic_extent`, or |
| 313 |
* `padding_value == OtherPaddingStride`. |
| 314 |
*/ |
| 315 |
MDSPAN_TEMPLATE_REQUIRES( |
| 316 |
class _Mapping, |
| 317 |
/* requires */ (detail::is_layout_left_padded_mapping<_Mapping>::value |
| 318 |
&&std::is_constructible_v< |
| 319 |
extents_type, typename _Mapping::extents_type>)) |
| 320 |
MDSPAN_CONDITIONAL_EXPLICIT((extents_type::rank() > 1 && |
| 321 |
(padding_value == dynamic_extent || |
| 322 |
_Mapping::padding_value == dynamic_extent))) |
| 323 |
MDSPAN_INLINE_FUNCTION |
| 324 |
constexpr mapping(const _Mapping &other_mapping) |
| 325 |
: padded_stride(padded_stride_type::init_padding( |
| 326 |
other_mapping, |
| 327 |
std::integral_constant<size_t, padded_stride_idx>{})), |
| 328 |
exts(other_mapping.extents()) { |
| 329 |
static_assert(padding_value == dynamic_extent || |
| 330 |
_Mapping::padding_value == dynamic_extent || |
| 331 |
padding_value == _Mapping::padding_value); |
| 332 |
} |
| 333 |
|
| 334 |
/** |
| 335 |
* Converting constructor from `layout_right_padded::mapping`. |
| 336 |
* |
| 337 |
* This overload participates in overload resolution only if |
| 338 |
* `extents_type::rank()` is 0 or 1 and `is_constructible_v<extents_type, |
| 339 |
* OtherExtents>` is `true`. |
| 340 |
*/ |
| 341 |
MDSPAN_TEMPLATE_REQUIRES( |
| 342 |
class _Mapping, |
| 343 |
/* requires */ (detail::is_layout_right_padded_mapping<_Mapping>::value |
| 344 |
&&extents_type::rank() <= 1 && |
| 345 |
std::is_constructible_v<extents_type, |
| 346 |
typename _Mapping::extents_type>)) |
| 347 |
MDSPAN_CONDITIONAL_EXPLICIT( |
| 348 |
(!std::is_convertible_v<typename _Mapping::extents_type, extents_type>)) |
| 349 |
MDSPAN_INLINE_FUNCTION |
| 350 |
constexpr mapping(const _Mapping &other_mapping) noexcept |
| 351 |
: padded_stride(padded_stride_type::init_padding( |
| 352 |
static_cast<extents_type>(other_mapping.extents()), |
| 353 |
other_mapping.extents().extent(extent_to_pad_idx))), |
| 354 |
exts(other_mapping.extents()) {} |
| 355 |
|
| 356 |
MDSPAN_INLINE_FUNCTION constexpr const extents_type & |
| 357 |
extents() const noexcept { |
| 358 |
return exts; |
| 359 |
} |
| 360 |
|
| 361 |
constexpr std::array<index_type, extents_type::rank()> |
| 362 |
strides() const noexcept { |
| 363 |
if constexpr (extents_type::rank() == 0) { |
| 364 |
return {}; |
| 365 |
} else if constexpr (extents_type::rank() == 1) { |
| 366 |
return {1}; |
| 367 |
} else { |
| 368 |
index_type value = 1; |
| 369 |
std::array<index_type, extents_type::rank()> s{}; |
| 370 |
s[extent_to_pad_idx] = value; |
| 371 |
value *= padded_stride.value(0); |
| 372 |
for (rank_type r = extent_to_pad_idx + 1; r < extents_type::rank() - 1; |
| 373 |
++r) { |
| 374 |
s[r] = value; |
| 375 |
value *= exts.extent(r); |
| 376 |
} |
| 377 |
s[extents_type::rank() - 1] = value; |
| 378 |
return s; |
| 379 |
} |
| 380 |
} |
| 381 |
|
| 382 |
MDSPAN_INLINE_FUNCTION constexpr index_type |
| 383 |
required_span_size() const noexcept { |
| 384 |
if constexpr (extents_type::rank() == 0) { |
| 385 |
return 1; |
| 386 |
} else if constexpr (extents_type::rank() == 1) { |
| 387 |
return exts.extent(0); |
| 388 |
} else { |
| 389 |
index_type value = padded_stride.value(0); |
| 390 |
for (rank_type r = 1; r < extents_type::rank(); ++r) { |
| 391 |
value *= exts.extent(r); |
| 392 |
} |
| 393 |
return value; |
| 394 |
} |
| 395 |
} |
| 396 |
|
| 397 |
/** |
| 398 |
* Return the mapping given the provided indices per rank. |
| 399 |
* |
| 400 |
* This overload participates in overload resolution only if: |
| 401 |
* - `sizeof...(Indices) == extents_type::rank()`, |
| 402 |
* - `(is_convertible_v<Indices, index_type> && ...) is true`, and |
| 403 |
* - (is_nothrow_constructible_v<index_type, Indices> && ...) is true. |
| 404 |
*/ |
| 405 |
MDSPAN_TEMPLATE_REQUIRES( |
| 406 |
class... _Indices, |
| 407 |
/* requires */ (sizeof...(_Indices) == extents_type::rank() && |
| 408 |
(::MDSPAN_IMPL_STANDARD_NAMESPACE::detail:: |
| 409 |
are_valid_indices<index_type, _Indices...>()))) |
| 410 |
MDSPAN_INLINE_FUNCTION constexpr size_t |
| 411 |
operator()(_Indices... idxs) const noexcept { |
| 412 |
#if !defined(NDEBUG) |
| 413 |
::MDSPAN_IMPL_STANDARD_NAMESPACE::detail::check_all_indices(this->extents(), |
| 414 |
idxs...); |
| 415 |
#endif // ! NDEBUG |
| 416 |
return compute_offset(std::index_sequence_for<_Indices...>{}, idxs...); |
| 417 |
} |
| 418 |
|
| 419 |
MDSPAN_INLINE_FUNCTION static constexpr bool is_always_unique() noexcept { |
| 420 |
return true; |
| 421 |
} |
| 422 |
MDSPAN_INLINE_FUNCTION static constexpr bool is_always_exhaustive() noexcept { |
| 423 |
return (extents_type::rank() <= rank_type(1)) || |
| 424 |
(extents_type::static_extent(extent_to_pad_idx) != dynamic_extent && |
| 425 |
extents_type::static_extent(extent_to_pad_idx) == |
| 426 |
padded_stride_type::static_value()); |
| 427 |
} |
| 428 |
MDSPAN_INLINE_FUNCTION static constexpr bool is_always_strided() noexcept { |
| 429 |
return true; |
| 430 |
} |
| 431 |
|
| 432 |
MDSPAN_INLINE_FUNCTION static constexpr bool is_unique() noexcept { |
| 433 |
return true; |
| 434 |
} |
| 435 |
MDSPAN_INLINE_FUNCTION constexpr bool is_exhaustive() const noexcept { |
| 436 |
return (extents_type::rank() < 2) || |
| 437 |
(exts.extent(extent_to_pad_idx) == padded_stride.value(0)); |
| 438 |
} |
| 439 |
MDSPAN_INLINE_FUNCTION static constexpr bool is_strided() noexcept { |
| 440 |
return true; |
| 441 |
} |
| 442 |
|
| 443 |
MDSPAN_INLINE_FUNCTION |
| 444 |
constexpr index_type stride(rank_type r) const noexcept { |
| 445 |
assert(r < extents_type::rank()); |
| 446 |
if (r == 0) |
| 447 |
return index_type(1); |
| 448 |
|
| 449 |
index_type value = padded_stride.value(0); |
| 450 |
for (rank_type k = 1; k < r; k++) |
| 451 |
value *= exts.extent(k); |
| 452 |
|
| 453 |
return value; |
| 454 |
} |
| 455 |
|
| 456 |
/** |
| 457 |
* Equality operator between `layout_left_padded`s |
| 458 |
* |
| 459 |
* This overload only participates in overload resolution if |
| 460 |
* `OtherExtents::rank() == extents_type::rank()`. |
| 461 |
* |
| 462 |
* \note There is currently a difference from p2642r2, where this function is |
| 463 |
* specified as taking `layout_left_padded< padding_value >::mapping< |
| 464 |
* Extents>`. However, this makes `padding_value` non-deducible. |
| 465 |
*/ |
| 466 |
MDSPAN_TEMPLATE_REQUIRES( |
| 467 |
class _Mapping, |
| 468 |
/* requires */ (detail::is_layout_left_padded_mapping<_Mapping>::value && |
| 469 |
(_Mapping::extents_type::rank() == extents_type::rank()))) |
| 470 |
MDSPAN_INLINE_FUNCTION friend constexpr bool |
| 471 |
operator==(const mapping &left, const _Mapping &right) noexcept { |
| 472 |
// Workaround for some compilers not short-circuiting properly with |
| 473 |
// compile-time checks i.e. we can't access stride(_padding_stride_idx) of a |
| 474 |
// rank 0 mapping |
| 475 |
bool strides_equal = true; |
| 476 |
if constexpr (extents_type::rank() > rank_type(1)) { |
| 477 |
strides_equal = |
| 478 |
left.stride(padded_stride_idx) == right.stride(padded_stride_idx); |
| 479 |
} |
| 480 |
return (left.extents() == right.extents()) && strides_equal; |
| 481 |
} |
| 482 |
|
| 483 |
#if !MDSPAN_HAS_CXX_20 |
| 484 |
/** |
| 485 |
* Inequality operator between `layout_left_padded`s |
| 486 |
* |
| 487 |
* This overload only participates in overload resolution if |
| 488 |
* `OtherExtents::rank() == extents_type::rank()`. |
| 489 |
*/ |
| 490 |
MDSPAN_TEMPLATE_REQUIRES( |
| 491 |
class _Mapping, |
| 492 |
/* requires */ (detail::is_layout_left_padded_mapping<_Mapping>::value && |
| 493 |
(_Mapping::extents_type::rank() == extents_type::rank()))) |
| 494 |
MDSPAN_INLINE_FUNCTION friend constexpr bool |
| 495 |
operator!=(const mapping &left, const _Mapping &right) noexcept { |
| 496 |
return !(left == right); |
| 497 |
} |
| 498 |
#endif |
| 499 |
|
| 500 |
// [mdspan.submdspan.mapping], submdspan mapping specialization |
| 501 |
template<class... SliceSpecifiers> |
| 502 |
MDSPAN_INLINE_FUNCTION |
| 503 |
constexpr auto submdspan_mapping_impl( |
| 504 |
SliceSpecifiers... slices) const; |
| 505 |
|
| 506 |
template<class... SliceSpecifiers> |
| 507 |
MDSPAN_INLINE_FUNCTION |
| 508 |
friend constexpr auto submdspan_mapping( |
| 509 |
const mapping& src, SliceSpecifiers... slices) { |
| 510 |
return src.submdspan_mapping_impl(slices...); |
| 511 |
} |
| 512 |
}; |
| 513 |
|
| 514 |
template <size_t PaddingValue> |
| 515 |
template <class Extents> |
| 516 |
class layout_right_padded<PaddingValue>::mapping { |
| 517 |
public: |
| 518 |
static constexpr size_t padding_value = PaddingValue; |
| 519 |
|
| 520 |
using extents_type = Extents; |
| 521 |
using index_type = typename extents_type::index_type; |
| 522 |
using size_type = typename extents_type::size_type; |
| 523 |
using rank_type = typename extents_type::rank_type; |
| 524 |
using layout_type = layout_right_padded<padding_value>; |
| 525 |
|
| 526 |
#ifndef MDSPAN_INTERNAL_TEST |
| 527 |
private: |
| 528 |
#endif // MDSPAN_INTERNAL_TEST |
| 529 |
|
| 530 |
static constexpr rank_type padded_stride_idx = detail::layout_padded_constants<layout_type, extents_type>::padded_stride_idx; |
| 531 |
static constexpr rank_type extent_to_pad_idx = detail::layout_padded_constants<layout_type, extents_type>::extent_to_pad_idx; |
| 532 |
|
| 533 |
static_assert((padding_value != 0) |
| 534 |
|| (extents_type::static_extent(extent_to_pad_idx) == 0) |
| 535 |
|| (extents_type::static_extent(extent_to_pad_idx) == dynamic_extent), |
| 536 |
"if padding stride is 0, static_extent(extent-to-pad-rank) must also be 0 or dynamic_extent"); |
| 537 |
|
| 538 |
using padded_stride_type = detail::padded_extent< padding_value, extents_type, extent_to_pad_idx >; |
| 539 |
static constexpr size_t static_padding_stride = padded_stride_type::static_value(); |
| 540 |
|
| 541 |
typename padded_stride_type::static_array_type padded_stride = {}; |
| 542 |
extents_type exts = {}; |
| 543 |
|
| 544 |
MDSPAN_INLINE_FUNCTION constexpr index_type |
| 545 |
compute_offset(std::index_sequence<>) const { |
| 546 |
return 0; |
| 547 |
} |
| 548 |
|
| 549 |
template <size_t Rank, class IndexOffset> |
| 550 |
MDSPAN_INLINE_FUNCTION constexpr index_type |
| 551 |
compute_offset(std::index_sequence<Rank>, IndexOffset index_offset) const { |
| 552 |
return index_offset; |
| 553 |
} |
| 554 |
|
| 555 |
template <size_t... Ranks, class... IndexOffsets> |
| 556 |
MDSPAN_INLINE_FUNCTION constexpr index_type |
| 557 |
compute_offset(std::index_sequence<Ranks...>, |
| 558 |
IndexOffsets... index_offsets) const { |
| 559 |
// self-recursive fold trick from |
| 560 |
// https://github.com/llvm/llvm-project/blob/4d9771741d40cc9cfcccb6b033f43689d36b705a/libcxx/include/mdspan/layout_right.h#L141 |
| 561 |
index_type res = 0; |
| 562 |
((res = static_cast<index_type>(index_offsets) + |
| 563 |
(Ranks == extent_to_pad_idx ? padded_stride.value(0) |
| 564 |
: exts.extent(Ranks)) * |
| 565 |
res), |
| 566 |
...); |
| 567 |
return res; |
| 568 |
} |
| 569 |
|
| 570 |
public: |
| 571 |
#if !MDSPAN_HAS_CXX_20 || defined(__NVCC__) |
| 572 |
MDSPAN_INLINE_FUNCTION |
| 573 |
constexpr mapping() |
| 574 |
: mapping(extents_type{}) |
| 575 |
{} |
| 576 |
#else |
| 577 |
MDSPAN_INLINE_FUNCTION_DEFAULTED |
| 578 |
constexpr mapping() |
| 579 |
requires(static_padding_stride != dynamic_extent) = default; |
| 580 |
|
| 581 |
MDSPAN_INLINE_FUNCTION |
| 582 |
constexpr mapping() |
| 583 |
requires(static_padding_stride == dynamic_extent) |
| 584 |
: mapping(extents_type{}) |
| 585 |
{} |
| 586 |
#endif |
| 587 |
|
| 588 |
MDSPAN_INLINE_FUNCTION_DEFAULTED constexpr mapping(const mapping&) noexcept = default; |
| 589 |
MDSPAN_INLINE_FUNCTION_DEFAULTED constexpr mapping& operator=(const mapping&) noexcept = default; |
| 590 |
|
| 591 |
/** |
| 592 |
* Initializes the mapping with the given extents. |
| 593 |
* |
| 594 |
* \param ext the given extents |
| 595 |
*/ |
| 596 |
MDSPAN_INLINE_FUNCTION |
| 597 |
constexpr mapping(const extents_type &ext) |
| 598 |
: padded_stride(padded_stride_type::init_padding(ext)), exts(ext) {} |
| 599 |
|
| 600 |
/** |
| 601 |
* Initializes the mapping with the given extents and the specified padding value. |
| 602 |
* |
| 603 |
* This overload participates in overload resolution only if `is_convertible_v<Size, index_type>` |
| 604 |
* is `true` and `is_nothrow_constructible_v<index_type, Size>` is `true` |
| 605 |
* |
| 606 |
* \param ext the given extents |
| 607 |
* \param padding_value the padding value |
| 608 |
*/ |
| 609 |
MDSPAN_TEMPLATE_REQUIRES( |
| 610 |
class _Size, |
| 611 |
/* requires */ ( |
| 612 |
std::is_convertible_v<_Size, index_type> |
| 613 |
&& std::is_nothrow_constructible_v<index_type, _Size> |
| 614 |
) |
| 615 |
) |
| 616 |
MDSPAN_INLINE_FUNCTION |
| 617 |
constexpr mapping(const extents_type &ext, _Size dynamic_padding_value) |
| 618 |
: padded_stride(padded_stride_type::init_padding(ext, static_cast<index_type>(dynamic_padding_value))), |
| 619 |
exts(ext) { |
| 620 |
assert((padding_value == dynamic_extent) || |
| 621 |
(static_cast<index_type>(padding_value) == static_cast<index_type>(dynamic_padding_value))); |
| 622 |
} |
| 623 |
|
| 624 |
/** |
| 625 |
* Converting constructor from `layout_right::mapping`. |
| 626 |
* |
| 627 |
* This overload participates in overload resolution only if `is_constructible_v<extents_type, OtherExtents>` is true. |
| 628 |
* If `OtherExtents::rank() > 1` then one of `padding_value`, `static_extent(0)`, or `OtherExtents::static_extent(0)` must be `dynamic_extent`; |
| 629 |
* otherwise, `OtherExtents::static_extent(0)` must be equal to the least multiple of `padding_value` greater than or equal to `extents_type::static_extent(0)` |
| 630 |
*/ |
| 631 |
MDSPAN_TEMPLATE_REQUIRES( |
| 632 |
class _OtherExtents, |
| 633 |
/* requires */ (std::is_constructible_v<extents_type, _OtherExtents>)) |
| 634 |
MDSPAN_CONDITIONAL_EXPLICIT( |
| 635 |
(!std::is_convertible_v<_OtherExtents, extents_type>)) |
| 636 |
MDSPAN_INLINE_FUNCTION |
| 637 |
constexpr mapping(const layout_right::mapping<_OtherExtents> &other_mapping) |
| 638 |
: padded_stride(padded_stride_type::init_padding( |
| 639 |
other_mapping, |
| 640 |
std::integral_constant<size_t, padded_stride_idx>{})), |
| 641 |
exts(other_mapping.extents()) { |
| 642 |
static_assert( |
| 643 |
(_OtherExtents::rank() > 1) || |
| 644 |
(padded_stride_type::static_value() != dynamic_extent) || |
| 645 |
(_OtherExtents::static_extent(extent_to_pad_idx) != dynamic_extent) || |
| 646 |
(padded_stride_type::static_value() == |
| 647 |
_OtherExtents::static_extent(extent_to_pad_idx))); |
| 648 |
} |
| 649 |
|
| 650 |
/** |
| 651 |
* Converting constructor from `layout_stride::mapping`. |
| 652 |
* |
| 653 |
* This overload participates in overload resolution only if |
| 654 |
* `is_constructible_v<extents_type, OtherExtents>` is true |
| 655 |
*/ |
| 656 |
MDSPAN_TEMPLATE_REQUIRES( |
| 657 |
class _OtherExtents, |
| 658 |
/* requires */ (std::is_constructible_v<extents_type, _OtherExtents>)) |
| 659 |
MDSPAN_CONDITIONAL_EXPLICIT((extents_type::rank() > 0)) |
| 660 |
MDSPAN_INLINE_FUNCTION |
| 661 |
constexpr mapping(const layout_stride::mapping<_OtherExtents> &other_mapping) |
| 662 |
: padded_stride(padded_stride_type::init_padding( |
| 663 |
other_mapping, |
| 664 |
std::integral_constant<size_t, padded_stride_idx>{})), |
| 665 |
exts(other_mapping.extents()) {} |
| 666 |
|
| 667 |
/** |
| 668 |
* Converting constructor from `layout_right_padded::mapping`. |
| 669 |
* |
| 670 |
* This overload participates in overload resolution only if |
| 671 |
* `is_constructible_v<extents_type, OtherExtents>` is true. Either |
| 672 |
* `padding_value` or `OtherPaddingStride` must be `std::dynamic_extent`, or |
| 673 |
* `padding_value == OtherPaddingStride`. |
| 674 |
*/ |
| 675 |
MDSPAN_TEMPLATE_REQUIRES( |
| 676 |
class _Mapping, |
| 677 |
/* requires */ (detail::is_layout_right_padded_mapping<_Mapping>::value |
| 678 |
&&std::is_constructible_v< |
| 679 |
extents_type, typename _Mapping::extents_type>)) |
| 680 |
MDSPAN_CONDITIONAL_EXPLICIT((extents_type::rank() > 1 && |
| 681 |
(padding_value == dynamic_extent || |
| 682 |
_Mapping::padding_value == dynamic_extent))) |
| 683 |
MDSPAN_INLINE_FUNCTION |
| 684 |
constexpr mapping(const _Mapping &other_mapping) |
| 685 |
: padded_stride(padded_stride_type::init_padding( |
| 686 |
other_mapping, |
| 687 |
std::integral_constant<size_t, padded_stride_idx>{})), |
| 688 |
exts(other_mapping.extents()) { |
| 689 |
static_assert(padding_value == dynamic_extent || |
| 690 |
_Mapping::padding_value == dynamic_extent || |
| 691 |
padding_value == _Mapping::padding_value); |
| 692 |
} |
| 693 |
|
| 694 |
/** |
| 695 |
* Converting constructor from `layout_left_padded::mapping`. |
| 696 |
* |
| 697 |
* This overload participates in overload resolution only if |
| 698 |
* `extents_type::rank()` is 0 or 1 and `is_constructible_v<extents_type, |
| 699 |
* OtherExtents>` is `true`. |
| 700 |
*/ |
| 701 |
MDSPAN_TEMPLATE_REQUIRES( |
| 702 |
class _Mapping, |
| 703 |
/* requires */ (detail::is_layout_left_padded_mapping<_Mapping>::value |
| 704 |
&&extents_type::rank() <= 1 && |
| 705 |
std::is_constructible_v<extents_type, |
| 706 |
typename _Mapping::extents_type>)) |
| 707 |
MDSPAN_CONDITIONAL_EXPLICIT( |
| 708 |
(!std::is_convertible_v<typename _Mapping::extents_type, extents_type>)) |
| 709 |
MDSPAN_INLINE_FUNCTION |
| 710 |
constexpr mapping(const _Mapping &other_mapping) noexcept |
| 711 |
: padded_stride(padded_stride_type::init_padding( |
| 712 |
static_cast<extents_type>(other_mapping.extents()), |
| 713 |
other_mapping.extents().extent(extent_to_pad_idx))), |
| 714 |
exts(other_mapping.extents()) {} |
| 715 |
|
| 716 |
MDSPAN_INLINE_FUNCTION constexpr const extents_type & |
| 717 |
extents() const noexcept { |
| 718 |
return exts; |
| 719 |
} |
| 720 |
|
| 721 |
constexpr std::array<index_type, extents_type::rank()> |
| 722 |
strides() const noexcept { |
| 723 |
if constexpr (extents_type::rank() == 0) { |
| 724 |
return {}; |
| 725 |
} else if constexpr (extents_type::rank() == 1) { |
| 726 |
return {1}; |
| 727 |
} else { |
| 728 |
index_type value = 1; |
| 729 |
std::array<index_type, extents_type::rank()> s{}; |
| 730 |
s[extent_to_pad_idx] = value; |
| 731 |
value *= padded_stride.value(0); |
| 732 |
for (rank_type r = extent_to_pad_idx - 1; r > 0; --r) { |
| 733 |
s[r] = value; |
| 734 |
value *= exts.extent(r); |
| 735 |
} |
| 736 |
s[0] = value; |
| 737 |
return s; |
| 738 |
} |
| 739 |
} |
| 740 |
|
| 741 |
MDSPAN_INLINE_FUNCTION constexpr index_type |
| 742 |
required_span_size() const noexcept { |
| 743 |
if constexpr (extents_type::rank() == 0) { |
| 744 |
return 1; |
| 745 |
} else if constexpr (extents_type::rank() == 1) { |
| 746 |
return exts.extent(0); |
| 747 |
} else { |
| 748 |
index_type value = 1; |
| 749 |
for (rank_type r = 0; r < extent_to_pad_idx; ++r) { |
| 750 |
value *= exts.extent(r); |
| 751 |
} |
| 752 |
return value * padded_stride.value(0); |
| 753 |
} |
| 754 |
} |
| 755 |
|
| 756 |
/** |
| 757 |
* Return the mapping given the provided indices per rank. |
| 758 |
* |
| 759 |
* This overload participates in overload resolution only if: |
| 760 |
* - `sizeof...(Indices) == extents_type::rank()`, |
| 761 |
* - `(is_convertible_v<Indices, index_type> && ...) is true`, and |
| 762 |
* - (is_nothrow_constructible_v<index_type, Indices> && ...) is true. |
| 763 |
*/ |
| 764 |
MDSPAN_TEMPLATE_REQUIRES( |
| 765 |
class... _Indices, |
| 766 |
/* requires */ (sizeof...(_Indices) == extents_type::rank() && |
| 767 |
(::MDSPAN_IMPL_STANDARD_NAMESPACE::detail:: |
| 768 |
are_valid_indices<index_type, _Indices...>()))) |
| 769 |
MDSPAN_INLINE_FUNCTION constexpr size_t |
| 770 |
operator()(_Indices... idxs) const noexcept { |
| 771 |
return compute_offset(std::index_sequence_for<_Indices...>{}, idxs...); |
| 772 |
} |
| 773 |
|
| 774 |
MDSPAN_INLINE_FUNCTION static constexpr bool is_always_unique() noexcept { |
| 775 |
return true; |
| 776 |
} |
| 777 |
MDSPAN_INLINE_FUNCTION static constexpr bool is_always_exhaustive() noexcept { |
| 778 |
return (extents_type::rank() <= rank_type(1)) || |
| 779 |
(extents_type::static_extent(extent_to_pad_idx) != dynamic_extent && |
| 780 |
extents_type::static_extent(extent_to_pad_idx) == |
| 781 |
padded_stride_type::static_value()); |
| 782 |
} |
| 783 |
MDSPAN_INLINE_FUNCTION static constexpr bool is_always_strided() noexcept { |
| 784 |
return true; |
| 785 |
} |
| 786 |
|
| 787 |
MDSPAN_INLINE_FUNCTION static constexpr bool is_unique() noexcept { |
| 788 |
return true; |
| 789 |
} |
| 790 |
MDSPAN_INLINE_FUNCTION constexpr bool is_exhaustive() const noexcept { |
| 791 |
return (extents_type::rank() < 2) || |
| 792 |
(exts.extent(extent_to_pad_idx) == padded_stride.value(0)); |
| 793 |
} |
| 794 |
MDSPAN_INLINE_FUNCTION static constexpr bool is_strided() noexcept { |
| 795 |
return true; |
| 796 |
} |
| 797 |
|
| 798 |
MDSPAN_INLINE_FUNCTION constexpr index_type |
| 799 |
stride(rank_type r) const noexcept { |
| 800 |
assert(r < extents_type::rank()); |
| 801 |
if (r == extents_type::rank() - 1) |
| 802 |
return index_type(1); |
| 803 |
|
| 804 |
index_type value = padded_stride.value(0); |
| 805 |
for (rank_type k = extents_type::rank() - 2; k > r; k--) |
| 806 |
value *= exts.extent(k); |
| 807 |
|
| 808 |
return value; |
| 809 |
} |
| 810 |
|
| 811 |
/** |
| 812 |
* Equality operator between `layout_right_padded`s |
| 813 |
* |
| 814 |
* This overload only participates in overload resolution if |
| 815 |
* `OtherExtents::rank() == extents_type::rank()`. |
| 816 |
* |
| 817 |
* \note There is currently a difference from p2642r2, where this function is |
| 818 |
* specified as taking `layout_right_padded< padding_value >::mapping< |
| 819 |
* Extents>`. However, this makes `padding_value` non-deducible. |
| 820 |
*/ |
| 821 |
MDSPAN_TEMPLATE_REQUIRES( |
| 822 |
class _Mapping, |
| 823 |
/* requires */ (detail::is_layout_right_padded_mapping<_Mapping>::value && |
| 824 |
(_Mapping::extents_type::rank() == extents_type::rank()))) |
| 825 |
MDSPAN_INLINE_FUNCTION friend constexpr bool |
| 826 |
operator==(const mapping &left, const _Mapping &right) noexcept { |
| 827 |
// Workaround for some compilers not short-circuiting properly with |
| 828 |
// compile-time checks i.e. we can't access stride(_padding_stride_idx) of a |
| 829 |
// rank 0 mapping |
| 830 |
bool strides_equal = true; |
| 831 |
if constexpr (extents_type::rank() > rank_type(1)) { |
| 832 |
strides_equal = |
| 833 |
left.stride(padded_stride_idx) == right.stride(padded_stride_idx); |
| 834 |
} |
| 835 |
return (left.extents() == right.extents()) && strides_equal; |
| 836 |
} |
| 837 |
|
| 838 |
#if !MDSPAN_HAS_CXX_20 |
| 839 |
/** |
| 840 |
* Inequality operator between `layout_right_padded`s |
| 841 |
* |
| 842 |
* This overload only participates in overload resolution if |
| 843 |
* `OtherExtents::rank() == extents_type::rank()`. |
| 844 |
*/ |
| 845 |
MDSPAN_TEMPLATE_REQUIRES( |
| 846 |
class _Mapping, |
| 847 |
/* requires */ (detail::is_layout_right_padded_mapping<_Mapping>::value && |
| 848 |
(_Mapping::extents_type::rank() == extents_type::rank()))) |
| 849 |
MDSPAN_INLINE_FUNCTION friend constexpr bool |
| 850 |
operator!=(const mapping &left, const _Mapping &right) noexcept { |
| 851 |
return !(left == right); |
| 852 |
} |
| 853 |
#endif |
| 854 |
|
| 855 |
// [mdspan.submdspan.mapping], submdspan mapping specialization |
| 856 |
template<class... SliceSpecifiers> |
| 857 |
MDSPAN_INLINE_FUNCTION |
| 858 |
constexpr auto submdspan_mapping_impl( |
| 859 |
SliceSpecifiers... slices) const; |
| 860 |
|
| 861 |
template<class... SliceSpecifiers> |
| 862 |
MDSPAN_INLINE_FUNCTION |
| 863 |
friend constexpr auto submdspan_mapping( |
| 864 |
const mapping& src, SliceSpecifiers... slices) { |
| 865 |
return src.submdspan_mapping_impl(slices...); |
| 866 |
} |
| 867 |
}; |
| 868 |
} |
| 869 |
} |