| 1 |
// @HEADER |
| 2 |
// ***************************************************************************** |
| 3 |
// Stokhos Package |
| 4 |
// |
| 5 |
// Copyright 2009 NTESS and the Stokhos contributors. |
| 6 |
// SPDX-License-Identifier: BSD-3-Clause |
| 7 |
// ***************************************************************************** |
| 8 |
// @HEADER |
| 9 |
|
| 10 |
#ifndef STOKHOS_TPETRA_UQ_PCE_HPP |
| 11 |
#define STOKHOS_TPETRA_UQ_PCE_HPP |
| 12 |
|
| 13 |
// This header file should be included whenever compiling any Tpetra |
| 14 |
// code with Stokhos scalar types |
| 15 |
|
| 16 |
// MP includes and specializations |
| 17 |
#include "Stokhos_Sacado_Kokkos_UQ_PCE.hpp" |
| 18 |
|
| 19 |
// Kokkos includes |
| 20 |
#include "Tpetra_ConfigDefs.hpp" |
| 21 |
#include "Tpetra_MultiVector_fwd.hpp" |
| 22 |
#include "Tpetra_Vector_fwd.hpp" |
| 23 |
#include "Tpetra_Access.hpp" |
| 24 |
#include "Kokkos_Core.hpp" |
| 25 |
#include <Tpetra_KokkosCompat_ClassicNodeAPI_Wrapper.hpp> |
| 26 |
#include "KokkosCompat_View.hpp" |
| 27 |
#include "KokkosCompat_View_def.hpp" |
| 28 |
|
| 29 |
// Hack for mean-based prec-setup where we get the PCE size from the first |
| 30 |
// entry in the Teuchos::ArrayView. This is clearly quite dangerous and is |
| 31 |
// likely to bite us in the ass at some point! |
| 32 |
namespace Kokkos { |
| 33 |
namespace Compat { |
| 34 |
template <typename D, typename S> |
| 35 |
Kokkos::View<Sacado::UQ::PCE<S>*,D> |
| 36 |
getKokkosViewDeepCopy(const Teuchos::ArrayView< Sacado::UQ::PCE<S> >& a) { |
| 37 |
typedef Sacado::UQ::PCE<S> T; |
| 38 |
typedef typename std::conditional< |
| 39 |
::Kokkos::SpaceAccessibility< D, Kokkos::HostSpace>::accessible, |
| 40 |
typename D::execution_space, Kokkos::HostSpace>::type |
| 41 |
HostDevice; |
| 42 |
typedef Kokkos::View<T*,D> view_type; |
| 43 |
typedef Kokkos::View<T*,typename view_type::array_layout,HostDevice,Kokkos::MemoryUnmanaged> unmanaged_host_view_type; |
| 44 |
if (a.size() == 0) |
| 45 |
return view_type(); |
| 46 |
view_type v("", a.size(), a[0].size()); |
| 47 |
unmanaged_host_view_type hv(a.getRawPtr(), a.size(), a[0].size()); |
| 48 |
Kokkos::deep_copy(v,hv); |
| 49 |
return v; |
| 50 |
} |
| 51 |
|
| 52 |
template <typename D, typename S> |
| 53 |
Kokkos::View<const Sacado::UQ::PCE<S>*,D> |
| 54 |
getKokkosViewDeepCopy(const Teuchos::ArrayView<const Sacado::UQ::PCE<S> >& a) { |
| 55 |
typedef Sacado::UQ::PCE<S> T; |
| 56 |
typedef typename std::conditional< |
| 57 |
::Kokkos::SpaceAccessibility< D, Kokkos::HostSpace>::accessible, |
| 58 |
typename D::execution_space, Kokkos::HostSpace>::type |
| 59 |
HostDevice; |
| 60 |
typedef Kokkos::View<T*,D> view_type; |
| 61 |
typedef Kokkos::View<const T*,typename view_type::array_layout,HostDevice,Kokkos::MemoryUnmanaged> unmanaged_host_view_type; |
| 62 |
if (a.size() == 0) |
| 63 |
return view_type(); |
| 64 |
view_type v("", a.size(), a[0].size()); |
| 65 |
unmanaged_host_view_type hv(a.getRawPtr(), a.size(), a[0].size()); |
| 66 |
Kokkos::deep_copy(v,hv); |
| 67 |
return v; |
| 68 |
} |
| 69 |
} |
| 70 |
} |
| 71 |
|
| 72 |
// Kokkos-Linalg |
| 73 |
#include "Kokkos_ArithTraits_UQ_PCE.hpp" |
| 74 |
#include "Kokkos_InnerProductSpaceTraits_UQ_PCE.hpp" |
| 75 |
#include "Kokkos_MV_UQ_PCE.hpp" |
| 76 |
#include "Kokkos_CrsMatrix_UQ_PCE.hpp" |
| 77 |
#include "Kokkos_CrsMatrix_UQ_PCE_Cuda.hpp" |
| 78 |
#include "Kokkos_TeuchosCommAdapters_UQ_PCE.hpp" |
| 79 |
#include "Tpetra_KokkosRefactor_Details_MultiVectorDistObjectKernels_UQ_PCE.hpp" |
| 80 |
#include "Tpetra_KokkosRefactor_Details_MultiVectorLocalDeepCopy_UQ_PCE.hpp" |
| 81 |
#include "Tpetra_Details_fill_UQ_PCE.hpp" |
| 82 |
#include "Kokkos_Random_UQ_PCE.hpp" |
| 83 |
|
| 84 |
namespace Stokhos { |
| 85 |
|
| 86 |
// Traits for determining device type from node type |
| 87 |
template <typename Node> |
| 88 |
struct DeviceForNode2 { |
| 89 |
// Prefer Serial execution space as the default, but if that's not |
| 90 |
// available, use the Host memory space's default execution space. |
| 91 |
#if defined(KOKKOS_ENABLE_SERIAL) |
| 92 |
typedef Kokkos::Serial type; |
| 93 |
#else |
| 94 |
typedef Kokkos::HostSpace::execution_space type; |
| 95 |
#endif // defined(KOKKOS_ENABLE_SERIAL) |
| 96 |
}; |
| 97 |
|
| 98 |
template <typename ExecSpace, typename MemSpace> |
| 99 |
struct DeviceForNode2< Tpetra::KokkosCompat::KokkosDeviceWrapperNode<ExecSpace, MemSpace> > { |
| 100 |
typedef typename Tpetra::KokkosCompat::KokkosDeviceWrapperNode<ExecSpace, MemSpace>::device_type type; |
| 101 |
}; |
| 102 |
|
| 103 |
} |
| 104 |
|
| 105 |
#include "Tpetra_Details_PackTraits.hpp" |
| 106 |
#include "Tpetra_Details_ScalarViewTraits.hpp" |
| 107 |
|
| 108 |
namespace Tpetra { |
| 109 |
namespace Details { |
| 110 |
|
| 111 |
/// \brief Partial specialization of PackTraits for Sacado's PCE UQ type. |
| 112 |
/// |
| 113 |
/// \tparam S The underlying scalar type in the PCE UQ type. |
| 114 |
template<class S> |
| 115 |
struct PackTraits<Sacado::UQ::PCE<S>> { |
| 116 |
using value_type = Sacado::UQ::PCE<S>; |
| 117 |
|
| 118 |
/// \brief Whether the number of bytes required to pack one instance |
| 119 |
/// of \c value_type is fixed at compile time. |
| 120 |
static const bool compileTimeSize = false; |
| 121 |
|
| 122 |
using input_buffer_type = Kokkos::View<const char*, Kokkos::AnonymousSpace>; |
| 123 |
using output_buffer_type = Kokkos::View<char*, Kokkos::AnonymousSpace>; |
| 124 |
using input_array_type = Kokkos::View<const value_type*, Kokkos::AnonymousSpace>; |
| 125 |
using output_array_type = Kokkos::View<value_type*, Kokkos::AnonymousSpace>; |
| 126 |
|
| 127 |
using scalar_value_type = typename value_type::value_type; |
| 128 |
using SPT = PackTraits<scalar_value_type>; |
| 129 |
using scalar_input_array_type = typename SPT::input_array_type; |
| 130 |
using scalar_output_array_type = typename SPT::output_array_type; |
| 131 |
|
| 132 |
KOKKOS_INLINE_FUNCTION |
| 133 |
static size_t numValuesPerScalar (const value_type& x) { |
| 134 |
return x.size (); |
| 135 |
} |
| 136 |
|
| 137 |
KOKKOS_INLINE_FUNCTION |
| 138 |
static Kokkos::pair<int, size_t> |
| 139 |
packArray (char outBuf[], |
| 140 |
const value_type inBuf[], |
| 141 |
const size_t numEnt) |
| 142 |
{ |
| 143 |
using return_type = Kokkos::pair<int, size_t>; |
| 144 |
size_t numBytes = 0; |
| 145 |
int errorCode = 0; |
| 146 |
|
| 147 |
if (numEnt == 0) { |
| 148 |
return return_type (errorCode, numBytes); |
| 149 |
} |
| 150 |
else { |
| 151 |
// Check whether input array is contiguously allocated based on the size |
| 152 |
// of the first entry. We can only pack contiguously allocated data |
| 153 |
// since that is the only way we can guarrantee all of the PCE arrays |
| 154 |
// are the same size and the buffer will allocated correctly. |
| 155 |
const size_t scalar_size = numValuesPerScalar (inBuf[0]); |
| 156 |
const scalar_value_type* last_coeff = inBuf[numEnt - 1].coeff (); |
| 157 |
const scalar_value_type* last_coeff_expected = |
| 158 |
inBuf[0].coeff () + (numEnt - 1)*scalar_size; |
| 159 |
const bool is_contiguous = (last_coeff == last_coeff_expected); |
| 160 |
if (! is_contiguous) { |
| 161 |
// Cannot pack non-contiguous PCE array since buffer size calculation |
| 162 |
// is likely wrong. |
| 163 |
errorCode = 3; |
| 164 |
return return_type (errorCode, numBytes); |
| 165 |
} |
| 166 |
|
| 167 |
// Check we are packing length-1 PCE arrays (mean-based preconditioner). |
| 168 |
// We can technically pack length > 1, but the unpack assumes the |
| 169 |
// output array is sized appropriately. Currently this is not the case |
| 170 |
// in Tpetra::CrsMatrix::transferAndFillComplete() which allocates a |
| 171 |
// local Teuchos::Array for the CSR values, which will only be length-1 |
| 172 |
// by default. |
| 173 |
if (scalar_size != 1) { |
| 174 |
// Cannot pack PCE array with pce_size > 1 since unpack array |
| 175 |
// may not be allocated correctly. |
| 176 |
errorCode = 4; |
| 177 |
return return_type (errorCode, numBytes); |
| 178 |
} |
| 179 |
|
| 180 |
const size_t flat_numEnt = numEnt * scalar_size; |
| 181 |
return SPT::packArray (outBuf, inBuf[0].coeff (), flat_numEnt); |
| 182 |
} |
| 183 |
} |
| 184 |
|
| 185 |
KOKKOS_INLINE_FUNCTION |
| 186 |
static Kokkos::pair<int, size_t> |
| 187 |
unpackArray (value_type outBuf[], |
| 188 |
const char inBuf[], |
| 189 |
const size_t numEnt) |
| 190 |
{ |
| 191 |
using return_type = Kokkos::pair<int, size_t>; |
| 192 |
size_t numBytes = 0; |
| 193 |
int errorCode = 0; |
| 194 |
|
| 195 |
if (numEnt == 0) { |
| 196 |
return return_type (errorCode, numBytes); |
| 197 |
} |
| 198 |
else { |
| 199 |
// Check whether output array is contiguously allocated based on the size |
| 200 |
// of the first entry. We have a simpler method to unpack in this case |
| 201 |
const size_t scalar_size = numValuesPerScalar (outBuf[0]); |
| 202 |
const scalar_value_type* last_coeff = outBuf[numEnt - 1].coeff (); |
| 203 |
const scalar_value_type* last_coeff_expected = |
| 204 |
outBuf[0].coeff () + (numEnt - 1) * scalar_size; |
| 205 |
const bool is_contiguous = (last_coeff == last_coeff_expected); |
| 206 |
|
| 207 |
if (is_contiguous) { |
| 208 |
// Unpack all of the PCE coefficients for the whole array |
| 209 |
const size_t flat_numEnt = numEnt * scalar_size; |
| 210 |
return SPT::unpackArray (outBuf[0].coeff (), inBuf, flat_numEnt); |
| 211 |
} |
| 212 |
else { |
| 213 |
// Unpack one entry at a time. This assumes each entry of outBuf |
| 214 |
// is the correct size based on the packing. This is is only |
| 215 |
// guarranteed to be true for pce_size == 1, hence the check in |
| 216 |
// packArray(). |
| 217 |
size_t numBytesTotal = 0; |
| 218 |
for (size_t i = 0; i < numEnt; ++i) { |
| 219 |
const char* inBufVal = inBuf + numBytesTotal; |
| 220 |
const size_t numBytes = unpackValue (outBuf[i], inBufVal); |
| 221 |
numBytesTotal += numBytes; |
| 222 |
} |
| 223 |
return return_type (errorCode, numBytesTotal); |
| 224 |
} |
| 225 |
} |
| 226 |
} |
| 227 |
|
| 228 |
KOKKOS_INLINE_FUNCTION |
| 229 |
static size_t |
| 230 |
packValueCount (const value_type& inVal) |
| 231 |
{ |
| 232 |
return inVal.size () * SPT::packValueCount (inVal.val ()); |
| 233 |
} |
| 234 |
|
| 235 |
KOKKOS_INLINE_FUNCTION |
| 236 |
static size_t |
| 237 |
packValue (char outBuf[], |
| 238 |
const value_type& inVal) |
| 239 |
{ |
| 240 |
const size_t numBytes = packValueCount (inVal); |
| 241 |
memcpy (outBuf, inVal.coeff (), numBytes); |
| 242 |
return numBytes; |
| 243 |
} |
| 244 |
|
| 245 |
KOKKOS_INLINE_FUNCTION |
| 246 |
static size_t |
| 247 |
packValue (char outBuf[], |
| 248 |
const size_t outBufIndex, |
| 249 |
const value_type& inVal) |
| 250 |
{ |
| 251 |
const size_t numBytes = packValueCount (inVal); |
| 252 |
const size_t offset = outBufIndex * numBytes; |
| 253 |
memcpy (outBuf + offset, inVal.coeff (), numBytes); |
| 254 |
return numBytes; |
| 255 |
} |
| 256 |
|
| 257 |
KOKKOS_INLINE_FUNCTION |
| 258 |
static size_t |
| 259 |
unpackValue (value_type& outVal, const char inBuf[]) |
| 260 |
{ |
| 261 |
const size_t numBytes = packValueCount (outVal); |
| 262 |
memcpy (outVal.coeff (), inBuf, numBytes); |
| 263 |
return numBytes; |
| 264 |
} |
| 265 |
}; // struct PackTraits |
| 266 |
|
| 267 |
/// \brief Partial specialization of ScalarViewTraits |
| 268 |
/// for Sacado's PCE UQ type. |
| 269 |
/// |
| 270 |
/// \tparam S The underlying scalar type in the PCE UQ type. |
| 271 |
/// \tparam D The Kokkos "device" type. |
| 272 |
template<typename S, typename D> |
| 273 |
struct ScalarViewTraits<Sacado::UQ::PCE<S>, D> { |
| 274 |
using value_type = Sacado::UQ::PCE<S>; |
| 275 |
using device_type = D; |
| 276 |
|
| 277 |
static Kokkos::View<value_type*, device_type> |
| 278 |
allocateArray (const value_type& x, |
| 279 |
const size_t numEnt, |
| 280 |
const std::string& label = "") |
| 281 |
{ |
| 282 |
const size_t numVals = PackTraits<value_type>::numValuesPerScalar (x); |
| 283 |
using view_type = Kokkos::View<value_type*, device_type>; |
| 284 |
return view_type (label, numEnt, numVals); |
| 285 |
} |
| 286 |
}; |
| 287 |
|
| 288 |
} // namespace Details |
| 289 |
} // namespace Tpetra |
| 290 |
|
| 291 |
namespace Kokkos { |
| 292 |
template <class S, class L, class G, class N> |
| 293 |
size_t dimension_scalar(const Tpetra::MultiVector<S,L,G,N>& mv) { |
| 294 |
if ( mv.need_sync_device() ) { |
| 295 |
// NOTE (mfh 02 Apr 2019) This doesn't look right. Shouldn't I |
| 296 |
// want the most recently updated View, which in this case would |
| 297 |
// be the host View? However, this is what I found when I |
| 298 |
// changed these lines not to call deprecated code, so I'm |
| 299 |
// leaving it. |
| 300 |
return dimension_scalar(mv.getLocalViewHost(Tpetra::Access::ReadOnly)); |
| 301 |
} |
| 302 |
return dimension_scalar(mv.getLocalViewDevice(Tpetra::Access::ReadOnly)); |
| 303 |
} |
| 304 |
|
| 305 |
template <class S, class L, class G, class N> |
| 306 |
size_t dimension_scalar(const Tpetra::Vector<S,L,G,N>& v) { |
| 307 |
if ( v.need_sync_device() ) { |
| 308 |
// NOTE (mfh 02 Apr 2019) This doesn't look right. Shouldn't I |
| 309 |
// want the most recently updated View, which in this case would |
| 310 |
// be the host View? However, this is what I found when I |
| 311 |
// changed these lines not to call deprecated code, so I'm |
| 312 |
// leaving it. |
| 313 |
return dimension_scalar(v.getLocalViewHost(Tpetra::Access::ReadOnly)); |
| 314 |
} |
| 315 |
return dimension_scalar(v.getLocalViewDevice(Tpetra::Access::ReadOnly)); |
| 316 |
} |
| 317 |
} |
| 318 |
|
| 319 |
#endif // STOKHOS_TPETRA_UQ_PCE_HPP |