| 1 |
// @HEADER |
| 2 |
// ***************************************************************************** |
| 3 |
// Stokhos Package |
| 4 |
// |
| 5 |
// Copyright 2009 NTESS and the Stokhos contributors. |
| 6 |
// SPDX-License-Identifier: BSD-3-Clause |
| 7 |
// ***************************************************************************** |
| 8 |
// @HEADER |
| 9 |
|
| 10 |
#ifndef BELOS_TPETRA_ADAPTER_UQ_PCE_HPP |
| 11 |
#define BELOS_TPETRA_ADAPTER_UQ_PCE_HPP |
| 12 |
|
| 13 |
#include "BelosTpetraAdapter.hpp" |
| 14 |
#include "Stokhos_Sacado_Kokkos_UQ_PCE.hpp" |
| 15 |
#include "KokkosBlas.hpp" |
| 16 |
|
| 17 |
#ifdef HAVE_BELOS_TSQR |
| 18 |
# include <Tpetra_TsqrAdaptor_UQ_PCE.hpp> |
| 19 |
#endif // HAVE_BELOS_TSQR |
| 20 |
|
| 21 |
namespace Belos { |
| 22 |
|
| 23 |
//////////////////////////////////////////////////////////////////// |
| 24 |
// |
| 25 |
// Implementation of Belos::MultiVecTraits for Tpetra::MultiVector. |
| 26 |
// |
| 27 |
//////////////////////////////////////////////////////////////////// |
| 28 |
|
| 29 |
/// \brief Partial specialization of MultiVecTraits for MV = Tpetra::MultiVector. |
| 30 |
/// |
| 31 |
/// This interface lets Belos' solvers work directly with |
| 32 |
/// Tpetra::MultiVector objects as the multivector type |
| 33 |
/// (corresponding to the MV template parameter). |
| 34 |
/// |
| 35 |
/// The four template parameters of this partial specialization |
| 36 |
/// correspond exactly to the four template parameters of |
| 37 |
/// Tpetra::MultiVector. See the Tpetra::MultiVector documentation |
| 38 |
/// for more information. |
| 39 |
template<class Storage, class LO, class GO, class Node> |
| 40 |
class MultiVecTraits<typename Storage::value_type, |
| 41 |
Tpetra::MultiVector< Sacado::UQ::PCE<Storage>, |
| 42 |
LO, GO, Node > > { |
| 43 |
public: |
| 44 |
typedef typename Storage::ordinal_type s_ordinal; |
| 45 |
typedef typename Storage::value_type BaseScalar; |
| 46 |
typedef Sacado::UQ::PCE<Storage> Scalar; |
| 47 |
typedef Tpetra::MultiVector<Scalar, LO, GO, Node> MV; |
| 48 |
public: |
| 49 |
typedef typename Tpetra::MultiVector<Scalar,LO,GO,Node>::dot_type dot_type; |
| 50 |
typedef typename Tpetra::MultiVector<Scalar,LO,GO,Node>::mag_type mag_type; |
| 51 |
|
| 52 |
/// \brief Create a new MultiVector with \c numVecs columns. |
| 53 |
/// |
| 54 |
/// The returned Tpetra::MultiVector has the same Tpetra::Map |
| 55 |
/// (distribution over one or more parallel processes) as \c X. |
| 56 |
/// Its entries are not initialized and have undefined values. |
| 57 |
static Teuchos::RCP<MV> Clone (const MV& X, const int numVecs) { |
| 58 |
Teuchos::RCP<MV> Y (new MV (X.getMap (), numVecs, false)); |
| 59 |
Y->setCopyOrView (Teuchos::View); |
| 60 |
return Y; |
| 61 |
} |
| 62 |
|
| 63 |
//! Create and return a deep copy of X. |
| 64 |
static Teuchos::RCP<MV> CloneCopy (const MV& X) |
| 65 |
{ |
| 66 |
// Make a deep copy of X. The one-argument copy constructor |
| 67 |
// does a shallow copy by default; the second argument tells it |
| 68 |
// to do a deep copy. |
| 69 |
Teuchos::RCP<MV> X_copy (new MV (X, Teuchos::Copy)); |
| 70 |
// Make Tpetra::MultiVector use the new view semantics. This is |
| 71 |
// a no-op for the Kokkos refactor version of Tpetra; it only |
| 72 |
// does something for the "classic" version of Tpetra. This |
| 73 |
// shouldn't matter because Belos only handles MV through RCP |
| 74 |
// and through this interface anyway, but it doesn't hurt to set |
| 75 |
// it and make sure that it works. |
| 76 |
X_copy->setCopyOrView (Teuchos::View); |
| 77 |
return X_copy; |
| 78 |
} |
| 79 |
|
| 80 |
/// \brief Create and return a deep copy of the given columns of mv. |
| 81 |
/// |
| 82 |
/// \pre \code mv.getNumVectors() != 0 || index.size() == 0 \endcode |
| 83 |
/// \pre For all k such that <tt>0 <= k < index.size()</tt>, |
| 84 |
/// \code |
| 85 |
/// 0 <= index[k] < mv.getNumVectors(); |
| 86 |
/// \endcode |
| 87 |
/// \post If this method returns Y: |
| 88 |
/// \code |
| 89 |
/// Y->isConstantStride() && Y->getNumVectors() == index.size(); |
| 90 |
/// \endcode |
| 91 |
static Teuchos::RCP<MV> |
| 92 |
CloneCopy (const MV& mv, const std::vector<int>& index) |
| 93 |
{ |
| 94 |
#ifdef HAVE_TPETRA_DEBUG |
| 95 |
const char fnName[] = "Belos::MultiVecTraits::CloneCopy(mv,index)"; |
| 96 |
const size_t inNumVecs = mv.getNumVectors (); |
| 97 |
TEUCHOS_TEST_FOR_EXCEPTION( |
| 98 |
index.size () > 0 && *std::min_element (index.begin (), index.end ()) < 0, |
| 99 |
std::runtime_error, fnName << ": All indices must be nonnegative."); |
| 100 |
TEUCHOS_TEST_FOR_EXCEPTION( |
| 101 |
index.size () > 0 && |
| 102 |
static_cast<size_t> (*std::max_element (index.begin (), index.end ())) >= inNumVecs, |
| 103 |
std::runtime_error, |
| 104 |
fnName << ": All indices must be strictly less than the number of " |
| 105 |
"columns " << inNumVecs << " of the input multivector mv."); |
| 106 |
#endif // HAVE_TPETRA_DEBUG |
| 107 |
|
| 108 |
// Tpetra wants an array of size_t, not of int. |
| 109 |
Teuchos::Array<size_t> columns (index.size ()); |
| 110 |
for (std::vector<int>::size_type j = 0; j < index.size (); ++j) { |
| 111 |
columns[j] = index[j]; |
| 112 |
} |
| 113 |
// mfh 14 Aug 2014: Tpetra already detects and optimizes for a |
| 114 |
// continuous column index range in MultiVector::subCopy, so we |
| 115 |
// don't have to check here. |
| 116 |
Teuchos::RCP<MV> X_copy = mv.subCopy (columns ()); |
| 117 |
X_copy->setCopyOrView (Teuchos::View); |
| 118 |
return X_copy; |
| 119 |
} |
| 120 |
|
| 121 |
/// \brief Create and return a deep copy of the given columns of mv. |
| 122 |
/// |
| 123 |
/// \post If this method returns Y: |
| 124 |
/// \code |
| 125 |
/// Y->isConstantStride() && Y->getNumVectors() == index.size(); |
| 126 |
/// \endcode |
| 127 |
static Teuchos::RCP<MV> |
| 128 |
CloneCopy (const MV& mv, const Teuchos::Range1D& index) |
| 129 |
{ |
| 130 |
const bool validRange = index.size() > 0 && |
| 131 |
index.lbound() >= 0 && |
| 132 |
index.ubound() < GetNumberVecs(mv); |
| 133 |
if (! validRange) { // invalid range; generate error message |
| 134 |
std::ostringstream os; |
| 135 |
os << "Belos::MultiVecTraits::CloneCopy(mv,index=[" |
| 136 |
<< index.lbound() << "," << index.ubound() << "]): "; |
| 137 |
TEUCHOS_TEST_FOR_EXCEPTION( |
| 138 |
index.size() == 0, std::invalid_argument, |
| 139 |
os.str() << "Empty index range is not allowed."); |
| 140 |
TEUCHOS_TEST_FOR_EXCEPTION( |
| 141 |
index.lbound() < 0, std::invalid_argument, |
| 142 |
os.str() << "Index range includes negative index/ices, which is not " |
| 143 |
"allowed."); |
| 144 |
TEUCHOS_TEST_FOR_EXCEPTION( |
| 145 |
index.ubound() >= GetNumberVecs(mv), std::invalid_argument, |
| 146 |
os.str() << "Index range exceeds number of vectors " |
| 147 |
<< mv.getNumVectors() << " in the input multivector."); |
| 148 |
TEUCHOS_TEST_FOR_EXCEPTION(true, std::logic_error, |
| 149 |
os.str() << "Should never get here!"); |
| 150 |
} |
| 151 |
Teuchos::RCP<MV> X_copy = mv.subCopy (index); |
| 152 |
X_copy->setCopyOrView (Teuchos::View); |
| 153 |
return X_copy; |
| 154 |
} |
| 155 |
|
| 156 |
|
| 157 |
static Teuchos::RCP<MV> |
| 158 |
CloneViewNonConst (MV& mv, const std::vector<int>& index) |
| 159 |
{ |
| 160 |
#ifdef HAVE_TPETRA_DEBUG |
| 161 |
const char fnName[] = "Belos::MultiVecTraits::CloneViewNonConst(mv,index)"; |
| 162 |
const size_t numVecs = mv.getNumVectors (); |
| 163 |
TEUCHOS_TEST_FOR_EXCEPTION( |
| 164 |
index.size () > 0 && *std::min_element (index.begin (), index.end ()) < 0, |
| 165 |
std::invalid_argument, |
| 166 |
fnName << ": All indices must be nonnegative."); |
| 167 |
TEUCHOS_TEST_FOR_EXCEPTION( |
| 168 |
index.size () > 0 && |
| 169 |
static_cast<size_t> (*std::max_element (index.begin (), index.end ())) >= numVecs, |
| 170 |
std::invalid_argument, |
| 171 |
fnName << ": All indices must be strictly less than the number of " |
| 172 |
"columns " << numVecs << " in the input MultiVector mv."); |
| 173 |
#endif // HAVE_TPETRA_DEBUG |
| 174 |
|
| 175 |
// Tpetra wants an array of size_t, not of int. |
| 176 |
Teuchos::Array<size_t> columns (index.size ()); |
| 177 |
for (std::vector<int>::size_type j = 0; j < index.size (); ++j) { |
| 178 |
columns[j] = index[j]; |
| 179 |
} |
| 180 |
// mfh 14 Aug 2014: Tpetra already detects and optimizes for a |
| 181 |
// continuous column index range in |
| 182 |
// MultiVector::subViewNonConst, so we don't have to check here. |
| 183 |
Teuchos::RCP<MV> X_view = mv.subViewNonConst (columns ()); |
| 184 |
X_view->setCopyOrView (Teuchos::View); |
| 185 |
return X_view; |
| 186 |
} |
| 187 |
|
| 188 |
static Teuchos::RCP<MV> |
| 189 |
CloneViewNonConst (MV& mv, const Teuchos::Range1D& index) |
| 190 |
{ |
| 191 |
// NOTE (mfh 11 Jan 2011) We really should check for possible |
| 192 |
// overflow of int here. However, the number of columns in a |
| 193 |
// multivector typically fits in an int. |
| 194 |
const int numCols = static_cast<int> (mv.getNumVectors()); |
| 195 |
const bool validRange = index.size() > 0 && |
| 196 |
index.lbound() >= 0 && index.ubound() < numCols; |
| 197 |
if (! validRange) { |
| 198 |
std::ostringstream os; |
| 199 |
os << "Belos::MultiVecTraits::CloneViewNonConst(mv,index=[" |
| 200 |
<< index.lbound() << ", " << index.ubound() << "]): "; |
| 201 |
TEUCHOS_TEST_FOR_EXCEPTION( |
| 202 |
index.size() == 0, std::invalid_argument, |
| 203 |
os.str() << "Empty index range is not allowed."); |
| 204 |
TEUCHOS_TEST_FOR_EXCEPTION( |
| 205 |
index.lbound() < 0, std::invalid_argument, |
| 206 |
os.str() << "Index range includes negative inde{x,ices}, which is " |
| 207 |
"not allowed."); |
| 208 |
TEUCHOS_TEST_FOR_EXCEPTION( |
| 209 |
index.ubound() >= numCols, std::invalid_argument, |
| 210 |
os.str() << "Index range exceeds number of vectors " << numCols |
| 211 |
<< " in the input multivector."); |
| 212 |
TEUCHOS_TEST_FOR_EXCEPTION( |
| 213 |
true, std::logic_error, |
| 214 |
os.str() << "Should never get here!"); |
| 215 |
} |
| 216 |
Teuchos::RCP<MV> X_view = mv.subViewNonConst (index); |
| 217 |
X_view->setCopyOrView (Teuchos::View); |
| 218 |
return X_view; |
| 219 |
} |
| 220 |
|
| 221 |
static Teuchos::RCP<const MV> |
| 222 |
CloneView (const MV& mv, const std::vector<int>& index) |
| 223 |
{ |
| 224 |
#ifdef HAVE_TPETRA_DEBUG |
| 225 |
const char fnName[] = "Belos::MultiVecTraits<Scalar, " |
| 226 |
"Tpetra::MultiVector<...> >::CloneView(mv,index)"; |
| 227 |
const size_t numVecs = mv.getNumVectors (); |
| 228 |
TEUCHOS_TEST_FOR_EXCEPTION( |
| 229 |
*std::min_element (index.begin (), index.end ()) < 0, |
| 230 |
std::invalid_argument, |
| 231 |
fnName << ": All indices must be nonnegative."); |
| 232 |
TEUCHOS_TEST_FOR_EXCEPTION( |
| 233 |
static_cast<size_t> (*std::max_element (index.begin (), index.end ())) >= numVecs, |
| 234 |
std::invalid_argument, |
| 235 |
fnName << ": All indices must be strictly less than the number of " |
| 236 |
"columns " << numVecs << " in the input MultiVector mv."); |
| 237 |
#endif // HAVE_TPETRA_DEBUG |
| 238 |
|
| 239 |
// Tpetra wants an array of size_t, not of int. |
| 240 |
Teuchos::Array<size_t> columns (index.size ()); |
| 241 |
for (std::vector<int>::size_type j = 0; j < index.size (); ++j) { |
| 242 |
columns[j] = index[j]; |
| 243 |
} |
| 244 |
// mfh 14 Aug 2014: Tpetra already detects and optimizes for a |
| 245 |
// continuous column index range in MultiVector::subView, so we |
| 246 |
// don't have to check here. |
| 247 |
Teuchos::RCP<const MV> X_view = mv.subView (columns); |
| 248 |
Teuchos::rcp_const_cast<MV> (X_view)->setCopyOrView (Teuchos::View); |
| 249 |
return X_view; |
| 250 |
} |
| 251 |
|
| 252 |
static Teuchos::RCP<const MV> |
| 253 |
CloneView (const MV& mv, const Teuchos::Range1D& index) |
| 254 |
{ |
| 255 |
// NOTE (mfh 11 Jan 2011) We really should check for possible |
| 256 |
// overflow of int here. However, the number of columns in a |
| 257 |
// multivector typically fits in an int. |
| 258 |
const int numCols = static_cast<int> (mv.getNumVectors()); |
| 259 |
const bool validRange = index.size() > 0 && |
| 260 |
index.lbound() >= 0 && index.ubound() < numCols; |
| 261 |
if (! validRange) { |
| 262 |
std::ostringstream os; |
| 263 |
os << "Belos::MultiVecTraits::CloneView(mv, index=[" |
| 264 |
<< index.lbound () << ", " << index.ubound() << "]): "; |
| 265 |
TEUCHOS_TEST_FOR_EXCEPTION(index.size() == 0, std::invalid_argument, |
| 266 |
os.str() << "Empty index range is not allowed."); |
| 267 |
TEUCHOS_TEST_FOR_EXCEPTION(index.lbound() < 0, std::invalid_argument, |
| 268 |
os.str() << "Index range includes negative index/ices, which is not " |
| 269 |
"allowed."); |
| 270 |
TEUCHOS_TEST_FOR_EXCEPTION( |
| 271 |
index.ubound() >= numCols, std::invalid_argument, |
| 272 |
os.str() << "Index range exceeds number of vectors " << numCols |
| 273 |
<< " in the input multivector."); |
| 274 |
TEUCHOS_TEST_FOR_EXCEPTION(true, std::logic_error, |
| 275 |
os.str() << "Should never get here!"); |
| 276 |
} |
| 277 |
Teuchos::RCP<const MV> X_view = mv.subView (index); |
| 278 |
Teuchos::rcp_const_cast<MV> (X_view)->setCopyOrView (Teuchos::View); |
| 279 |
return X_view; |
| 280 |
} |
| 281 |
|
| 282 |
static ptrdiff_t GetGlobalLength (const MV& mv) { |
| 283 |
return static_cast<ptrdiff_t> (mv.getGlobalLength ()); |
| 284 |
} |
| 285 |
|
| 286 |
static int GetNumberVecs (const MV& mv) { |
| 287 |
return static_cast<int> (mv.getNumVectors ()); |
| 288 |
} |
| 289 |
|
| 290 |
static bool HasConstantStride (const MV& mv) { |
| 291 |
return mv.isConstantStride (); |
| 292 |
} |
| 293 |
|
| 294 |
static void |
| 295 |
MvTimesMatAddMv (const dot_type& alpha, |
| 296 |
const Tpetra::MultiVector<Scalar,LO,GO,Node>& A, |
| 297 |
const Teuchos::SerialDenseMatrix<int,dot_type>& B, |
| 298 |
const dot_type& beta, |
| 299 |
Tpetra::MultiVector<Scalar,LO,GO,Node>& C) |
| 300 |
{ |
| 301 |
using Teuchos::RCP; |
| 302 |
using Teuchos::rcp; |
| 303 |
|
| 304 |
// Check if numRowsB == numColsB == 1, in which case we can call update() |
| 305 |
const int numRowsB = B.numRows (); |
| 306 |
const int numColsB = B.numCols (); |
| 307 |
const int strideB = B.stride (); |
| 308 |
if (numRowsB == 1 && numColsB == 1) { |
| 309 |
C.update (alpha*B(0,0), A, beta); |
| 310 |
return; |
| 311 |
} |
| 312 |
|
| 313 |
// Ensure A and C have constant stride |
| 314 |
RCP<const MV> Atmp; |
| 315 |
RCP<MV> Ctmp; |
| 316 |
if (A.isConstantStride() == false) Atmp = rcp (new MV (A, Teuchos::Copy)); |
| 317 |
else Atmp = rcp(&A,false); |
| 318 |
|
| 319 |
if (C.isConstantStride() == false) Ctmp = rcp (new MV (C, Teuchos::Copy)); |
| 320 |
else Ctmp = rcp(&C,false); |
| 321 |
|
| 322 |
// Create flattened view's |
| 323 |
typedef Tpetra::MultiVector<dot_type,LO,GO,Node> FMV; |
| 324 |
typedef Tpetra::MultiVector<const dot_type,LO,GO,Node> CFMV; |
| 325 |
typedef typename FMV::dual_view_type::t_dev flat_view_type; |
| 326 |
typedef typename CFMV::dual_view_type::t_dev const_flat_view_type; |
| 327 |
typedef typename flat_view_type::execution_space execution_space; |
| 328 |
const_flat_view_type flat_A_view = Atmp->template getLocalView<execution_space>(Tpetra::Access::ReadOnly); |
| 329 |
flat_view_type flat_C_view = Ctmp->template getLocalView<execution_space>(Tpetra::Access::ReadWrite); |
| 330 |
|
| 331 |
// Create a view for B on the host |
| 332 |
typedef Kokkos::View<dot_type**, Kokkos::LayoutLeft, Kokkos::HostSpace> b_host_view_type; |
| 333 |
b_host_view_type B_view_host_input( B.values(), strideB, numColsB); |
| 334 |
auto B_view_host = Kokkos::subview( B_view_host_input, |
| 335 |
Kokkos::pair<int,int>(0, numRowsB), |
| 336 |
Kokkos::pair<int,int>(0, numColsB)); |
| 337 |
|
| 338 |
// Create view for B on the device -- need to be careful to get the |
| 339 |
// right stride to match B |
| 340 |
typedef Kokkos::View<dot_type**, Kokkos::LayoutLeft, execution_space> b_view_type; |
| 341 |
typedef Kokkos::View<dot_type*, Kokkos::LayoutLeft, execution_space> b_1d_view_type; |
| 342 |
b_1d_view_type B_1d_view_dev(Kokkos::ViewAllocateWithoutInitializing("B"), numRowsB*numColsB); |
| 343 |
b_view_type B_view_dev( B_1d_view_dev.data(), numRowsB, numColsB); |
| 344 |
//Kokkos::deep_copy(B_view_dev, B_view_host); |
| 345 |
// Device-to-host copies requires dest to be contiguous, which |
| 346 |
// C_view_host may not be. So do 1 column at a time. |
| 347 |
for (int j=0; j<numColsB; ++j) |
| 348 |
Kokkos::deep_copy(Kokkos::subview(B_view_dev,Kokkos::ALL,j), |
| 349 |
Kokkos::subview(B_view_host,Kokkos::ALL,j)); |
| 350 |
|
| 351 |
// Do local multiply |
| 352 |
{ |
| 353 |
const char ctransA = 'N', ctransB = 'N'; |
| 354 |
KokkosBlas::gemm ( |
| 355 |
&ctransA, &ctransB, |
| 356 |
alpha, flat_A_view, B_view_dev, beta, flat_C_view); |
| 357 |
} |
| 358 |
// Copy back to C if we made a copy |
| 359 |
if (C.isConstantStride() == false) |
| 360 |
C.assign(*Ctmp); |
| 361 |
} |
| 362 |
|
| 363 |
/// \brief <tt>mv := alpha*A + beta*B</tt> |
| 364 |
/// |
| 365 |
/// The Tpetra specialization of this method ignores and |
| 366 |
/// completely overwrites any NaN or Inf entries in A. Thus, it |
| 367 |
/// does <i>not</i> mean the same thing as <tt>mv := 0*mv + |
| 368 |
/// alpha*A + beta*B</tt> in IEEE 754 floating-point arithmetic. |
| 369 |
/// (Remember that NaN*0 = NaN.) |
| 370 |
static void |
| 371 |
MvAddMv (Scalar alpha, |
| 372 |
const Tpetra::MultiVector<Scalar,LO,GO,Node>& A, |
| 373 |
Scalar beta, |
| 374 |
const Tpetra::MultiVector<Scalar,LO,GO,Node>& B, |
| 375 |
Tpetra::MultiVector<Scalar,LO,GO,Node>& mv) |
| 376 |
{ |
| 377 |
mv.update (alpha, A, beta, B, Teuchos::ScalarTraits<Scalar>::zero ()); |
| 378 |
} |
| 379 |
|
| 380 |
static void |
| 381 |
MvScale (Tpetra::MultiVector<Scalar,LO,GO,Node>& mv, |
| 382 |
const Scalar& alpha) |
| 383 |
{ |
| 384 |
mv.scale (alpha); |
| 385 |
} |
| 386 |
|
| 387 |
static void |
| 388 |
MvScale (Tpetra::MultiVector<Scalar,LO,GO,Node>& mv, |
| 389 |
const std::vector<BaseScalar>& alphas) |
| 390 |
{ |
| 391 |
std::vector<Scalar> alphas_mp(alphas.size()); |
| 392 |
const size_t sz = alphas.size(); |
| 393 |
for (size_t i=0; i<sz; ++i) |
| 394 |
alphas_mp[i] = alphas[i]; |
| 395 |
mv.scale (alphas_mp); |
| 396 |
} |
| 397 |
|
| 398 |
static void |
| 399 |
MvScale (Tpetra::MultiVector<Scalar,LO,GO,Node>& mv, |
| 400 |
const std::vector<Scalar>& alphas) |
| 401 |
{ |
| 402 |
mv.scale (alphas); |
| 403 |
} |
| 404 |
|
| 405 |
static void |
| 406 |
MvTransMv (dot_type alpha, |
| 407 |
const Tpetra::MultiVector<Scalar,LO,GO,Node>& A, |
| 408 |
const Tpetra::MultiVector<Scalar,LO,GO,Node>& B, |
| 409 |
Teuchos::SerialDenseMatrix<int,dot_type>& C) |
| 410 |
{ |
| 411 |
using Teuchos::Comm; |
| 412 |
using Teuchos::RCP; |
| 413 |
using Teuchos::rcp; |
| 414 |
using Teuchos::REDUCE_SUM; |
| 415 |
using Teuchos::reduceAll; |
| 416 |
|
| 417 |
// Check if numRowsC == numColsC == 1, in which case we can call dot() |
| 418 |
const int numRowsC = C.numRows (); |
| 419 |
const int numColsC = C.numCols (); |
| 420 |
const int strideC = C.stride (); |
| 421 |
if (numRowsC == 1 && numColsC == 1) { |
| 422 |
if (alpha == Teuchos::ScalarTraits<Scalar>::zero ()) { |
| 423 |
// Short-circuit, as required by BLAS semantics. |
| 424 |
C(0,0) = alpha; |
| 425 |
return; |
| 426 |
} |
| 427 |
A.dot (B, Teuchos::ArrayView<dot_type> (C.values (), 1)); |
| 428 |
if (alpha != Teuchos::ScalarTraits<Scalar>::one ()) { |
| 429 |
C(0,0) *= alpha; |
| 430 |
} |
| 431 |
return; |
| 432 |
} |
| 433 |
|
| 434 |
// Ensure A and B have constant stride |
| 435 |
RCP<const MV> Atmp, Btmp; |
| 436 |
if (A.isConstantStride() == false) Atmp = rcp (new MV (A, Teuchos::Copy)); |
| 437 |
else Atmp = rcp(&A,false); |
| 438 |
|
| 439 |
if (B.isConstantStride() == false) Btmp = rcp (new MV (B, Teuchos::Copy)); |
| 440 |
else Btmp = rcp(&B,false); |
| 441 |
|
| 442 |
// Create flattened Kokkos::MultiVector's |
| 443 |
typedef Tpetra::MultiVector<const dot_type,LO,GO,Node> FMV; |
| 444 |
typedef typename FMV::dual_view_type::t_dev flat_view_type; |
| 445 |
typedef typename flat_view_type::execution_space execution_space; |
| 446 |
flat_view_type flat_A_view = Atmp->template getLocalView<execution_space>(Tpetra::Access::ReadOnly); |
| 447 |
flat_view_type flat_B_view = Btmp->template getLocalView<execution_space>(Tpetra::Access::ReadOnly); |
| 448 |
|
| 449 |
// Create a view for C on the host |
| 450 |
typedef Kokkos::View<dot_type**, Kokkos::LayoutLeft, Kokkos::HostSpace> c_host_view_type; |
| 451 |
c_host_view_type C_view_host_input( C.values(), strideC, numColsC); |
| 452 |
auto C_view_host = Kokkos::subview(C_view_host_input, |
| 453 |
Kokkos::pair<int,int>(0, numRowsC), |
| 454 |
Kokkos::pair<int,int>(0, numColsC)); |
| 455 |
|
| 456 |
// Create view for C on the device -- need to be careful to get the |
| 457 |
// right stride to match C (allow setting to 0 for first-touch) |
| 458 |
typedef Kokkos::View<dot_type**, Kokkos::LayoutLeft, execution_space> c_view_type; |
| 459 |
typedef Kokkos::View<dot_type*, Kokkos::LayoutLeft, execution_space> c_1d_view_type; |
| 460 |
c_1d_view_type C_1d_view_dev("C", numRowsC*numColsC); |
| 461 |
c_view_type C_view_dev( C_1d_view_dev.data(), numRowsC, numColsC); |
| 462 |
|
| 463 |
// Do local multiply |
| 464 |
{ |
| 465 |
const char ctransA = 'C', ctransB = 'N'; |
| 466 |
KokkosBlas::gemm ( |
| 467 |
&ctransA, &ctransB, |
| 468 |
alpha, flat_A_view, flat_B_view, |
| 469 |
Kokkos::ArithTraits<dot_type>::zero(), |
| 470 |
C_view_dev); |
| 471 |
} |
| 472 |
// reduce across processors -- could check for RDMA |
| 473 |
RCP<const Comm<int> > pcomm = A.getMap()->getComm (); |
| 474 |
if (pcomm->getSize () == 1) { |
| 475 |
//Kokkos::deep_copy(C_view_host, C_view_dev); |
| 476 |
// Device-to-host copies requires dest to be contiguous, which |
| 477 |
// C_view_host may not be. So do 1 column at a time. |
| 478 |
for (int j=0; j<numColsC; ++j) |
| 479 |
Kokkos::deep_copy(Kokkos::subview(C_view_host,Kokkos::ALL,j), |
| 480 |
Kokkos::subview(C_view_dev,Kokkos::ALL,j)); |
| 481 |
} |
| 482 |
else { |
| 483 |
typedef Kokkos::View<dot_type*, Kokkos::LayoutLeft, Kokkos::HostSpace> c_1d_host_view_type; |
| 484 |
c_1d_host_view_type C_1d_view_tmp(Kokkos::ViewAllocateWithoutInitializing("C_tmp"), strideC*numColsC); |
| 485 |
c_host_view_type C_view_tmp( C_1d_view_tmp.data(), |
| 486 |
strideC, numColsC); |
| 487 |
for (int j=0; j<numColsC; ++j) |
| 488 |
Kokkos::deep_copy(Kokkos::subview(C_view_tmp, |
| 489 |
Kokkos::pair<int,int>(0, numRowsC), |
| 490 |
j), |
| 491 |
Kokkos::subview(C_view_dev, Kokkos::ALL, j)); |
| 492 |
reduceAll<int> (*pcomm, REDUCE_SUM, strideC*numColsC, |
| 493 |
C_view_tmp.data(), |
| 494 |
C_view_host.data()); |
| 495 |
} |
| 496 |
} |
| 497 |
|
| 498 |
//! For all columns j of A, set <tt>dots[j] := A[j]^T * B[j]</tt>. |
| 499 |
static void |
| 500 |
MvDot (const Tpetra::MultiVector<Scalar,LO,GO,Node>& A, |
| 501 |
const Tpetra::MultiVector<Scalar,LO,GO,Node>& B, |
| 502 |
std::vector<dot_type>& dots) |
| 503 |
{ |
| 504 |
const size_t numVecs = A.getNumVectors (); |
| 505 |
|
| 506 |
TEUCHOS_TEST_FOR_EXCEPTION( |
| 507 |
numVecs != B.getNumVectors (), std::invalid_argument, |
| 508 |
"Belos::MultiVecTraits<Scalar,Tpetra::MultiVector>::MvDot(A,B,dots): " |
| 509 |
"A and B must have the same number of columns. " |
| 510 |
"A has " << numVecs << " column(s), " |
| 511 |
"but B has " << B.getNumVectors () << " column(s)."); |
| 512 |
#ifdef HAVE_TPETRA_DEBUG |
| 513 |
TEUCHOS_TEST_FOR_EXCEPTION( |
| 514 |
dots.size() < numVecs, std::invalid_argument, |
| 515 |
"Belos::MultiVecTraits<Scalar,Tpetra::MultiVector>::MvDot(A,B,dots): " |
| 516 |
"The output array 'dots' must have room for all dot products. " |
| 517 |
"A and B each have " << numVecs << " column(s), " |
| 518 |
"but 'dots' only has " << dots.size() << " entry(/ies)."); |
| 519 |
#endif // HAVE_TPETRA_DEBUG |
| 520 |
|
| 521 |
Teuchos::ArrayView<dot_type> av (dots); |
| 522 |
A.dot (B, av (0, numVecs)); |
| 523 |
} |
| 524 |
|
| 525 |
//! For all columns j of mv, set <tt>normvec[j] = norm(mv[j])</tt>. |
| 526 |
static void |
| 527 |
MvNorm (const Tpetra::MultiVector<Scalar,LO,GO,Node>& mv, |
| 528 |
std::vector<mag_type> &normvec, |
| 529 |
NormType type=TwoNorm) |
| 530 |
{ |
| 531 |
|
| 532 |
#ifdef HAVE_TPETRA_DEBUG |
| 533 |
typedef std::vector<int>::size_type size_type; |
| 534 |
TEUCHOS_TEST_FOR_EXCEPTION( |
| 535 |
normvec.size () < static_cast<size_type> (mv.getNumVectors ()), |
| 536 |
std::invalid_argument, |
| 537 |
"Belos::MultiVecTraits::MvNorm(mv,normvec): The normvec output " |
| 538 |
"argument must have at least as many entries as the number of vectors " |
| 539 |
"(columns) in the MultiVector mv. normvec.size() = " << normvec.size () |
| 540 |
<< " < mv.getNumVectors() = " << mv.getNumVectors () << "."); |
| 541 |
#endif |
| 542 |
|
| 543 |
Teuchos::ArrayView<mag_type> av(normvec); |
| 544 |
switch (type) { |
| 545 |
case OneNorm: |
| 546 |
mv.norm1(av(0,mv.getNumVectors())); |
| 547 |
break; |
| 548 |
case TwoNorm: |
| 549 |
mv.norm2(av(0,mv.getNumVectors())); |
| 550 |
break; |
| 551 |
case InfNorm: |
| 552 |
mv.normInf(av(0,mv.getNumVectors())); |
| 553 |
break; |
| 554 |
default: |
| 555 |
// Throw logic_error rather than invalid_argument, because if |
| 556 |
// we get here, it's probably the fault of a Belos solver, |
| 557 |
// rather than a user giving Belos an invalid input. |
| 558 |
TEUCHOS_TEST_FOR_EXCEPTION( |
| 559 |
true, std::logic_error, |
| 560 |
"Belos::MultiVecTraits::MvNorm: Invalid NormType value " << type |
| 561 |
<< ". Valid values are OneNorm=" << OneNorm << ", TwoNorm=" |
| 562 |
<< TwoNorm <<", and InfNorm=" << InfNorm << ". If you are a Belos " |
| 563 |
"user and have not modified Belos in any way, and you get this " |
| 564 |
"message, then this is probably a bug in the Belos solver you were " |
| 565 |
"using. Please report this to the Belos developers."); |
| 566 |
} |
| 567 |
} |
| 568 |
|
| 569 |
static void |
| 570 |
SetBlock (const MV& A, const std::vector<int>& index, MV& mv) |
| 571 |
{ |
| 572 |
using Teuchos::Range1D; |
| 573 |
using Teuchos::RCP; |
| 574 |
const size_t inNumVecs = A.getNumVectors (); |
| 575 |
#ifdef HAVE_TPETRA_DEBUG |
| 576 |
TEUCHOS_TEST_FOR_EXCEPTION( |
| 577 |
inNumVecs < static_cast<size_t> (index.size ()), std::invalid_argument, |
| 578 |
"Belos::MultiVecTraits::SetBlock(A,index,mv): 'index' argument must " |
| 579 |
"have no more entries as the number of columns in the input MultiVector" |
| 580 |
" A. A.getNumVectors() = " << inNumVecs << " < index.size () = " |
| 581 |
<< index.size () << "."); |
| 582 |
#endif // HAVE_TPETRA_DEBUG |
| 583 |
RCP<MV> mvsub = CloneViewNonConst (mv, index); |
| 584 |
if (inNumVecs > static_cast<size_t> (index.size ())) { |
| 585 |
RCP<const MV> Asub = A.subView (Range1D (0, index.size () - 1)); |
| 586 |
::Tpetra::deep_copy (*mvsub, *Asub); |
| 587 |
} else { |
| 588 |
::Tpetra::deep_copy (*mvsub, A); |
| 589 |
} |
| 590 |
} |
| 591 |
|
| 592 |
static void |
| 593 |
SetBlock (const MV& A, const Teuchos::Range1D& index, MV& mv) |
| 594 |
{ |
| 595 |
// Range1D bounds are signed; size_t is unsigned. |
| 596 |
// Assignment of Tpetra::MultiVector is a deep copy. |
| 597 |
|
| 598 |
// Tpetra::MultiVector::getNumVectors() returns size_t. It's |
| 599 |
// fair to assume that the number of vectors won't overflow int, |
| 600 |
// since the typical use case of multivectors involves few |
| 601 |
// columns, but it's friendly to check just in case. |
| 602 |
const size_t maxInt = |
| 603 |
static_cast<size_t> (Teuchos::OrdinalTraits<int>::max ()); |
| 604 |
const bool overflow = |
| 605 |
maxInt < A.getNumVectors () && maxInt < mv.getNumVectors (); |
| 606 |
if (overflow) { |
| 607 |
std::ostringstream os; |
| 608 |
os << "Belos::MultiVecTraits::SetBlock(A, index=[" << index.lbound () |
| 609 |
<< ", " << index.ubound () << "], mv): "; |
| 610 |
TEUCHOS_TEST_FOR_EXCEPTION( |
| 611 |
maxInt < A.getNumVectors (), std::range_error, os.str () << "Number " |
| 612 |
"of columns (size_t) in the input MultiVector 'A' overflows int."); |
| 613 |
TEUCHOS_TEST_FOR_EXCEPTION( |
| 614 |
maxInt < mv.getNumVectors (), std::range_error, os.str () << "Number " |
| 615 |
"of columns (size_t) in the output MultiVector 'mv' overflows int."); |
| 616 |
} |
| 617 |
// We've already validated the static casts above. |
| 618 |
const int numColsA = static_cast<int> (A.getNumVectors ()); |
| 619 |
const int numColsMv = static_cast<int> (mv.getNumVectors ()); |
| 620 |
// 'index' indexes into mv; it's the index set of the target. |
| 621 |
const bool validIndex = |
| 622 |
index.lbound () >= 0 && index.ubound () < numColsMv; |
| 623 |
// We can't take more columns out of A than A has. |
| 624 |
const bool validSource = index.size () <= numColsA; |
| 625 |
|
| 626 |
if (! validIndex || ! validSource) { |
| 627 |
std::ostringstream os; |
| 628 |
os << "Belos::MultiVecTraits::SetBlock(A, index=[" << index.lbound () |
| 629 |
<< ", " << index.ubound () << "], mv): "; |
| 630 |
TEUCHOS_TEST_FOR_EXCEPTION( |
| 631 |
index.lbound() < 0, std::invalid_argument, |
| 632 |
os.str() << "Range lower bound must be nonnegative."); |
| 633 |
TEUCHOS_TEST_FOR_EXCEPTION( |
| 634 |
index.ubound() >= numColsMv, std::invalid_argument, |
| 635 |
os.str() << "Range upper bound must be less than the number of " |
| 636 |
"columns " << numColsA << " in the 'mv' output argument."); |
| 637 |
TEUCHOS_TEST_FOR_EXCEPTION( |
| 638 |
index.size() > numColsA, std::invalid_argument, |
| 639 |
os.str() << "Range must have no more elements than the number of " |
| 640 |
"columns " << numColsA << " in the 'A' input argument."); |
| 641 |
TEUCHOS_TEST_FOR_EXCEPTION( |
| 642 |
true, std::logic_error, "Should never get here!"); |
| 643 |
} |
| 644 |
|
| 645 |
// View of the relevant column(s) of the target multivector mv. |
| 646 |
// We avoid view creation overhead by only creating a view if |
| 647 |
// the index range is different than [0, (# columns in mv) - 1]. |
| 648 |
Teuchos::RCP<MV> mv_view; |
| 649 |
if (index.lbound () == 0 && index.ubound () + 1 == numColsMv) { |
| 650 |
mv_view = Teuchos::rcpFromRef (mv); // Non-const, non-owning RCP |
| 651 |
} else { |
| 652 |
mv_view = CloneViewNonConst (mv, index); |
| 653 |
} |
| 654 |
|
| 655 |
// View of the relevant column(s) of the source multivector A. |
| 656 |
// If A has fewer columns than mv_view, then create a view of |
| 657 |
// the first index.size() columns of A. |
| 658 |
Teuchos::RCP<const MV> A_view; |
| 659 |
if (index.size () == numColsA) { |
| 660 |
A_view = Teuchos::rcpFromRef (A); // Const, non-owning RCP |
| 661 |
} else { |
| 662 |
A_view = CloneView (A, Teuchos::Range1D (0, index.size () - 1)); |
| 663 |
} |
| 664 |
|
| 665 |
::Tpetra::deep_copy (*mv_view, *A_view); |
| 666 |
} |
| 667 |
|
| 668 |
static void Assign (const MV& A, MV& mv) |
| 669 |
{ |
| 670 |
const char errPrefix[] = "Belos::MultiVecTraits::Assign(A, mv): "; |
| 671 |
|
| 672 |
// Range1D bounds are signed; size_t is unsigned. |
| 673 |
// Assignment of Tpetra::MultiVector is a deep copy. |
| 674 |
|
| 675 |
// Tpetra::MultiVector::getNumVectors() returns size_t. It's |
| 676 |
// fair to assume that the number of vectors won't overflow int, |
| 677 |
// since the typical use case of multivectors involves few |
| 678 |
// columns, but it's friendly to check just in case. |
| 679 |
const size_t maxInt = |
| 680 |
static_cast<size_t> (Teuchos::OrdinalTraits<int>::max ()); |
| 681 |
const bool overflow = |
| 682 |
maxInt < A.getNumVectors () && maxInt < mv.getNumVectors (); |
| 683 |
if (overflow) { |
| 684 |
TEUCHOS_TEST_FOR_EXCEPTION( |
| 685 |
maxInt < A.getNumVectors(), std::range_error, |
| 686 |
errPrefix << "Number of columns in the input multivector 'A' " |
| 687 |
"(a size_t) overflows int."); |
| 688 |
TEUCHOS_TEST_FOR_EXCEPTION( |
| 689 |
maxInt < mv.getNumVectors(), std::range_error, |
| 690 |
errPrefix << "Number of columns in the output multivector 'mv' " |
| 691 |
"(a size_t) overflows int."); |
| 692 |
TEUCHOS_TEST_FOR_EXCEPTION( |
| 693 |
true, std::logic_error, "Should never get here!"); |
| 694 |
} |
| 695 |
// We've already validated the static casts above. |
| 696 |
const int numColsA = static_cast<int> (A.getNumVectors ()); |
| 697 |
const int numColsMv = static_cast<int> (mv.getNumVectors ()); |
| 698 |
if (numColsA > numColsMv) { |
| 699 |
TEUCHOS_TEST_FOR_EXCEPTION( |
| 700 |
numColsA > numColsMv, std::invalid_argument, |
| 701 |
errPrefix << "Input multivector 'A' has " << numColsA << " columns, " |
| 702 |
"but output multivector 'mv' has only " << numColsMv << " columns."); |
| 703 |
TEUCHOS_TEST_FOR_EXCEPTION(true, std::logic_error, "Should never get here!"); |
| 704 |
} |
| 705 |
if (numColsA == numColsMv) { |
| 706 |
::Tpetra::deep_copy (mv, A); |
| 707 |
} else { |
| 708 |
Teuchos::RCP<MV> mv_view = |
| 709 |
CloneViewNonConst (mv, Teuchos::Range1D (0, numColsA - 1)); |
| 710 |
::Tpetra::deep_copy (*mv_view, A); |
| 711 |
} |
| 712 |
} |
| 713 |
|
| 714 |
|
| 715 |
static void MvRandom (MV& mv) { |
| 716 |
mv.randomize (); |
| 717 |
} |
| 718 |
|
| 719 |
static void |
| 720 |
MvInit (MV& mv, const Scalar alpha = Teuchos::ScalarTraits<Scalar>::zero ()) |
| 721 |
{ |
| 722 |
mv.putScalar (alpha); |
| 723 |
} |
| 724 |
|
| 725 |
static void MvPrint (const MV& mv, std::ostream& os) { |
| 726 |
Teuchos::FancyOStream fos (Teuchos::rcpFromRef (os)); |
| 727 |
mv.describe (fos, Teuchos::VERB_EXTREME); |
| 728 |
} |
| 729 |
|
| 730 |
#ifdef HAVE_BELOS_TSQR |
| 731 |
/// \typedef tsqr_adaptor_type |
| 732 |
/// \brief TsqrAdaptor specialization for Tpetra::MultiVector |
| 733 |
/// |
| 734 |
typedef Tpetra::TsqrAdaptor< Tpetra::MultiVector< Scalar, LO, GO, Node > > tsqr_adaptor_type; |
| 735 |
#endif // HAVE_BELOS_TSQR |
| 736 |
}; |
| 737 |
|
| 738 |
//////////////////////////////////////////////////////////////////// |
| 739 |
// |
| 740 |
// Implementation of the Belos::OperatorTraits for Tpetra::Operator. |
| 741 |
// |
| 742 |
//////////////////////////////////////////////////////////////////// |
| 743 |
|
| 744 |
/// \brief Partial specialization of OperatorTraits for Tpetra::Operator. |
| 745 |
template <class Storage, class LO, class GO, class Node> |
| 746 |
class OperatorTraits <typename Storage::value_type, |
| 747 |
Tpetra::MultiVector<Sacado::UQ::PCE<Storage>, |
| 748 |
LO,GO,Node>, |
| 749 |
Tpetra::Operator<Sacado::UQ::PCE<Storage>, |
| 750 |
LO,GO,Node> > |
| 751 |
{ |
| 752 |
public: |
| 753 |
typedef Sacado::UQ::PCE<Storage> Scalar; |
| 754 |
static void |
| 755 |
Apply (const Tpetra::Operator<Scalar,LO,GO,Node>& Op, |
| 756 |
const Tpetra::MultiVector<Scalar,LO,GO,Node>& X, |
| 757 |
Tpetra::MultiVector<Scalar,LO,GO,Node>& Y, |
| 758 |
ETrans trans=NOTRANS) |
| 759 |
{ |
| 760 |
switch (trans) { |
| 761 |
case NOTRANS: |
| 762 |
Op.apply(X,Y,Teuchos::NO_TRANS); |
| 763 |
break; |
| 764 |
case TRANS: |
| 765 |
Op.apply(X,Y,Teuchos::TRANS); |
| 766 |
break; |
| 767 |
case CONJTRANS: |
| 768 |
Op.apply(X,Y,Teuchos::CONJ_TRANS); |
| 769 |
break; |
| 770 |
default: |
| 771 |
const std::string scalarName = Teuchos::TypeNameTraits<Scalar>::name(); |
| 772 |
const std::string loName = Teuchos::TypeNameTraits<LO>::name(); |
| 773 |
const std::string goName = Teuchos::TypeNameTraits<GO>::name(); |
| 774 |
const std::string nodeName = Teuchos::TypeNameTraits<Node>::name(); |
| 775 |
const std::string otName = "Belos::OperatorTraits<" + scalarName |
| 776 |
+ "," + loName + "," + goName + "," + nodeName + ">"; |
| 777 |
TEUCHOS_TEST_FOR_EXCEPTION(true, std::logic_error, otName << ": Should never " |
| 778 |
"get here; fell through a switch statement. " |
| 779 |
"Please report this bug to the Belos developers."); |
| 780 |
} |
| 781 |
} |
| 782 |
|
| 783 |
static bool |
| 784 |
HasApplyTranspose (const Tpetra::Operator<Scalar,LO,GO,Node>& Op) |
| 785 |
{ |
| 786 |
return Op.hasTransposeApply (); |
| 787 |
} |
| 788 |
}; |
| 789 |
|
| 790 |
} // end of Belos namespace |
| 791 |
|
| 792 |
#endif |
| 793 |
// end of file BELOS_TPETRA_ADAPTER_UQ_PCE_HPP |