| 1 |
// @HEADER |
| 2 |
// ***************************************************************************** |
| 3 |
// Anasazi: Block Eigensolvers Package |
| 4 |
// |
| 5 |
// Copyright 2004 NTESS and the Anasazi contributors. |
| 6 |
// SPDX-License-Identifier: BSD-3-Clause |
| 7 |
// ***************************************************************************** |
| 8 |
// @HEADER |
| 9 |
|
| 10 |
#ifndef __TSQR_Tsqr_TwoLevelDistTsqr_hpp |
| 11 |
#define __TSQR_Tsqr_TwoLevelDistTsqr_hpp |
| 12 |
|
| 13 |
#include <Tsqr_DistTsqr.hpp> |
| 14 |
#include <Tsqr_MpiCommFactory.hpp> |
| 15 |
#include <sstream> |
| 16 |
|
| 17 |
//////////////////////////////////////////////////////////////////////////////// |
| 18 |
//////////////////////////////////////////////////////////////////////////////// |
| 19 |
|
| 20 |
namespace TSQR { |
| 21 |
|
| 22 |
/// \class TwoLevelDistTsqr |
| 23 |
/// \brief Interprocess part of TSQR |
| 24 |
/// |
| 25 |
/// Interprocess part of TSQR, composed of an internode part and an |
| 26 |
/// intranode part (but only working between MPI processes, not |
| 27 |
/// within a process). |
| 28 |
template< class LocalOrdinal, |
| 29 |
class Scalar, |
| 30 |
class DistTsqrType = DistTsqr< LocalOrdinal, Scalar > > |
| 31 |
class TwoLevelDistTsqr { |
| 32 |
public: |
| 33 |
typedef Scalar scalar_type; |
| 34 |
typedef LocalOrdinal ordinal_type; |
| 35 |
typedef DistTsqrType dist_tsqr_type; |
| 36 |
typedef typename dist_tsqr_type::rank_type rank_type; |
| 37 |
typedef typename Teuchos::RCP< dist_tsqr_type > dist_tsqr_ptr; |
| 38 |
typedef typename dist_tsqr_type::FactorOutput DistTsqrFactorOutput; |
| 39 |
typedef std::pair< DistTsqrFactorOutput, DistTsqrFactorOutput > FactorOutput; |
| 40 |
|
| 41 |
/// \brief Constructor |
| 42 |
/// |
| 43 |
TwoLevelDistTsqr () : |
| 44 |
worldMess_ (TSQR::MPI::makeMpiCommWorld()), |
| 45 |
nodeDistTsqr_ (TSQR::MPI::makeMpiCommNode()), |
| 46 |
networkDistTsqr_ (TSQR::MPI::makeMpiCommNetwork()) |
| 47 |
{} |
| 48 |
|
| 49 |
/// |
| 50 |
/// \brief Destructor |
| 51 |
/// |
| 52 |
~TwoLevelDistTsqr () {} |
| 53 |
|
| 54 |
/// Whether or not all diagonal entries of the R factor computed |
| 55 |
/// by the QR factorization are guaranteed to be nonnegative. |
| 56 |
bool QR_produces_R_factor_with_nonnegative_diagonal () const { |
| 57 |
return nodeDistTsqr_->QR_produces_R_factor_with_nonnegative_diagonal() && |
| 58 |
networkDistTsqr_->QR_produces_R_factor_with_nonnegative_diagonal(); |
| 59 |
} |
| 60 |
|
| 61 |
/// \brief Compute QR factorization of R factors, one per MPI process |
| 62 |
/// |
| 63 |
/// Compute the QR factorization of the P*ncols by ncols matrix |
| 64 |
/// consisting of all P nodes' R_mine upper triangular matrices |
| 65 |
/// stacked on top of each other. Generally these upper triangular |
| 66 |
/// matrices should come from the QR factorization (perhaps computed |
| 67 |
/// by sequential or node-parallel TSQR) of a general matrix on each |
| 68 |
/// node. |
| 69 |
/// |
| 70 |
/// \note "ncols" below is the number of columns in the matrix to |
| 71 |
/// factor. Should be the same on all nodes. |
| 72 |
/// |
| 73 |
/// \param R_mine [in,out] On input, an ncols by ncols upper triangular |
| 74 |
/// matrix with leading dimension ncols, stored unpacked (as a general |
| 75 |
/// matrix). Elements below the diagonal are ignored. On output, the |
| 76 |
/// final R factor of the QR factorization of all nodes' different |
| 77 |
/// R_mine inputs. The final R factor is replicated over all nodes. |
| 78 |
FactorOutput |
| 79 |
factor (MatView< LocalOrdinal, Scalar > R_mine) |
| 80 |
{ |
| 81 |
DistTsqrFactorOutput nodeOutput = nodeDistTsqr_->factor (R_mine); |
| 82 |
DistTsqrFactorOutput networkOutput = networkDistTsqr_->factor (R_mine); |
| 83 |
return std::make_pair (nodeOutput, networkOutput); |
| 84 |
} |
| 85 |
|
| 86 |
void |
| 87 |
apply (const ApplyType& applyType, |
| 88 |
const LocalOrdinal ncols_C, |
| 89 |
const LocalOrdinal ncols_Q, |
| 90 |
Scalar C_mine[], |
| 91 |
const LocalOrdinal ldc_mine, |
| 92 |
const FactorOutput& factorOutput) |
| 93 |
{ |
| 94 |
if (applyType.transposed()) |
| 95 |
{ |
| 96 |
nodeDistTsqr_->apply (applyType, ncols_C, ncols_Q, |
| 97 |
C_mine, ldc_mine, factorOutput.first); |
| 98 |
networkDistTsqr_->apply (applyType, ncols_C, ncols_Q, |
| 99 |
C_mine, ldc_mine, factorOutput.second); |
| 100 |
} |
| 101 |
else |
| 102 |
{ |
| 103 |
networkDistTsqr_->apply (applyType, ncols_C, ncols_Q, |
| 104 |
C_mine, ldc_mine, factorOutput.second); |
| 105 |
nodeDistTsqr_->apply (applyType, ncols_C, ncols_Q, |
| 106 |
C_mine, ldc_mine, factorOutput.first); |
| 107 |
} |
| 108 |
} |
| 109 |
|
| 110 |
void |
| 111 |
explicit_Q (const LocalOrdinal ncols_Q, |
| 112 |
Scalar Q_mine[], |
| 113 |
const LocalOrdinal ldq_mine, |
| 114 |
const FactorOutput& factorOutput) |
| 115 |
{ |
| 116 |
typedef MatView< LocalOrdinal, Scalar > matview_type; |
| 117 |
matview_type Q_view (ncols_Q, ncols_Q, Q_mine, ldq_mine, Scalar(0)); |
| 118 |
Q_view.fill (Scalar(0)); |
| 119 |
|
| 120 |
const rank_type myRank = worldMess_->rank(); |
| 121 |
if (myRank == 0) |
| 122 |
{ |
| 123 |
if (networkMess_->rank() != 0) |
| 124 |
{ |
| 125 |
std::ostringstream os; |
| 126 |
os << "My rank with respect to MPI_COMM_WORLD is 0, but my rank " |
| 127 |
"with respect to MPI_COMM_NETWORK is nonzero (= " |
| 128 |
<< networkMess_->rank() << "). We could deal with this by " |
| 129 |
"swapping data between those two ranks, but we haven\'t " |
| 130 |
"implemented that fix yet."; |
| 131 |
throw std::logic_error (os.str()); |
| 132 |
} |
| 133 |
for (LocalOrdinal j = 0; j < ncols_Q; ++j) |
| 134 |
Q_view(j, j) = Scalar (1); |
| 135 |
} |
| 136 |
apply (ApplyType::NoTranspose, ncols_Q, ncols_Q, |
| 137 |
Q_mine, ldq_mine, factorOutput); |
| 138 |
} |
| 139 |
|
| 140 |
private: |
| 141 |
Teuchos::RCP< MessengerBase< Scalar > > worldMess_; |
| 142 |
dist_tsqr_ptr nodeDistTsqr_, networkDistTsqr_; |
| 143 |
}; |
| 144 |
|
| 145 |
} // namespace TSQR |
| 146 |
|
| 147 |
#endif // __TSQR_Tsqr_TwoLevelDistTsqr_hpp |