// @HEADER // ***************************************************************************** // Xpetra: A linear algebra interface package // // Copyright 2012 NTESS and the Xpetra contributors. // SPDX-License-Identifier: BSD-3-Clause // ***************************************************************************** // @HEADER #ifndef PACKAGES_XPETRA_SUP_UTILS_XPETRA_IO_HPP_ #define PACKAGES_XPETRA_SUP_UTILS_XPETRA_IO_HPP_ #include #include "Xpetra_ConfigDefs.hpp" #ifdef HAVE_XPETRA_EPETRA #ifdef HAVE_MPI #include "Epetra_MpiComm.h" #endif #endif #if defined(HAVE_XPETRA_EPETRA) && defined(HAVE_XPETRA_EPETRAEXT) #include #include #include #include #include #include #include #include #include #endif #ifdef HAVE_XPETRA_TPETRA #include #include #include #include #include #include #include #include "Tpetra_Util.hpp" #endif #ifdef HAVE_XPETRA_EPETRA #include #endif #include "Xpetra_Matrix.hpp" #include "Xpetra_MatrixMatrix.hpp" #include "Xpetra_Helpers.hpp" #include "Xpetra_CrsGraph.hpp" #include "Xpetra_CrsMatrixWrap.hpp" #include "Xpetra_BlockedCrsMatrix.hpp" #include "Xpetra_Map.hpp" #include "Xpetra_StridedMap.hpp" #include "Xpetra_StridedMapFactory.hpp" #include "Xpetra_MapExtractor.hpp" #include "Xpetra_MatrixFactory.hpp" #include #include #include #include namespace Xpetra { #ifdef HAVE_XPETRA_EPETRA // This non-member templated function exists so that the matrix-matrix multiply will compile if Epetra, Tpetra, and ML are enabled. template RCP> Convert_Epetra_CrsMatrix_ToXpetra_CrsMatrixWrap(RCP& /* epAB */) { TEUCHOS_TEST_FOR_EXCEPTION(true, Exceptions::RuntimeError, "Convert_Epetra_CrsMatrix_ToXpetra_CrsMatrixWrap cannot be used with Scalar != double, LocalOrdinal != int, GlobalOrdinal != int"); TEUCHOS_UNREACHABLE_RETURN(Teuchos::null); } // specialization for the case of ScalarType=double and LocalOrdinal=GlobalOrdinal=int template <> inline RCP> Convert_Epetra_CrsMatrix_ToXpetra_CrsMatrixWrap(RCP& epAB) { typedef double SC; typedef int LO; typedef int GO; typedef Xpetra::EpetraNode NO; RCP> tmpC1 = rcp(new Xpetra::EpetraCrsMatrixT(epAB)); RCP> tmpC2 = Teuchos::rcp_implicit_cast>(tmpC1); RCP> tmpC3 = rcp(new Xpetra::CrsMatrixWrap(tmpC2)); return tmpC3; } template RCP> Convert_Epetra_MultiVector_ToXpetra_MultiVector(RCP& epX) { TEUCHOS_TEST_FOR_EXCEPTION(true, Exceptions::RuntimeError, "Convert_Epetra_MultiVector_ToXpetra_MultiVector cannot be used with Scalar != double, LocalOrdinal != int, GlobalOrdinal != int"); TEUCHOS_UNREACHABLE_RETURN(Teuchos::null); } // specialization for the case of ScalarType=double and LocalOrdinal=GlobalOrdinal=int template <> inline RCP> Convert_Epetra_MultiVector_ToXpetra_MultiVector(RCP& epX) { typedef double SC; typedef int LO; typedef int GO; typedef Xpetra::EpetraNode NO; RCP> tmp = Xpetra::toXpetra(epX); return tmp; } #endif /*! @class IO @brief Xpetra utility class containing IO routines to read/write vectors, matrices etc... */ template class IO { private: #undef XPETRA_IO_SHORT #include "Xpetra_UseShortNames.hpp" public: #ifdef HAVE_XPETRA_EPETRA //! Helper utility to pull out the underlying Epetra objects from an Xpetra object // @{ /*static RCP MV2EpetraMV(RCP const vec); static RCP< Epetra_MultiVector> MV2NonConstEpetraMV(RCP vec); static const Epetra_MultiVector& MV2EpetraMV(const MultiVector& vec); static Epetra_MultiVector& MV2NonConstEpetraMV(MultiVector& vec); static RCP Op2EpetraCrs(RCP Op); static RCP< Epetra_CrsMatrix> Op2NonConstEpetraCrs(RCP Op); static const Epetra_CrsMatrix& Op2EpetraCrs(const Matrix& Op); static Epetra_CrsMatrix& Op2NonConstEpetraCrs(Matrix& Op);*/ static const Epetra_Map& Map2EpetraMap(const Xpetra::Map& map); // @} #endif #ifdef HAVE_XPETRA_TPETRA //! Helper utility to pull out the underlying Tpetra objects from an Xpetra object // @{ /*static RCP > MV2TpetraMV(RCP const vec); static RCP< Tpetra::MultiVector > MV2NonConstTpetraMV(RCP vec); static RCP< Tpetra::MultiVector > MV2NonConstTpetraMV2(MultiVector& vec); static const Tpetra::MultiVector& MV2TpetraMV(const MultiVector& vec); static Tpetra::MultiVector& MV2NonConstTpetraMV(MultiVector& vec); static RCP > Op2TpetraCrs(RCP Op); static RCP< Tpetra::CrsMatrix > Op2NonConstTpetraCrs(RCP Op); static const Tpetra::CrsMatrix& Op2TpetraCrs(const Matrix& Op); static Tpetra::CrsMatrix& Op2NonConstTpetraCrs(Matrix& Op); static RCP > Op2TpetraRow(RCP Op); static RCP< Tpetra::RowMatrix > Op2NonConstTpetraRow(RCP Op);*/ static const RCP> Map2TpetraMap(const Xpetra::Map& map); #endif //! Read/Write methods //@{ /*! @brief Save map to file. */ static void Write(const std::string& fileName, const Xpetra::Map& M); /*! @brief Save multivector to file in Matrix Market format. */ static void Write(const std::string& fileName, const Xpetra::MultiVector& vec); // CAG: // The class is templated on the usual SC-LO-GO-NO. // Instead of instantiating the entire class using Scalar=LO or GO and then dealing with the headaches that integer-valued CrsMatrix creates, we use these two methods. /*! @brief Save multivector with Scalar=LocalOrdinal to file in Matrix Market format. */ static void WriteLOMV(const std::string& fileName, const Xpetra::MultiVector& vec); /*! @brief Save multivector with Scalar=GlobalOrdinal to file in Matrix Market format. */ static void WriteGOMV(const std::string& fileName, const Xpetra::MultiVector& vec); /*! @brief Save matrix to file in Matrix Market format. */ static void Write(const std::string& fileName, const Xpetra::Matrix& Op, const bool& writeAllMaps = false); /*! @brief Save local parts of matrix to files in Matrix Market format. */ static void WriteLocal(const std::string& fileName, const Xpetra::Matrix& Op); /*! @brief Save block matrix to one file per block in Matrix Market format. We write one MatrixMarket file for each block of the given BlockedCrsMatrix. The block (row,col) indicators are included into the file name, such that filename02.m indicates the block in row = 0 and col = 2, for example. While the user can enable/disable the output of all maps of each matrix block, we always write the range and domain maps of each block as well of the full blocked operator in order to enable reconstruction of the MapExtractor objects for range and domain maps. @param fileName String to indicate file name of files to be written @param Op BlockedCrsMatrix to be written @param writeAllMaps Flag to control output of maps to separate files (defaults to \c false ) */ static void WriteBlockedCrsMatrix(const std::string& fileName, const Xpetra::BlockedCrsMatrix& Op, const bool& writeAllMaps = false); //! @brief Read matrix from file in Matrix Market or binary format. static Teuchos::RCP> Read(const std::string& fileName, Xpetra::UnderlyingLib lib, const RCP>& comm, bool binary = false); /*! @brief Read matrix from file in Matrix Market or binary format. If only rowMap is specified, then it is used for the domainMap and rangeMap, as well. */ static Teuchos::RCP> Read(const std::string& filename, const RCP> rowMap, RCP> colMap = Teuchos::null, const RCP> domainMap = Teuchos::null, const RCP> rangeMap = Teuchos::null, const bool callFillComplete = true, const bool binary = false, const bool tolerant = false, const bool debug = false); /*! @brief Read matrix from local files in Matrix Market or binary format. The file name format is filename.SIZE.RANK, where SIZE is the size of the communicator of the rowMap and RANK is the MPI ranks of the calling process. If only rowMap is specified, then it is used for the domainMap and rangeMap, as well. */ static Teuchos::RCP> ReadLocal(const std::string& filename, const RCP> rowMap, RCP> colMap, const RCP> domainMap = Teuchos::null, const RCP> rangeMap = Teuchos::null, const bool callFillComplete = true, const bool binary = false, const bool tolerant = false, const bool debug = false); //@} //! @brief Read a MultiVector from file in Matrix Matrix or binary format. static RCP ReadMultiVector(const std::string& fileName, const RCP& map, const bool binary = false); //! @brief Read a MultiVector with Scalar=LocalOrdinal from file in Matrix Matrix or binary format. static RCP> ReadMultiVectorLO(const std::string& fileName, const RCP& map, const bool binary = false); static RCP ReadMap(const std::string& fileName, Xpetra::UnderlyingLib lib, const RCP>& comm, const bool binary = false); /*! @brief Read block matrix from one file per block in Matrix Market format. We read one MatrixMarket file for each block of the desired BlockedCrsMatrix. The block (row,col) indicators are included into the file name, such that filename02.m indicates the block in row = 0 and col = 2, for example. We also read the range and domain maps of each block as well of the full blocked operator in order to enable reconstruction of the MapExtractor objects for range and domain maps. @param fileName String to indicate file name of files to be written @param lib Underlying type of linear algebra package @param comm Communicator */ static RCP> ReadBlockedCrsMatrix(const std::string& fileName, Xpetra::UnderlyingLib lib, const RCP>& comm); //! Little helper function to convert non-string types to strings template static std::string toString(const T& what); }; #ifdef HAVE_XPETRA_EPETRA /*! @class IO @brief Xpetra utility class containing IO routines to read/write vectors, matrices. Specialization for LO=GO=int TODO: do we need specialization for SC=double and std::complex<>??? TODO: need specialization for */ template class IO { public: typedef int LocalOrdinal; typedef int GlobalOrdinal; typedef EpetraNode Node; #ifdef HAVE_XPETRA_EPETRA //! Helper utility to pull out the underlying Epetra objects from an Xpetra object // @{ static const Epetra_Map& Map2EpetraMap(const Xpetra::Map& map) { RCP> xeMap = Teuchos::rcp_dynamic_cast>(Teuchos::rcpFromRef(map)); if (xeMap == Teuchos::null) throw Exceptions::BadCast("IO::Map2EpetraMap : Cast from Xpetra::Map to Xpetra::EpetraMap failed"); return xeMap->getEpetra_Map(); } // @} #endif #ifdef HAVE_XPETRA_TPETRA //! Helper utility to pull out the underlying Tpetra objects from an Xpetra object // @{ static const RCP> Map2TpetraMap(const Xpetra::Map& map) { const RCP>& tmp_TMap = Teuchos::rcp_dynamic_cast>(rcpFromRef(map)); if (tmp_TMap == Teuchos::null) throw Exceptions::BadCast("IO::Map2TpetraMap : Cast from Xpetra::Map to Xpetra::TpetraMap failed"); return tmp_TMap->getTpetra_Map(); } #endif //! Read/Write methods //@{ /*! @brief Save map to file. */ static void Write(const std::string& fileName, const Xpetra::Map& M) { RCP> tmp_Map = rcpFromRef(M); #if defined(HAVE_XPETRA_EPETRA) && defined(HAVE_XPETRA_EPETRAEXT) const RCP>& tmp_EMap = Teuchos::rcp_dynamic_cast>(tmp_Map); if (tmp_EMap != Teuchos::null) { int rv = EpetraExt::BlockMapToMatrixMarketFile(fileName.c_str(), tmp_EMap->getEpetra_Map()); if (rv != 0) throw Exceptions::RuntimeError("EpetraExt::BlockMapToMatrixMarketFile() return value of " + Teuchos::toString(rv)); return; } #endif // HAVE_XPETRA_EPETRA #ifdef HAVE_XPETRA_TPETRA #if ((defined(EPETRA_HAVE_OMP) && (!defined(HAVE_TPETRA_INST_OPENMP) || !defined(HAVE_TPETRA_INST_INT_INT))) || \ (!defined(EPETRA_HAVE_OMP) && (!defined(HAVE_TPETRA_INST_SERIAL) || !defined(HAVE_TPETRA_INST_INT_INT)))) // do nothing #else const RCP>& tmp_TMap = Teuchos::rcp_dynamic_cast>(tmp_Map); if (tmp_TMap != Teuchos::null) { RCP> TMap = tmp_TMap->getTpetra_Map(); Tpetra::MatrixMarket::Writer>::writeMapFile(fileName, *TMap); return; } #endif #endif // HAVE_XPETRA_TPETRA throw Exceptions::BadCast("Could not cast to EpetraMap or TpetraMap in map writing"); } static void Write(const std::string& fileName, const Xpetra::MultiVector& vec) { std::string mapfile = "map_" + fileName; Write(mapfile, *(vec.getMap())); RCP> tmp_Vec = Teuchos::rcpFromRef(vec); #if defined(HAVE_XPETRA_EPETRA) && defined(HAVE_XPETRA_EPETRAEXT) const RCP>& tmp_EVec = Teuchos::rcp_dynamic_cast>(tmp_Vec); if (tmp_EVec != Teuchos::null) { int rv = EpetraExt::MultiVectorToMatrixMarketFile(fileName.c_str(), *(tmp_EVec->getEpetra_MultiVector())); if (rv != 0) throw Exceptions::RuntimeError("EpetraExt::RowMatrixToMatrixMarketFile return value of " + Teuchos::toString(rv)); return; } #endif // HAVE_XPETRA_EPETRAEXT #ifdef HAVE_XPETRA_TPETRA #if ((defined(EPETRA_HAVE_OMP) && (!defined(HAVE_TPETRA_INST_OPENMP) || !defined(HAVE_TPETRA_INST_INT_INT))) || \ (!defined(EPETRA_HAVE_OMP) && (!defined(HAVE_TPETRA_INST_SERIAL) || !defined(HAVE_TPETRA_INST_INT_INT)))) // do nothin #else const RCP>& tmp_TVec = Teuchos::rcp_dynamic_cast>(tmp_Vec); if (tmp_TVec != Teuchos::null) { RCP> TVec = tmp_TVec->getTpetra_MultiVector(); Tpetra::MatrixMarket::Writer>::writeDenseFile(fileName, TVec); return; } #endif #endif // HAVE_XPETRA_TPETRA throw Exceptions::BadCast("Could not cast to EpetraMultiVector or TpetraMultiVector in multivector writing"); } static void WriteLOMV(const std::string& fileName, const Xpetra::MultiVector& vec) { std::string mapfile = "map_" + fileName; Write(mapfile, *(vec.getMap())); RCP> tmp_Vec = Teuchos::rcpFromRef(vec); #ifdef HAVE_XPETRA_TPETRA const RCP>& tmp_TVec = Teuchos::rcp_dynamic_cast>(tmp_Vec); if (tmp_TVec != Teuchos::null) { RCP> TVec = tmp_TVec->getTpetra_MultiVector(); Tpetra::MatrixMarket::Writer>::writeDenseFile(fileName, TVec); return; } else #endif // HAVE_XPETRA_TPETRA { throw Exceptions::RuntimeError("Xpetra cannot write MV when the underlying library is Epetra."); } throw Exceptions::BadCast("Could not cast to EpetraMultiVector or TpetraMultiVector in multivector writing"); } static void WriteGOMV(const std::string& fileName, const Xpetra::MultiVector& vec) { std::string mapfile = "map_" + fileName; Write(mapfile, *(vec.getMap())); RCP> tmp_Vec = Teuchos::rcpFromRef(vec); #ifdef HAVE_XPETRA_TPETRA const RCP>& tmp_TVec = Teuchos::rcp_dynamic_cast>(tmp_Vec); if (tmp_TVec != Teuchos::null) { RCP> TVec = tmp_TVec->getTpetra_MultiVector(); Tpetra::MatrixMarket::Writer>::writeDenseFile(fileName, TVec); return; } else #endif // HAVE_XPETRA_TPETRA { throw Exceptions::RuntimeError("Xpetra cannot write MV when the underlying library is Epetra."); } throw Exceptions::BadCast("Could not cast to EpetraMultiVector or TpetraMultiVector in multivector writing"); } static void Write(const std::string& fileName, const Xpetra::Matrix& Op, const bool& writeAllMaps = false) { Write("rowmap_" + fileName, *(Op.getRowMap())); if (!Op.getDomainMap()->isSameAs(*(Op.getRowMap())) || writeAllMaps) Write("domainmap_" + fileName, *(Op.getDomainMap())); if (!Op.getRangeMap()->isSameAs(*(Op.getRowMap())) || writeAllMaps) Write("rangemap_" + fileName, *(Op.getRangeMap())); if (!Op.getColMap()->isSameAs(*(Op.getDomainMap())) || writeAllMaps) Write("colmap_" + fileName, *(Op.getColMap())); const Xpetra::CrsMatrixWrap& crsOp = dynamic_cast&>(Op); RCP> tmp_CrsMtx = crsOp.getCrsMatrix(); #if defined(HAVE_XPETRA_EPETRA) && defined(HAVE_XPETRA_EPETRAEXT) const RCP>& tmp_ECrsMtx = Teuchos::rcp_dynamic_cast>(tmp_CrsMtx); if (tmp_ECrsMtx != Teuchos::null) { RCP A = tmp_ECrsMtx->getEpetra_CrsMatrix(); int rv = EpetraExt::RowMatrixToMatrixMarketFile(fileName.c_str(), *A); if (rv != 0) throw Exceptions::RuntimeError("EpetraExt::RowMatrixToMatrixMarketFile return value of " + Teuchos::toString(rv)); return; } #endif // endif HAVE_XPETRA_EPETRA #ifdef HAVE_XPETRA_TPETRA #if ((defined(EPETRA_HAVE_OMP) && (!defined(HAVE_TPETRA_INST_OPENMP) || !defined(HAVE_TPETRA_INST_INT_INT))) || \ (!defined(EPETRA_HAVE_OMP) && (!defined(HAVE_TPETRA_INST_SERIAL) || !defined(HAVE_TPETRA_INST_INT_INT)))) // do nothin #else const RCP>& tmp_TCrsMtx = Teuchos::rcp_dynamic_cast>(tmp_CrsMtx); if (tmp_TCrsMtx != Teuchos::null) { RCP> A = tmp_TCrsMtx->getTpetra_CrsMatrix(); Tpetra::MatrixMarket::Writer>::writeSparseFile(fileName, A); return; } const RCP>& tmp_BlockCrs = Teuchos::rcp_dynamic_cast>(tmp_CrsMtx); if (tmp_BlockCrs != Teuchos::null) { std::ofstream outstream(fileName, std::ofstream::out); Teuchos::FancyOStream ofs(Teuchos::rcpFromRef(outstream)); tmp_BlockCrs->getTpetra_BlockCrsMatrix()->describe(ofs, Teuchos::VERB_EXTREME); return; } #endif #endif // HAVE_XPETRA_TPETRA throw Exceptions::BadCast("Could not cast to EpetraCrsMatrix or TpetraCrsMatrix in matrix writing"); } static void Write(const std::string& fileName, const Xpetra::CrsGraph& graph, const bool& writeAllMaps = false) { Write("rowmap_" + fileName, *(graph.getRowMap())); if (!graph.getDomainMap()->isSameAs(*(graph.getRowMap())) || writeAllMaps) Write("domainmap_" + fileName, *(graph.getDomainMap())); if (!graph.getRangeMap()->isSameAs(*(graph.getRowMap())) || writeAllMaps) Write("rangemap_" + fileName, *(graph.getRangeMap())); if (!graph.getColMap()->isSameAs(*(graph.getDomainMap())) || writeAllMaps) Write("colmap_" + fileName, *(graph.getColMap())); RCP> tmp_Graph = rcpFromRef(graph); #if defined(HAVE_XPETRA_EPETRA) && defined(HAVE_XPETRA_EPETRAEXT) const RCP>& tmp_ECrsGraph = Teuchos::rcp_dynamic_cast>(tmp_Graph); if (tmp_ECrsGraph != Teuchos::null) { throw Exceptions::BadCast("Writing not implemented for EpetraCrsGraphT"); } #endif // endif HAVE_XPETRA_EPETRA #ifdef HAVE_XPETRA_TPETRA #if ((defined(EPETRA_HAVE_OMP) && (!defined(HAVE_TPETRA_INST_OPENMP) || !defined(HAVE_TPETRA_INST_INT_INT))) || \ (!defined(EPETRA_HAVE_OMP) && (!defined(HAVE_TPETRA_INST_SERIAL) || !defined(HAVE_TPETRA_INST_INT_INT)))) // do nothin #else RCP> tmp_TCrsGraph = Teuchos::rcp_dynamic_cast>(tmp_Graph); if (tmp_TCrsGraph != Teuchos::null) { RCP> G = tmp_TCrsGraph->getTpetra_CrsGraph(); Tpetra::MatrixMarket::Writer>::writeSparseGraphFile(fileName, G); return; } #endif #endif // HAVE_XPETRA_TPETRA throw Exceptions::BadCast("Could not cast to EpetraCrsMatrix or TpetraCrsMatrix in matrix writing"); } static void WriteLocal(const std::string& fileName, const Xpetra::Matrix& Op) { const Xpetra::CrsMatrixWrap& crsOp = dynamic_cast&>(Op); RCP> tmp_CrsMtx = crsOp.getCrsMatrix(); ArrayRCP rowptr_RCP; ArrayRCP rowptr2_RCP; ArrayRCP colind_RCP; ArrayRCP vals_RCP; tmp_CrsMtx->getAllValues(rowptr_RCP, colind_RCP, vals_RCP); ArrayView rowptr = rowptr_RCP(); ArrayView colind = colind_RCP(); ArrayView vals = vals_RCP(); rowptr2_RCP.resize(rowptr.size()); ArrayView rowptr2 = rowptr2_RCP(); for (size_t j = 0; j < Teuchos::as(rowptr.size()); j++) rowptr2[j] = rowptr[j]; Teuchos::MatrixMarket::Raw::Writer writer; writer.writeFile(fileName + "." + std::to_string(Op.getRowMap()->getComm()->getSize()) + "." + std::to_string(Op.getRowMap()->getComm()->getRank()), rowptr2, colind, vals, rowptr.size() - 1, Op.getColMap()->getLocalNumElements()); } static void WriteBlockedCrsMatrix(const std::string& fileName, const Xpetra::BlockedCrsMatrix& Op, const bool& writeAllMaps = false) { // write all matrices with their maps for (size_t row = 0; row < Op.Rows(); ++row) { for (size_t col = 0; col < Op.Cols(); ++col) { auto m = Op.getMatrix(row, col); if (m != Teuchos::null) { // skip empty blocks const bool cond = Teuchos::rcp_dynamic_cast>(m) == Teuchos::null; TEUCHOS_TEST_FOR_EXCEPTION(cond, Exceptions::BadCast, "Sub block matrix (" << row << "," << col << ") is not of type CrsMatrixWrap."); Write(fileName + toString(row) + toString(col) + ".m", *m, writeAllMaps); } } } // write map information of map extractors auto rangeMapExtractor = Op.getRangeMapExtractor(); auto domainMapExtractor = Op.getDomainMapExtractor(); for (size_t row = 0; row < rangeMapExtractor->NumMaps(); ++row) { auto map = rangeMapExtractor->getMap(row); Write("subRangeMap_" + fileName + toString(row) + ".m", *map); } Write("fullRangeMap_" + fileName + ".m", *(rangeMapExtractor->getFullMap())); for (size_t col = 0; col < domainMapExtractor->NumMaps(); ++col) { auto map = domainMapExtractor->getMap(col); Write("subDomainMap_" + fileName + toString(col) + ".m", *map); } Write("fullDomainMap_" + fileName + ".m", *(domainMapExtractor->getFullMap())); } static Teuchos::RCP> Read(const std::string& fileName, Xpetra::UnderlyingLib lib, const RCP>& comm, bool binary = false) { if (!binary) { // Matrix Market file format (ASCII) if (lib == Xpetra::UseEpetra) { #if defined(HAVE_XPETRA_EPETRA) && defined(HAVE_XPETRA_EPETRAEXT) Epetra_CrsMatrix* eA; const RCP epcomm = Xpetra::toEpetra(comm); int rv = EpetraExt::MatrixMarketFileToCrsMatrix(fileName.c_str(), *epcomm, eA); if (rv != 0) throw Exceptions::RuntimeError("EpetraExt::MatrixMarketFileToCrsMatrix return value of " + Teuchos::toString(rv)); RCP tmpA = rcp(eA); RCP> A = Convert_Epetra_CrsMatrix_ToXpetra_CrsMatrixWrap(tmpA); return A; #else throw Exceptions::RuntimeError("Xpetra has not been compiled with Epetra and EpetraExt support."); #endif } else if (lib == Xpetra::UseTpetra) { #ifdef HAVE_XPETRA_TPETRA #if ((defined(EPETRA_HAVE_OMP) && (!defined(HAVE_TPETRA_INST_OPENMP) || !defined(HAVE_TPETRA_INST_INT_INT))) || \ (!defined(EPETRA_HAVE_OMP) && (!defined(HAVE_TPETRA_INST_SERIAL) || !defined(HAVE_TPETRA_INST_INT_INT)))) throw Exceptions::RuntimeError("Xpetra has not been compiled with Tpetra GO=int enabled."); #else typedef Tpetra::CrsMatrix sparse_matrix_type; typedef Tpetra::MatrixMarket::Reader reader_type; bool callFillComplete = true; RCP tA = reader_type::readSparseFile(fileName, comm, callFillComplete); if (tA.is_null()) throw Exceptions::RuntimeError("The Tpetra::CrsMatrix returned from readSparseFile() is null."); RCP> tmpA1 = rcp(new Xpetra::TpetraCrsMatrix(tA)); RCP> tmpA2 = Teuchos::rcp_implicit_cast>(tmpA1); RCP> A = rcp(new Xpetra::CrsMatrixWrap(tmpA2)); return A; #endif #else throw Exceptions::RuntimeError("Xpetra has not been compiled with Tpetra support."); #endif } else { throw Exceptions::RuntimeError("Xpetra:IO: you must specify Xpetra::UseEpetra or Xpetra::UseTpetra."); } } else { // Custom file format (binary) std::ifstream ifs(fileName.c_str(), std::ios::binary); TEUCHOS_TEST_FOR_EXCEPTION(!ifs.good(), Exceptions::RuntimeError, "Can not read \"" << fileName << "\""); int m, n, nnz; ifs.read(reinterpret_cast(&m), sizeof(m)); ifs.read(reinterpret_cast(&n), sizeof(n)); ifs.read(reinterpret_cast(&nnz), sizeof(nnz)); int myRank = comm->getRank(); GlobalOrdinal indexBase = 0; RCP> rowMap = Xpetra::MapFactory::Build(lib, m, (myRank == 0 ? m : 0), indexBase, comm), rangeMap = rowMap; RCP> colMap = Xpetra::MapFactory::Build(lib, n, (myRank == 0 ? n : 0), indexBase, comm), domainMap = colMap; RCP> A; if (myRank == 0) { Teuchos::Array inds; Teuchos::Array vals; // Scan matrix to determine the exact nnz per row. Teuchos::ArrayRCP numEntriesPerRow(m); for (int i = 0; i < m; i++) { int row, rownnz; ifs.read(reinterpret_cast(&row), sizeof(row)); ifs.read(reinterpret_cast(&rownnz), sizeof(rownnz)); numEntriesPerRow[i] = rownnz; for (int j = 0; j < rownnz; j++) { int index; ifs.read(reinterpret_cast(&index), sizeof(index)); } for (int j = 0; j < rownnz; j++) { double value; ifs.read(reinterpret_cast(&value), sizeof(value)); } } A = Xpetra::MatrixFactory::Build(rowMap, colMap, numEntriesPerRow); // Now that nnz per row are known, reread and store the matrix. ifs.seekg(0, ifs.beg); // rewind to beginning of file int junk; // skip header info ifs.read(reinterpret_cast(&m), sizeof(junk)); ifs.read(reinterpret_cast(&n), sizeof(junk)); ifs.read(reinterpret_cast(&nnz), sizeof(junk)); for (int i = 0; i < m; i++) { int row, rownnz; ifs.read(reinterpret_cast(&row), sizeof(row)); ifs.read(reinterpret_cast(&rownnz), sizeof(rownnz)); inds.resize(rownnz); vals.resize(rownnz); for (int j = 0; j < rownnz; j++) { int index; ifs.read(reinterpret_cast(&index), sizeof(index)); inds[j] = Teuchos::as(index); } for (int j = 0; j < rownnz; j++) { double value; ifs.read(reinterpret_cast(&value), sizeof(value)); vals[j] = Teuchos::as(value); } A->insertGlobalValues(row, inds, vals); } } // if (myRank == 0) else { Teuchos::ArrayRCP numEntriesPerRow(0); A = Xpetra::MatrixFactory::Build(rowMap, colMap, numEntriesPerRow); } A->fillComplete(domainMap, rangeMap); return A; } TEUCHOS_UNREACHABLE_RETURN(Teuchos::null); } static Teuchos::RCP> Read(const std::string& filename, const RCP> rowMap, RCP> colMap = Teuchos::null, const RCP> domainMap = Teuchos::null, const RCP> rangeMap = Teuchos::null, const bool callFillComplete = true, const bool binary = false, const bool tolerant = false, const bool debug = false) { TEUCHOS_TEST_FOR_EXCEPTION(rowMap.is_null(), Exceptions::RuntimeError, "Utils::Read() : rowMap cannot be null"); RCP> domain = (domainMap.is_null() ? rowMap : domainMap); RCP> range = (rangeMap.is_null() ? rowMap : rangeMap); const Xpetra::UnderlyingLib lib = rowMap->lib(); if (binary == false) { if (lib == Xpetra::UseEpetra) { #if defined(HAVE_XPETRA_EPETRA) && defined(HAVE_XPETRA_EPETRAEXT) Epetra_CrsMatrix* eA; const RCP epcomm = Xpetra::toEpetra(rowMap->getComm()); const Epetra_Map& epetraRowMap = Xpetra::IO::Map2EpetraMap(*rowMap); const Epetra_Map& epetraDomainMap = (domainMap.is_null() ? epetraRowMap : Xpetra::IO::Map2EpetraMap(*domainMap)); const Epetra_Map& epetraRangeMap = (rangeMap.is_null() ? epetraRowMap : Xpetra::IO::Map2EpetraMap(*rangeMap)); int rv; if (colMap.is_null()) { rv = EpetraExt::MatrixMarketFileToCrsMatrix(filename.c_str(), epetraRowMap, epetraRangeMap, epetraDomainMap, eA); } else { const Epetra_Map& epetraColMap = Map2EpetraMap(*colMap); rv = EpetraExt::MatrixMarketFileToCrsMatrix(filename.c_str(), epetraRowMap, epetraColMap, epetraRangeMap, epetraDomainMap, eA); } if (rv != 0) throw Exceptions::RuntimeError("EpetraExt::MatrixMarketFileToCrsMatrix return value of " + Teuchos::toString(rv)); RCP tmpA = rcp(eA); RCP> A = Convert_Epetra_CrsMatrix_ToXpetra_CrsMatrixWrap(tmpA); return A; #else throw Exceptions::RuntimeError("Xpetra has not been compiled with Epetra and EpetraExt support."); #endif } else if (lib == Xpetra::UseTpetra) { #ifdef HAVE_XPETRA_TPETRA #if ((defined(EPETRA_HAVE_OMP) && (!defined(HAVE_TPETRA_INST_OPENMP) || !defined(HAVE_TPETRA_INST_INT_INT))) || \ (!defined(EPETRA_HAVE_OMP) && (!defined(HAVE_TPETRA_INST_SERIAL) || !defined(HAVE_TPETRA_INST_INT_INT)))) throw Exceptions::RuntimeError("Xpetra has not been compiled with Tpetra GO=int support."); #else typedef Tpetra::CrsMatrix sparse_matrix_type; typedef Tpetra::MatrixMarket::Reader reader_type; typedef Tpetra::Map map_type; const RCP tpetraRowMap = Map2TpetraMap(*rowMap); RCP tpetraColMap = (colMap.is_null() ? Teuchos::null : Map2TpetraMap(*colMap)); const RCP tpetraRangeMap = (rangeMap.is_null() ? tpetraRowMap : Map2TpetraMap(*rangeMap)); const RCP tpetraDomainMap = (domainMap.is_null() ? tpetraRowMap : Map2TpetraMap(*domainMap)); RCP tA = reader_type::readSparseFile(filename, tpetraRowMap, tpetraColMap, tpetraDomainMap, tpetraRangeMap, callFillComplete, tolerant, debug); if (tA.is_null()) throw Exceptions::RuntimeError("The Tpetra::CrsMatrix returned from readSparseFile() is null."); RCP> tmpA1 = rcp(new Xpetra::TpetraCrsMatrix(tA)); RCP> tmpA2 = Teuchos::rcp_implicit_cast>(tmpA1); RCP> A = rcp(new Xpetra::CrsMatrixWrap(tmpA2)); return A; #endif #else throw Exceptions::RuntimeError("Xpetra has not been compiled with Tpetra support."); #endif } else { throw Exceptions::RuntimeError("Utils::Read : you must specify Xpetra::UseEpetra or Xpetra::UseTpetra."); } } else { // Read in on rank 0. auto tempA = Read(filename, lib, rowMap->getComm(), binary); auto A = Xpetra::MatrixFactory::Build(rowMap, colMap, 0); auto importer = Xpetra::ImportFactory::Build(tempA->getRowMap(), rowMap); A->doImport(*tempA, *importer, Xpetra::INSERT); if (callFillComplete) A->fillComplete(domainMap, rangeMap); return A; } TEUCHOS_UNREACHABLE_RETURN(Teuchos::null); } static Teuchos::RCP> ReadLocal(const std::string& filename, const RCP> rowMap, RCP> colMap, const RCP> domainMap = Teuchos::null, const RCP> rangeMap = Teuchos::null, const bool callFillComplete = true, const bool binary = false, const bool tolerant = false, const bool debug = false) { TEUCHOS_TEST_FOR_EXCEPTION(rowMap.is_null(), Exceptions::RuntimeError, "Utils::ReadLocal() : rowMap cannot be null"); TEUCHOS_TEST_FOR_EXCEPTION(colMap.is_null(), Exceptions::RuntimeError, "Utils::ReadLocal() : colMap cannot be null"); using matrix_type = Xpetra::Matrix; using crs_wrap_type = Xpetra::CrsMatrixWrap; using crs_type = Xpetra::CrsMatrix; RCP> domain = (domainMap.is_null() ? rowMap : domainMap); RCP> range = (rangeMap.is_null() ? rowMap : rangeMap); std::string rankFilename = filename + "." + std::to_string(rowMap->getComm()->getSize()) + "." + std::to_string(rowMap->getComm()->getRank()); RCP A = rcp(new crs_wrap_type(rowMap, colMap, 0)); if (binary == false) { RCP params = rcp(new Teuchos::ParameterList()); params->set("Parse tolerantly", tolerant); params->set("Debug mode", debug); LocalOrdinal numRows = rowMap->getLocalNumElements(); LocalOrdinal numCols = colMap->getLocalNumElements(); ArrayRCP rowptr2_RCP; ArrayRCP colind2_RCP; ArrayRCP vals2_RCP; Teuchos::MatrixMarket::Raw::Reader reader; reader.readFile(rowptr2_RCP, colind2_RCP, vals2_RCP, numRows, numCols, rankFilename); RCP ACrs = Teuchos::rcp_dynamic_cast(A)->getCrsMatrix(); ArrayRCP rowptr_RCP; ArrayRCP colind_RCP; ArrayRCP vals_RCP; ACrs->allocateAllValues(colind2_RCP.size(), rowptr_RCP, colind_RCP, vals_RCP); rowptr_RCP.assign(rowptr2_RCP.begin(), rowptr2_RCP.end()); colind_RCP = colind2_RCP; vals_RCP = vals2_RCP; ACrs->setAllValues(rowptr_RCP, colind_RCP, vals_RCP); } else { // Custom file format (binary) std::ifstream ifs = std::ifstream(rankFilename.c_str(), std::ios::binary); TEUCHOS_TEST_FOR_EXCEPTION(!ifs.good(), Exceptions::RuntimeError, "Can not read \"" << filename << "\""); int m, n, nnz; ifs.read(reinterpret_cast(&m), sizeof(m)); ifs.read(reinterpret_cast(&n), sizeof(n)); ifs.read(reinterpret_cast(&nnz), sizeof(nnz)); TEUCHOS_ASSERT_EQUALITY(Teuchos::as(rowMap->getLocalNumElements()), m); Teuchos::ArrayRCP rowptrRCP; Teuchos::ArrayRCP indicesRCP; Teuchos::ArrayRCP valuesRCP; RCP ACrs = Teuchos::rcp_dynamic_cast(A)->getCrsMatrix(); ACrs->allocateAllValues(nnz, rowptrRCP, indicesRCP, valuesRCP); Teuchos::ArrayView rowptr = rowptrRCP(); Teuchos::ArrayView indices = indicesRCP(); Teuchos::ArrayView values = valuesRCP(); bool sorted = true; // Read in rowptr for (int i = 0; i < m; i++) { int row, rownnz; ifs.read(reinterpret_cast(&row), sizeof(row)); ifs.read(reinterpret_cast(&rownnz), sizeof(rownnz)); rowptr[row + 1] += rownnz; ifs.seekg(sizeof(int) * rownnz + sizeof(double) * rownnz, ifs.cur); } for (int i = 0; i < m; i++) rowptr[i + 1] += rowptr[i]; TEUCHOS_ASSERT(Teuchos::as(rowptr[m]) == nnz); // reset to where the data starts ifs.seekg(sizeof(int) * 3, ifs.beg); // read in entries for (int i = 0; i < m; i++) { int row, rownnz; ifs.read(reinterpret_cast(&row), sizeof(row)); ifs.read(reinterpret_cast(&rownnz), sizeof(rownnz)); size_t ptr = rowptr[row]; for (int j = 0; j < rownnz; j++) { int index; ifs.read(reinterpret_cast(&index), sizeof(index)); indices[ptr] = Teuchos::as(index); if (j > 0) sorted = sorted & (indices[ptr - 1] < indices[ptr]); ++ptr; } ptr = rowptr[row]; for (int j = 0; j < rownnz; j++) { double value; ifs.read(reinterpret_cast(&value), sizeof(value)); values[ptr] = Teuchos::as(value); ++ptr; } rowptr[row] += rownnz; } for (int i = m; i > 0; i--) rowptr[i] = rowptr[i - 1]; rowptr[0] = 0; #ifdef HAVE_XPETRA_TPETRA if (!sorted) { for (LocalOrdinal lclRow = 0; lclRow < m; lclRow++) { size_t rowBegin = rowptr[lclRow]; size_t rowEnd = rowptr[lclRow + 1]; Tpetra::sort2(&indices[rowBegin], &indices[rowEnd], &values[rowBegin]); } } #else TEUCHOS_ASSERT(sorted); #endif ACrs->setAllValues(rowptrRCP, indicesRCP, valuesRCP); } if (callFillComplete) A->fillComplete(domainMap, rangeMap); return A; } static RCP> ReadMultiVector(const std::string& fileName, const RCP>& map, const bool binary = false) { Xpetra::UnderlyingLib lib = map->lib(); if (lib == Xpetra::UseEpetra) { // taw: Oct 9 2015: do we need a specialization for ?? // TEUCHOS_TEST_FOR_EXCEPTION(true, ::Xpetra::Exceptions::BadCast, "Epetra can only be used with Scalar=double and Ordinal=int"); #if defined(HAVE_XPETRA_EPETRA) && defined(HAVE_XPETRA_EPETRAEXT) TEUCHOS_ASSERT(!binary); Epetra_MultiVector* MV; int rv = EpetraExt::MatrixMarketFileToMultiVector(fileName.c_str(), toEpetra(map), MV); if (rv != 0) throw Exceptions::RuntimeError("EpetraExt::MatrixMarketFileToMultiVector failed"); RCP MVrcp = rcp(MV); return Convert_Epetra_MultiVector_ToXpetra_MultiVector(MVrcp); #else throw Exceptions::RuntimeError("Xpetra has not been compiled with Epetra and EpetraExt support."); #endif } else if (lib == Xpetra::UseTpetra) { #ifdef HAVE_XPETRA_TPETRA #if ((defined(EPETRA_HAVE_OMP) && (!defined(HAVE_TPETRA_INST_OPENMP) || !defined(HAVE_TPETRA_INST_INT_INT))) || \ (!defined(EPETRA_HAVE_OMP) && (!defined(HAVE_TPETRA_INST_SERIAL) || !defined(HAVE_TPETRA_INST_INT_INT)))) throw Exceptions::RuntimeError("Xpetra has not been compiled with Tpetra GO=int support."); #else typedef Tpetra::CrsMatrix sparse_matrix_type; typedef Tpetra::MatrixMarket::Reader reader_type; typedef Tpetra::Map map_type; typedef Tpetra::MultiVector multivector_type; RCP temp = toTpetra(map); RCP TMV = reader_type::readDenseFile(fileName, map->getComm(), temp, false, false, binary); RCP> rmv = Xpetra::toXpetra(TMV); return rmv; #endif #else throw Exceptions::RuntimeError("Xpetra has not been compiled with Tpetra support."); #endif } else { throw Exceptions::RuntimeError("Utils::Read : you must specify Xpetra::UseEpetra or Xpetra::UseTpetra."); } TEUCHOS_UNREACHABLE_RETURN(Teuchos::null); } static RCP> ReadMultiVectorLO(const std::string& fileName, const RCP>& map, const bool binary = false) { Xpetra::UnderlyingLib lib = map->lib(); if (lib == Xpetra::UseTpetra) { #ifdef HAVE_XPETRA_TPETRA typedef Tpetra::CrsMatrix sparse_matrix_type; typedef Tpetra::MatrixMarket::Reader reader_type; typedef Tpetra::Map map_type; typedef Tpetra::MultiVector multivector_type; RCP temp = toTpetra(map); RCP TMV = reader_type::readDenseFile(fileName, map->getComm(), temp, false, false, binary); RCP> rmv = Xpetra::toXpetra(TMV); return rmv; #else throw Exceptions::RuntimeError("Xpetra has not been compiled with Tpetra support."); #endif } else { throw Exceptions::RuntimeError("Utils::ReadMultiVectorLO : only implemented for Tpetra"); } TEUCHOS_UNREACHABLE_RETURN(Teuchos::null); } static RCP> ReadMap(const std::string& fileName, Xpetra::UnderlyingLib lib, const RCP>& comm, const bool binary = false) { if (lib == Xpetra::UseEpetra) { // do we need another specialization for ?? // TEUCHOS_TEST_FOR_EXCEPTION(true, ::Xpetra::Exceptions::BadCast, "Epetra can only be used with Scalar=double and Ordinal=int"); #if defined(HAVE_XPETRA_EPETRA) && defined(HAVE_XPETRA_EPETRAEXT) TEUCHOS_ASSERT(!binary); Epetra_Map* eMap; int rv = EpetraExt::MatrixMarketFileToMap(fileName.c_str(), *(Xpetra::toEpetra(comm)), eMap); if (rv != 0) throw Exceptions::RuntimeError("Error reading map from file " + fileName + " with EpetraExt::MatrixMarketToMap (returned " + Teuchos::toString(rv) + ")"); RCP eMap1 = rcp(new Epetra_Map(*eMap)); return Xpetra::toXpetra(*eMap1); #else throw Exceptions::RuntimeError("Xpetra has not been compiled with Epetra and EpetraExt support."); #endif } else if (lib == Xpetra::UseTpetra) { #ifdef HAVE_XPETRA_TPETRA #if ((defined(EPETRA_HAVE_OMP) && (!defined(HAVE_TPETRA_INST_OPENMP) || !defined(HAVE_TPETRA_INST_INT_INT))) || \ (!defined(EPETRA_HAVE_OMP) && (!defined(HAVE_TPETRA_INST_SERIAL) || !defined(HAVE_TPETRA_INST_INT_INT)))) throw Exceptions::RuntimeError("Xpetra has not been compiled with Tpetra GO=int support."); #else typedef Tpetra::CrsMatrix sparse_matrix_type; typedef Tpetra::MatrixMarket::Reader reader_type; RCP> tMap = reader_type::readMapFile(fileName, comm, false, false, binary); if (tMap.is_null()) throw Exceptions::RuntimeError("The Tpetra::Map returned from readSparseFile() is null."); return Xpetra::toXpetra(tMap); #endif #else throw Exceptions::RuntimeError("Xpetra has not been compiled with Tpetra support."); #endif } else { throw Exceptions::RuntimeError("Utils::Read : you must specify Xpetra::UseEpetra or Xpetra::UseTpetra."); } TEUCHOS_UNREACHABLE_RETURN(Teuchos::null); } static RCP> ReadBlockedCrsMatrix(const std::string& fileName, Xpetra::UnderlyingLib lib, const RCP>& comm) { size_t numBlocks = 2; // TODO user parameter? std::vector>> rangeMapVec; for (size_t row = 0; row < numBlocks; ++row) { auto map = ReadMap("subRangeMap_" + fileName + toString(row) + ".m", lib, comm); rangeMapVec.push_back(map); } auto fullRangeMap = ReadMap("fullRangeMap_" + fileName + ".m", lib, comm); std::vector>> domainMapVec; for (size_t col = 0; col < numBlocks; ++col) { auto map = ReadMap("subDomainMap_" + fileName + toString(col) + ".m", lib, comm); domainMapVec.push_back(map); } auto fullDomainMap = ReadMap("fullDomainMap_" + fileName + ".m", lib, comm); /*std::vector > testRgMapVec; for(size_t r = 0; r < numBlocks; ++r) { RCP map = ReadMap("rangemap_" + fileName + XpIO::toString(r) + "0.m", lib, comm); testRgMapVec.push_back(map); } std::vector > testDoMapVec; for(size_t c = 0; c < numBlocks; ++c) { RCP map = ReadMap("domainmap_" + fileName + "0" + XpIO::toString(c) + ".m", lib, comm); testDoMapVec.push_back(map); }*/ // create map extractors // range map extractor bool bRangeUseThyraStyleNumbering = false; /* GlobalOrdinal gMinGids = 0; for(size_t v = 0; v < testRgMapVec.size(); ++v) { gMinGids += testRgMapVec[v]->getMinAllGlobalIndex(); } if ( gMinGids==0 && testRgMapVec.size() > 1 ) bRangeUseThyraStyleNumbering = true;*/ RCP> rangeMapExtractor = rcp(new Xpetra::MapExtractor(fullRangeMap, rangeMapVec, bRangeUseThyraStyleNumbering)); // domain map extractor bool bDomainUseThyraStyleNumbering = false; /*gMinGids = 0; for(size_t v = 0; v < testDoMapVec.size(); ++v) { gMinGids += testDoMapVec[v]->getMinAllGlobalIndex(); } if ( gMinGids==0 && testDoMapVec.size() > 1) bDomainUseThyraStyleNumbering = true;*/ RCP> domainMapExtractor = rcp(new Xpetra::MapExtractor(fullDomainMap, domainMapVec, bDomainUseThyraStyleNumbering)); auto bOp = Teuchos::rcp(new Xpetra::BlockedCrsMatrix(rangeMapExtractor, domainMapExtractor, 33)); // Read all matrices with their maps and create the BlockedCrsMatrix for (size_t row = 0; row < numBlocks; ++row) { for (size_t col = 0; col < numBlocks; ++col) { auto rowSubMap = ReadMap("rowmap_" + fileName + toString(row) + toString(col) + ".m", lib, comm); auto colSubMap = ReadMap("colmap_" + fileName + toString(row) + toString(col) + ".m", lib, comm); auto domSubMap = ReadMap("domainmap_" + fileName + toString(row) + toString(col) + ".m", lib, comm); auto ranSubMap = ReadMap("rangemap_" + fileName + toString(row) + toString(col) + ".m", lib, comm); auto mat = Read(fileName + toString(row) + toString(col) + ".m", rowSubMap, colSubMap, domSubMap, ranSubMap); bOp->setMatrix(row, col, mat); } } bOp->fillComplete(); return bOp; } //! Little helper function to convert non-string types to strings template static std::string toString(const T& what) { std::ostringstream buf; buf << what; return buf.str(); } }; #endif // HAVE_XPETRA_EPETRA } // end namespace Xpetra #define XPETRA_IO_SHORT #endif /* PACKAGES_XPETRA_SUP_UTILS_XPETRA_IO_HPP_ */