// @HEADER // ***************************************************************************** // Amesos2: Templated Direct Sparse Solver Package // // Copyright 2011 NTESS and the Amesos2 contributors. // SPDX-License-Identifier: BSD-3-Clause // ***************************************************************************** // @HEADER /** \file Amesos2_EpetraMultiVecAdapter_def.hpp \author Eric T Bavier \date \brief Amesos2::MultiVecAdapter specialization for the Epetra_MultiVector class. */ #ifndef AMESOS2_EPETRA_MULTIVEC_ADAPTER_DEF_HPP #define AMESOS2_EPETRA_MULTIVEC_ADAPTER_DEF_HPP #include #include #ifdef HAVE_MPI #include #endif #include #include #include #include "Amesos2_EpetraMultiVecAdapter_decl.hpp" #include "Amesos2_Util.hpp" namespace Amesos2 { MultiVecAdapter::MultiVecAdapter(const MultiVecAdapter& adapter) : mv_(adapter.mv_) , mv_map_(adapter.mv_map_) { } MultiVecAdapter::MultiVecAdapter(const Teuchos::RCP& m) : mv_(m) { mv_map_ = Teuchos::rcpFromRef(mv_->Map()); } Teuchos::RCP MultiVecAdapter::clone() const { Teuchos::RCP Y (new Epetra_MultiVector(*mv_map_, mv_->NumVectors(), false)); return Y; } bool MultiVecAdapter::isLocallyIndexed() const { return !mv_->DistributedGlobal(); } bool MultiVecAdapter::isGloballyIndexed() const { return mv_->DistributedGlobal(); } const Teuchos::RCP > MultiVecAdapter::getComm() const { return Util::to_teuchos_comm(Teuchos::rcpFromRef(mv_->Comm())); } size_t MultiVecAdapter::getLocalLength() const { return Teuchos::as(mv_->MyLength()); } size_t MultiVecAdapter::getLocalNumVectors() const { return Teuchos::as(mv_->NumVectors()); } MultiVecAdapter::global_size_t MultiVecAdapter::getGlobalLength() const { return Teuchos::as(mv_->GlobalLength()); } size_t MultiVecAdapter::getGlobalNumVectors() const { return Teuchos::as(mv_->NumVectors()); } size_t MultiVecAdapter::getStride() const { return Teuchos::as(mv_->Stride()); } bool MultiVecAdapter::isConstantStride() const { return mv_->ConstantStride(); } Teuchos::RCP::scalar_t, MultiVecAdapter::local_ordinal_t, MultiVecAdapter::global_ordinal_t, MultiVecAdapter::node_t> > MultiVecAdapter::getVector( size_t j ) const { using Teuchos::RCP; using Teuchos::rcp; using Teuchos::ArrayRCP; using Tpetra::MultiVector; typedef scalar_t st; typedef local_ordinal_t lot; typedef global_ordinal_t got; typedef node_t nt; RCP > vector = rcp(new MultiVector(this->getMap(),1)); // Copy vector contents into Tpetra multi-vector ArrayRCP it = vector->getDataNonConst(0); double* vector_data = mv_->operator[](Teuchos::as(j)); // values from j^th vector Tpetra::global_size_t size = vector->getGlobalLength(); for( Tpetra::global_size_t i = 0; i < size; ++i ){ *it = vector_data[i]; } return vector->getVector(j); } // Implementation is essentially the same as getVector() Teuchos::RCP::scalar_t, MultiVecAdapter::local_ordinal_t, MultiVecAdapter::global_ordinal_t, MultiVecAdapter::node_t> > MultiVecAdapter::getVectorNonConst( size_t j ) { using Teuchos::RCP; using Teuchos::rcp; using Teuchos::ArrayRCP; using Tpetra::MultiVector; typedef scalar_t st; typedef local_ordinal_t lot; typedef global_ordinal_t got; typedef node_t nt; RCP > vector = rcp(new MultiVector(this->getMap(),1)); // Copy vector contents into Tpetra multi-vector ArrayRCP it = vector->getDataNonConst(0); double* vector_data = mv_->operator[](Teuchos::as(j)); // values from j^th vector Tpetra::global_size_t size = vector->getGlobalLength(); for( Tpetra::global_size_t i = 0; i < size; ++i ){ *it = vector_data[i]; } return vector->getVectorNonConst(j); } double * MultiVecAdapter::getMVPointer_impl() const { TEUCHOS_TEST_FOR_EXCEPTION( this->getGlobalNumVectors() != 1, std::invalid_argument, "Amesos2_EpetraMultiVectorAdapter: getMVPointer_impl should only be called for case with a single vector and single MPI process" ); double* vector_data = mv_->operator[](Teuchos::as(0)); // raw pointer to data from 0^th vector return vector_data; } void MultiVecAdapter::get1dCopy( const Teuchos::ArrayView::scalar_t>& av, size_t lda, Teuchos::Ptr< const Tpetra::Map::local_ordinal_t, MultiVecAdapter::global_ordinal_t, MultiVecAdapter::node_t> > distribution_map, EDistribution /* distribution */) const { using Teuchos::rcpFromPtr; using Teuchos::as; const size_t num_vecs = getGlobalNumVectors(); #ifdef HAVE_AMESOS2_DEBUG const size_t requested_vector_length = distribution_map->getLocalNumElements(); TEUCHOS_TEST_FOR_EXCEPTION( lda < requested_vector_length, std::invalid_argument, "Given stride is not large enough for local vector length" ); TEUCHOS_TEST_FOR_EXCEPTION( as(av.size()) < (num_vecs-1) * lda + requested_vector_length, std::invalid_argument, "MultiVector storage not large enough given leading dimension " "and number of vectors" ); #endif // Optimization for ROOTED and single MPI process if ( num_vecs == 1 && mv_->Comm().MyPID() == 0 && mv_->Comm().NumProc() == 1 ) { mv_->ExtractCopy(av.getRawPtr(), lda); } else { Epetra_Map e_dist_map = *Util::tpetra_map_to_epetra_map(*distribution_map); multivec_t redist_mv(e_dist_map, as(num_vecs)); const Epetra_Import importer(e_dist_map, *mv_map_); // Note, target/source order is reversed in Tpetra redist_mv.Import(*mv_, importer, Insert); // Finally, do copy redist_mv.ExtractCopy(av.getRawPtr(), lda); } } void MultiVecAdapter::get1dCopy_kokkos_view_host( Kokkos::View & host_view, size_t lda, Teuchos::Ptr< const Tpetra::Map::local_ordinal_t, MultiVecAdapter::global_ordinal_t, MultiVecAdapter::node_t> > distribution_map, EDistribution /* distribution */) const { using Teuchos::rcpFromPtr; using Teuchos::as; const size_t num_vecs = getGlobalNumVectors(); #ifdef HAVE_AMESOS2_DEBUG const size_t requested_vector_length = distribution_map->getLocalNumElements(); TEUCHOS_TEST_FOR_EXCEPTION( lda < requested_vector_length, std::invalid_argument, "Given stride is not large enough for local vector length" ); #endif // First make a host view host_view = Kokkos::View( Kokkos::ViewAllocateWithoutInitializing("get1dCopy_kokkos_view"), distribution_map->getLocalNumElements(), num_vecs); // Optimization for ROOTED and single MPI process if ( num_vecs == 1 && this->mv_->Comm().MyPID() == 0 && this->mv_->Comm().NumProc() == 1 ) { mv_->ExtractCopy(host_view.data(), lda); } else { Epetra_Map e_dist_map = *Util::tpetra_map_to_epetra_map(*distribution_map); multivec_t redist_mv(e_dist_map, as(num_vecs)); const Epetra_Import importer(e_dist_map, *mv_map_); // Note, target/source order is reversed in Tpetra redist_mv.Import(*mv_, importer, Insert); // Finally, access data // Can we consider direct ptr usage with ExtractView? // For now I will just copy - this was discussed as low priority for now. redist_mv.ExtractCopy(host_view.data(), lda); } } Teuchos::ArrayRCP::scalar_t> MultiVecAdapter::get1dViewNonConst(bool local) { ((void) local); // TEUCHOS_TEST_FOR_EXCEPTION( !this->isConstantStride(), // std::logic_error, // "get1dViewNonConst() : can only get 1d view if stride is constant"); // if( local ){ // TEUCHOS_TEST_FOR_EXCEPTION( // true, // std::logic_error, // "Amesos2::MultiVecAdapter : 1D views not yet supported for local-local Epetra multi-vectors"); // // localize(); // // /* Use the global element list returned by // // * mv_->getMap()->getLocalElementList() to get a subCopy of mv_ which we // // * assign to l_l_mv_, then return get1dViewNonConst() of l_l_mv_ // // */ // // l_l_mv_ = Teuchos::null; // // Teuchos::Array nodeElements_go(mv_->Map().NumMyElements()); // // mv_->Map().MyGlobalElements(nodeElements_go.getRawPtr()); // // Teuchos::Array nodeElements_st(nodeElements_go.size()); // // // Convert the node element to a list of size_t type objects // // typename Teuchos::ArrayView::iterator it_go; // // Teuchos::Array::iterator it_st = nodeElements_st.begin(); // // for( it_go = nodeElements_go.begin(); it_go != nodeElements_go.end(); ++it_go ){ // // *(it_st++) = Teuchos::as(*it_go); // // } // // l_l_mv_ = l_mv_->subViewNonConst(nodeElements_st); // // return(l_l_mv_->get1dViewNonConst()); // } else { // scalar_t* values; // int lda; // if( !isLocal() ){ // this->localize(); // l_mv_->ExtractView(&values, &lda); // } else { // mv_->ExtractView(&values, &lda); // } // TEUCHOS_TEST_FOR_EXCEPTION( lda != Teuchos::as(this->getStride()), // std::logic_error, // "Stride reported during extraction not consistent with what multivector reports"); // return Teuchos::arcp(values,0,lda*this->getGlobalNumVectors(),false); // } return Teuchos::null; } void MultiVecAdapter::put1dData( const Teuchos::ArrayView::scalar_t>& new_data, size_t lda, Teuchos::Ptr< const Tpetra::Map::local_ordinal_t, MultiVecAdapter::global_ordinal_t, MultiVecAdapter::node_t> > source_map, EDistribution /* distribution */) { using Teuchos::rcpFromPtr; using Teuchos::as; const size_t num_vecs = getGlobalNumVectors(); // TODO: check that the following const_cast is safe double* data_ptr = const_cast(new_data.getRawPtr()); // Optimization for ROOTED and single MPI process if ( num_vecs == 1 && mv_->Comm().MyPID() == 0 && mv_->Comm().NumProc() == 1 ) { // First, functioning impl //const multivec_t source_mv(Copy, *mv_map_, data_ptr, as(lda), as(num_vecs)); //const Epetra_Import importer(*mv_map_, *mv_map_); //trivial - map does not change //mv_->Import(source_mv, importer, Insert); // Element-wise copy rather than using importer auto vector = mv_->Pointers(); for ( size_t i = 0; i < lda; ++i ) { vector[0][i] = data_ptr[i]; } } else { const Epetra_BlockMap e_source_map = *Util::tpetra_map_to_epetra_map(*source_map); const multivec_t source_mv(Copy, e_source_map, data_ptr, as(lda), as(num_vecs)); const Epetra_Import importer(*mv_map_, e_source_map); mv_->Import(source_mv, importer, Insert); } } void MultiVecAdapter::put1dData_kokkos_view_host( Kokkos::View & host_new_data, size_t lda, Teuchos::Ptr< const Tpetra::Map::local_ordinal_t, MultiVecAdapter::global_ordinal_t, MultiVecAdapter::node_t> > source_map, EDistribution /* distribution */) { using Teuchos::rcpFromPtr; using Teuchos::as; const size_t num_vecs = getGlobalNumVectors(); double* data_ptr = host_new_data.data(); // Optimization for ROOTED and single MPI process if ( num_vecs == 1 && mv_->Comm().MyPID() == 0 && mv_->Comm().NumProc() == 1 ) { auto vector = mv_->Pointers(); for ( size_t i = 0; i < lda; ++i ) { vector[0][i] = data_ptr[i]; } } else { const Epetra_BlockMap e_source_map = *Util::tpetra_map_to_epetra_map(*source_map); const multivec_t source_mv(Copy, e_source_map, data_ptr, as(lda), as(num_vecs)); const Epetra_Import importer(*mv_map_, e_source_map); mv_->Import(source_mv, importer, Insert); } } std::string MultiVecAdapter::description() const { std::ostringstream oss; oss << "Amesos2 adapter wrapping: Epetra_MultiVector"; return oss.str(); } void MultiVecAdapter::describe( Teuchos::FancyOStream& os, const Teuchos::EVerbosityLevel verbLevel) const { // TODO: implement! if(verbLevel != Teuchos::VERB_NONE) { os << "TODO: implement! "; } } Teuchos::RCP::local_ordinal_t, MultiVecAdapter::global_ordinal_t, MultiVecAdapter::node_t> > MultiVecAdapter::getMap() const { return Util::epetra_map_to_tpetra_map(*mv_map_); } const char* MultiVecAdapter::name = "Amesos2 adapter for Epetra_MultiVector"; } // end namespace Amesos2 #endif // AMESOS2_EPETRA_MULTIVEC_ADAPTER_DEF_HPP