| 1 |
// SPDX-FileCopyrightText: Copyright © DUNE Project contributors, see file LICENSE.md in module root |
| 2 |
// SPDX-License-Identifier: LicenseRef-GPL-2.0-only-with-DUNE-exception |
| 3 |
// -*- tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- |
| 4 |
// vi: set et ts=4 sw=2 sts=2: |
| 5 |
|
| 6 |
#define DEBUG_REPART |
| 7 |
|
| 8 |
#include <dune/istl/matrixredistribute.hh> |
| 9 |
#include <iostream> |
| 10 |
#include <dune/istl/paamg/test/anisotropic.hh> |
| 11 |
#include <dune/istl/bcrsmatrix.hh> |
| 12 |
#include <dune/istl/matrixutils.hh> |
| 13 |
#include <dune/istl/paamg/graph.hh> |
| 14 |
#include <dune/istl/io.hh> |
| 15 |
#include <dune/common/exceptions.hh> |
| 16 |
#include <dune/common/bigunsignedint.hh> |
| 17 |
|
| 18 |
class MPIError { |
| 19 |
public: |
| 20 |
/** @brief Constructor. */ |
| 21 |
MPIError(std::string s, int e) : errorstring(s), errorcode(e){} |
| 22 |
/** @brief The error string. */ |
| 23 |
std::string errorstring; |
| 24 |
/** @brief The mpi error code. */ |
| 25 |
int errorcode; |
| 26 |
}; |
| 27 |
|
| 28 |
void MPI_err_handler(MPI_Comm *, int *err_code, ...){ |
| 29 |
char *err_string=new char[MPI_MAX_ERROR_STRING]; |
| 30 |
int err_length; |
| 31 |
MPI_Error_string(*err_code, err_string, &err_length); |
| 32 |
std::string s(err_string, err_length); |
| 33 |
std::cerr << "An MPI Error occurred:"<<std::endl<<s<<std::endl; |
| 34 |
delete[] err_string; |
| 35 |
throw MPIError(s, *err_code); |
| 36 |
} |
| 37 |
|
| 38 |
template<class MatrixBlock> |
| 39 |
int testRepart(int N, int coarsenTarget) |
| 40 |
{ |
| 41 |
|
| 42 |
std::cout<<"==================================================="<<std::endl; |
| 43 |
std::cout<<" N="<<N<<" coarsenTarget="<<coarsenTarget<<std::endl; |
| 44 |
|
| 45 |
int procs, rank; |
| 46 |
MPI_Comm_rank(MPI_COMM_WORLD, &rank); |
| 47 |
MPI_Comm_size(MPI_COMM_WORLD, &procs); |
| 48 |
|
| 49 |
typedef Dune::BCRSMatrix<MatrixBlock> BCRSMat; |
| 50 |
typedef Dune::bigunsignedint<56> GlobalId; |
| 51 |
typedef Dune::OwnerOverlapCopyCommunication<GlobalId> Communication; |
| 52 |
int n; |
| 53 |
|
| 54 |
Communication comm(MPI_COMM_WORLD); |
| 55 |
|
| 56 |
BCRSMat mat = setupAnisotropic2d<MatrixBlock>(N, comm.indexSet(), comm.communicator(), &n, 1); |
| 57 |
typedef typename Dune::Amg::MatrixGraph<BCRSMat> MatrixGraph; |
| 58 |
|
| 59 |
MatrixGraph graph(mat); |
| 60 |
std::shared_ptr<Communication> coarseComm; |
| 61 |
|
| 62 |
comm.remoteIndices().template rebuild<false>(); |
| 63 |
|
| 64 |
std::cout<<comm.communicator().rank()<<comm.indexSet()<<std::endl; |
| 65 |
|
| 66 |
Dune::RedistributeInformation<Communication> ri; |
| 67 |
Dune::graphRepartition(graph, comm, coarsenTarget, |
| 68 |
coarseComm, ri.getInterface()); |
| 69 |
|
| 70 |
std::cout<<coarseComm->communicator().rank()<<coarseComm->indexSet()<<std::endl; |
| 71 |
BCRSMat newMat; |
| 72 |
|
| 73 |
if(comm.communicator().rank()==0) |
| 74 |
std::cout<<"Original matrix"<<std::endl; |
| 75 |
comm.communicator().barrier(); |
| 76 |
printGlobalSparseMatrix(mat, comm, std::cout); |
| 77 |
|
| 78 |
|
| 79 |
redistributeMatrix(mat, newMat, comm, *coarseComm, ri); |
| 80 |
|
| 81 |
std::cout<<comm.communicator().rank()<<": redist interface "<<ri.getInterface()<<std::endl; |
| 82 |
|
| 83 |
if(comm.communicator().rank()==0) |
| 84 |
std::cout<<"Redistributed matrix"<<std::endl; |
| 85 |
comm.communicator().barrier(); |
| 86 |
if(coarseComm->communicator().size()>0) |
| 87 |
printGlobalSparseMatrix(newMat, *coarseComm, std::cout); |
| 88 |
comm.communicator().barrier(); |
| 89 |
|
| 90 |
// Check for symmetry |
| 91 |
int ret=0; |
| 92 |
typedef typename BCRSMat::ConstRowIterator RIter; |
| 93 |
for(RIter row=newMat.begin(), rend=newMat.end(); row != rend; ++row) { |
| 94 |
typedef typename BCRSMat::ConstColIterator CIter; |
| 95 |
for(CIter col=row->begin(), cend=row->end(); col!=cend; ++col) |
| 96 |
{ |
| 97 |
if(col.index()<=row.index()) |
| 98 |
try{ |
| 99 |
newMat[col.index()][row.index()]; |
| 100 |
}catch(const Dune::ISTLError&) { |
| 101 |
std::cerr<<coarseComm->communicator().rank()<<": entry (" |
| 102 |
<<col.index()<<","<<row.index()<<") missing!"<<std::endl; |
| 103 |
ret=1; |
| 104 |
|
| 105 |
} |
| 106 |
else |
| 107 |
break; |
| 108 |
} |
| 109 |
} |
| 110 |
|
| 111 |
//if(coarseComm->communicator().rank()==0) |
| 112 |
//Dune::printmatrix(std::cout, newMat, "redist", "row"); |
| 113 |
return ret; |
| 114 |
} |
| 115 |
|
| 116 |
int main(int argc, char** argv) |
| 117 |
{ |
| 118 |
MPI_Init(&argc, &argv); |
| 119 |
MPI_Errhandler handler; |
| 120 |
MPI_Comm_create_errhandler(MPI_err_handler, &handler); |
| 121 |
MPI_Comm_set_errhandler(MPI_COMM_WORLD, handler); |
| 122 |
int procs; |
| 123 |
MPI_Comm_size(MPI_COMM_WORLD, &procs); |
| 124 |
|
| 125 |
int N=4*procs; |
| 126 |
|
| 127 |
int coarsenTarget=1; |
| 128 |
|
| 129 |
if(argc>1) |
| 130 |
N = atoi(argv[1]); |
| 131 |
if(argc>2) |
| 132 |
coarsenTarget = atoi(argv[2]); |
| 133 |
|
| 134 |
if(N<procs*2) { |
| 135 |
std::cerr<<"Problem size insufficient for process number"<<std::endl; |
| 136 |
return 1; |
| 137 |
} |
| 138 |
|
| 139 |
testRepart<Dune::FieldMatrix<double, 1, 1>>(N,coarsenTarget); |
| 140 |
testRepart<Dune::FieldMatrix<double, 2, 2>>(N/2,coarsenTarget); |
| 141 |
testRepart<double>(N,coarsenTarget); |
| 142 |
MPI_Finalize(); |
| 143 |
} |