| 1 |
// ------------------------------------------------------------------------ |
| 2 |
// |
| 3 |
// SPDX-License-Identifier: LGPL-2.1-or-later |
| 4 |
// Copyright (C) 2022 - 2024 by the deal.II authors |
| 5 |
// |
| 6 |
// This file is part of the deal.II library. |
| 7 |
// |
| 8 |
// Part of the source code is dual licensed under Apache-2.0 WITH |
| 9 |
// LLVM-exception OR LGPL-2.1-or-later. Detailed license information |
| 10 |
// governing the source code and code contributions can be found in |
| 11 |
// LICENSE.md and CONTRIBUTING.md at the top level directory of deal.II. |
| 12 |
// |
| 13 |
// ------------------------------------------------------------------------ |
| 14 |
|
| 15 |
#ifndef dealii_trilinos_utilities_h |
| 16 |
#define dealii_trilinos_utilities_h |
| 17 |
|
| 18 |
#include <deal.II/base/config.h> |
| 19 |
|
| 20 |
#include <deal.II/base/exceptions.h> |
| 21 |
|
| 22 |
#ifdef DEAL_II_WITH_TRILINOS |
| 23 |
|
| 24 |
DEAL_II_DISABLE_EXTRA_DIAGNOSTICS |
| 25 |
# include <Epetra_Comm.h> |
| 26 |
# include <Epetra_Map.h> |
| 27 |
# include <Teuchos_Comm.hpp> |
| 28 |
# include <Teuchos_RCP.hpp> |
| 29 |
|
| 30 |
# ifdef DEAL_II_WITH_MPI |
| 31 |
# include <Epetra_MpiComm.h> |
| 32 |
# else |
| 33 |
# include <Epetra_SerialComm.h> |
| 34 |
# endif |
| 35 |
|
| 36 |
# ifdef DEAL_II_TRILINOS_WITH_TPETRA |
| 37 |
# include <Teuchos_RCPDecl.hpp> |
| 38 |
# endif // DEAL_II_TRILINOS_WITH_TPETRA |
| 39 |
DEAL_II_ENABLE_EXTRA_DIAGNOSTICS |
| 40 |
#endif |
| 41 |
|
| 42 |
|
| 43 |
DEAL_II_NAMESPACE_OPEN |
| 44 |
|
| 45 |
/** |
| 46 |
* A namespace for utility functions that are not particularly specific to |
| 47 |
* finite element computing or numerical programs, but nevertheless are needed |
| 48 |
* in various contexts when writing applications. |
| 49 |
* |
| 50 |
* @ingroup utilities |
| 51 |
*/ |
| 52 |
namespace Utilities |
| 53 |
{ |
| 54 |
#ifdef DEAL_II_WITH_TRILINOS |
| 55 |
/** |
| 56 |
* This namespace provides some of the basic structures used in the |
| 57 |
* initialization of the Trilinos objects (e.g., matrices, vectors, and |
| 58 |
* preconditioners). |
| 59 |
*/ |
| 60 |
namespace Trilinos |
| 61 |
{ |
| 62 |
/** |
| 63 |
* Return a Trilinos Epetra_Comm object needed for creation of |
| 64 |
* Epetra_Maps. |
| 65 |
* |
| 66 |
* If deal.II has been configured to use a compiler that does not support |
| 67 |
* MPI then the resulting communicator will be a serial one. Otherwise, |
| 68 |
* the communicator will correspond to MPI_COMM_WORLD, i.e. a communicator |
| 69 |
* that encompasses all processes within this MPI universe. |
| 70 |
*/ |
| 71 |
const Epetra_Comm & |
| 72 |
comm_world(); |
| 73 |
|
| 74 |
/** |
| 75 |
* Return a Trilinos Epetra_Comm object needed for creation of |
| 76 |
* Epetra_Maps. |
| 77 |
* |
| 78 |
* If deal.II has been configured to use a compiler that does not support |
| 79 |
* MPI then the resulting communicator will be a serial one. Otherwise, |
| 80 |
* the communicator will correspond to MPI_COMM_SELF, i.e. a communicator |
| 81 |
* that comprises only this one processor. |
| 82 |
*/ |
| 83 |
const Epetra_Comm & |
| 84 |
comm_self(); |
| 85 |
|
| 86 |
/** |
| 87 |
* Return a Teuchos::Comm object needed for creation of Tpetra::Maps. |
| 88 |
* |
| 89 |
* If deal.II has been configured to use a compiler that does not support |
| 90 |
* MPI then the resulting communicator will be a serial one. Otherwise, |
| 91 |
* the communicator will correspond to MPI_COMM_SELF, i.e. a communicator |
| 92 |
* that comprises only this one processor. |
| 93 |
*/ |
| 94 |
const Teuchos::RCP<const Teuchos::Comm<int>> & |
| 95 |
tpetra_comm_self(); |
| 96 |
|
| 97 |
/** |
| 98 |
* Given a communicator, duplicate it. If the given communicator is |
| 99 |
* serial, that means to just return a copy of itself. On the other hand, |
| 100 |
* if it is %parallel, we duplicate the underlying MPI_Comm object: we |
| 101 |
* create a separate MPI communicator that contains the same processors |
| 102 |
* and in the same order but has a separate identifier distinct from the |
| 103 |
* given communicator. The function returns a pointer to a new object of a |
| 104 |
* class derived from Epetra_Comm. The caller of this function needs to |
| 105 |
* assume ownership of this function. The returned object should be |
| 106 |
* destroyed using the destroy_communicator() function. |
| 107 |
* |
| 108 |
* This facility is used to separate streams of communication. For |
| 109 |
* example, a program could simply use MPI_Comm_World for everything. But |
| 110 |
* it is easy to come up with scenarios where sometimes not all processors |
| 111 |
* participate in a communication that is intended to be global -- for |
| 112 |
* example if we assemble a matrix on a coarse mesh with fewer cells than |
| 113 |
* there are processors, some processors may not sync their matrices with |
| 114 |
* the rest because they haven't written into it because they own no |
| 115 |
* cells. That's clearly a bug. However, if these processors just continue |
| 116 |
* their work, and the next %parallel operation happens to be a sync on a |
| 117 |
* different matrix, then the sync could succeed -- by accident, since |
| 118 |
* different processors are talking about different matrices. |
| 119 |
* |
| 120 |
* This kind of situation can be avoided if we use different communicators |
| 121 |
* for different matrices which reduces the likelihood that communications |
| 122 |
* meant to be separate aren't recognized as such just because they happen |
| 123 |
* on the same communicator. In addition, it is conceivable that some MPI |
| 124 |
* operations can be parallelized using multiple threads because their |
| 125 |
* communicators identifies the communication in question, not their |
| 126 |
* relative timing as is the case in a sequential program that just uses a |
| 127 |
* single communicator. |
| 128 |
*/ |
| 129 |
Epetra_Comm * |
| 130 |
duplicate_communicator(const Epetra_Comm &communicator); |
| 131 |
|
| 132 |
/** |
| 133 |
* Given an Epetra communicator that was created by the |
| 134 |
* duplicate_communicator() function, destroy the underlying MPI |
| 135 |
* communicator object and reset the Epetra_Comm object to a the result of |
| 136 |
* comm_self(). |
| 137 |
* |
| 138 |
* It is necessary to call this function at the time when the result of |
| 139 |
* duplicate_communicator() is no longer needed. The reason is that in |
| 140 |
* that function, we first create a new MPI_Comm object and then create an |
| 141 |
* Epetra_Comm around it. While we can take care of destroying the latter, |
| 142 |
* it doesn't destroy the communicator since it can only assume that it |
| 143 |
* may also be still used by other objects in the program. Consequently, |
| 144 |
* we have to take care of destroying it ourselves, explicitly. |
| 145 |
* |
| 146 |
* This function does exactly that. Because this has to happen while the |
| 147 |
* Epetra_Comm object is still around, it first resets the latter and then |
| 148 |
* destroys the communicator object. |
| 149 |
* |
| 150 |
* @note If you call this function on an Epetra_Comm object that is not |
| 151 |
* created by duplicate_communicator(), you are likely doing something |
| 152 |
* quite wrong. Don't do this. |
| 153 |
*/ |
| 154 |
void |
| 155 |
destroy_communicator(Epetra_Comm &communicator); |
| 156 |
|
| 157 |
/** |
| 158 |
* Return the number of MPI processes there exist in the given |
| 159 |
* @ref GlossMPICommunicator "communicator" |
| 160 |
* object. If this is a sequential job (i.e., the program |
| 161 |
* is not using MPI at all, or is using MPI but has been started with |
| 162 |
* only one MPI process), then the communicator necessarily involves |
| 163 |
* only one process and the function returns 1. |
| 164 |
*/ |
| 165 |
unsigned int |
| 166 |
get_n_mpi_processes(const Epetra_Comm &mpi_communicator); |
| 167 |
|
| 168 |
/** |
| 169 |
* Return the number of the present MPI process in the space of processes |
| 170 |
* described by the given communicator. This will be a unique value for |
| 171 |
* each process between zero and (less than) the number of all processes |
| 172 |
* (given by get_n_mpi_processes()). |
| 173 |
*/ |
| 174 |
unsigned int |
| 175 |
get_this_mpi_process(const Epetra_Comm &mpi_communicator); |
| 176 |
|
| 177 |
/** |
| 178 |
* Given a Trilinos Epetra map, create a new map that has the same |
| 179 |
* subdivision of elements to processors but uses the given communicator |
| 180 |
* object instead of the one stored in the first argument. In essence, |
| 181 |
* this means that we create a map that communicates among the same |
| 182 |
* processors in the same way, but using a separate channel. |
| 183 |
* |
| 184 |
* This function is typically used with a communicator that has been |
| 185 |
* obtained by the duplicate_communicator() function. |
| 186 |
*/ |
| 187 |
Epetra_Map |
| 188 |
duplicate_map(const Epetra_BlockMap &map, const Epetra_Comm &comm); |
| 189 |
} // namespace Trilinos |
| 190 |
#endif |
| 191 |
|
| 192 |
|
| 193 |
#ifdef DEAL_II_TRILINOS_WITH_TPETRA |
| 194 |
namespace Trilinos |
| 195 |
{ |
| 196 |
/** |
| 197 |
* Return the underlying MPI_Comm communicator from the |
| 198 |
* <a |
| 199 |
* href="https://docs.trilinos.org/dev/packages/teuchos/doc/html/classTeuchos_1_1Comm.html">Teuchos::Comm</a> |
| 200 |
* communicator. |
| 201 |
*/ |
| 202 |
MPI_Comm |
| 203 |
teuchos_comm_to_mpi_comm( |
| 204 |
const Teuchos::RCP<const Teuchos::Comm<int>> &teuchos_comm); |
| 205 |
|
| 206 |
namespace internal |
| 207 |
{ |
| 208 |
/** |
| 209 |
* Creates and returns a |
| 210 |
* <a |
| 211 |
* href="https://docs.trilinos.org/dev/packages/teuchos/doc/html/classTeuchos_1_1RCP.html">Teuchos::RCP</a> |
| 212 |
* object for type T. |
| 213 |
* |
| 214 |
* @note In Trilinos 14.0.0, the function |
| 215 |
* <a |
| 216 |
* href="https://docs.trilinos.org/dev/packages/teuchos/doc/html/namespaceTeuchos.html#a280c0ab8c9ee8d0481114d4edf5a3393">Teuchos::make_rcp()</a> |
| 217 |
* was introduced, which should be preferred to this function. |
| 218 |
*/ |
| 219 |
# if defined(DOXYGEN) || !DEAL_II_TRILINOS_VERSION_GTE(14, 0, 0) |
| 220 |
template <class T, class... Args> |
| 221 |
Teuchos::RCP<T> |
| 222 |
make_rcp(Args &&...args); |
| 223 |
# else |
| 224 |
using Teuchos::make_rcp; |
| 225 |
# endif // defined DOXYGEN || !DEAL_II_TRILINOS_VERSION_GTE(14, 0, 0) |
| 226 |
} // namespace internal |
| 227 |
|
| 228 |
|
| 229 |
|
| 230 |
/* ------------------------- Inline functions ---------------------- */ |
| 231 |
namespace internal |
| 232 |
{ |
| 233 |
# if !DEAL_II_TRILINOS_VERSION_GTE(14, 0, 0) |
| 234 |
template <class T, class... Args> |
| 235 |
inline Teuchos::RCP<T> |
| 236 |
make_rcp(Args &&...args) |
| 237 |
{ |
| 238 |
return Teuchos::RCP<T>(new T(std::forward<Args>(args)...)); |
| 239 |
} |
| 240 |
# endif // !DEAL_II_TRILINOS_VERSION_GTE(14, 0, 0) |
| 241 |
} // namespace internal |
| 242 |
|
| 243 |
} // namespace Trilinos |
| 244 |
#endif // DEAL_II_TRILINOS_WITH_TPETRA |
| 245 |
|
| 246 |
} // namespace Utilities |
| 247 |
|
| 248 |
DEAL_II_NAMESPACE_CLOSE |
| 249 |
|
| 250 |
#endif |