| 1 |
// ------------------------------------------------------------------------ |
| 2 |
// |
| 3 |
// SPDX-License-Identifier: LGPL-2.1-or-later |
| 4 |
// Copyright (C) 2023 - 2025 by the deal.II authors |
| 5 |
// |
| 6 |
// This file is part of the deal.II library. |
| 7 |
// |
| 8 |
// Part of the source code is dual licensed under Apache-2.0 WITH |
| 9 |
// LLVM-exception OR LGPL-2.1-or-later. Detailed license information |
| 10 |
// governing the source code and code contributions can be found in |
| 11 |
// LICENSE.md and CONTRIBUTING.md at the top level directory of deal.II. |
| 12 |
// |
| 13 |
// ------------------------------------------------------------------------ |
| 14 |
|
| 15 |
#ifndef dealii_init_finalize_h |
| 16 |
#define dealii_init_finalize_h |
| 17 |
|
| 18 |
#include <deal.II/base/config.h> |
| 19 |
|
| 20 |
#include <deal.II/base/mpi_stub.h> |
| 21 |
#include <deal.II/base/numbers.h> |
| 22 |
#include <deal.II/base/types.h> |
| 23 |
|
| 24 |
#include <boost/signals2.hpp> |
| 25 |
|
| 26 |
#include <set> |
| 27 |
|
| 28 |
DEAL_II_NAMESPACE_OPEN |
| 29 |
|
| 30 |
/** |
| 31 |
* The enum type given to the constructor of InitFinalize telling which |
| 32 |
* external libraries should be initialized and finalized by deal.II. |
| 33 |
*/ |
| 34 |
enum class InitializeLibrary |
| 35 |
{ |
| 36 |
/** |
| 37 |
* No external library will be initialized/finalized. |
| 38 |
*/ |
| 39 |
None = 0, |
| 40 |
/** |
| 41 |
* Initialize/finalize MPI. |
| 42 |
*/ |
| 43 |
MPI = 1, |
| 44 |
/** |
| 45 |
* Initialize/finalize Kokkos. |
| 46 |
*/ |
| 47 |
Kokkos = 2, |
| 48 |
/** |
| 49 |
* Initialize/finalize both PETSc and SLEPc. |
| 50 |
*/ |
| 51 |
SLEPc = 4, |
| 52 |
/** |
| 53 |
* Initialize/finalize PETSc. |
| 54 |
*/ |
| 55 |
PETSc = 8, |
| 56 |
/** |
| 57 |
* Initialize/finalize Zoltan. |
| 58 |
*/ |
| 59 |
Zoltan = 16, |
| 60 |
/** |
| 61 |
* Initialize/finalize P4EST and SC. |
| 62 |
*/ |
| 63 |
P4EST = 32 |
| 64 |
}; |
| 65 |
|
| 66 |
|
| 67 |
|
| 68 |
/** |
| 69 |
* Global operator which returns an object in which all bits are set which are |
| 70 |
* either set in the first or the second argument. |
| 71 |
*/ |
| 72 |
inline InitializeLibrary |
| 73 |
operator|(const InitializeLibrary f1, const InitializeLibrary f2) |
| 74 |
{ |
| 75 |
return static_cast<InitializeLibrary>(static_cast<unsigned int>(f1) | |
| 76 |
static_cast<unsigned int>(f2)); |
| 77 |
} |
| 78 |
|
| 79 |
|
| 80 |
|
| 81 |
/** |
| 82 |
* Global operator which returns an object in which all bits are set that are |
| 83 |
* set both in the first and second argument. |
| 84 |
*/ |
| 85 |
inline InitializeLibrary |
| 86 |
operator&(const InitializeLibrary f1, const InitializeLibrary f2) |
| 87 |
{ |
| 88 |
return static_cast<InitializeLibrary>(static_cast<unsigned int>(f1) & |
| 89 |
static_cast<unsigned int>(f2)); |
| 90 |
} |
| 91 |
|
| 92 |
|
| 93 |
|
| 94 |
/** |
| 95 |
* A class that is used to initialize and finalize deal.II and the external |
| 96 |
* libraries requested by the user. |
| 97 |
*/ |
| 98 |
class InitFinalize |
| 99 |
{ |
| 100 |
public: |
| 101 |
/** |
| 102 |
* Initialize deal.II and the requested external libraries. |
| 103 |
* |
| 104 |
* @note This function calls MultithreadInfo::set_thread_limit() with |
| 105 |
* either @p max_num_threads or, following the discussion above, a |
| 106 |
* number of threads equal to the number of cores allocated to this MPI |
| 107 |
* process. However, MultithreadInfo::set_thread_limit() in turn also |
| 108 |
* evaluates the environment variable DEAL_II_NUM_THREADS. Finally, the |
| 109 |
* worker threads can only be created on cores to which the current MPI |
| 110 |
* process has access to; some MPI implementations limit the number of |
| 111 |
* cores each process may access to one or a subset of cores in order to |
| 112 |
* ensure better cache behavior. Consequently, the number of threads |
| 113 |
* that will really be created will be the minimum of the argument |
| 114 |
* passed here, the environment variable (if set), and the number of |
| 115 |
* cores accessible to the thread. |
| 116 |
* |
| 117 |
* @note MultithreadInfo::set_thread_limit() can only work if it is |
| 118 |
* called before any threads are created. The safest place for a call to |
| 119 |
* it is therefore at the beginning of <code>main()</code>. |
| 120 |
* Consequently, this extends to the current class: the best place to |
| 121 |
* create an object of this type is also at or close to the top of |
| 122 |
* <code>main()</code>. |
| 123 |
*/ |
| 124 |
InitFinalize( |
| 125 |
int &argc, |
| 126 |
char **&argv, |
| 127 |
const InitializeLibrary &libraries, |
| 128 |
const unsigned int max_num_threads = numbers::invalid_unsigned_int); |
| 129 |
|
| 130 |
/** |
| 131 |
* Destructor. Finalize deal.II and the external libraries initialize by |
| 132 |
* deal.II. |
| 133 |
*/ |
| 134 |
~InitFinalize(); |
| 135 |
|
| 136 |
/** |
| 137 |
* Finalize deal.II and the external libraries initialize by |
| 138 |
* deal.II. |
| 139 |
*/ |
| 140 |
void |
| 141 |
finalize(); |
| 142 |
|
| 143 |
/** |
| 144 |
* Register a reference to an MPI_Request |
| 145 |
* on which we need to call `MPI_Wait` before calling `MPI_Finalize`. |
| 146 |
* |
| 147 |
* The object @p request needs to exist when MPI_Finalize is called, which means the |
| 148 |
* request is typically statically allocated. Otherwise, you need to call |
| 149 |
* unregister_request() before the request goes out of scope. Note that it |
| 150 |
* is acceptable for a request to be already waited on (and consequently |
| 151 |
* reset to MPI_REQUEST_NULL). |
| 152 |
* |
| 153 |
* It is acceptable to call this function more than once with the same |
| 154 |
* instance (as it is done in the example below). |
| 155 |
* |
| 156 |
* Typically, this function is used by CollectiveMutex and not directly, |
| 157 |
* but it can also be used directly like this: |
| 158 |
* @code |
| 159 |
* void my_fancy_communication() |
| 160 |
* { |
| 161 |
* static MPI_Request request = MPI_REQUEST_NULL; |
| 162 |
* MPI_InitFinalize::register_request(request); |
| 163 |
* MPI_Wait(&request, MPI_STATUS_IGNORE); |
| 164 |
* // [some algorithm that is not safe to be executed twice in a row.] |
| 165 |
* MPI_IBarrier(comm, &request); |
| 166 |
* } |
| 167 |
* @endcode |
| 168 |
*/ |
| 169 |
static void |
| 170 |
register_request(MPI_Request &request); |
| 171 |
|
| 172 |
/** |
| 173 |
* Unregister a request previously added using register_request(). |
| 174 |
*/ |
| 175 |
static void |
| 176 |
unregister_request(MPI_Request &request); |
| 177 |
|
| 178 |
/** |
| 179 |
* A structure that has boost::signal objects to register a call back |
| 180 |
* to run after MPI init or finalize. |
| 181 |
* |
| 182 |
* For documentation on signals, see |
| 183 |
* http://www.boost.org/doc/libs/release/libs/signals2 . |
| 184 |
*/ |
| 185 |
struct Signals |
| 186 |
{ |
| 187 |
/** |
| 188 |
* A signal that is triggered immediately after we have |
| 189 |
* initialized the MPI context with <code>MPI_Init()</code>. |
| 190 |
*/ |
| 191 |
boost::signals2::signal<void()> at_mpi_init; |
| 192 |
|
| 193 |
/** |
| 194 |
* A signal that is triggered just before we close the MPI context |
| 195 |
* with <code>MPI_Finalize()</code>. It can be used to deallocate |
| 196 |
* statically allocated MPI resources that need to be deallocated |
| 197 |
* before <code>MPI_Finalize()</code> is called. |
| 198 |
*/ |
| 199 |
boost::signals2::signal<void()> at_mpi_finalize; |
| 200 |
}; |
| 201 |
|
| 202 |
static Signals signals; |
| 203 |
|
| 204 |
private: |
| 205 |
InitializeLibrary libraries; |
| 206 |
|
| 207 |
/** |
| 208 |
* Requests to MPI_Wait before finalizing |
| 209 |
*/ |
| 210 |
static std::set<MPI_Request *> requests; |
| 211 |
|
| 212 |
bool is_finalized = false; |
| 213 |
|
| 214 |
#ifdef DEAL_II_WITH_PETSC |
| 215 |
bool finalize_petscslepc; |
| 216 |
#endif |
| 217 |
}; |
| 218 |
|
| 219 |
DEAL_II_NAMESPACE_CLOSE |
| 220 |
|
| 221 |
#endif |