| 1 |
// ------------------------------------------------------------------------ |
| 2 |
// |
| 3 |
// SPDX-License-Identifier: LGPL-2.1-or-later |
| 4 |
// Copyright (C) 2017 - 2024 by the deal.II authors |
| 5 |
// |
| 6 |
// This file is part of the deal.II library. |
| 7 |
// |
| 8 |
// Part of the source code is dual licensed under Apache-2.0 WITH |
| 9 |
// LLVM-exception OR LGPL-2.1-or-later. Detailed license information |
| 10 |
// governing the source code and code contributions can be found in |
| 11 |
// LICENSE.md and CONTRIBUTING.md at the top level directory of deal.II. |
| 12 |
// |
| 13 |
// ------------------------------------------------------------------------ |
| 14 |
|
| 15 |
#ifndef dealii_process_grid_h |
| 16 |
#define dealii_process_grid_h |
| 17 |
|
| 18 |
#include <deal.II/base/config.h> |
| 19 |
|
| 20 |
#include <deal.II/base/exceptions.h> |
| 21 |
#include <deal.II/base/mpi.h> |
| 22 |
|
| 23 |
#ifdef DEAL_II_WITH_SCALAPACK |
| 24 |
|
| 25 |
DEAL_II_NAMESPACE_OPEN |
| 26 |
|
| 27 |
|
| 28 |
// Forward declaration of class ScaLAPACKMatrix for ProcessGrid |
| 29 |
# ifndef DOXYGEN |
| 30 |
template <typename NumberType> |
| 31 |
class ScaLAPACKMatrix; |
| 32 |
# endif |
| 33 |
|
| 34 |
namespace Utilities |
| 35 |
{ |
| 36 |
namespace MPI |
| 37 |
{ |
| 38 |
/** |
| 39 |
* A class taking care of setting up a two-dimensional processor grid. |
| 40 |
* For example an MPI communicator with 5 processes can be arranged into a |
| 41 |
* 2x2 grid with the 5-th processor being inactive: |
| 42 |
* @code |
| 43 |
* | 0 | 1 |
| 44 |
* -----| ------- |----- |
| 45 |
* 0 | P0 | P1 |
| 46 |
* | | |
| 47 |
* -----| ------- |----- |
| 48 |
* 1 | P2 | P3 |
| 49 |
* @endcode |
| 50 |
* |
| 51 |
* A shared pointer to this class is provided to ScaLAPACKMatrix matrices to |
| 52 |
* perform block-cyclic distribution. |
| 53 |
* |
| 54 |
* Note that this class allows to set up a process grid which has fewer |
| 55 |
* MPI cores than the total number of cores in the communicator. |
| 56 |
* |
| 57 |
* Currently the only place where one would use a ProcessGrid object is |
| 58 |
* in connection with a ScaLAPACKMatrix object. |
| 59 |
*/ |
| 60 |
class ProcessGrid |
| 61 |
{ |
| 62 |
public: |
| 63 |
// Declare class ScaLAPACK as friend to provide access to private members. |
| 64 |
template <typename NumberType> |
| 65 |
friend class dealii::ScaLAPACKMatrix; |
| 66 |
|
| 67 |
/** |
| 68 |
* Constructor for a process grid with @p n_rows and @p n_columns for a given @p mpi_communicator. |
| 69 |
* The product of rows and columns should be less or equal to the total |
| 70 |
* number of cores |
| 71 |
* in the @p mpi_communicator. |
| 72 |
*/ |
| 73 |
ProcessGrid(const MPI_Comm mpi_communicator, |
| 74 |
const unsigned int n_rows, |
| 75 |
const unsigned int n_columns); |
| 76 |
|
| 77 |
/** |
| 78 |
* Constructor for a process grid for a given @p mpi_communicator. |
| 79 |
* In this case the process grid is heuristically chosen based on the |
| 80 |
* dimensions and block-cyclic distribution of a target matrix provided |
| 81 |
* in @p n_rows_matrix, @p n_columns_matrix, @p row_block_size and @p column_block_size. |
| 82 |
* |
| 83 |
* The maximum number of MPI cores one can utilize is |
| 84 |
* $\min\{\frac{M}{MB}\frac{N}{NB}, Np\}$, where $M,N$ are the matrix |
| 85 |
* dimension and $MB,NB$ are the block sizes and $Np$ is the number of |
| 86 |
* processes in the @p mpi_communicator. This function then creates a 2d processor grid |
| 87 |
* assuming the ratio between number of process row $p$ and columns $q$ to |
| 88 |
* be equal the ratio between matrix dimensions $M$ and $N$. |
| 89 |
* |
| 90 |
* For example, a square matrix $640x640$ with the block size $32$ |
| 91 |
* and the @p mpi_communicator with 11 cores will result in the $3x3$ |
| 92 |
* process grid. |
| 93 |
*/ |
| 94 |
ProcessGrid(const MPI_Comm mpi_communicator, |
| 95 |
const unsigned int n_rows_matrix, |
| 96 |
const unsigned int n_columns_matrix, |
| 97 |
const unsigned int row_block_size, |
| 98 |
const unsigned int column_block_size); |
| 99 |
|
| 100 |
/** |
| 101 |
* Destructor. |
| 102 |
*/ |
| 103 |
~ProcessGrid(); |
| 104 |
|
| 105 |
/** |
| 106 |
* Return the number of rows in the processes grid. |
| 107 |
*/ |
| 108 |
unsigned int |
| 109 |
get_process_grid_rows() const; |
| 110 |
|
| 111 |
/** |
| 112 |
* Return the number of columns in the processes grid. |
| 113 |
*/ |
| 114 |
unsigned int |
| 115 |
get_process_grid_columns() const; |
| 116 |
|
| 117 |
/** |
| 118 |
* Return row of this process in the process grid. |
| 119 |
* |
| 120 |
* It's negative for inactive processes. |
| 121 |
*/ |
| 122 |
int |
| 123 |
get_this_process_row() const; |
| 124 |
|
| 125 |
/** |
| 126 |
* Return column of this process in the process grid. |
| 127 |
* |
| 128 |
* It's negative for inactive processes. |
| 129 |
*/ |
| 130 |
int |
| 131 |
get_this_process_column() const; |
| 132 |
|
| 133 |
/** |
| 134 |
* Send @p count values stored consequently starting at @p value from |
| 135 |
* the process with rank zero to processes which |
| 136 |
* are not in the process grid. |
| 137 |
*/ |
| 138 |
template <typename NumberType> |
| 139 |
void |
| 140 |
send_to_inactive(NumberType *value, const int count = 1) const; |
| 141 |
|
| 142 |
/** |
| 143 |
* Return <code>true</code> if the process is active within the grid. |
| 144 |
*/ |
| 145 |
bool |
| 146 |
is_process_active() const; |
| 147 |
|
| 148 |
private: |
| 149 |
/** |
| 150 |
* A private constructor which takes grid dimensions as an |
| 151 |
* <code>std::pair</code>. |
| 152 |
*/ |
| 153 |
ProcessGrid(const MPI_Comm mpi_communicator, |
| 154 |
const std::pair<unsigned int, unsigned int> &grid_dimensions); |
| 155 |
|
| 156 |
/** |
| 157 |
* An MPI communicator with all processes (active and inactive). |
| 158 |
*/ |
| 159 |
MPI_Comm mpi_communicator; |
| 160 |
|
| 161 |
/** |
| 162 |
* An MPI communicator with inactive processes and the process with rank |
| 163 |
* zero. |
| 164 |
*/ |
| 165 |
MPI_Comm mpi_communicator_inactive_with_root; |
| 166 |
|
| 167 |
/** |
| 168 |
* BLACS context. This is equivalent to MPI communicators and is |
| 169 |
* used by ScaLAPACK. |
| 170 |
*/ |
| 171 |
int blacs_context; |
| 172 |
|
| 173 |
/** |
| 174 |
* Rank of this MPI process. |
| 175 |
*/ |
| 176 |
const unsigned int this_mpi_process; |
| 177 |
|
| 178 |
/** |
| 179 |
* Total number of MPI processes. |
| 180 |
*/ |
| 181 |
const unsigned int n_mpi_processes; |
| 182 |
|
| 183 |
/** |
| 184 |
* Number of rows in the processes grid. |
| 185 |
*/ |
| 186 |
int n_process_rows; |
| 187 |
|
| 188 |
/** |
| 189 |
* Number of columns in the processes grid. |
| 190 |
*/ |
| 191 |
int n_process_columns; |
| 192 |
|
| 193 |
/** |
| 194 |
* Row of this process in the grid. |
| 195 |
* |
| 196 |
* It's negative for in-active processes. |
| 197 |
*/ |
| 198 |
int this_process_row; |
| 199 |
|
| 200 |
/** |
| 201 |
* Column of this process in the grid. |
| 202 |
* |
| 203 |
* It's negative for in-active processes. |
| 204 |
*/ |
| 205 |
int this_process_column; |
| 206 |
|
| 207 |
/** |
| 208 |
* A flag which is true for processes within the 2d process grid. |
| 209 |
*/ |
| 210 |
bool mpi_process_is_active; |
| 211 |
}; |
| 212 |
|
| 213 |
/*--------------------- Inline functions --------------------------------*/ |
| 214 |
|
| 215 |
# ifndef DOXYGEN |
| 216 |
|
| 217 |
inline unsigned int |
| 218 |
ProcessGrid::get_process_grid_rows() const |
| 219 |
{ |
| 220 |
return n_process_rows; |
| 221 |
} |
| 222 |
|
| 223 |
|
| 224 |
|
| 225 |
inline unsigned int |
| 226 |
ProcessGrid::get_process_grid_columns() const |
| 227 |
{ |
| 228 |
return n_process_columns; |
| 229 |
} |
| 230 |
|
| 231 |
|
| 232 |
|
| 233 |
inline int |
| 234 |
ProcessGrid::get_this_process_row() const |
| 235 |
{ |
| 236 |
return this_process_row; |
| 237 |
} |
| 238 |
|
| 239 |
|
| 240 |
|
| 241 |
inline int |
| 242 |
ProcessGrid::get_this_process_column() const |
| 243 |
{ |
| 244 |
return this_process_column; |
| 245 |
} |
| 246 |
|
| 247 |
|
| 248 |
|
| 249 |
inline bool |
| 250 |
ProcessGrid::is_process_active() const |
| 251 |
{ |
| 252 |
return mpi_process_is_active; |
| 253 |
} |
| 254 |
|
| 255 |
|
| 256 |
# endif // ifndef DOXYGEN |
| 257 |
|
| 258 |
} // end of namespace MPI |
| 259 |
|
| 260 |
} // end of namespace Utilities |
| 261 |
|
| 262 |
|
| 263 |
DEAL_II_NAMESPACE_CLOSE |
| 264 |
|
| 265 |
#else |
| 266 |
|
| 267 |
// Make sure the scripts that create the C++20 module input files have |
| 268 |
// something to latch on if the preprocessor #ifdef above would |
| 269 |
// otherwise lead to an empty content of the file. |
| 270 |
DEAL_II_NAMESPACE_OPEN |
| 271 |
DEAL_II_NAMESPACE_CLOSE |
| 272 |
|
| 273 |
#endif // DEAL_II_WITH_SCALAPACK |
| 274 |
|
| 275 |
#endif |