Debian Codegraph @

include/deal.II/base/process_grid.h

raw
1 // ------------------------------------------------------------------------
2 //
3 // SPDX-License-Identifier: LGPL-2.1-or-later
4 // Copyright (C) 2017 - 2024 by the deal.II authors
5 //
6 // This file is part of the deal.II library.
7 //
8 // Part of the source code is dual licensed under Apache-2.0 WITH
9 // LLVM-exception OR LGPL-2.1-or-later. Detailed license information
10 // governing the source code and code contributions can be found in
11 // LICENSE.md and CONTRIBUTING.md at the top level directory of deal.II.
12 //
13 // ------------------------------------------------------------------------
14
15 #ifndef dealii_process_grid_h
16 #define dealii_process_grid_h
17
18 #include <deal.II/base/config.h>
19
20 #include <deal.II/base/exceptions.h>
21 #include <deal.II/base/mpi.h>
22
23 #ifdef DEAL_II_WITH_SCALAPACK
24
25 DEAL_II_NAMESPACE_OPEN
26
27
28 // Forward declaration of class ScaLAPACKMatrix for ProcessGrid
29 # ifndef DOXYGEN
30 template <typename NumberType>
31 class ScaLAPACKMatrix;
32 # endif
33
34 namespace Utilities
35 {
36 namespace MPI
37 {
38 /**
39 * A class taking care of setting up a two-dimensional processor grid.
40 * For example an MPI communicator with 5 processes can be arranged into a
41 * 2x2 grid with the 5-th processor being inactive:
42 * @code
43 * | 0 | 1
44 * -----| ------- |-----
45 * 0 | P0 | P1
46 * | |
47 * -----| ------- |-----
48 * 1 | P2 | P3
49 * @endcode
50 *
51 * A shared pointer to this class is provided to ScaLAPACKMatrix matrices to
52 * perform block-cyclic distribution.
53 *
54 * Note that this class allows to set up a process grid which has fewer
55 * MPI cores than the total number of cores in the communicator.
56 *
57 * Currently the only place where one would use a ProcessGrid object is
58 * in connection with a ScaLAPACKMatrix object.
59 */
60 class ProcessGrid
61 {
62 public:
63 // Declare class ScaLAPACK as friend to provide access to private members.
64 template <typename NumberType>
65 friend class dealii::ScaLAPACKMatrix;
66
67 /**
68 * Constructor for a process grid with @p n_rows and @p n_columns for a given @p mpi_communicator.
69 * The product of rows and columns should be less or equal to the total
70 * number of cores
71 * in the @p mpi_communicator.
72 */
73 ProcessGrid(const MPI_Comm mpi_communicator,
74 const unsigned int n_rows,
75 const unsigned int n_columns);
76
77 /**
78 * Constructor for a process grid for a given @p mpi_communicator.
79 * In this case the process grid is heuristically chosen based on the
80 * dimensions and block-cyclic distribution of a target matrix provided
81 * in @p n_rows_matrix, @p n_columns_matrix, @p row_block_size and @p column_block_size.
82 *
83 * The maximum number of MPI cores one can utilize is
84 * $\min\{\frac{M}{MB}\frac{N}{NB}, Np\}$, where $M,N$ are the matrix
85 * dimension and $MB,NB$ are the block sizes and $Np$ is the number of
86 * processes in the @p mpi_communicator. This function then creates a 2d processor grid
87 * assuming the ratio between number of process row $p$ and columns $q$ to
88 * be equal the ratio between matrix dimensions $M$ and $N$.
89 *
90 * For example, a square matrix $640x640$ with the block size $32$
91 * and the @p mpi_communicator with 11 cores will result in the $3x3$
92 * process grid.
93 */
94 ProcessGrid(const MPI_Comm mpi_communicator,
95 const unsigned int n_rows_matrix,
96 const unsigned int n_columns_matrix,
97 const unsigned int row_block_size,
98 const unsigned int column_block_size);
99
100 /**
101 * Destructor.
102 */
103 ~ProcessGrid();
104
105 /**
106 * Return the number of rows in the processes grid.
107 */
108 unsigned int
109 get_process_grid_rows() const;
110
111 /**
112 * Return the number of columns in the processes grid.
113 */
114 unsigned int
115 get_process_grid_columns() const;
116
117 /**
118 * Return row of this process in the process grid.
119 *
120 * It's negative for inactive processes.
121 */
122 int
123 get_this_process_row() const;
124
125 /**
126 * Return column of this process in the process grid.
127 *
128 * It's negative for inactive processes.
129 */
130 int
131 get_this_process_column() const;
132
133 /**
134 * Send @p count values stored consequently starting at @p value from
135 * the process with rank zero to processes which
136 * are not in the process grid.
137 */
138 template <typename NumberType>
139 void
140 send_to_inactive(NumberType *value, const int count = 1) const;
141
142 /**
143 * Return <code>true</code> if the process is active within the grid.
144 */
145 bool
146 is_process_active() const;
147
148 private:
149 /**
150 * A private constructor which takes grid dimensions as an
151 * <code>std::pair</code>.
152 */
153 ProcessGrid(const MPI_Comm mpi_communicator,
154 const std::pair<unsigned int, unsigned int> &grid_dimensions);
155
156 /**
157 * An MPI communicator with all processes (active and inactive).
158 */
159 MPI_Comm mpi_communicator;
160
161 /**
162 * An MPI communicator with inactive processes and the process with rank
163 * zero.
164 */
165 MPI_Comm mpi_communicator_inactive_with_root;
166
167 /**
168 * BLACS context. This is equivalent to MPI communicators and is
169 * used by ScaLAPACK.
170 */
171 int blacs_context;
172
173 /**
174 * Rank of this MPI process.
175 */
176 const unsigned int this_mpi_process;
177
178 /**
179 * Total number of MPI processes.
180 */
181 const unsigned int n_mpi_processes;
182
183 /**
184 * Number of rows in the processes grid.
185 */
186 int n_process_rows;
187
188 /**
189 * Number of columns in the processes grid.
190 */
191 int n_process_columns;
192
193 /**
194 * Row of this process in the grid.
195 *
196 * It's negative for in-active processes.
197 */
198 int this_process_row;
199
200 /**
201 * Column of this process in the grid.
202 *
203 * It's negative for in-active processes.
204 */
205 int this_process_column;
206
207 /**
208 * A flag which is true for processes within the 2d process grid.
209 */
210 bool mpi_process_is_active;
211 };
212
213 /*--------------------- Inline functions --------------------------------*/
214
215 # ifndef DOXYGEN
216
217 inline unsigned int
218 ProcessGrid::get_process_grid_rows() const
219 {
220 return n_process_rows;
221 }
222
223
224
225 inline unsigned int
226 ProcessGrid::get_process_grid_columns() const
227 {
228 return n_process_columns;
229 }
230
231
232
233 inline int
234 ProcessGrid::get_this_process_row() const
235 {
236 return this_process_row;
237 }
238
239
240
241 inline int
242 ProcessGrid::get_this_process_column() const
243 {
244 return this_process_column;
245 }
246
247
248
249 inline bool
250 ProcessGrid::is_process_active() const
251 {
252 return mpi_process_is_active;
253 }
254
255
256 # endif // ifndef DOXYGEN
257
258 } // end of namespace MPI
259
260 } // end of namespace Utilities
261
262
263 DEAL_II_NAMESPACE_CLOSE
264
265 #else
266
267 // Make sure the scripts that create the C++20 module input files have
268 // something to latch on if the preprocessor #ifdef above would
269 // otherwise lead to an empty content of the file.
270 DEAL_II_NAMESPACE_OPEN
271 DEAL_II_NAMESPACE_CLOSE
272
273 #endif // DEAL_II_WITH_SCALAPACK
274
275 #endif