Debian Codegraph @

include/deal.II/base/mpi_noncontiguous_partitioner.h

raw
1 // ------------------------------------------------------------------------
2 //
3 // SPDX-License-Identifier: LGPL-2.1-or-later
4 // Copyright (C) 2020 - 2025 by the deal.II authors
5 //
6 // This file is part of the deal.II library.
7 //
8 // Part of the source code is dual licensed under Apache-2.0 WITH
9 // LLVM-exception OR LGPL-2.1-or-later. Detailed license information
10 // governing the source code and code contributions can be found in
11 // LICENSE.md and CONTRIBUTING.md at the top level directory of deal.II.
12 //
13 // ------------------------------------------------------------------------
14
15 #ifndef dealii_mpi_noncontiguous_partitioner_h
16 #define dealii_mpi_noncontiguous_partitioner_h
17
18 #include <deal.II/base/config.h>
19
20 #include <deal.II/base/array_view.h>
21 #include <deal.II/base/communication_pattern_base.h>
22 #include <deal.II/base/mpi_stub.h>
23 #include <deal.II/base/types.h>
24
25 #include <deal.II/lac/vector_operation.h>
26
27
28 DEAL_II_NAMESPACE_OPEN
29
30 namespace Utilities
31 {
32 namespace MPI
33 {
34 /**
35 * A flexible Partitioner class, which does not impose restrictions
36 * regarding the order of the underlying index sets. In other words,
37 * this class implements the interface of the
38 * Utilities::MPI::CommunicationPatternBase base class with no
39 * assumption that every process stores a contiguous part of the
40 * array of objects, but that indeed the locally owned indices
41 * can be an arbitrary subset of all indices of elements of the array
42 * to which they refer.
43 *
44 * If you want to store only contiguous parts of these arrays on
45 * each process, take a look at Utilities::MPI::Partitioner.
46 */
47 class NoncontiguousPartitioner
48 : public Utilities::MPI::CommunicationPatternBase
49 {
50 public:
51 /**
52 * Default constructor. Requires calling one of the reinit() functions
53 * to create a valid object.
54 */
55 NoncontiguousPartitioner() = default;
56
57 /**
58 * Constructor. Set up point-to-point communication pattern based on the
59 * IndexSets arguments @p indexset_locally_owned and @p indexset_ghost for the MPI
60 * communicator @p communicator.
61 */
62 NoncontiguousPartitioner(const IndexSet &indexset_locally_owned,
63 const IndexSet &indexset_ghost,
64 const MPI_Comm communicator);
65
66 /**
67 * Constructor. Same as above but for vectors of indices @p indices_locally_owned
68 * and @p indices_ghost. This allows the indices to not be sorted and the
69 * values are read and written automatically at the right position of
70 * the vector during update_values(), update_values_start(), and
71 * update_values_finish(). It is allowed to include entries with the
72 * value numbers::invalid_dof_index which do not take part of the index
73 * exchange but are present in the data vectors as padding.
74 */
75 NoncontiguousPartitioner(
76 const std::vector<types::global_dof_index> &indices_locally_owned,
77 const std::vector<types::global_dof_index> &indices_ghost,
78 const MPI_Comm communicator);
79
80 /**
81 * Fill the vector @p ghost_array according to the precomputed communication
82 * pattern with values from @p locally_owned_array.
83 *
84 * In the default case, only one object is communicated per entry
85 * (`n_components_templated == 1'). If you want to communicate more
86 * entries, you can increase the value of @p n_components_templated in the
87 * case that you know the size at compile time. If you want to set the
88 * size during runtime, you can set @p n_components. However,
89 * @p n_components_templated has to be set to `0` in this case. Either
90 * @p n_components_templated or @p n_components can be set.
91 *
92 * @pre The vectors only have to provide a method begin(), which allows
93 * to access their raw data.
94 *
95 * @pre The size of both vectors must be at least as large as the number
96 * of entries in the index sets passed to the constructors or the
97 * reinit() functions.
98 *
99 * @note This function calls the methods update_values_start() and
100 * update_values_finish() in sequence. Users can call these two
101 * functions separately and hereby overlap communication and
102 * computation.
103 */
104 template <typename Number, unsigned int n_components_templated = 1>
105 void
106 export_to_ghosted_array(
107 const ArrayView<const Number> &locally_owned_array,
108 const ArrayView<Number> &ghost_array,
109 const unsigned int n_components = 0) const;
110
111 /**
112 * Same as above but with an interface similar to
113 * Utilities::MPI::Partitioner::export_to_ghosted_array_start and
114 * Utilities::MPI::Partitioner::export_to_ghosted_array_finish. In this
115 * function, the user can provide the temporary data structures to be
116 * used.
117 *
118 * @pre The size of the @p temporary_storage vector has to be at least
119 * temporary_storage_size. The reason for this is that this vector is
120 * used as buffer for both sending and receiving data.
121 *
122 * @note Any value less than 10 is a valid value of
123 * @p communication_channel.
124 */
125 template <typename Number, unsigned int n_components_templated = 1>
126 void
127 export_to_ghosted_array(
128 const unsigned int communication_channel,
129 const ArrayView<const Number> &locally_owned_array,
130 const ArrayView<Number> &temporary_storage,
131 const ArrayView<Number> &ghost_array,
132 std::vector<MPI_Request> &requests,
133 const unsigned int n_components = 0) const;
134
135 /**
136 * Start update: Data is packed, non-blocking send and receives
137 * are started.
138 *
139 * @note In contrast to the function
140 * Utilities::MPI::Partitioner::export_to_ghosted_array_start, the user
141 * does not pass a reference to the destination vector, since the data
142 * is received into a designated part of the buffer @p temporary_storage. This
143 * allows for padding and other post-processing of the received data.
144 *
145 * @pre The required size of the vectors are the same as in the functions
146 * above.
147 *
148 * @note Any value less than 10 is a valid value of
149 * @p communication_channel.
150 */
151 template <typename Number, unsigned int n_components_templated = 1>
152 void
153 export_to_ghosted_array_start(
154 const unsigned int communication_channel,
155 const ArrayView<const Number> &locally_owned_array,
156 const ArrayView<Number> &temporary_storage,
157 std::vector<MPI_Request> &requests,
158 const unsigned int n_components = 0) const;
159
160 /**
161 * Finish update. The method waits until all data has been sent and
162 * received. Once data from any process is received it is processed and
163 * placed at the right position of the vector @p dst.
164 *
165 * @note In contrast to the function
166 * Utilities::MPI::Partitioner::export_to_ghosted_array_finish, the user
167 * also has to pass a reference to the buffer @p temporary_storage,
168 * since the data has been received into the buffer and not into the
169 * destination vector.
170 *
171 * @pre The required size of the vectors are the same as in the functions
172 * above.
173 */
174 template <typename Number, unsigned int n_components_templated = 1>
175 void
176 export_to_ghosted_array_finish(
177 const ArrayView<const Number> &temporary_storage,
178 const ArrayView<Number> &ghost_array,
179 std::vector<MPI_Request> &requests,
180 const unsigned int n_components = 0) const;
181
182 /**
183 * Similar to the above functions but for importing vector entries
184 * from @p ghost_array to @p locally_owned_storage.
185 *
186 * @note In contrast to the functions in
187 * Utilities::MPI::Partitioner, this function expects that
188 * locally_owned_storage is empty.
189 */
190 template <typename Number>
191 void
192 import_from_ghosted_array(
193 const VectorOperation::values vector_operation,
194 const ArrayView<Number> &ghost_array,
195 const ArrayView<Number> &locally_owned_storage) const;
196
197 /**
198 * Similar to the above function with the difference that
199 * users can provide temporaty arrays. This function calls
200 * import_from_ghosted_array_start() and
201 * import_from_ghosted_array_finish() in sequence.
202 */
203 template <typename Number>
204 void
205 import_from_ghosted_array(const VectorOperation::values vector_operation,
206 const unsigned int communication_channel,
207 const ArrayView<Number> &ghost_array,
208 const ArrayView<Number> &temporary_storage,
209 const ArrayView<Number> &locally_owned_storage,
210 std::vector<MPI_Request> &requests) const;
211
212 /**
213 * Start update for importig values: Data is packed, non-blocking send
214 * and receives are started.
215 */
216 template <typename Number>
217 void
218 import_from_ghosted_array_start(
219 const VectorOperation::values vector_operation,
220 const unsigned int communication_channel,
221 const ArrayView<Number> &ghost_array,
222 const ArrayView<Number> &temporary_storage,
223 std::vector<MPI_Request> &requests) const;
224
225 /**
226 * Finish update for importing values. The method waits until all data has
227 * been sent and received. Once data from any process is received it is
228 * processed and placed at the right position of the vector
229 * @p locally_owned_storage.
230 */
231 template <typename Number>
232 void
233 import_from_ghosted_array_finish(
234 const VectorOperation::values vector_operation,
235 const ArrayView<const Number> &temporary_storage,
236 const ArrayView<Number> &locally_owned_storage,
237 std::vector<MPI_Request> &requests) const;
238
239 /**
240 * Returns the number of processes this process sends data to and the
241 * number of processes this process receives data from.
242 */
243 std::pair<unsigned int, unsigned int>
244 n_targets() const;
245
246 /**
247 * Return the size of the temporary storage needed by the
248 * export_to_ghosted_array() functions, if the temporary storage is
249 * handled by the user code.
250 */
251 unsigned int
252 temporary_storage_size() const;
253
254 /**
255 * Return memory consumption in Byte.
256 */
257 types::global_dof_index
258 memory_consumption();
259
260 /**
261 * Return the underlying MPI communicator.
262 */
263 MPI_Comm
264 get_mpi_communicator() const override;
265
266 void
267 reinit(const IndexSet &locally_owned_indices,
268 const IndexSet &ghost_indices,
269 const MPI_Comm communicator) override;
270
271 /**
272 * Initialize the inner data structures using explicit sets of
273 * indices. See the documentation of the other reinit() function for
274 * what the function does.
275 */
276 void
277 reinit(const std::vector<types::global_dof_index> &locally_owned_indices,
278 const std::vector<types::global_dof_index> &ghost_indices,
279 const MPI_Comm communicator);
280
281 private:
282 /**
283 * MPI communicator.
284 */
285 MPI_Comm communicator;
286
287 /**
288 * The ranks this process sends data to.
289 */
290 std::vector<unsigned int> send_ranks;
291
292 /**
293 * Offset of each process within send_buffer.
294 *
295 * @note Together with `send_indices` this forms a CRS data structure.
296 */
297 std::vector<types::global_dof_index> send_ptr;
298
299 /**
300 * Local index of each entry in send_buffer within the destination
301 * vector.
302 *
303 * @note Together with `send_ptr` this forms a CRS data structure.
304 */
305 std::vector<types::global_dof_index> send_indices;
306
307 /**
308 * The ranks this process receives data from.
309 */
310 std::vector<unsigned int> recv_ranks;
311
312 /**
313 * Offset of each process within recv_buffer.
314 *
315 * @note Together with `recv_indices` this forms a CRS data structure.
316 */
317 std::vector<types::global_dof_index> recv_ptr;
318
319 /**
320 * Local index of each entry in recv_buffer within the destination
321 * vector.
322 *
323 * In the case that an entry is more than once requested by the
324 * same rank, local index of each entry in recv_buffer within
325 * recv_indices_duplicates_ptr.
326 *
327 * @note Together with `recv_ptr` this forms a CRS data structure.
328 */
329 std::vector<types::global_dof_index> recv_indices;
330
331 /**
332 * In the case that an entry is more than once requested by the
333 * same rank, offset of each index in recv_indices_duplicates.
334 */
335 std::vector<unsigned int> recv_indices_duplicates_ptr;
336
337 /**
338 * Local index of each entry within the destination vector in the
339 * case that an entry is more than once requested by the same rank.
340 *
341 * @note Together with `recv_indices_duplicates_ptr`
342 * this forms a CRS data structure.
343 */
344 std::vector<unsigned int> recv_indices_duplicates;
345
346 /**
347 * Buffer containing the values sorted by rank for sending and receiving.
348 *
349 * @note Only allocated if not provided externally by user.
350 *
351 * @note At this place we do not know the type of the data to be sent. So
352 * we use an arbitrary type of size 1 byte. The type is cast to the
353 * requested type in the relevant functions.
354 */
355 mutable std::vector<std::uint8_t> buffers;
356
357 /**
358 * MPI requests for sending and receiving.
359 *
360 * @note Only allocated if not provided externally by user.
361 */
362 mutable std::vector<MPI_Request> requests;
363 };
364
365 } // namespace MPI
366 } // namespace Utilities
367
368 DEAL_II_NAMESPACE_CLOSE
369
370 #endif