Debian Codegraph @

include/deal.II/lac/petsc_communication_pattern.h

raw
1 // ------------------------------------------------------------------------
2 //
3 // SPDX-License-Identifier: LGPL-2.1-or-later
4 // Copyright (C) 2023 - 2024 by the deal.II authors
5 //
6 // This file is part of the deal.II library.
7 //
8 // Part of the source code is dual licensed under Apache-2.0 WITH
9 // LLVM-exception OR LGPL-2.1-or-later. Detailed license information
10 // governing the source code and code contributions can be found in
11 // LICENSE.md and CONTRIBUTING.md at the top level directory of deal.II.
12 //
13 // ------------------------------------------------------------------------
14
15 #ifndef dealii_petsc_communication_pattern_h
16 #define dealii_petsc_communication_pattern_h
17
18 #include <deal.II/base/config.h>
19
20 #ifdef DEAL_II_WITH_PETSC
21
22 # include <deal.II/base/array_view.h>
23 # include <deal.II/base/communication_pattern_base.h>
24 # include <deal.II/base/index_set.h>
25
26 # include <deal.II/lac/vector_operation.h>
27
28 # include <petscsf.h>
29
30 # include <vector>
31
32 DEAL_II_NAMESPACE_OPEN
33
34 /**
35 * @addtogroup PETScWrappers
36 * @{
37 */
38 namespace PETScWrappers
39 {
40 /**
41 * CommunicationPattern implementation based on the PetscSF object.
42 * This class implements the same communication patterns of
43 * Utilities::MPI::NoncontiguousPartitioner, internally using PetscSF
44 * API calls.
45 *
46 * For additional information, see the paper @cite zhang2022petscsf
47 */
48 class CommunicationPattern : public Utilities::MPI::CommunicationPatternBase
49 {
50 public:
51 /**
52 * Default constructor.
53 */
54 CommunicationPattern();
55
56 /**
57 * Destructor.
58 */
59 virtual ~CommunicationPattern() override;
60
61 virtual void
62 reinit(const IndexSet &locally_owned_indices,
63 const IndexSet &ghost_indices,
64 const MPI_Comm communicator) override;
65
66 /**
67 * Reinitialize the communication pattern. The argument @p indices_locally_owned
68 * and @p indices_want indicates the owned and required dofs, respectively.
69 * This allows the indices to not be sorted and to include entries with
70 * the value numbers::invalid_dof_index which do not take part of the index
71 * exchange but are present in the data vectors as padding.
72 *
73 * The export_to_ghost_array will populate an array with the values
74 * associated to the @p indices_want only.
75 *
76 * This emulates the corresponding constructor in
77 * Utilities::MPI::NoncontiguousPartitioner.
78 */
79 void
80 reinit(const std::vector<types::global_dof_index> &indices_locally_owned,
81 const std::vector<types::global_dof_index> &indices_want,
82 const MPI_Comm communicator);
83
84 /**
85 * Reinitialization that takes the number of locally-owned degrees of
86 * freedom @p local_size and an index set for the required ghost indices
87 * @p ghost_indices.
88 *
89 * The local index range is translated to global indices in an ascending
90 * and one-to-one fashion, i.e., the indices of process $p$ sit exactly
91 * between the indices of the processes $p-1$ and $p+1$, respectively.
92 *
93 * The export_to_ghost_array will populate an array containing
94 * values from locally-owned AND ghost indices, as for the relevant
95 * set of dofs of a usual FEM simulation.
96 */
97 void
98 reinit(const types::global_dof_index local_size,
99 const IndexSet &ghost_indices,
100 const MPI_Comm communicator);
101
102 /**
103 * Fill the vector @p ghost_array according to the precomputed communication
104 * pattern with values from @p locally_owned_array.
105 */
106 template <typename Number>
107 void
108 export_to_ghosted_array(const ArrayView<const Number> &locally_owned_array,
109 const ArrayView<Number> &ghost_array) const;
110
111 /**
112 * Start the communication round to fill the vector @p ghost_array according
113 * to the precomputed communication pattern with values from
114 * @p locally_owned_array. It can be overlapped with other communications.
115 */
116 template <typename Number>
117 void
118 export_to_ghosted_array_start(
119 const ArrayView<const Number> &locally_owned_array,
120 const ArrayView<Number> &ghost_array) const;
121
122 /**
123 * Finish the communication round to fill the vector @p ghost_array according
124 * to the precomputed communication pattern with values from
125 * @p locally_owned_array. It can be overlapped with other communications.
126 */
127 template <typename Number>
128 void
129 export_to_ghosted_array_finish(
130 const ArrayView<const Number> &locally_owned_array,
131 const ArrayView<Number> &ghost_array) const;
132
133 /**
134 * Modify the vector @p locally_owned_array according to the precomputed communication
135 * pattern and the operation @p op with values from @p ghost_array.
136 */
137 template <typename Number>
138 void
139 import_from_ghosted_array(
140 const VectorOperation::values op,
141 const ArrayView<const Number> &ghost_array,
142 const ArrayView<Number> &locally_owned_array) const;
143
144 /**
145 * Start the communication round to modify the vector @p locally_owned_array according
146 * to the precomputed communication pattern and the operation @p op with
147 * values from
148 * @p ghost_array. It can be overlapped with other communications.
149 */
150 template <typename Number>
151 void
152 import_from_ghosted_array_start(
153 const VectorOperation::values op,
154 const ArrayView<const Number> &ghost_array,
155 const ArrayView<Number> &locally_owned_array) const;
156
157 /**
158 * Finish the communication round to modify the vector @p locally_owned_array according
159 * to the precomputed communication pattern and the operation @p op with
160 * values from
161 * @p ghost_array. It can be overlapped with other communications.
162 */
163 template <typename Number>
164 void
165 import_from_ghosted_array_finish(
166 const VectorOperation::values op,
167 const ArrayView<const Number> &ghost_array,
168 const ArrayView<Number> &locally_owned_array) const;
169
170 /**
171 * Return the underlying MPI communicator.
172 */
173 MPI_Comm
174 get_mpi_communicator() const override;
175
176 /**
177 * Conversion operator to gain access to the underlying PETSc object.
178 */
179 operator const PetscSF &() const
180 {
181 return sf;
182 }
183
184 /**
185 * Reset the object.
186 */
187 void
188 clear();
189
190 protected:
191 /**
192 * A generic PetscSF object that will perform the communication.
193 */
194 PetscSF sf;
195
196 /**
197 * General setup
198 */
199 void
200 do_reinit(const std::vector<PetscInt> &inidx,
201 const std::vector<PetscInt> &inloc,
202 const std::vector<PetscInt> &outidx,
203 const std::vector<PetscInt> &outloc,
204 const MPI_Comm communicator);
205 };
206
207 /**
208 * Partitioner implementation based on the PetscSF object.
209 * This class implements the same communication patterns of
210 * Utilities::MPI::Partitioner, internally using PetscSF
211 * API calls.
212 * Differently from the Utilities::MPI::Partitioner implementation, here we
213 * don't need to specify the communication channel, the temporary storage,
214 * and the MPI requests within export and import functions. Moreover,
215 * the import API does not zero the input ghost array.
216 *
217 * For additional information, see the paper @cite zhang2022petscsf
218 */
219 class Partitioner : public Utilities::MPI::CommunicationPatternBase
220 {
221 public:
222 /**
223 * Default constructor.
224 */
225 Partitioner();
226
227 /**
228 * Destructor.
229 */
230 virtual ~Partitioner() override = default;
231
232 /**
233 * Reinitialize the partitioner. As for the Utilities::MPI::Partitioner,
234 * any entry of @p ghost_indices that is also present in
235 * @p locally_owned_indices is discarded.
236 */
237 virtual void
238 reinit(const IndexSet &locally_owned_indices,
239 const IndexSet &ghost_indices,
240 const MPI_Comm communicator) override;
241
242 /**
243 * Reinitialize the partitioner. As for the Utilities::MPI::Partitioner,
244 * any entry of @p ghost_indices that is also present in
245 * @p locally_owned_indices is discarded. This reinitialization will allow
246 * to perform communications either using a ghost data array of the size
247 * of @p ghost_indices or of @p larger_ghost_indices.
248 */
249 void
250 reinit(const IndexSet &locally_owned_indices,
251 const IndexSet &ghost_indices,
252 const IndexSet &larger_ghost_indices,
253 const MPI_Comm communicator);
254
255 /**
256 * Return the actual number of ghost indices.
257 */
258 unsigned int
259 n_ghost_indices() const
260 {
261 return n_ghost_indices_data;
262 }
263
264 /**
265 * Return an IndexSet representation of the actual ghost indices.
266 */
267 const IndexSet &
268 ghost_indices() const
269 {
270 return ghost_indices_data;
271 }
272
273 /**
274 * Fill the vector @p ghost_array according to the precomputed communication
275 * pattern with values from @p locally_owned_array.
276 */
277 template <typename Number>
278 void
279 export_to_ghosted_array(const ArrayView<const Number> &locally_owned_array,
280 const ArrayView<Number> &ghost_array) const;
281
282 /**
283 * Start the communication round to fill the vector @p ghost_array according
284 * to the precomputed communication pattern with values from
285 * @p locally_owned_array. It can be overlapped with other communications.
286 * Differently from the Utilities::MPI::Partitioner implementation, here we
287 * don't need to specify the communication channel, the temporary storage,
288 * and the MPI requests.
289 */
290 template <typename Number>
291 void
292 export_to_ghosted_array_start(
293 const ArrayView<const Number> &locally_owned_array,
294 const ArrayView<Number> &ghost_array) const;
295
296 /**
297 * Finish the communication round to fill the vector @p ghost_array according
298 * to the precomputed communication pattern with values from
299 * @p locally_owned_array. It can be overlapped with other communications.
300 * Differently from the Utilities::MPI::Partitioner implementation, here we
301 * don't need to specify the communication channel, the temporary storage,
302 * and the MPI requests.
303 */
304 template <typename Number>
305 void
306 export_to_ghosted_array_finish(
307 const ArrayView<const Number> &locally_owned_array,
308 const ArrayView<Number> &ghost_array) const;
309
310 /**
311 * Modify the vector @p locally_owned_array according to the precomputed communication
312 * pattern and the operation @p op with values from @p ghost_array.
313 */
314 template <typename Number>
315 void
316 import_from_ghosted_array(
317 const VectorOperation::values op,
318 const ArrayView<const Number> &ghost_array,
319 const ArrayView<Number> &locally_owned_array) const;
320
321 /**
322 * Start the communication round to modify the vector @p locally_owned_array according
323 * to the precomputed communication pattern and the operation @p op with
324 * values from
325 * @p ghost_array. It can be overlapped with other communications.
326 * Differently from the Utilities::MPI::Partitioner implementation, here we
327 * don't need to specify the communication channel, the temporary storage,
328 * and the MPI requests.
329 */
330 template <typename Number>
331 void
332 import_from_ghosted_array_start(
333 const VectorOperation::values op,
334 const ArrayView<const Number> &ghost_array,
335 const ArrayView<Number> &locally_owned_array) const;
336
337 /**
338 * Finish the communication round to modify the vector @p locally_owned_array according
339 * to the precomputed communication pattern and the operation @p op with
340 * values from
341 * @p ghost_array. It can be overlapped with other communications.
342 * Differently from the Utilities::MPI::Partitioner implementation, here we
343 * don't need to specify the communication channel, the temporary storage,
344 * and the MPI requests.
345 */
346 template <typename Number>
347 void
348 import_from_ghosted_array_finish(
349 const VectorOperation::values op,
350 const ArrayView<const Number> &ghost_array,
351 const ArrayView<Number> &locally_owned_array) const;
352
353 /**
354 * Return the underlying MPI communicator.
355 */
356 MPI_Comm
357 get_mpi_communicator() const override;
358
359 protected:
360 CommunicationPattern ghost, larger_ghost;
361 IndexSet ghost_indices_data;
362 types::global_dof_index n_ghost_indices_data;
363 types::global_dof_index n_ghost_indices_larger;
364 };
365
366 } // namespace PETScWrappers
367
368 /** @} */
369
370 DEAL_II_NAMESPACE_CLOSE
371
372 #else
373
374 // Make sure the scripts that create the C++20 module input files have
375 // something to latch on if the preprocessor #ifdef above would
376 // otherwise lead to an empty content of the file.
377 DEAL_II_NAMESPACE_OPEN
378 DEAL_II_NAMESPACE_CLOSE
379
380 #endif
381
382 #endif