Debian Codegraph @

include/deal.II/lac/petsc_vector.h

raw
1 // ------------------------------------------------------------------------
2 //
3 // SPDX-License-Identifier: LGPL-2.1-or-later
4 // Copyright (C) 2004 - 2024 by the deal.II authors
5 //
6 // This file is part of the deal.II library.
7 //
8 // Part of the source code is dual licensed under Apache-2.0 WITH
9 // LLVM-exception OR LGPL-2.1-or-later. Detailed license information
10 // governing the source code and code contributions can be found in
11 // LICENSE.md and CONTRIBUTING.md at the top level directory of deal.II.
12 //
13 // ------------------------------------------------------------------------
14
15 #ifndef dealii_petsc_vector_h
16 #define dealii_petsc_vector_h
17
18
19 #include <deal.II/base/config.h>
20
21 #ifdef DEAL_II_WITH_PETSC
22
23 # include <deal.II/base/enable_observer_pointer.h>
24 # include <deal.II/base/index_set.h>
25 # include <deal.II/base/partitioner.h>
26
27 # include <deal.II/lac/exceptions.h>
28 # include <deal.II/lac/petsc_vector_base.h>
29 # include <deal.II/lac/vector.h>
30 # include <deal.II/lac/vector_operation.h>
31 # include <deal.II/lac/vector_type_traits.h>
32
33 DEAL_II_NAMESPACE_OPEN
34
35
36 /**
37 * @addtogroup PETScWrappers
38 * @{
39 */
40 namespace PETScWrappers
41 {
42 /**
43 * Namespace for PETSc classes that work in parallel over MPI, such as
44 * distributed vectors and matrices.
45 *
46 * @ingroup PETScWrappers
47 */
48 namespace MPI
49 {
50 /**
51 * Implementation of a parallel vector class based on PETSC and using MPI
52 * communication to synchronize distributed operations. All the
53 * functionality is actually in the base class, except for the calls to
54 * generate a parallel vector. This is possible since PETSc only works on
55 * an abstract vector type and internally distributes to functions that do
56 * the actual work depending on the actual vector type (much like using
57 * virtual functions). Only the functions creating a vector of specific
58 * type differ, and are implemented in this particular class.
59 *
60 *
61 * <h3>Parallel communication model</h3>
62 *
63 * The parallel functionality of PETSc is built on top of the Message
64 * Passing Interface (MPI). MPI's communication model is built on
65 * collective communications: if one process wants something from another,
66 * that other process has to be willing to accept this communication. A
67 * process cannot query data from another process by calling a remote
68 * function, without that other process expecting such a transaction. The
69 * consequence is that most of the operations in the base class of this
70 * class have to be called collectively. For example, if you want to
71 * compute the l2 norm of a parallel vector, @em all processes across
72 * which this vector is shared have to call the @p l2_norm function. If
73 * you don't do this, but instead only call the @p l2_norm function on one
74 * process, then the following happens: This one process will call one of
75 * the collective MPI functions and wait for all the other processes to
76 * join in on this. Since the other processes don't call this function,
77 * you will either get a time-out on the first process, or, worse, by the
78 * time the next a call to a PETSc function generates an MPI message on
79 * the other processes, you will get a cryptic message that only a subset
80 * of processes attempted a communication. These bugs can be very hard to
81 * figure out, unless you are well-acquainted with the communication model
82 * of MPI, and know which functions may generate MPI messages.
83 *
84 * One particular case, where an MPI message may be generated unexpectedly
85 * is discussed below.
86 *
87 *
88 * <h3>Accessing individual elements of a vector</h3>
89 *
90 * PETSc does allow read access to individual elements of a vector, but in
91 * the distributed case only to elements that are stored locally. We
92 * implement this through calls like <tt>d=vec(i)</tt>. However, if you
93 * access an element outside the locally stored range, an exception is
94 * generated.
95 *
96 * In contrast to read access, PETSc (and the respective deal.II wrapper
97 * classes) allow to write (or add) to individual elements of vectors,
98 * even if they are stored on a different process. You can do this
99 * writing, for example, <tt>vec(i)=d</tt> or <tt>vec(i)+=d</tt>, or
100 * similar operations. There is one catch, however, that may lead to very
101 * confusing error messages: PETSc requires application programs to call
102 * the compress() function when they switch from adding, to elements to
103 * writing to elements. The reasoning is that all processes might
104 * accumulate addition operations to elements, even if multiple processes
105 * write to the same elements. By the time we call compress() the next
106 * time, all these additions are executed. However, if one process adds to
107 * an element, and another overwrites to it, the order of execution would
108 * yield non-deterministic behavior if we don't make sure that a
109 * synchronization with compress() happens in between.
110 *
111 * In order to make sure these calls to compress() happen at the
112 * appropriate time, the deal.II wrappers keep a state variable that store
113 * which is the presently allowed operation: additions or writes. If it
114 * encounters an operation of the opposite kind, it calls compress() and
115 * flips the state. This can sometimes lead to very confusing behavior, in
116 * code that may for example look like this:
117 * @code
118 * PETScWrappers::MPI::Vector vector;
119 * ...
120 * // do some write operations on the vector
121 * for (unsigned int i=0; i<vector.size(); ++i)
122 * vector(i) = i;
123 *
124 * // do some additions to vector elements, but only for some elements
125 * for (unsigned int i=0; i<vector.size(); ++i)
126 * if (some_condition(i) == true)
127 * vector(i) += 1;
128 *
129 * // do another collective operation
130 * const double norm = vector.l2_norm();
131 * @endcode
132 *
133 * This code can run into trouble: by the time we see the first addition
134 * operation, we need to flush the overwrite buffers for the vector, and
135 * the deal.II library will do so by calling compress(). However, it will
136 * only do so for all processes that actually do an addition -- if the
137 * condition is never true for one of the processes, then this one will
138 * not get to the actual compress() call, whereas all the other ones do.
139 * This gets us into trouble, since all the other processes hang in the
140 * call to flush the write buffers, while the one other process advances
141 * to the call to compute the l2 norm. At this time, you will get an error
142 * that some operation was attempted by only a subset of processes. This
143 * behavior may seem surprising, unless you know that write/addition
144 * operations on single elements may trigger this behavior.
145 *
146 * The problem described here may be avoided by placing additional calls
147 * to compress(), or making sure that all processes do the same type of
148 * operations at the same time, for example by placing zero additions if
149 * necessary.
150 *
151 * @see
152 * @ref GlossGhostedVector "vectors with ghost elements"
153 *
154 * @ingroup PETScWrappers
155 * @ingroup Vectors
156 */
157 class Vector : public VectorBase
158 {
159 public:
160 /**
161 * Declare type for container size.
162 */
163 using size_type = types::global_dof_index;
164
165 /**
166 * Default constructor. Initialize the vector as empty.
167 */
168 Vector();
169
170 /**
171 * Import VectorBase constructors, including from a PETSc Vec object.
172 */
173 using VectorBase::VectorBase;
174
175 /**
176 * Constructor. Set dimension to @p n and initialize all elements with
177 * zero.
178 *
179 * @arg locally_owned_size denotes the size of the chunk that shall be
180 * stored on the present process.
181 *
182 * @arg communicator denotes the MPI communicator over which the
183 * different parts of the vector shall communicate
184 *
185 * The constructor is made explicit to avoid accidents like this:
186 * <tt>v=0;</tt>. Presumably, the user wants to set every element of the
187 * vector to zero, but instead, what happens is this call:
188 * <tt>v=Vector@<number@>(0);</tt>, i.e. the vector is replaced by one
189 * of length zero.
190 */
191 explicit Vector(const MPI_Comm communicator,
192 const size_type n,
193 const size_type locally_owned_size);
194
195 /**
196 * Copy-constructor from deal.II vectors. Sets the dimension to that of
197 * the given vector, and copies all elements.
198 *
199 * @arg locally_owned_size denotes the size of the chunk that shall be
200 * stored on the present process.
201 *
202 * @arg communicator denotes the MPI communicator over which the
203 * different parts of the vector shall communicate
204 */
205 template <typename Number>
206 explicit Vector(const MPI_Comm communicator,
207 const dealii::Vector<Number> &v,
208 const size_type locally_owned_size);
209
210 /**
211 * Construct a new parallel ghosted PETSc vector from IndexSets.
212 *
213 * Note that @p local must be ascending and 1:1, see
214 * IndexSet::is_ascending_and_one_to_one(). In particular, the DoFs in
215 * @p local need to be contiguous, meaning you can only create vectors
216 * from a DoFHandler with several finite element components if they are
217 * not reordered by component (use a PETScWrappers::BlockVector
218 * otherwise). The global size of the vector is determined by
219 * local.size(). The global indices in @p ghost are supplied as ghost
220 * indices so that they can be read locally.
221 *
222 * Note that the @p ghost IndexSet may be empty and that any indices
223 * already contained in @p local are ignored during construction. That
224 * way, the ghost parameter can equal the set of locally relevant
225 * degrees of freedom, see step-32.
226 *
227 * @note This operation always creates a ghosted vector, which is considered
228 * read-only.
229 *
230 * @see
231 * @ref GlossGhostedVector "vectors with ghost elements"
232 */
233 Vector(const IndexSet &local,
234 const IndexSet &ghost,
235 const MPI_Comm communicator);
236
237 /**
238 * Construct a new parallel PETSc vector without ghost elements from an
239 * IndexSet.
240 *
241 * Note that @p local must be ascending and 1:1, see
242 * IndexSet::is_ascending_and_one_to_one(). In particular, the DoFs in
243 * @p local need to be contiguous, meaning you can only create vectors
244 * from a DoFHandler with several finite element components if they are
245 * not reordered by component (use a PETScWrappers::BlockVector
246 * otherwise).
247 */
248 explicit Vector(const IndexSet &local, const MPI_Comm communicator);
249
250 /**
251 * Copy constructor.
252 */
253 Vector(const Vector &v);
254
255 /**
256 * Release all memory and return to a state just like after having
257 * called the default constructor.
258 */
259 virtual void
260 clear() override;
261
262 /**
263 * Copy the given vector. Resize the present vector if necessary. Also
264 * take over the MPI communicator of @p v.
265 *
266 * The semantics of this operator are complex. If the two vectors have
267 * the same size, and
268 * if either the left or right hand side vector of the assignment (i.e.,
269 * either the input vector on the right hand side, or the calling vector
270 * to the left of the assignment operator) currently has ghost elements,
271 * then the left hand side vector will also have ghost values and will
272 * consequently be a read-only vector (see also the
273 * @ref GlossGhostedVector "glossary entry" on the issue). Otherwise, the
274 * left hand vector will be a writable vector after this operation.
275 * These semantics facilitate having a vector with ghost elements on the
276 * left hand side of the assignment, and a vector without ghost elements
277 * on the right hand side, with the resulting left hand side vector
278 * having the correct values in both its locally owned and its ghost
279 * elements.
280 *
281 * On the other hand, if the left hand side vector does not have the
282 * correct size yet, or is perhaps an entirely uninitialized vector,
283 * then the assignment is simply a copy operation in the usual sense:
284 * In that case, if the right hand side has no ghost elements (i.e.,
285 * is a completely distributed vector), then the left hand side will
286 * have no ghost elements either. And if the right hand side has
287 * ghost elements (and is consequently read-only), then the left
288 * hand side will have these same properties after the operation.
289 */
290 Vector &
291 operator=(const Vector &v);
292
293 /**
294 * Set all components of the vector to the given number @p s. Simply
295 * pass this down to the base class, but we still need to declare this
296 * function to make the example given in the discussion about making the
297 * constructor explicit work.
298 */
299 Vector &
300 operator=(const PetscScalar s);
301
302 /**
303 * Copy the values of a deal.II vector (as opposed to those of the PETSc
304 * vector wrapper class) into this object.
305 *
306 * Contrary to the case of sequential vectors, this operators requires
307 * that the present vector already has the correct size, since we need
308 * to have a partition and a communicator present which we otherwise
309 * can't get from the source vector.
310 */
311 template <typename number>
312 Vector &
313 operator=(const dealii::Vector<number> &v);
314
315 using VectorBase::reinit;
316
317 /**
318 * Change the dimension of the vector to @p N. It is unspecified how
319 * resizing the vector affects the memory allocation of this object;
320 * i.e., it is not guaranteed that resizing it to a smaller size
321 * actually also reduces memory consumption, or if for efficiency the
322 * same amount of memory is used
323 *
324 * @p locally_owned_size denotes how many of the @p N values shall be
325 * stored locally on the present process. for less data.
326 *
327 * @p communicator denotes the MPI communicator henceforth to be used
328 * for this vector.
329 *
330 * If @p omit_zeroing_entries is false, the vector is filled by zeros.
331 * Otherwise, the elements are left an unspecified state.
332 */
333 void
334 reinit(const MPI_Comm communicator,
335 const size_type N,
336 const size_type locally_owned_size,
337 const bool omit_zeroing_entries = false);
338
339 /**
340 * Change the dimension to that of the vector @p v, and also take over
341 * the partitioning into local sizes as well as the MPI communicator.
342 * The same applies as for the other @p reinit function.
343 *
344 * The elements of @p v are not copied, i.e. this function is the same
345 * as calling <tt>reinit(v.size(), v.locally_owned_size(),
346 * omit_zeroing_entries)</tt>.
347 */
348 void
349 reinit(const Vector &v, const bool omit_zeroing_entries = false);
350
351 /**
352 * Reinit as a vector with ghost elements. See the constructor with
353 * same signature for more details.
354 *
355 * @see
356 * @ref GlossGhostedVector "vectors with ghost elements"
357 */
358 void
359 reinit(const IndexSet &local,
360 const IndexSet &ghost,
361 const MPI_Comm communicator);
362
363 /**
364 * Reinit as a vector without ghost elements. See constructor with same
365 * signature for more details.
366 *
367 * @see
368 * @ref GlossGhostedVector "vectors with ghost elements"
369 */
370 void
371 reinit(const IndexSet &local, const MPI_Comm communicator);
372
373 /**
374 * Initialize the vector given to the parallel partitioning described in
375 * @p partitioner.
376 *
377 * You can decide whether your vector will contain ghost elements with
378 * @p make_ghosted.
379 */
380 void
381 reinit(
382 const std::shared_ptr<const Utilities::MPI::Partitioner> &partitioner,
383 const bool make_ghosted = true);
384
385 /**
386 * Print to a stream. @p precision denotes the desired precision with
387 * which values shall be printed, @p scientific whether scientific
388 * notation shall be used. If @p across is @p true then the vector is
389 * printed in a line, while if @p false then the elements are printed on
390 * a separate line each.
391 *
392 * @note This function overloads the one in the base class to ensure
393 * that the right thing happens for parallel vectors that are
394 * distributed across processors.
395 */
396 void
397 print(std::ostream &out,
398 const unsigned int precision = 3,
399 const bool scientific = true,
400 const bool across = true) const;
401
402 /**
403 * @copydoc PETScWrappers::VectorBase::all_zero()
404 *
405 * @note This function overloads the one in the base class to make this
406 * a @ref GlossCollectiveOperation "collective operation".
407 */
408 bool
409 all_zero() const;
410
411 protected:
412 /**
413 * Create a vector of length @p n. For this class, we create a parallel
414 * vector. @p n denotes the total size of the vector to be created. @p
415 * locally_owned_size denotes how many of these elements shall be stored
416 * locally.
417 */
418 virtual void
419 create_vector(const MPI_Comm comm,
420 const size_type n,
421 const size_type locally_owned_size);
422
423
424
425 /**
426 * Create a vector of global length @p n, local size @p
427 * locally_owned_size and with the specified ghost indices. Note that
428 * you need to call update_ghost_values() before accessing those.
429 */
430 virtual void
431 create_vector(const MPI_Comm comm,
432 const size_type n,
433 const size_type locally_owned_size,
434 const IndexSet &ghostnodes);
435 };
436
437
438 // ------------------ template and inline functions -------------
439
440
441 /**
442 * Global function @p swap which overloads the default implementation of
443 * the C++ standard library which uses a temporary object. The function
444 * simply exchanges the data of the two vectors.
445 *
446 * @relatesalso PETScWrappers::MPI::Vector
447 */
448 inline void
449 swap(Vector &u, Vector &v) noexcept
450 {
451 u.swap(v);
452 }
453
454
455 # ifndef DOXYGEN
456
457 template <typename number>
458 Vector::Vector(const MPI_Comm communicator,
459 const dealii::Vector<number> &v,
460 const size_type locally_owned_size)
461 {
462 Vector::create_vector(communicator, v.size(), locally_owned_size);
463
464 *this = v;
465 }
466
467
468
469 inline Vector &
470 Vector::operator=(const PetscScalar s)
471 {
472 VectorBase::operator=(s);
473
474 return *this;
475 }
476
477
478
479 template <typename number>
480 inline Vector &
481 Vector::operator=(const dealii::Vector<number> &v)
482 {
483 Assert(size() == v.size(), ExcDimensionMismatch(size(), v.size()));
484
485 // FIXME: the following isn't necessarily fast, but this is due to
486 // the fact that PETSc doesn't offer an inlined access operator.
487 //
488 // if someone wants to contribute some code: to make this code
489 // faster, one could either first convert all values to PetscScalar,
490 // and then set them all at once using VecSetValues. This has the
491 // drawback that it could take quite some memory, if the vector is
492 // large, and it would in addition allocate memory on the heap, which
493 // is expensive. an alternative would be to split the vector into
494 // chunks of, say, 128 elements, convert a chunk at a time and set it
495 // in the output vector using VecSetValues. since 128 elements is
496 // small enough, this could easily be allocated on the stack (as a
497 // local variable) which would make the whole thing much more
498 // efficient.
499 //
500 // a second way to make things faster is for the special case that
501 // number==PetscScalar. we could then declare a specialization of
502 // this template, and omit the conversion. the problem with this is
503 // that the best we can do is to use VecSetValues, but this isn't
504 // very efficient either: it wants to see an array of indices, which
505 // in this case a) again takes up a whole lot of memory on the heap,
506 // and b) is totally dumb since its content would simply be the
507 // sequence 0,1,2,3,...,n. the best of all worlds would probably be a
508 // function in PETSc that would take a pointer to an array of
509 // PetscScalar values and simply copy n elements verbatim into the
510 // vector...
511 for (size_type i = 0; i < v.size(); ++i)
512 (*this)(i) = v(i);
513
514 compress(VectorOperation::insert);
515
516 return *this;
517 }
518
519
520
521 # endif // DOXYGEN
522 } // namespace MPI
523 } // namespace PETScWrappers
524
525 namespace internal
526 {
527 namespace LinearOperatorImplementation
528 {
529 template <typename>
530 class ReinitHelper;
531
532 /**
533 * A helper class used internally in linear_operator.h. Specialization for
534 * PETScWrappers::MPI::Vector.
535 */
536 template <>
537 class ReinitHelper<PETScWrappers::MPI::Vector>
538 {
539 public:
540 template <typename Matrix>
541 static void
542 reinit_range_vector(const Matrix &matrix,
543 PETScWrappers::MPI::Vector &v,
544 bool /*omit_zeroing_entries*/)
545 {
546 v.reinit(matrix.locally_owned_range_indices(),
547 matrix.get_mpi_communicator());
548 }
549
550 template <typename Matrix>
551 static void
552 reinit_domain_vector(const Matrix &matrix,
553 PETScWrappers::MPI::Vector &v,
554 bool /*omit_zeroing_entries*/)
555 {
556 v.reinit(matrix.locally_owned_domain_indices(),
557 matrix.get_mpi_communicator());
558 }
559 };
560
561 } // namespace LinearOperatorImplementation
562 } /* namespace internal */
563
564 /** @} */
565
566
567 /**
568 * Declare dealii::PETScWrappers::MPI::Vector as distributed vector.
569 */
570 template <>
571 struct is_serial_vector<PETScWrappers::MPI::Vector> : std::false_type
572 {};
573
574
575 DEAL_II_NAMESPACE_CLOSE
576
577 #else
578
579 // Make sure the scripts that create the C++20 module input files have
580 // something to latch on if the preprocessor #ifdef above would
581 // otherwise lead to an empty content of the file.
582 DEAL_II_NAMESPACE_OPEN
583 DEAL_II_NAMESPACE_CLOSE
584
585 #endif // DEAL_II_WITH_PETSC
586
587 #endif