Debian Codegraph @

include/deal.II/lac/petsc_sparse_matrix.h

raw
1 // ------------------------------------------------------------------------
2 //
3 // SPDX-License-Identifier: LGPL-2.1-or-later
4 // Copyright (C) 2004 - 2025 by the deal.II authors
5 //
6 // This file is part of the deal.II library.
7 //
8 // Part of the source code is dual licensed under Apache-2.0 WITH
9 // LLVM-exception OR LGPL-2.1-or-later. Detailed license information
10 // governing the source code and code contributions can be found in
11 // LICENSE.md and CONTRIBUTING.md at the top level directory of deal.II.
12 //
13 // ------------------------------------------------------------------------
14
15 #ifndef dealii_petsc_sparse_matrix_h
16 #define dealii_petsc_sparse_matrix_h
17
18
19 #include <deal.II/base/config.h>
20
21 #ifdef DEAL_II_WITH_PETSC
22
23 # include <deal.II/lac/exceptions.h>
24 # include <deal.II/lac/petsc_matrix_base.h>
25 # include <deal.II/lac/petsc_vector.h>
26
27 # include <petscis.h>
28 # include <petscistypes.h>
29
30 # include <cstddef>
31 # include <vector>
32
33 DEAL_II_NAMESPACE_OPEN
34 // forward declaration
35 # ifndef DOXYGEN
36 template <typename MatrixType>
37 class BlockMatrixBase;
38 # endif
39
40 namespace PETScWrappers
41 {
42 /**
43 * Implementation of a sequential sparse matrix class based on PETSc. All
44 * the functionality is actually in the base class, except for the calls to
45 * generate a sequential sparse matrix. This is possible since PETSc only
46 * works on an abstract matrix type and internally distributes to functions
47 * that do the actual work depending on the actual matrix type (much like
48 * using virtual functions). Only the functions creating a matrix of
49 * specific type differ, and are implemented in this particular class.
50 *
51 * @ingroup PETScWrappers
52 * @ingroup Matrix1
53 */
54 class SparseMatrix : public MatrixBase
55 {
56 public:
57 /**
58 * A structure that describes some of the traits of this class in terms of
59 * its run-time behavior. Some other classes (such as the block matrix
60 * classes) that take one or other of the matrix classes as its template
61 * parameters can tune their behavior based on the variables in this
62 * class.
63 */
64 struct Traits
65 {
66 /**
67 * It is safe to elide additions of zeros to individual elements of this
68 * matrix.
69 */
70 static const bool zero_addition_can_be_elided = true;
71 };
72
73 /**
74 * Default constructor. Create an empty matrix.
75 */
76 SparseMatrix();
77
78 /**
79 * Initialize a Matrix from a PETSc Mat object. Note that we do not copy
80 * the matrix. The Mat object is referenced by the newly created instance
81 * of the class using PetscObjectReference. This is in line with the PETSc
82 * approach to object ownership, which mimics std::shared_ptr.
83 */
84 explicit SparseMatrix(const Mat &);
85
86 /**
87 * Create a sparse matrix of dimensions @p m times @p n, with an initial
88 * guess of @p n_nonzero_per_row nonzero elements per row. PETSc is able
89 * to cope with the situation that more than this number of elements is
90 * later allocated for a row, but this involves copying data, and is thus
91 * expensive.
92 *
93 * The @p is_symmetric flag determines whether we should tell PETSc that
94 * the matrix is going to be symmetric (as indicated by the call
95 * <tt>MatSetOption(mat, MAT_SYMMETRIC)</tt>. Note that the PETSc
96 * documentation states that one cannot form an ILU decomposition of a
97 * matrix for which this flag has been set to @p true, only an ICC. The
98 * default value of this flag is @p false.
99 */
100 SparseMatrix(const size_type m,
101 const size_type n,
102 const size_type n_nonzero_per_row,
103 const bool is_symmetric = false);
104
105 /**
106 * Initialize a rectangular matrix with @p m rows and @p n columns. The
107 * maximal number of nonzero entries for each row separately is given by
108 * the @p row_lengths array.
109 *
110 * Just as for the other constructors: PETSc is able to cope with the
111 * situation that more than this number of elements is later allocated for
112 * a row, but this involves copying data, and is thus expensive.
113 *
114 * The @p is_symmetric flag determines whether we should tell PETSc that
115 * the matrix is going to be symmetric (as indicated by the call
116 * <tt>MatSetOption(mat, MAT_SYMMETRIC)</tt>. Note that the PETSc
117 * documentation states that one cannot form an ILU decomposition of a
118 * matrix for which this flag has been set to @p true, only an ICC. The
119 * default value of this flag is @p false.
120 */
121 SparseMatrix(const size_type m,
122 const size_type n,
123 const std::vector<size_type> &row_lengths,
124 const bool is_symmetric = false);
125
126 /**
127 * Initialize a sparse matrix using the given sparsity pattern.
128 *
129 * Note that PETSc can be very slow if you do not provide it with a good
130 * estimate of the lengths of rows. Using the present function is a very
131 * efficient way to do this, as it uses the exact number of nonzero
132 * entries for each row of the matrix by using the given sparsity pattern
133 * argument. If the @p preset_nonzero_locations flag is @p true, this
134 * function in addition not only sets the correct row sizes up front, but
135 * also pre-allocated the correct nonzero entries in the matrix.
136 *
137 * PETsc allows to later add additional nonzero entries to a matrix, by
138 * simply writing to these elements. However, this will then lead to
139 * additional memory allocations which are very inefficient and will
140 * greatly slow down your program. It is therefore significantly more
141 * efficient to get memory allocation right from the start.
142 */
143 template <typename SparsityPatternType>
144 explicit SparseMatrix(const SparsityPatternType &sparsity_pattern,
145 const bool preset_nonzero_locations = true);
146
147 /**
148 * This operator assigns a scalar to a matrix. Since this does usually not
149 * make much sense (should we set all matrix entries to this value? Only
150 * the nonzero entries of the sparsity pattern?), this operation is only
151 * allowed if the actual value to be assigned is zero. This operator only
152 * exists to allow for the obvious notation <tt>matrix=0</tt>, which sets
153 * all elements of the matrix to zero, but keep the sparsity pattern
154 * previously used.
155 */
156 SparseMatrix &
157 operator=(const double d);
158
159 /**
160 * The copy constructor is deleted.
161 */
162 SparseMatrix(const SparseMatrix &) = delete;
163
164 /**
165 * The copy assignment operator is deleted.
166 */
167 SparseMatrix &
168 operator=(const SparseMatrix &) = delete;
169
170 /**
171 * Throw away the present matrix and generate one that has the same
172 * properties as if it were created by the constructor of this class with
173 * the same argument list as the present function.
174 */
175 void
176 reinit(const size_type m,
177 const size_type n,
178 const size_type n_nonzero_per_row,
179 const bool is_symmetric = false);
180
181 /**
182 * Throw away the present matrix and generate one that has the same
183 * properties as if it were created by the constructor of this class with
184 * the same argument list as the present function.
185 */
186 void
187 reinit(const size_type m,
188 const size_type n,
189 const std::vector<size_type> &row_lengths,
190 const bool is_symmetric = false);
191
192 /**
193 * Initialize a sparse matrix using the given sparsity pattern.
194 *
195 * Note that PETSc can be very slow if you do not provide it with a good
196 * estimate of the lengths of rows. Using the present function is a very
197 * efficient way to do this, as it uses the exact number of nonzero
198 * entries for each row of the matrix by using the given sparsity pattern
199 * argument. If the @p preset_nonzero_locations flag is @p true, this
200 * function in addition not only sets the correct row sizes up front, but
201 * also pre-allocated the correct nonzero entries in the matrix.
202 *
203 * PETsc allows to later add additional nonzero entries to a matrix, by
204 * simply writing to these elements. However, this will then lead to
205 * additional memory allocations which are very inefficient and will
206 * greatly slow down your program. It is therefore significantly more
207 * efficient to get memory allocation right from the start.
208 *
209 * Despite the fact that it would seem to be an obvious win, setting the
210 * @p preset_nonzero_locations flag to @p true doesn't seem to accelerate
211 * program. Rather on the contrary, it seems to be able to slow down
212 * entire programs somewhat. This is surprising, since we can use
213 * efficient function calls into PETSc that allow to create multiple
214 * entries at once; nevertheless, given the fact that it is inefficient,
215 * the respective flag has a default value equal to @p false.
216 */
217 template <typename SparsityPatternType>
218 void
219 reinit(const SparsityPatternType &sparsity_pattern,
220 const bool preset_nonzero_locations = true);
221
222 /**
223 * Return the number of rows of this matrix.
224 */
225 std::size_t
226 m() const;
227
228 /**
229 * Return the number of columns of this matrix.
230 */
231 std::size_t
232 n() const;
233
234 /**
235 * Perform the matrix-matrix multiplication $C = AB$, or,
236 * $C = A \text{diag}(V) B$ given a compatible vector $V$.
237 *
238 * This function calls MatrixBase::mmult() to do the actual work.
239 */
240 void
241 mmult(SparseMatrix &C,
242 const SparseMatrix &B,
243 const MPI::Vector &V = MPI::Vector()) const;
244
245 /**
246 * Perform the matrix-matrix multiplication with the transpose of
247 * <tt>this</tt>, i.e., $C = A^T B$, or,
248 * $C = A^T \text{diag}(V) B$ given a compatible vector $V$.
249 *
250 * This function calls MatrixBase::Tmmult() to do the actual work.
251 */
252 void
253 Tmmult(SparseMatrix &C,
254 const SparseMatrix &B,
255 const MPI::Vector &V = MPI::Vector()) const;
256
257 private:
258 /**
259 * Do the actual work for the respective reinit() function and the
260 * matching constructor, i.e. create a matrix.
261 */
262 void
263 do_reinit(const size_type m,
264 const size_type n,
265 const size_type n_nonzero_per_row,
266 const bool is_symmetric = false);
267
268 /**
269 * Same as previous function.
270 */
271 void
272 do_reinit(const size_type m,
273 const size_type n,
274 const std::vector<size_type> &row_lengths,
275 const bool is_symmetric = false);
276
277 /**
278 * Same as previous function.
279 */
280 template <typename SparsityPatternType>
281 void
282 do_reinit(const SparsityPatternType &sparsity_pattern,
283 const bool preset_nonzero_locations);
284
285 // To allow calling protected prepare_add() and prepare_set().
286 friend class BlockMatrixBase<SparseMatrix>;
287 };
288
289 namespace MPI
290 {
291 /**
292 * Implementation of a parallel sparse matrix class based on PETSc, with
293 * rows of the matrix distributed across an MPI network. All the
294 * functionality is actually in the base class, except for the calls to
295 * generate a parallel sparse matrix. This is possible since PETSc only
296 * works on an abstract matrix type and internally distributes to
297 * functions that do the actual work depending on the actual matrix type
298 * (much like using virtual functions). Only the functions creating a
299 * matrix of specific type differ, and are implemented in this particular
300 * class.
301 *
302 * There are a number of comments on the communication model as well as
303 * access to individual elements in the documentation to the parallel
304 * vector class. These comments apply here as well.
305 *
306 *
307 * <h3>Partitioning of matrices</h3>
308 *
309 * PETSc partitions parallel matrices so that each MPI process "owns" a
310 * certain number of rows (i.e. only this process stores the respective
311 * entries in these rows). The number of rows each process owns has to be
312 * passed to the constructors and reinit() functions via the argument @p
313 * local_rows. The individual values passed as @p local_rows on all the
314 * MPI processes of course have to add up to the global number of rows of
315 * the matrix.
316 *
317 * In addition to this, PETSc also partitions the rectangular chunk of the
318 * matrix it owns (i.e. the @p local_rows times n() elements in the
319 * matrix), so that matrix vector multiplications can be performed
320 * efficiently. This column-partitioning therefore has to match the
321 * partitioning of the vectors with which the matrix is multiplied, just
322 * as the row-partitioning has to match the partitioning of destination
323 * vectors. This partitioning is passed to the constructors and reinit()
324 * functions through the @p local_columns variable, which again has to add
325 * up to the global number of columns in the matrix. The name @p
326 * local_columns may be named inappropriately since it does not reflect
327 * that only these columns are stored locally, but it reflects the fact
328 * that these are the columns for which the elements of incoming vectors
329 * are stored locally.
330 *
331 * To make things even more complicated, PETSc needs a very good estimate
332 * of the number of elements to be stored in each row to be efficient.
333 * Otherwise it spends most of the time with allocating small chunks of
334 * memory, a process that can slow down programs to a crawl if it happens
335 * to often. As if a good estimate of the number of entries per row isn't
336 * even, it even needs to split this as follows: for each row it owns, it
337 * needs an estimate for the number of elements in this row that fall into
338 * the columns that are set apart for this process (see above), and the
339 * number of elements that are in the rest of the columns.
340 *
341 * Since in general this information is not readily available, most of the
342 * initializing functions of this class assume that all of the number of
343 * elements you give as an argument to @p n_nonzero_per_row or by @p
344 * row_lengths fall into the columns "owned" by this process, and none
345 * into the other ones. This is a fair guess for most of the rows, since
346 * in a good domain partitioning, nodes only interact with nodes that are
347 * within the same subdomain. It does not hold for nodes on the interfaces
348 * of subdomain, however, and for the rows corresponding to these nodes,
349 * PETSc will have to allocate additional memory, a costly process.
350 *
351 * The only way to avoid this is to tell PETSc where the actual entries of
352 * the matrix will be. For this, there are constructors and reinit()
353 * functions of this class that take a DynamicSparsityPattern object
354 * containing all this information. While in the general case it is
355 * sufficient if the constructors and reinit() functions know the number
356 * of local rows and columns, the functions getting a sparsity pattern
357 * also need to know the number of local rows (@p local_rows_per_process)
358 * and columns (@p local_columns_per_process) for all other processes, in
359 * order to compute which parts of the matrix are which. Thus, it is not
360 * sufficient to just count the number of degrees of freedom that belong
361 * to a particular process, but you have to have the numbers for all
362 * processes available at all processes.
363 *
364 * @ingroup PETScWrappers
365 * @ingroup Matrix1
366 */
367 class SparseMatrix : public MatrixBase
368 {
369 public:
370 /**
371 * Declare type for container size.
372 */
373 using size_type = types::global_dof_index;
374
375 /**
376 * A structure that describes some of the traits of this class in terms
377 * of its run-time behavior. Some other classes (such as the block
378 * matrix classes) that take one or other of the matrix classes as its
379 * template parameters can tune their behavior based on the variables in
380 * this class.
381 */
382 struct Traits
383 {
384 /**
385 * It is not safe to elide additions of zeros to individual elements
386 * of this matrix. The reason is that additions to the matrix may
387 * trigger @ref GlossCollectiveOperation "collective operations" synchronizing buffers on multiple
388 * processes. If an addition is elided on one process, this may lead
389 * to other processes hanging in an infinite waiting loop.
390 */
391 static const bool zero_addition_can_be_elided = false;
392 };
393
394 /**
395 * Default constructor. Create an empty matrix.
396 */
397 SparseMatrix();
398
399 /**
400 * Initialize a SparseMatrix from a PETSc Mat object. Note that we do not
401 * copy the matrix. The Mat object is referenced by the newly created
402 * instance of the class using PetscObjectReference. This is in line with
403 * the PETSc approach to object ownership, which mimics std::shared_ptr.
404 */
405 explicit SparseMatrix(const Mat &);
406
407 /**
408 * Destructor to free the PETSc object.
409 */
410 ~SparseMatrix() override;
411
412 /**
413 * Initialize using the given sparsity pattern with communication
414 * happening over the provided @p communicator.
415 *
416 * For the meaning of the @p local_rows_per_process and @p
417 * local_columns_per_process parameters, see the class documentation.
418 *
419 * Note that PETSc can be very slow if you do not provide it with a good
420 * estimate of the lengths of rows. Using the present function is a very
421 * efficient way to do this, as it uses the exact number of nonzero
422 * entries for each row of the matrix by using the given sparsity
423 * pattern argument. If the @p preset_nonzero_locations flag is @p true,
424 * this function in addition not only sets the correct row sizes up
425 * front, but also pre-allocated the correct nonzero entries in the
426 * matrix.
427 *
428 * PETsc allows to later add additional nonzero entries to a matrix, by
429 * simply writing to these elements. However, this will then lead to
430 * additional memory allocations which are very inefficient and will
431 * greatly slow down your program. It is therefore significantly more
432 * efficient to get memory allocation right from the start.
433 */
434 template <typename SparsityPatternType>
435 SparseMatrix(const MPI_Comm communicator,
436 const SparsityPatternType &sparsity_pattern,
437 const std::vector<size_type> &local_rows_per_process,
438 const std::vector<size_type> &local_columns_per_process,
439 const unsigned int this_process,
440 const bool preset_nonzero_locations = true);
441
442 /**
443 * This operator assigns a scalar to a matrix. Since this does usually
444 * not make much sense (should we set all matrix entries to this value?
445 * Only the nonzero entries of the sparsity pattern?), this operation is
446 * only allowed if the actual value to be assigned is zero. This
447 * operator only exists to allow for the obvious notation
448 * <tt>matrix=0</tt>, which sets all elements of the matrix to zero, but
449 * keep the sparsity pattern previously used.
450 */
451 SparseMatrix &
452 operator=(const value_type d);
453
454
455 /**
456 * Make a copy of the PETSc matrix @p other. It is assumed that both
457 * matrices have the same SparsityPattern.
458 */
459 void
460 copy_from(const SparseMatrix &other);
461
462 /**
463 * Initialize using the given sparsity pattern with communication
464 * happening over the provided @p communicator.
465 *
466 * Note that PETSc can be very slow if you do not provide it with a good
467 * estimate of the lengths of rows. Using the present function is a very
468 * efficient way to do this, as it uses the exact number of nonzero
469 * entries for each row of the matrix by using the given sparsity
470 * pattern argument. If the @p preset_nonzero_locations flag is @p true,
471 * this function in addition not only sets the correct row sizes up
472 * front, but also pre-allocated the correct nonzero entries in the
473 * matrix.
474 *
475 * PETsc allows to later add additional nonzero entries to a matrix, by
476 * simply writing to these elements. However, this will then lead to
477 * additional memory allocations which are very inefficient and will
478 * greatly slow down your program. It is therefore significantly more
479 * efficient to get memory allocation right from the start.
480 */
481 template <typename SparsityPatternType>
482 void
483 reinit(const MPI_Comm communicator,
484 const SparsityPatternType &sparsity_pattern,
485 const std::vector<size_type> &local_rows_per_process,
486 const std::vector<size_type> &local_columns_per_process,
487 const unsigned int this_process,
488 const bool preset_nonzero_locations = true);
489
490 /**
491 * Create a square matrix where the size() of the IndexSet determines the
492 * global number of rows and columns and the entries of the IndexSet
493 * give the rows and columns for the calling processor. Note that only
494 * ascending, 1:1 IndexSets are supported.
495 */
496 template <typename SparsityPatternType>
497 void
498 reinit(const IndexSet &local_partitioning,
499 const SparsityPatternType &sparsity_pattern,
500 const MPI_Comm communicator);
501
502 /**
503 * Create a matrix where the size() of the IndexSets determine the
504 * global number of rows and columns and the entries of the IndexSet
505 * give the rows and columns for the calling processor. Note that only
506 * ascending, 1:1 IndexSets are supported.
507 */
508 template <typename SparsityPatternType>
509 void
510 reinit(const IndexSet &local_rows,
511 const IndexSet &local_columns,
512 const SparsityPatternType &sparsity_pattern,
513 const MPI_Comm communicator);
514
515 /**
516 * Initialize this matrix to have the same structure as @p other. This
517 * will not copy the values of the other matrix, but you can use
518 * copy_from() for this.
519 */
520 void
521 reinit(const SparseMatrix &other);
522
523 /**
524 * Create a matrix where the size of the IndexSets determine the
525 * global number of rows and columns and the entries of the IndexSet
526 * give the rows and columns for the calling processor. Note that only
527 * ascending, 1:1 IndexSets are supported. The additional call to the
528 * local to global mappings is required to create the matrix of type
529 * IS (see DoFTools::extract_locally_active_dofs).
530 * This is required by the BDDC preconditioner.
531 */
532 template <typename SparsityPatternType>
533 void
534 reinit(const IndexSet &local_rows,
535 const IndexSet &local_active_rows,
536 const IndexSet &local_columns,
537 const IndexSet &local_active_columns,
538 const SparsityPatternType &sparsity_pattern,
539 const MPI_Comm communicator);
540
541 /**
542 * @addtogroup Exceptions
543 * @{
544 */
545 /**
546 * Exception
547 */
548 DeclException2(ExcLocalRowsTooLarge,
549 int,
550 int,
551 << "The number of local rows " << arg1
552 << " must be larger than the total number of rows "
553 << arg2);
554 /** @} */
555
556 /**
557 * Return the square of the norm of the vector $v$ with respect to the
558 * norm induced by this matrix, i.e. $\left(v^\ast,Mv\right)$. This is
559 * useful, e.g. in the finite element context, where the $L_2$ norm of a
560 * function equals the matrix norm with respect to the @ref GlossMassMatrix "mass matrix" of
561 * the vector representing the nodal values of the finite element
562 * function.
563 *
564 * Obviously, the matrix needs to be quadratic for this operation.
565 *
566 * The implementation of this function is not as efficient as the one in
567 * the @p MatrixBase class used in deal.II (i.e. the original one, not
568 * the PETSc wrapper class) since PETSc doesn't support this operation
569 * and needs a temporary vector.
570 */
571 PetscScalar
572 matrix_norm_square(const Vector &v) const;
573
574 /**
575 * Compute the matrix scalar product $\left(u^\ast,Mv\right)$.
576 *
577 * The implementation of this function is not as efficient as the one in
578 * the @p MatrixBase class used in deal.II (i.e. the original one, not
579 * the PETSc wrapper class) since PETSc doesn't support this operation
580 * and needs a temporary vector.
581 */
582 PetscScalar
583 matrix_scalar_product(const Vector &u, const Vector &v) const;
584
585 /**
586 * Return the partitioning of the domain space of this matrix, i.e., the
587 * partitioning of the vectors this matrix has to be multiplied with.
588 */
589 IndexSet
590 locally_owned_domain_indices() const;
591
592 /**
593 * Return the partitioning of the range space of this matrix, i.e., the
594 * partitioning of the vectors that result from matrix-vector
595 * products.
596 */
597 IndexSet
598 locally_owned_range_indices() const;
599
600 /**
601 * Perform the matrix-matrix multiplication $C = AB$, or,
602 * $C = A \text{diag}(V) B$ given a compatible vector $V$.
603 *
604 * This function calls MatrixBase::mmult() to do the actual work.
605 */
606 void
607 mmult(SparseMatrix &C,
608 const SparseMatrix &B,
609 const MPI::Vector &V = MPI::Vector()) const;
610
611 /**
612 * Perform the matrix-matrix multiplication with the transpose of
613 * <tt>this</tt>, i.e., $C = A^T B$, or,
614 * $C = A^T \text{diag}(V) B$ given a compatible vector $V$.
615 *
616 * This function calls MatrixBase::Tmmult() to do the actual work.
617 */
618 void
619 Tmmult(SparseMatrix &C,
620 const SparseMatrix &B,
621 const MPI::Vector &V = MPI::Vector()) const;
622
623 private:
624 /**
625 * Same as previous functions.
626 */
627 template <typename SparsityPatternType>
628 void
629 do_reinit(const MPI_Comm comm,
630 const SparsityPatternType &sparsity_pattern,
631 const std::vector<size_type> &local_rows_per_process,
632 const std::vector<size_type> &local_columns_per_process,
633 const unsigned int this_process,
634 const bool preset_nonzero_locations);
635
636 /**
637 * Same as previous functions.
638 */
639 template <typename SparsityPatternType>
640 void
641 do_reinit(const MPI_Comm comm,
642 const IndexSet &local_rows,
643 const IndexSet &local_columns,
644 const SparsityPatternType &sparsity_pattern);
645
646 /**
647 * Same as previous functions, but here we consider active dofs for
648 * matrices of IS type.
649 */
650 template <typename SparsityPatternType>
651 void
652 do_reinit(const MPI_Comm comm,
653 const IndexSet &local_rows,
654 const IndexSet &local_active_rows,
655 const IndexSet &local_columns,
656 const IndexSet &local_active_columns,
657 const SparsityPatternType &sparsity_pattern);
658
659 // To allow calling protected prepare_add() and prepare_set().
660 friend class BlockMatrixBase<SparseMatrix>;
661 };
662
663 } // namespace MPI
664 } // namespace PETScWrappers
665
666 DEAL_II_NAMESPACE_CLOSE
667
668 #else
669
670 // Make sure the scripts that create the C++20 module input files have
671 // something to latch on if the preprocessor #ifdef above would
672 // otherwise lead to an empty content of the file.
673 DEAL_II_NAMESPACE_OPEN
674 DEAL_II_NAMESPACE_CLOSE
675
676 #endif // DEAL_II_WITH_PETSC
677
678 #endif