Debian Codegraph @

include/deal.II/lac/scalapack.h

raw
1 // ------------------------------------------------------------------------
2 //
3 // SPDX-License-Identifier: LGPL-2.1-or-later
4 // Copyright (C) 2017 - 2024 by the deal.II authors
5 //
6 // This file is part of the deal.II library.
7 //
8 // Part of the source code is dual licensed under Apache-2.0 WITH
9 // LLVM-exception OR LGPL-2.1-or-later. Detailed license information
10 // governing the source code and code contributions can be found in
11 // LICENSE.md and CONTRIBUTING.md at the top level directory of deal.II.
12 //
13 // ------------------------------------------------------------------------
14
15 #ifndef dealii_scalapack_h
16 #define dealii_scalapack_h
17
18 #include <deal.II/base/config.h>
19
20 #ifdef DEAL_II_WITH_SCALAPACK
21
22 # include <deal.II/base/exceptions.h>
23 # include <deal.II/base/mpi.h>
24 # include <deal.II/base/mpi_stub.h>
25 # include <deal.II/base/mutex.h>
26 # include <deal.II/base/process_grid.h>
27
28 # include <deal.II/lac/full_matrix.h>
29 # include <deal.II/lac/lapack_full_matrix.h>
30 # include <deal.II/lac/lapack_support.h>
31
32 # include <limits>
33 # include <memory>
34
35 DEAL_II_NAMESPACE_OPEN
36
37 /**
38 * A wrapper class around ScaLAPACK parallel dense linear algebra.
39 *
40 * ScaLAPACK assumes that matrices are distributed according to the
41 * block-cyclic decomposition scheme. An $M$ by $N$ matrix is first decomposed
42 * into $\lceil M / MB \rceil$ by $\lceil N / NB \rceil$ blocks which are then
43 * uniformly distributed across the 2d process grid with $p q \le Np$ processes,
44 * where $p,q$ are grid dimensions and $Np$ is the total number of processes.
45 * The parameters MB and NB are referred to as row and column block size and
46 * determine the granularity of the block-cyclic distribution.
47 *
48 * In the following the block-cyclic distribution of a $10 \times 9$ matrix
49 * onto a $3\times 3$ Cartesian process grid with block sizes
50 * $\text{MB}=\text{NB}=2$ is displayed.
51 *
52 * \htmlonly <style>div.image
53 * img[src="scalapack_block_cycling.png"]{width:35%;}</style>
54 * \endhtmlonly
55 * @image html scalapack_block_cycling.png "Block-Cyclic Distribution"
56 *
57 * Note that the odd number of columns of the local matrices owned by the
58 * processes P2, P5 and P8 accounts for $N=9$ not being an integral multiple of
59 * $\text{NB}=2$.
60 *
61 * The choice of the block sizes is a compromise between a sufficiently large
62 * size for efficient local/serial BLAS, but one that is also small enough to
63 * achieve good parallel load balance.
64 *
65 * Below we show a strong scaling example of ScaLAPACKMatrix::invert()
66 * on up to 5 nodes each composed of two Intel Xeon 2660v2 IvyBridge sockets
67 * 2.20GHz, 10 cores/socket. Calculations are performed on square processor
68 * grids 1x1, 2x2, 3x3, 4x4, 5x5, 6x6, 7x7, 8x8, 9x9, 10x10.
69 *
70 * @image html scalapack_invert.png
71 *
72 * @ingroup Matrix1
73 */
74 template <typename NumberType>
75 class ScaLAPACKMatrix : protected TransposeTable<NumberType>
76 {
77 public:
78 /**
79 * Declare the type for container size.
80 */
81 using size_type = unsigned int;
82
83 /**
84 * Constructor for a rectangular matrix with @p n_rows and @p n_cols
85 * and distributed using the grid @p process_grid.
86 *
87 * The parameters @p row_block_size and @p column_block_size are the block sizes used
88 * for the block-cyclic distribution of the matrix.
89 * In general, it is recommended to use powers of $2$, e.g. $16,32,64, \dots$.
90 */
91 ScaLAPACKMatrix(
92 const size_type n_rows,
93 const size_type n_columns,
94 const std::shared_ptr<const Utilities::MPI::ProcessGrid> &process_grid,
95 const size_type row_block_size = 32,
96 const size_type column_block_size = 32,
97 const LAPACKSupport::Property property = LAPACKSupport::Property::general);
98
99 /**
100 * Constructor for a square matrix of size @p size, and distributed
101 * using the process grid in @p process_grid.
102 *
103 * The parameter @p block_size is used for the block-cyclic distribution of the matrix.
104 * An identical block size is used for the rows and columns of the matrix.
105 * In general, it is recommended to use powers of $2$, e.g. $16,32,64, \dots$.
106 */
107 ScaLAPACKMatrix(
108 const size_type size,
109 const std::shared_ptr<const Utilities::MPI::ProcessGrid> &process_grid,
110 const size_type block_size = 32,
111 const LAPACKSupport::Property property =
112 LAPACKSupport::Property::symmetric);
113
114 /**
115 * Constructor for a general rectangular matrix that is read from
116 * the file @p filename and distributed using the grid @p process_grid.
117 *
118 * Loads the matrix from file @p filename using HDF5.
119 * In case that deal.II was built without HDF5
120 * a call to this function will cause an exception to be thrown.
121 *
122 * The parameters @p row_block_size and @p column_block_size are the block sizes used
123 * for the block-cyclic distribution of the matrix.
124 * In general, it is recommended to use powers of $2$, e.g. $16,32,64, \dots$.
125 */
126 ScaLAPACKMatrix(
127 const std::string &filename,
128 const std::shared_ptr<const Utilities::MPI::ProcessGrid> &process_grid,
129 const size_type row_block_size = 32,
130 const size_type column_block_size = 32);
131
132 /**
133 * Destructor
134 */
135 ~ScaLAPACKMatrix() override = default;
136
137 /**
138 * Initialize the rectangular matrix with @p n_rows and @p n_cols
139 * and distributed using the grid @p process_grid.
140 *
141 * The parameters @p row_block_size and @p column_block_size are the block sizes used
142 * for the block-cyclic distribution of the matrix.
143 * In general, it is recommended to use powers of $2$, e.g. $16,32,64, \dots$.
144 */
145 void
146 reinit(
147 const size_type n_rows,
148 const size_type n_columns,
149 const std::shared_ptr<const Utilities::MPI::ProcessGrid> &process_grid,
150 const size_type row_block_size = 32,
151 const size_type column_block_size = 32,
152 const LAPACKSupport::Property property = LAPACKSupport::Property::general);
153
154 /**
155 * Initialize the square matrix of size @p size and distributed using the grid @p process_grid.
156 *
157 * The parameter @p block_size is used for the block-cyclic distribution of the matrix.
158 * An identical block size is used for the rows and columns of the matrix.
159 * In general, it is recommended to use powers of $2$, e.g. $16,32,64, \dots$.
160 */
161 void
162 reinit(const size_type size,
163 const std::shared_ptr<const Utilities::MPI::ProcessGrid> &process_grid,
164 const size_type block_size = 32,
165 const LAPACKSupport::Property property =
166 LAPACKSupport::Property::symmetric);
167
168 /**
169 * Assign @p property to this matrix.
170 */
171 void
172 set_property(const LAPACKSupport::Property property);
173
174 /**
175 * Return current @p property of this matrix
176 */
177 LAPACKSupport::Property
178 get_property() const;
179
180 /**
181 * Return current @p state of this matrix
182 */
183 LAPACKSupport::State
184 get_state() const;
185
186 /**
187 * Assignment operator from a regular FullMatrix.
188 *
189 * @note This function should only be used for relatively small matrix
190 * dimensions. It is primarily intended for debugging purposes.
191 */
192 ScaLAPACKMatrix<NumberType> &
193 operator=(const FullMatrix<NumberType> &);
194
195 /**
196 * Copies the content of the locally owned @p matrix to the distributed matrix.
197 * The distributed matrix and @p matrix on process @p rank must have matching dimensions.
198 *
199 * For all processes except the process with rank @p rank the serial @p matrix is not referenced.
200 * The user has to ensure that all processes call this with identical @p rank.
201 * The @p rank refers to a process of the MPI communicator used to create the process grid
202 * of the distributed matrix.
203 */
204 void
205 copy_from(const LAPACKFullMatrix<NumberType> &matrix,
206 const unsigned int rank);
207
208 /**
209 * Copy the contents of the distributed matrix into @p matrix.
210 *
211 * @note This function should only be used for relatively small matrix
212 * dimensions. It is primarily intended for debugging purposes.
213 */
214 void
215 copy_to(FullMatrix<NumberType> &matrix) const;
216
217 /**
218 * Copies the content of the distributed matrix into the locally replicated @p matrix
219 * on the process with rank @p rank. For all processes except @p rank @p matrix is not referenced.
220 * The distributed matrix and @p matrix on the process @p rank must have matching dimensions.
221 *
222 * The user has to ensure that all processes call this with identical @p rank.
223 * The @p rank refers to a process of the MPI communicator used to create the process grid
224 * of the distributed matrix.
225 */
226 void
227 copy_to(LAPACKFullMatrix<NumberType> &matrix, const unsigned int rank) const;
228
229 /**
230 * Copy the contents of the distributed matrix into a differently distributed matrix @p dest.
231 * The function also works for matrices with different process grids
232 * or block-cyclic distributions.
233 */
234 void
235 copy_to(ScaLAPACKMatrix<NumberType> &dest) const;
236
237 /**
238 * Copy a submatrix (subset) of the distributed matrix A to a submatrix of the distributed matrix @p B.
239 *
240 * - The global row and column index of the first element of the submatrix A is provided by @p offset_A
241 * with row index=<code>offset_A.first</code> and column
242 * index=<code>offset_A.second</code>.
243 *
244 * - The global row and column index of the first element of the submatrix B is provided by @p offset_B
245 * with row index=<code>offset_B.first</code> and column
246 * index=<code>offset_B.second</code>.
247 *
248 * - The dimension of the submatrix to be copied is given by @p submatrix_size
249 * with number of rows=<code>submatrix_size.first</code> and number of
250 * columns=<code>submatrix_size.second</code>.
251 *
252 *
253 * If it is necessary to copy complete matrices with an identical block-cyclic
254 * distribution, use
255 * ScaLAPACKMatrix<NumberType>::copy_to(ScaLAPACKMatrix<NumberType> &dest)
256 * with only one argument to avoid communication.
257 *
258 * The underlying process grids of the matrices @p A and @p B must have been built
259 * with the same MPI communicator.
260 */
261 void
262 copy_to(ScaLAPACKMatrix<NumberType> &B,
263 const std::pair<unsigned int, unsigned int> &offset_A,
264 const std::pair<unsigned int, unsigned int> &offset_B,
265 const std::pair<unsigned int, unsigned int> &submatrix_size) const;
266
267 /**
268 * Transposing assignment: $\mathbf{A} = \mathbf{B}^T$
269 *
270 * The matrices $\mathbf{A}$ and $\mathbf{B}$ must have the same process grid.
271 *
272 * The following alignment conditions have to be fulfilled: $MB_A=NB_B$ and
273 * $NB_A=MB_B$.
274 */
275 void
276 copy_transposed(const ScaLAPACKMatrix<NumberType> &B);
277
278 /**
279 * The operations based on the input parameter @p transpose_B and the
280 * alignment conditions are summarized in the following table:
281 *
282 * | transpose_B | Block Sizes | Operation |
283 * | :---------: | :--------------------------: | :-------------------------------------------: |
284 * | false | $MB_A=MB_B$ <br> $NB_A=NB_B$ | $\mathbf{A} = a \mathbf{A} + b \mathbf{B}$ |
285 * | true | $MB_A=NB_B$ <br> $NB_A=MB_B$ | $\mathbf{A} = a \mathbf{A} + b \mathbf{B}^T$ |
286 *
287 * The matrices $\mathbf{A}$ and $\mathbf{B}$ must have the same process grid.
288 */
289 void
290 add(const ScaLAPACKMatrix<NumberType> &B,
291 const NumberType a = 0.,
292 const NumberType b = 1.,
293 const bool transpose_B = false);
294
295 /**
296 * Matrix-addition:
297 * $\mathbf{A} = \mathbf{A} + b\, \mathbf{B}$
298 *
299 * The matrices $\mathbf{A}$ and $\mathbf{B}$ must have the same process grid.
300 *
301 * The following alignment conditions have to be fulfilled: $MB_A=MB_B$ and
302 * $NB_A=NB_B$.
303 */
304 void
305 add(const NumberType b, const ScaLAPACKMatrix<NumberType> &B);
306
307 /**
308 * Matrix-addition:
309 * $\mathbf{A} = \mathbf{A} + b\, \mathbf{B}^T$
310 *
311 * The matrices $\mathbf{A}$ and $\mathbf{B}$ must have the same process grid.
312 *
313 * The following alignment conditions have to be fulfilled: $MB_A=NB_B$ and
314 * $NB_A=MB_B$.
315 */
316 void
317 Tadd(const NumberType b, const ScaLAPACKMatrix<NumberType> &B);
318
319 /**
320 * Matrix-matrix-multiplication:
321 *
322 * The operations based on the input parameters and the alignment conditions
323 * are summarized in the following table:
324 *
325 * | transpose_A | transpose_B | Block Sizes | Operation |
326 * | :---------: | :---------: | :-------------------------------------------: | :-------------------------------------------------------------: |
327 * | false | false | $MB_A=MB_C$ <br> $NB_A=MB_B$ <br> $NB_B=NB_C$ | $\mathbf{C} = b \mathbf{A} \cdot \mathbf{B} + c \mathbf{C}$ |
328 * | false | true | $MB_A=MB_C$ <br> $NB_A=NB_B$ <br> $MB_B=NB_C$ | $\mathbf{C} = b \mathbf{A} \cdot \mathbf{B}^T + c \mathbf{C}$ |
329 * | true | false | $MB_A=MB_B$ <br> $NB_A=MB_C$ <br> $NB_B=NB_C$ | $\mathbf{C} = b \mathbf{A}^T \cdot \mathbf{B} + c \mathbf{C}$ |
330 * | true | true | $MB_A=NB_B$ <br> $NB_A=MB_C$ <br> $MB_B=NB_C$ | $\mathbf{C} = b \mathbf{A}^T \cdot \mathbf{B}^T + c \mathbf{C}$ |
331 *
332 * It is assumed that $\mathbf{A}$ and $\mathbf{B}$ have compatible sizes and
333 * that
334 * $\mathbf{C}$ already has the right size.
335 *
336 * The matrices $\mathbf{A}$, $\mathbf{B}$ and $\mathbf{C}$ must have the same
337 * process grid.
338 */
339 void
340 mult(const NumberType b,
341 const ScaLAPACKMatrix<NumberType> &B,
342 const NumberType c,
343 ScaLAPACKMatrix<NumberType> &C,
344 const bool transpose_A = false,
345 const bool transpose_B = false) const;
346
347 /**
348 * Matrix-matrix-multiplication.
349 *
350 * The optional parameter @p adding determines whether the result is
351 * stored in $\mathbf{C}$ or added to $\mathbf{C}$.
352 *
353 * if (@p adding) $\mathbf{C} = \mathbf{C} + \mathbf{A} \cdot \mathbf{B}$
354 *
355 * else $\mathbf{C} = \mathbf{A} \cdot \mathbf{B}$
356 *
357 * It is assumed that $\mathbf{A}$ and $\mathbf{B}$ have compatible sizes and
358 * that
359 * $\mathbf{C}$ already has the right size.
360 *
361 * The following alignment conditions have to be fulfilled: $MB_A=MB_C$,
362 * $NB_A=MB_B$ and $NB_B=NB_C$.
363 */
364 void
365 mmult(ScaLAPACKMatrix<NumberType> &C,
366 const ScaLAPACKMatrix<NumberType> &B,
367 const bool adding = false) const;
368
369 /**
370 * Matrix-matrix-multiplication using transpose of $\mathbf{A}$.
371 *
372 * The optional parameter @p adding determines whether the result is
373 * stored in $\mathbf{C}$ or added to $\mathbf{C}$.
374 *
375 * if (@p adding) $\mathbf{C} = \mathbf{C} + \mathbf{A}^T \cdot \mathbf{B}$
376 *
377 * else $\mathbf{C} = \mathbf{A}^T \cdot \mathbf{B}$
378 *
379 * It is assumed that $\mathbf{A}$ and $\mathbf{B}$ have compatible sizes and
380 * that
381 * $\mathbf{C}$ already has the right size.
382 *
383 * The following alignment conditions have to be fulfilled: $MB_A=MB_B$,
384 * $NB_A=MB_C$ and $NB_B=NB_C$.
385 */
386 void
387 Tmmult(ScaLAPACKMatrix<NumberType> &C,
388 const ScaLAPACKMatrix<NumberType> &B,
389 const bool adding = false) const;
390
391 /**
392 * Matrix-matrix-multiplication using the transpose of $\mathbf{B}$.
393 *
394 * The optional parameter @p adding determines whether the result is
395 * stored in $\mathbf{C}$ or added to $\mathbf{C}$.
396 *
397 * if (@p adding) $\mathbf{C} = \mathbf{C} + \mathbf{A} \cdot \mathbf{B}^T$
398 *
399 * else $\mathbf{C} = \mathbf{A} \cdot \mathbf{B}^T$
400 *
401 * It is assumed that $\mathbf{A}$ and $\mathbf{B}$ have compatible sizes and
402 * that
403 * $\mathbf{C}$ already has the right size.
404 *
405 * The following alignment conditions have to be fulfilled: $MB_A=MB_C$,
406 * $NB_A=NB_B$ and $MB_B=NB_C$.
407 */
408 void
409 mTmult(ScaLAPACKMatrix<NumberType> &C,
410 const ScaLAPACKMatrix<NumberType> &B,
411 const bool adding = false) const;
412
413 /**
414 * Matrix-matrix-multiplication using transpose of $\mathbf{A}$ and
415 * $\mathbf{B}$.
416 *
417 * The optional parameter @p adding determines whether the result is
418 * stored in $\mathbf{C}$ or added to $\mathbf{C}$.
419 *
420 * if (@p adding) $\mathbf{C} = \mathbf{C} + \mathbf{A}^T \cdot \mathbf{B}^T$
421 *
422 * else $\mathbf{C} = \mathbf{A}^T \cdot \mathbf{B}^T$
423 *
424 * It is assumed that $\mathbf{A}$ and $\mathbf{B}$ have compatible sizes and
425 * that
426 * $\mathbf{C}$ already has the right size.
427 *
428 * The following alignment conditions have to be fulfilled: $MB_A=NB_B$,
429 * $NB_A=MB_C$ and $MB_B=NB_C$.
430 */
431 void
432 TmTmult(ScaLAPACKMatrix<NumberType> &C,
433 const ScaLAPACKMatrix<NumberType> &B,
434 const bool adding = false) const;
435
436 /**
437 * Stores the distributed matrix in @p filename using HDF5.
438 *
439 * In case that deal.II was built without HDF5
440 * a call to this function will cause an exception to be thrown.
441 *
442 * If HDF5 was built with MPI, parallel I/O is used to save the matrix.
443 * Otherwise, just one process will do the output. This means that
444 * internally the distributed matrix is copied to one process, which
445 * does the output. Therefore, the matrix has to fit into the memory
446 * of one process.
447 *
448 * To tweak the I/O performance, especially for parallel I/O, the user may define the optional parameter @p chunk_size.
449 * All MPI processes need to call the function with the same value.
450 * The matrix is written in chunks to the file, therefore the properties of
451 * the system define the optimal chunk size. Internally, HDF5 splits the
452 * matrix into <tt>chunk_size.first</tt> x <tt>chunk_size.second</tt> sized
453 * blocks, with <tt>chunk_size.first</tt> being the number of rows of a chunk
454 * and <tt>chunk_size.second</tt> the number of columns.
455 */
456 void
457 save(const std::string &filename,
458 const std::pair<unsigned int, unsigned int> &chunk_size =
459 std::make_pair(numbers::invalid_unsigned_int,
460 numbers::invalid_unsigned_int)) const;
461
462 /**
463 * Loads the distributed matrix from file @p filename using HDF5.
464 * In case that deal.II was built without HDF5
465 * a call to this function will cause an exception to be thrown.
466 *
467 * The matrix must have the same dimensions as the matrix stored in the file.
468 *
469 * If HDF5 was build with MPI, parallel I/O is used to load the matrix.
470 * Otherwise, just one process will load the matrix from storage
471 * and distribute the content to the other processes subsequently.
472 */
473 void
474 load(const std::string &filename);
475
476 /**
477 * Compute the Cholesky factorization of the matrix using ScaLAPACK
478 * function <code>pXpotrf</code>. The result of the factorization is stored in
479 * this object.
480 */
481 void
482 compute_cholesky_factorization();
483
484 /**
485 * Compute the LU factorization of the matrix using ScaLAPACK
486 * function <code>pXgetrf</code> and partial pivoting with row interchanges.
487 * The result of the factorization is stored in this object.
488 */
489 void
490 compute_lu_factorization();
491
492 /**
493 * Invert the matrix by first computing a Cholesky for symmetric matrices
494 * or a LU factorization for general matrices and then
495 * building the actual inverse using <code>pXpotri</code> or
496 * <code>pXgetri</code>. If the matrix is triangular, the LU factorization
497 * step is skipped, and <code>pXtrtri</code> is used directly.
498 *
499 * If a Cholesky or LU factorization has been applied previously,
500 * <code>pXpotri</code> or <code>pXgetri</code> are called directly.
501 *
502 * The inverse is stored in this object.
503 */
504 void
505 invert();
506
507 /**
508 * Computing selected eigenvalues and, optionally, the eigenvectors of the
509 * real symmetric matrix $\mathbf{A} \in \mathbb{R}^{M \times M}$.
510 *
511 * The eigenvalues/eigenvectors are selected by prescribing a range of indices @p index_limits.
512 *
513 * If successful, the computed eigenvalues are arranged in ascending order.
514 * The eigenvectors are stored in the columns of the matrix, thereby
515 * overwriting the original content of the matrix.
516 *
517 * If all eigenvalues/eigenvectors have to be computed, pass the closed interval $ \left[ 0, M-1 \right] $ in @p index_limits.
518 *
519 * Pass the closed interval $ \left[ M-r, M-1 \right] $ if the $r$ largest
520 * eigenvalues/eigenvectors are desired.
521 */
522 std::vector<NumberType>
523 eigenpairs_symmetric_by_index(
524 const std::pair<unsigned int, unsigned int> &index_limits,
525 const bool compute_eigenvectors);
526
527 /**
528 * Computing selected eigenvalues and, optionally, the eigenvectors.
529 * The eigenvalues/eigenvectors are selected by prescribing a range of values @p value_limits for the eigenvalues.
530 *
531 * If successful, the computed eigenvalues are arranged in ascending order.
532 * The eigenvectors are stored in the columns of the matrix, thereby
533 * overwriting the original content of the matrix.
534 */
535 std::vector<NumberType>
536 eigenpairs_symmetric_by_value(
537 const std::pair<NumberType, NumberType> &value_limits,
538 const bool compute_eigenvectors);
539
540 /**
541 * Computing selected eigenvalues and, optionally, the eigenvectors of the
542 * real symmetric matrix $\mathbf{A} \in \mathbb{R}^{M \times M}$ using the
543 * MRRR algorithm.
544 *
545 * The eigenvalues/eigenvectors are selected by prescribing a range of indices @p index_limits.
546 *
547 * If successful, the computed eigenvalues are arranged in ascending order.
548 * The eigenvectors are stored in the columns of the matrix, thereby
549 * overwriting the original content of the matrix.
550 *
551 * If all eigenvalues/eigenvectors have to be computed, pass the closed interval $ \left[ 0, M-1 \right] $ in @p index_limits.
552 *
553 * Pass the closed interval $ \left[ M-r, M-1 \right] $ if the $r$ largest
554 * eigenvalues/eigenvectors are desired.
555 */
556 std::vector<NumberType>
557 eigenpairs_symmetric_by_index_MRRR(
558 const std::pair<unsigned int, unsigned int> &index_limits,
559 const bool compute_eigenvectors);
560
561 /**
562 * Computing selected eigenvalues and, optionally, the eigenvectors of the
563 * real symmetric matrix $\mathbf{A} \in \mathbb{R}^{M \times M}$ using the
564 * MRRR algorithm.
565 * The eigenvalues/eigenvectors are selected by prescribing a range of values @p value_limits for the eigenvalues.
566 *
567 * If successful, the computed eigenvalues are arranged in ascending order.
568 * The eigenvectors are stored in the columns of the matrix, thereby
569 * overwriting the original content of the matrix.
570 */
571 std::vector<NumberType>
572 eigenpairs_symmetric_by_value_MRRR(
573 const std::pair<NumberType, NumberType> &value_limits,
574 const bool compute_eigenvectors);
575
576 /**
577 * Computing the singular value decomposition (SVD) of a
578 * matrix $\mathbf{A} \in \mathbb{R}^{M \times N}$, optionally computing the
579 * left and/or right singular vectors. The SVD is written as $\mathbf{A} =
580 * \mathbf{U} \cdot \mathbf{\Sigma} \cdot \mathbf{V}^T$ with $\mathbf{\Sigma}
581 * \in \mathbb{R}^{M \times N}$ as a diagonal matrix,
582 * $\mathbf{U} \in \mathbb{R}^{M \times M}$ and $\mathbf{V} \in \mathbb{R}^{M
583 * \times M}$ as orthogonal matrices. The diagonal elements of
584 * $\mathbf{\Sigma}$ are the singular values of $A$ and the columns of
585 * $\mathbf{U}$ and $\mathbf{V}$ are the corresponding left and right singular
586 * vectors, respectively. The singular values are returned in decreasing order
587 * and only the first $\min(M,N)$ columns of $\mathbf{U}$ and rows of
588 * $\mathbf{V}^T$ are computed.
589 *
590 * Upon return the content of the matrix is unusable.
591 * The matrix $\mathbf{A}$ must have identical block cyclic distribution for
592 * the rows and column.
593 *
594 * If left singular vectors are required matrices $\mathbf{A}$ and
595 * $\mathbf{U}$ have to be constructed with the same process grid and block
596 * cyclic distribution. If right singular vectors are required matrices
597 * $\mathbf{A}$ and $\mathbf{V}^T$ have to be constructed with the same
598 * process grid and block cyclic distribution.
599 *
600 * To avoid computing the left and/or right singular vectors the function
601 * accepts <code>nullptr</code>
602 * for @p U and/or @p VT.
603 */
604 std::vector<NumberType>
605 compute_SVD(ScaLAPACKMatrix<NumberType> *U = nullptr,
606 ScaLAPACKMatrix<NumberType> *VT = nullptr);
607
608 /**
609 * Solving overdetermined or underdetermined real linear
610 * systems involving matrix $\mathbf{A} \in \mathbb{R}^{M \times N}$, or its
611 * transpose $\mathbf{A}^T$, using a QR or LQ factorization of $\mathbf{A}$
612 * for $N_{\rm RHS}$ RHS vectors in the columns of matrix $\mathbf{B}$
613 *
614 * It is assumed that $\mathbf{A}$ has full rank: $\rm{rank}(\mathbf{A}) =
615 * \min(M,N)$.
616 *
617 * The following options are supported:
618 * -# If(!transpose) and $M \geq N$: least squares solution of overdetermined
619 * system
620 * $\min \Vert \mathbf{B} - \mathbf{A}\cdot \mathbf{X}\Vert$.\n
621 * Upon exit the rows $0$ to $N-1$ of $\mathbf{B}$ contain the least square
622 * solution vectors. The residual sum of squares for each column is given by
623 * the sum of squares of elements $N$ to $M-1$ in that column.
624 *
625 * -# If(!transpose) and $M < N$: find minimum norm solutions of
626 * underdetermined systems
627 * $\mathbf{A} \cdot \mathbf{X} = \mathbf{B}$.\n
628 * Upon exit the columns of $\mathbf{B}$ contain the minimum norm solution
629 * vectors.
630 *
631 * -# If(transpose) and $M \geq N$: find minimum norm solutions of
632 * underdetermined system $ \mathbf{A}^\top \cdot \mathbf{X} = \mathbf{B}$.\n
633 * Upon exit the columns of $\mathbf{B}$ contain the minimum norm solution
634 * vectors.
635 *
636 * -# If(transpose) and $M < N$: least squares solution of overdetermined
637 * system
638 * $\min \Vert \mathbf{B} - \mathbf{A}^\top \cdot \mathbf{X}\Vert$.\n
639 * Upon exit the rows $0$ to $M-1$ contain the least square solution
640 * vectors. The residual sum of squares for each column is given by the sum of
641 * squares of elements $M$ to $N-1$ in that column.
642 *
643 * If(!transpose) then $\mathbf{B} \in \mathbb{R}^{M \times N_{\rm RHS}}$,
644 * otherwise $\mathbf{B} \in \mathbb{R}^{N \times N_{\rm RHS}}$.
645 * The matrices $\mathbf{A}$ and $\mathbf{B}$ must have an identical block
646 * cyclic distribution for rows and columns.
647 */
648 void
649 least_squares(ScaLAPACKMatrix<NumberType> &B, const bool transpose = false);
650
651 /**
652 * Compute the pseudoinverse $\mathbf{A}^+ \in \mathbb{R}^{N \times M}$
653 * (Moore-Penrose inverse) of a real matrix $\mathbf{A} \in \mathbb{R}^{M
654 * \times N}$ using the singular value decomposition
655 * $\mathbf{A} = \mathbf{U} \cdot \mathbf{\Sigma} \cdot \mathbf{V}^T$.
656 *
657 * Unlike the inverse, the pseudoinverse $\mathbf{A}^+ = \mathbf{V} \cdot
658 * \mathbf{\Sigma}^+ \cdot \mathbf{U}^T$ exists for both rectangular as well
659 * as singular matrices $\mathbf{A}$.
660 *
661 * For a rectangular $\mathbf{\Sigma}$ the pseudoinverse is computed by taking
662 * the reciprocal of each non-zero element on the diagonal, leaving the zeros
663 * in place, and then transposing $\mathbf{\Sigma}$. For the numerical
664 * computation only the singular values $\sigma_i > \sigma_{\text{max}} \,
665 * \text{ratio}$ are taken into account. Upon successful exit, the function
666 * returns the number of singular values fulfilling that condition. That value
667 * can be interpreted as the rank of $\mathbf{A}$.
668 *
669 * Upon return this object contains the pseudoinverse $\mathbf{A}^+ \in
670 * \mathbb{R}^{N \times M}$.
671 *
672 * The following alignment conditions have to be fulfilled: $MB_A = NB_A$.
673 */
674 unsigned int
675 pseudoinverse(const NumberType ratio);
676
677 /**
678 * Estimate the condition number of a SPD matrix in the $l_1$-norm.
679 * The matrix has to be in the Cholesky state (see
680 * compute_cholesky_factorization()). The reciprocal of the condition number
681 * is returned in order to avoid the possibility of overflow when the
682 * condition number is very large.
683 *
684 * @p a_norm must contain the $l_1$-norm of the matrix prior to calling
685 * Cholesky factorization (see l1_norm()).
686 *
687 * @note An alternative is to compute the inverse of the matrix
688 * explicitly and manually construct $k_1 = ||\mathbf{A}||_1 \,
689 * ||\mathbf{A}^{-1}||_1$.
690 */
691 NumberType
692 reciprocal_condition_number(const NumberType a_norm) const;
693
694 /**
695 * Compute the $l_1$-norm of the matrix.
696 */
697 NumberType
698 l1_norm() const;
699
700 /**
701 * Compute the $l_{\infty}$ norm of the matrix.
702 */
703 NumberType
704 linfty_norm() const;
705
706 /**
707 * Compute the Frobenius norm of the matrix.
708 */
709 NumberType
710 frobenius_norm() const;
711
712 /**
713 * Number of rows of the $M \times N$ matrix.
714 */
715 size_type
716 m() const;
717
718 /**
719 * Number of columns of the $M \times N$ matrix.
720 */
721 size_type
722 n() const;
723
724 /**
725 * Number of local rows on this MPI processes.
726 */
727 unsigned int
728 local_m() const;
729
730 /**
731 * Number of local columns on this MPI process.
732 */
733 unsigned int
734 local_n() const;
735
736 /**
737 * Return the global row number for the given local row @p loc_row .
738 */
739 unsigned int
740 global_row(const unsigned int loc_row) const;
741
742 /**
743 * Return the global column number for the given local column @p loc_column.
744 */
745 unsigned int
746 global_column(const unsigned int loc_column) const;
747
748 /**
749 * Read access to local element.
750 */
751 NumberType
752 local_el(const unsigned int loc_row, const unsigned int loc_column) const;
753
754 /**
755 * Write access to local element.
756 */
757 NumberType &
758 local_el(const unsigned int loc_row, const unsigned int loc_column);
759
760 /**
761 * Scale the columns of the distributed matrix by the scalars provided in the array @p factors.
762 *
763 * The array @p factors must have as many entries as the matrix columns.
764 *
765 * Copies of @p factors have to be available on all processes of the underlying MPI communicator.
766 *
767 * @note The fundamental prerequisite for the @p InputVector is that it must be possible to
768 * create an ArrayView from it; this is satisfied by the @p std::vector and Vector classes.
769 */
770 template <class InputVector>
771 void
772 scale_columns(const InputVector &factors);
773
774 /**
775 * Scale the rows of the distributed matrix by the scalars provided in the array @p factors.
776 *
777 * The array @p factors must have as many entries as the matrix rows.
778 *
779 * Copies of @p factors have to be available on all processes of the underlying MPI communicator.
780 *
781 * @note The fundamental prerequisite for the @p InputVector is that it must be possible to
782 * create an ArrayView from it; this is satisfied by the @p std::vector and Vector classes.
783 */
784 template <class InputVector>
785 void
786 scale_rows(const InputVector &factors);
787
788 private:
789 /**
790 * Calculate the norm of a distributed symmetric dense matrix using
791 * ScaLAPACK's internal function.
792 */
793 NumberType
794 norm_symmetric(const char type) const;
795
796 /**
797 * Calculate the norm of a distributed dense matrix using ScaLAPACK's
798 * internal function.
799 */
800 NumberType
801 norm_general(const char type) const;
802
803 /**
804 * Computing selected eigenvalues and, optionally, the eigenvectors.
805 * The eigenvalues/eigenvectors are selected by either prescribing a range of indices @p index_limits
806 * or a range of values @p value_limits for the eigenvalues. The function will throw an exception
807 * if both ranges are prescribed (meaning that both ranges differ from the
808 * default value) as this ambiguity is prohibited. If successful, the computed
809 * eigenvalues are arranged in ascending order. The eigenvectors are stored in
810 * the columns of the matrix, thereby overwriting the original content of the
811 * matrix.
812 */
813 std::vector<NumberType>
814 eigenpairs_symmetric(
815 const bool compute_eigenvectors,
816 const std::pair<unsigned int, unsigned int> &index_limits =
817 std::make_pair(numbers::invalid_unsigned_int,
818 numbers::invalid_unsigned_int),
819 const std::pair<NumberType, NumberType> &value_limits =
820 std::make_pair(std::numeric_limits<NumberType>::quiet_NaN(),
821 std::numeric_limits<NumberType>::quiet_NaN()));
822
823 /**
824 * Computing selected eigenvalues and, optionally, the eigenvectors of the
825 * real symmetric matrix $\mathbf{A} \in \mathbb{R}^{M \times M}$ using the
826 * MRRR algorithm.
827 * The eigenvalues/eigenvectors are selected by either prescribing a range of indices @p index_limits
828 * or a range of values @p value_limits for the eigenvalues. The function will throw an exception
829 * if both ranges are prescribed (meaning that both ranges differ from the
830 * default value) as this ambiguity is prohibited.
831 *
832 * By calling this function the original content of the matrix will be
833 * overwritten. If requested, the eigenvectors are stored in the columns of
834 * the matrix. Also in the case that just the eigenvalues are required, the
835 * content of the matrix will be overwritten.
836 *
837 * If successful, the computed eigenvalues are arranged in ascending order.
838 *
839 * @note Due to a bug in Netlib-ScaLAPACK, either all or no eigenvectors can be computed.
840 * Therefore, the input @p index_limits has to be set accordingly. Using Intel-MKL this restriction is not required.
841 */
842 std::vector<NumberType>
843 eigenpairs_symmetric_MRRR(
844 const bool compute_eigenvectors,
845 const std::pair<unsigned int, unsigned int> &index_limits =
846 std::make_pair(numbers::invalid_unsigned_int,
847 numbers::invalid_unsigned_int),
848 const std::pair<NumberType, NumberType> &value_limits =
849 std::make_pair(std::numeric_limits<NumberType>::quiet_NaN(),
850 std::numeric_limits<NumberType>::quiet_NaN()));
851
852 /*
853 * Stores the distributed matrix in @p filename
854 * using serial routines
855 */
856 void
857 save_serial(const std::string &filename,
858 const std::pair<unsigned int, unsigned int> &chunk_size) const;
859
860 /*
861 * Loads the distributed matrix from file @p filename
862 * using serial routines
863 */
864 void
865 load_serial(const std::string &filename);
866
867 /*
868 * Stores the distributed matrix in @p filename
869 * using parallel routines
870 */
871 void
872 save_parallel(const std::string &filename,
873 const std::pair<unsigned int, unsigned int> &chunk_size) const;
874
875 /*
876 * Loads the distributed matrix from file @p filename
877 * using parallel routines
878 */
879 void
880 load_parallel(const std::string &filename);
881
882 /**
883 * Since ScaLAPACK operations notoriously change the meaning of the matrix
884 * entries, we record the current state after the last operation here.
885 */
886 LAPACKSupport::State state;
887
888 /**
889 * Additional property of the matrix which may help to select more
890 * efficient ScaLAPACK functions.
891 */
892 LAPACKSupport::Property property;
893
894 /**
895 * A shared pointer to a Utilities::MPI::ProcessGrid object which contains a
896 * BLACS context and a MPI communicator, as well as other necessary data
897 * structures.
898 */
899 std::shared_ptr<const Utilities::MPI::ProcessGrid> grid;
900
901 /**
902 * Number of rows in the matrix.
903 */
904 int n_rows;
905
906 /**
907 * Number of columns in the matrix.
908 */
909 int n_columns;
910
911 /**
912 * Row block size.
913 */
914 int row_block_size;
915
916 /**
917 * Column block size.
918 */
919 int column_block_size;
920
921 /**
922 * Number of rows in the matrix owned by the current process.
923 */
924 int n_local_rows;
925
926 /**
927 * Number of columns in the matrix owned by the current process.
928 */
929 int n_local_columns;
930
931 /**
932 * ScaLAPACK description vector.
933 */
934 int descriptor[9];
935
936 /**
937 * Workspace array.
938 */
939 mutable std::vector<NumberType> work;
940
941 /**
942 * Integer workspace array.
943 */
944 mutable std::vector<int> iwork;
945
946 /**
947 * Integer array holding pivoting information required
948 * by ScaLAPACK's matrix factorization routines.
949 */
950 std::vector<int> ipiv;
951
952 /**
953 * A character to define where elements are stored in case
954 * ScaLAPACK operations support this.
955 */
956 const char uplo;
957
958 /**
959 * The process row of the process grid over which the first row
960 * of the global matrix is distributed.
961 */
962 const int first_process_row;
963
964 /**
965 * The process column of the process grid over which the first column
966 * of the global matrix is distributed.
967 */
968 const int first_process_column;
969
970 /**
971 * Global row index that determines where to start a submatrix.
972 * Currently this equals unity, as we don't use submatrices.
973 */
974 const int submatrix_row;
975
976 /**
977 * Global column index that determines where to start a submatrix.
978 * Currently this equals unity, as we don't use submatrices.
979 */
980 const int submatrix_column;
981
982 /**
983 * Thread mutex.
984 */
985 mutable Threads::Mutex mutex;
986 };
987
988 // ----------------------- inline functions ----------------------------
989
990 # ifndef DOXYGEN
991
992 template <typename NumberType>
993 inline NumberType
994 ScaLAPACKMatrix<NumberType>::local_el(const unsigned int loc_row,
995 const unsigned int loc_column) const
996 {
997 return (*this)(loc_row, loc_column);
998 }
999
1000
1001
1002 template <typename NumberType>
1003 inline NumberType &
1004 ScaLAPACKMatrix<NumberType>::local_el(const unsigned int loc_row,
1005 const unsigned int loc_column)
1006 {
1007 return (*this)(loc_row, loc_column);
1008 }
1009
1010
1011 template <typename NumberType>
1012 inline unsigned int
1013 ScaLAPACKMatrix<NumberType>::m() const
1014 {
1015 return n_rows;
1016 }
1017
1018
1019
1020 template <typename NumberType>
1021 inline unsigned int
1022 ScaLAPACKMatrix<NumberType>::n() const
1023 {
1024 return n_columns;
1025 }
1026
1027
1028
1029 template <typename NumberType>
1030 unsigned int
1031 ScaLAPACKMatrix<NumberType>::local_m() const
1032 {
1033 return n_local_rows;
1034 }
1035
1036
1037
1038 template <typename NumberType>
1039 unsigned int
1040 ScaLAPACKMatrix<NumberType>::local_n() const
1041 {
1042 return n_local_columns;
1043 }
1044
1045
1046 # endif // DOXYGEN
1047
1048 DEAL_II_NAMESPACE_CLOSE
1049
1050 #else
1051
1052 // Make sure the scripts that create the C++20 module input files have
1053 // something to latch on if the preprocessor #ifdef above would
1054 // otherwise lead to an empty content of the file.
1055 DEAL_II_NAMESPACE_OPEN
1056 DEAL_II_NAMESPACE_CLOSE
1057
1058 #endif // DEAL_II_WITH_SCALAPACK
1059
1060 #endif