Debian Codegraph @

include/deal.II/matrix_free/matrix_free.h

raw
1 // ------------------------------------------------------------------------
2 //
3 // SPDX-License-Identifier: LGPL-2.1-or-later
4 // Copyright (C) 2012 - 2025 by the deal.II authors
5 //
6 // This file is part of the deal.II library.
7 //
8 // Part of the source code is dual licensed under Apache-2.0 WITH
9 // LLVM-exception OR LGPL-2.1-or-later. Detailed license information
10 // governing the source code and code contributions can be found in
11 // LICENSE.md and CONTRIBUTING.md at the top level directory of deal.II.
12 //
13 // ------------------------------------------------------------------------
14
15
16 #ifndef dealii_matrix_free_h
17 #define dealii_matrix_free_h
18
19 #include <deal.II/base/config.h>
20
21 #include <deal.II/base/aligned_vector.h>
22 #include <deal.II/base/exceptions.h>
23 #include <deal.II/base/numbers.h>
24 #include <deal.II/base/quadrature.h>
25 #include <deal.II/base/template_constraints.h>
26 #include <deal.II/base/thread_local_storage.h>
27 #include <deal.II/base/vectorization.h>
28
29 #include <deal.II/dofs/dof_handler.h>
30
31 #include <deal.II/fe/fe.h>
32 #include <deal.II/fe/mapping.h>
33
34 #include <deal.II/hp/mapping_collection.h>
35 #include <deal.II/hp/q_collection.h>
36
37 #include <deal.II/lac/affine_constraints.h>
38 #include <deal.II/lac/block_vector_base.h>
39 #include <deal.II/lac/la_parallel_vector.h>
40 #include <deal.II/lac/vector_operation.h>
41
42 #include <deal.II/matrix_free/dof_info.h>
43 #include <deal.II/matrix_free/mapping_info.h>
44 #include <deal.II/matrix_free/shape_info.h>
45 #include <deal.II/matrix_free/task_info.h>
46 #include <deal.II/matrix_free/type_traits.h>
47 #include <deal.II/matrix_free/vector_data_exchange.h>
48
49 #include <cstdlib>
50 #include <limits>
51 #include <list>
52 #include <memory>
53
54
55 DEAL_II_NAMESPACE_OPEN
56
57
58
59 /**
60 * This class collects all the data that is stored for the matrix free
61 * implementation. The storage scheme is tailored towards several loops
62 * performed with the same data, i.e., typically doing many matrix-vector
63 * products or residual computations on the same mesh. The class is used in
64 * step-37 and step-48.
65 *
66 * This class does not implement any operations involving finite element basis
67 * functions, i.e., regarding the operation performed on the cells. For these
68 * operations, the class FEEvaluation is designed to use the data collected in
69 * this class.
70 *
71 * The stored data can be subdivided into three main components:
72 *
73 * - DoFInfo: It stores how local degrees of freedom relate to global degrees
74 * of freedom. It includes a description of constraints that are evaluated as
75 * going through all local degrees of freedom on a cell.
76 *
77 * - MappingInfo: It stores the transformations from real to unit cells that
78 * are necessary in order to build derivatives of finite element functions and
79 * find location of quadrature weights in physical space.
80 *
81 * - ShapeInfo: It contains the shape functions of the finite element,
82 * evaluated on the unit cell.
83 *
84 * Besides the initialization routines, this class implements only a single
85 * operation, namely a loop over all cells (cell_loop()). This loop is
86 * scheduled in such a way that cells that share degrees of freedom are not
87 * worked on simultaneously, which implies that it is possible to write to
88 * vectors (or matrices) in parallel without having to explicitly synchronize
89 * access to these vectors and matrices. This class does not implement any
90 * shape values, all it does is to cache the respective data. To implement
91 * finite element operations, use the class FEEvaluation (or some of the
92 * related classes).
93 *
94 * This class traverses the cells in a different order than the usual
95 * Triangulation class in deal.II, in order to be flexible with respect to
96 * parallelization in shared memory and vectorization.
97 *
98 * Vectorization is implemented by merging several topological cells into one
99 * so-called `cell batch`. This enables the application of all cell-related
100 * operations for several cells with one CPU instruction and is one of the
101 * main features of this framework.
102 *
103 * For details on usage of this class, see the description of FEEvaluation or
104 * the
105 * @ref matrixfree "matrix-free topic".
106 *
107 * @ingroup matrixfree
108 */
109
110 template <int dim,
111 typename Number = double,
112 typename VectorizedArrayType = VectorizedArray<Number>>
113 class MatrixFree : public EnableObserverPointer
114 {
115 static_assert(
116 std::is_same_v<Number, typename VectorizedArrayType::value_type>,
117 "Type of Number and of VectorizedArrayType do not match.");
118
119 public:
120 /**
121 * An alias for the underlying number type specified by the template
122 * argument.
123 */
124 using value_type = Number;
125 using vectorized_value_type = VectorizedArrayType;
126
127 /**
128 * The dimension set by the template argument `dim`.
129 */
130 static constexpr unsigned int dimension = dim;
131
132 /**
133 * Collects the options for initialization of the MatrixFree class. The
134 * parameter @p tasks_parallel_scheme specifies the
135 * parallelization options in shared memory (task-based parallelism, where
136 * one can choose between no parallelism and three schemes that avoid that
137 * cells with access to the same vector entries are accessed
138 * simultaneously), and the parameter @p tasks_block_size the block size for
139 * task parallel scheduling. The parameters @p mapping_update_flags,
140 * @p mapping_update_flags_boundary_faces, @p mapping_update_flags_inner_faces,
141 * and @p mapping_update_flags_faces_by_cells specify the update flags that
142 * should be stored by this class.
143 *
144 * The parameter @p mg_level specifies the level in the triangulation from which
145 * the indices are to be used. If the level is set to
146 * `numbers::invalid_unsigned_int`, the active cells are traversed, and
147 * otherwise the cells in the given level. This option has no effect in case
148 * a DoFHandler is given.
149 *
150 * The parameter @p store_plain_indices indicates whether the DoFInfo
151 * class should also allow for access to vectors without resolving
152 * constraints.
153 *
154 * The two parameters @p initialize_indices and @p initialize_mapping allow
155 * the user to disable some of the initialization processes. For example, if
156 * only the scheduling that avoids touching the same vector/matrix indices
157 * simultaneously is to be found, the mapping needs not be
158 * initialized. Likewise, if the mapping has changed from one iteration to
159 * the next but the topology has not (like when using a deforming mesh with
160 * MappingQEulerian), it suffices to initialize the mapping only.
161 *
162 * The two parameters @p cell_vectorization_categories and
163 * @p cell_vectorization_categories_strict control the formation of batches
164 * for vectorization over several cells. It is used implicitly when working
165 * with hp-adaptivity but can also be useful in other contexts, such as in
166 * local time stepping where one would like to control which elements
167 * together form a batch of cells. The array @p cell_vectorization_categories
168 * is accessed by the number given by cell->active_cell_index() when working
169 * on the active cells with @p mg_level set to `numbers::invalid_unsigned_int`
170 * and by cell->index() for the level cells. By default, the different
171 * categories in @p cell_vectorization_category can be mixed and the algorithm
172 * is allowed to merge lower category numbers with the next higher categories
173 * if it is necessary inside the algorithm, in order to avoid partially
174 * filled SIMD lanes as much as possible. This gives a better utilization of
175 * the vectorization but might need special treatment, in particular for
176 * face integrals. If set to `true', the algorithm will instead keep
177 * different categories separate and not mix them in a single vectorized
178 * array.
179 *
180 * Finally, @p allow_ghosted_vectors_in_loops allows to enable and disable
181 * checks and @p communicator_sm gives the MPI communicator to be used
182 * if MPI-3.0 shared-memory features should be used.
183 */
184 struct AdditionalData
185 {
186 /**
187 * Provide the type of the surrounding MatrixFree class.
188 */
189 using MatrixFreeType = MatrixFree<dim, Number, VectorizedArrayType>;
190
191 /**
192 * Collects options for task parallelism. See the documentation of the
193 * member variable MatrixFree::AdditionalData::tasks_parallel_scheme for a
194 * thorough description.
195 */
196 enum TasksParallelScheme
197 {
198 /**
199 * Perform application in serial.
200 */
201 none = internal::MatrixFreeFunctions::TaskInfo::none,
202 /**
203 * Partition the cells into two levels and afterwards form chunks.
204 */
205 partition_partition =
206 internal::MatrixFreeFunctions::TaskInfo::partition_partition,
207 /**
208 * Partition on the global level and color cells within the partitions.
209 */
210 partition_color =
211 internal::MatrixFreeFunctions::TaskInfo::partition_color,
212 /**
213 * Use the traditional coloring algorithm: this is like
214 * TasksParallelScheme::partition_color, but only uses one partition.
215 */
216 color = internal::MatrixFreeFunctions::TaskInfo::color
217 };
218
219 /**
220 * Constructor for AdditionalData.
221 */
222 AdditionalData(
223 const TasksParallelScheme tasks_parallel_scheme = partition_partition,
224 const unsigned int tasks_block_size = 0,
225 const UpdateFlags mapping_update_flags = update_gradients |
226 update_JxW_values,
227 const UpdateFlags mapping_update_flags_boundary_faces = update_default,
228 const UpdateFlags mapping_update_flags_inner_faces = update_default,
229 const UpdateFlags mapping_update_flags_faces_by_cells = update_default,
230 const unsigned int mg_level = numbers::invalid_unsigned_int,
231 const bool store_plain_indices = true,
232 const bool initialize_indices = true,
233 const bool initialize_mapping = true,
234 const bool overlap_communication_computation = true,
235 const bool hold_all_faces_to_owned_cells = false,
236 const bool cell_vectorization_categories_strict = false,
237 const bool allow_ghosted_vectors_in_loops = true)
238 : tasks_parallel_scheme(tasks_parallel_scheme)
239 , tasks_block_size(tasks_block_size)
240 , mapping_update_flags(mapping_update_flags)
241 , mapping_update_flags_boundary_faces(mapping_update_flags_boundary_faces)
242 , mapping_update_flags_inner_faces(mapping_update_flags_inner_faces)
243 , mapping_update_flags_faces_by_cells(mapping_update_flags_faces_by_cells)
244 , mg_level(mg_level)
245 , store_plain_indices(store_plain_indices)
246 , initialize_indices(initialize_indices)
247 , initialize_mapping(initialize_mapping)
248 , overlap_communication_computation(overlap_communication_computation)
249 , hold_all_faces_to_owned_cells(hold_all_faces_to_owned_cells)
250 , cell_vectorization_categories_strict(
251 cell_vectorization_categories_strict)
252 , allow_ghosted_vectors_in_loops(allow_ghosted_vectors_in_loops)
253 , store_ghost_cells(false)
254 , communicator_sm(MPI_COMM_SELF)
255 {}
256
257 /**
258 * Copy constructor.
259 */
260 AdditionalData(const AdditionalData &other)
261 : tasks_parallel_scheme(other.tasks_parallel_scheme)
262 , tasks_block_size(other.tasks_block_size)
263 , mapping_update_flags(other.mapping_update_flags)
264 , mapping_update_flags_boundary_faces(
265 other.mapping_update_flags_boundary_faces)
266 , mapping_update_flags_inner_faces(other.mapping_update_flags_inner_faces)
267 , mapping_update_flags_faces_by_cells(
268 other.mapping_update_flags_faces_by_cells)
269 , mg_level(other.mg_level)
270 , store_plain_indices(other.store_plain_indices)
271 , initialize_indices(other.initialize_indices)
272 , initialize_mapping(other.initialize_mapping)
273 , overlap_communication_computation(
274 other.overlap_communication_computation)
275 , hold_all_faces_to_owned_cells(other.hold_all_faces_to_owned_cells)
276 , cell_vectorization_category(other.cell_vectorization_category)
277 , cell_vectorization_categories_strict(
278 other.cell_vectorization_categories_strict)
279 , allow_ghosted_vectors_in_loops(other.allow_ghosted_vectors_in_loops)
280 , store_ghost_cells(other.store_ghost_cells)
281 , communicator_sm(other.communicator_sm)
282 {}
283
284 /**
285 * Copy assignment.
286 */
287 AdditionalData &
288 operator=(const AdditionalData &other) = default;
289
290 /**
291 * Set the scheme for task parallelism. There are four options available.
292 * If set to @p none, the operator application is done in serial without
293 * shared memory parallelism. If this class is used together with MPI and
294 * MPI is also used for parallelism within the nodes, this flag should be
295 * set to @p none. The default value is @p partition_partition, i.e. we
296 * actually use multithreading with the first option described below.
297 *
298 * The first option @p partition_partition is to partition the cells on
299 * two levels in onion-skin-like partitions and forming chunks of
300 * tasks_block_size after the partitioning. The partitioning finds sets of
301 * independent cells that enable working in parallel without accessing the
302 * same vector entries at the same time.
303 *
304 * The second option @p partition_color is to use a partition on the
305 * global level and color cells within the partitions (where all chunks
306 * within a color are independent). Here, the subdivision into chunks of
307 * cells is done before the partitioning, which might give worse
308 * partitions but better cache performance if degrees of freedom are not
309 * renumbered.
310 *
311 * The third option @p color is to use a traditional algorithm of coloring
312 * on the global level. This scheme is a special case of the second option
313 * where only one partition is present. Note that for problems with
314 * hanging nodes, there are quite many colors (50 or more in 3d), which
315 * might degrade parallel performance (bad cache behavior, many
316 * synchronization points).
317 *
318 * @note Threading support is currently experimental for the case inner
319 * face integrals are performed and it is recommended to use MPI
320 * parallelism if possible. While the scheme has been verified to work
321 * with the `partition_partition` option in case of usual DG elements, no
322 * comprehensive tests have been performed for systems of more general
323 * elements, like combinations of continuous and discontinuous elements
324 * that add face integrals to all terms.
325 */
326 TasksParallelScheme tasks_parallel_scheme;
327
328 /**
329 * Set the number of so-called cell batches that should form one
330 * partition. If zero size is given, the class tries to find a good size
331 * for the blocks based on MultithreadInfo::n_threads() and the number of
332 * cells present. Otherwise, the given number is used. If the given number
333 * is larger than one third of the number of total cells, this means no
334 * parallelism. Note that in the case vectorization is used, a cell batch
335 * consists of more than one physical cell.
336 */
337 unsigned int tasks_block_size;
338
339 /**
340 * This flag determines what data needs to be computed and cached on cells.
341 *
342 * If your computations require operations like quadrature point locations
343 * or Hessians, these need to specified here (update_quadrature_points
344 * or update_hessians, respectively). Note that values, gradients, and
345 * Jacobian determinants (JxW values) are always computed regardless of the
346 * flags specified here.
347 *
348 * Note that some additional flags might be set automatically (for example
349 * second derivatives might be evaluated on Cartesian cells since there
350 * the Jacobian describes the mapping completely).
351 */
352 UpdateFlags mapping_update_flags;
353
354 /**
355 * This flag determines the mapping data on boundary faces to be
356 * cached. Note that MatrixFree uses a separate loop layout for face
357 * integrals in order to effectively vectorize also in the case of hanging
358 * nodes (which require different subface settings on the two sides) or
359 * some cells in the batch of a VectorizedArray of cells that are adjacent
360 * to the boundary and others that are not.
361 *
362 * If set to a value different from update_general (default), the face
363 * information is explicitly built. Currently, MatrixFree supports to
364 * cache the following data on faces: inverse Jacobians, Jacobian
365 * determinants (JxW), quadrature points, data for Hessians (derivative of
366 * Jacobians), and normal vectors.
367 *
368 * @note In order to be able to perform a `boundary_face_operation` in the
369 * MatrixFree::loop(), this field must be set to a value different from
370 * UpdateFlags::update_default.
371 */
372 UpdateFlags mapping_update_flags_boundary_faces;
373
374 /**
375 * This flag determines the mapping data on interior faces to be
376 * cached. Note that MatrixFree uses a separate loop layout for face
377 * integrals in order to effectively vectorize also in the case of hanging
378 * nodes (which require different subface settings on the two sides) or
379 * some cells in the batch of a VectorizedArray of cells that are adjacent
380 * to the boundary and others that are not.
381 *
382 * If set to a value different from update_general (default), the face
383 * information is explicitly built. Currently, MatrixFree supports to
384 * cache the following data on faces: inverse Jacobians, Jacobian
385 * determinants (JxW), quadrature points, data for Hessians (derivative of
386 * Jacobians), and normal vectors.
387 *
388 * @note In order to be able to perform a `inner_face_operation`
389 * in the MatrixFree::loop(), this field must be set to a value different
390 * from UpdateFlags::update_default.
391 */
392 UpdateFlags mapping_update_flags_inner_faces;
393
394 /**
395 * This flag determines the mapping data for faces in a different layout
396 * with respect to vectorizations. Whereas
397 * `mapping_update_flags_inner_faces` and
398 * `mapping_update_flags_boundary_faces` trigger building the data in a
399 * face-centric way with proper vectorization, the current data field
400 * attaches the face information to the cells and their way of
401 * vectorization. This is only needed in special situations, as for
402 * example for block-Jacobi methods where the full operator to a cell
403 * including its faces are evaluated. This data is accessed by
404 * <code>FEFaceEvaluation::reinit(cell_batch_index,
405 * face_number)</code>. However, currently no coupling terms to neighbors
406 * can be computed with this approach because the neighbors are not laid
407 * out by the VectorizedArray data layout with an
408 * array-of-struct-of-array-type data structures.
409 *
410 * Note that you should only compute this data field in case you really
411 * need it as it more than doubles the memory required by the mapping data
412 * on faces.
413 *
414 * If set to a value different from update_general (default), the face
415 * information is explicitly built. Currently, MatrixFree supports to
416 * cache the following data on faces: inverse Jacobians, Jacobian
417 * determinants (JxW), quadrature points, data for Hessians (derivative of
418 * Jacobians), and normal vectors.
419 */
420 UpdateFlags mapping_update_flags_faces_by_cells;
421
422 /**
423 * This option can be used to define whether we work on a certain level of
424 * the mesh, and not the active cells. If set to invalid_unsigned_int
425 * (which is the default value), the active cells are gone through,
426 * otherwise the level given by this parameter. Note that if you specify
427 * to work on a level, its dofs must be distributed by using
428 * <code>dof_handler.distribute_mg_dofs(fe);</code>.
429 */
430 unsigned int mg_level;
431
432 /**
433 * Controls whether to enable reading from vectors without resolving
434 * constraints, i.e., just read the local values of the vector. By
435 * default, this option is enabled. In case you want to use
436 * FEEvaluationBase::read_dof_values_plain, this flag needs to be set.
437 */
438 bool store_plain_indices;
439
440 /**
441 * Option to control whether the indices stored in the DoFHandler
442 * should be read and the pattern for task parallelism should be
443 * set up in the initialize method of MatrixFree. The default
444 * value is true. Can be disabled in case the mapping should be
445 * recomputed (e.g. when using a deforming mesh described through
446 * MappingEulerian) but the topology of cells has remained the
447 * same.
448 */
449 bool initialize_indices;
450
451 /**
452 * Option to control whether the mapping information should be
453 * computed in the initialize method of MatrixFree. The default
454 * value is true. Can be disabled when only some indices should be
455 * set up (e.g. when only a set of independent cells should be
456 * computed).
457 */
458 bool initialize_mapping;
459
460 /**
461 * Option to control whether the loops should overlap communications and
462 * computations as far as possible in case the vectors passed to the loops
463 * support non-blocking data exchange. In most situations, overlapping is
464 * faster in case the amount of data to be sent is more than a few
465 * kilobytes. If less data is sent, the communication is latency bound on
466 * most clusters (point-to-point latency is around 1 microsecond on good
467 * clusters by 2016 standards). Depending on the MPI implementation and
468 * the fabric, it may be faster to not overlap and wait for the data to
469 * arrive. The default is true, i.e., communication and computation are
470 * overlapped.
471 */
472 bool overlap_communication_computation;
473
474 /**
475 * By default, the face part will only hold those faces (and ghost
476 * elements behind faces) that are going to be processed locally. In case
477 * MatrixFree should have access to all neighbors on locally owned cells,
478 * this option enables adding the respective faces at the end of the face
479 * range.
480 */
481 bool hold_all_faces_to_owned_cells;
482
483 /**
484 * This data structure allows to assign a fraction of cells to different
485 * categories when building the information for vectorization. It is used
486 * implicitly when working with hp-adaptivity (with each active index
487 * being a category) but can also be useful in other contexts where one
488 * would like to control which cells together can form a batch of cells.
489 * Such an example is "local time stepping", where cells of different
490 * categories progress with different time-step sizes and, as a
491 * consequence, can only processed together with cells with the same
492 * category.
493 *
494 * This array is accessed by the number given by cell->active_cell_index()
495 * when working on the active cells (with
496 * @p mg_level set to numbers::invalid_unsigned_int) and by cell->index()
497 * for the level cells.
498 *
499 * @note This field is empty upon construction of AdditionalData. It is
500 * the responsibility of the user to resize this field to
501 * `triangulation.n_active_cells()` or `triangulation.n_cells(level)` when
502 * filling data.
503 */
504 std::vector<unsigned int> cell_vectorization_category;
505
506 /**
507 * By default, the different categories in @p cell_vectorization_category
508 * can be mixed and the algorithm is allowed to merge lower categories with
509 * the next higher categories if it is necessary inside the algorithm. This
510 * gives a better utilization of the vectorization but might need special
511 * treatment, in particular for face integrals. If set to @p true, the
512 * algorithm will instead keep different categories separate and not mix
513 * them in a single vectorized array.
514 */
515 bool cell_vectorization_categories_strict;
516
517 /**
518 * Assert that vectors passed to the MatrixFree loops are not ghosted.
519 * This variable is primarily intended to reveal bugs or performance
520 * problems caused by vectors that are involuntarily in ghosted mode,
521 * by adding a check that this is not the case. In terms of correctness,
522 * the MatrixFree::loop() and MatrixFree::cell_loop() methods support
523 * both cases and perform similar operations. In particular, ghost values
524 * are always updated on the source vector within the loop, and the
525 * difference is only in whether the initial non-ghosted state is restored.
526 */
527 bool allow_ghosted_vectors_in_loops;
528
529 /**
530 * Option to control whether data should be generated on ghost cells.
531 * If set to true, the data on ghost cells will be generated.
532 * The default value is false.
533 */
534 bool store_ghost_cells;
535
536 /**
537 * Shared-memory MPI communicator. Default: MPI_COMM_SELF.
538 */
539 MPI_Comm communicator_sm;
540 };
541
542 /**
543 * @name Construction and initialization
544 */
545 /** @{ */
546 /**
547 * Default empty constructor. Does nothing.
548 */
549 MatrixFree();
550
551 /**
552 * Copy constructor, calls copy_from
553 */
554 MatrixFree(const MatrixFree<dim, Number, VectorizedArrayType> &other);
555
556 /**
557 * Destructor.
558 */
559 ~MatrixFree() override = default;
560
561 /**
562 * Extracts the information needed to perform loops over cells. The
563 * DoFHandler and AffineConstraints objects describe the layout of degrees
564 * of freedom, the DoFHandler and the mapping describe the
565 * transformations from unit to real cell, and the finite element
566 * underlying the DoFHandler together with the quadrature formula
567 * describe the local operations. Note that the finite element underlying
568 * the DoFHandler must either be scalar or contain several copies of the
569 * same element. Mixing several different elements into one FESystem is
570 * not allowed. In that case, use the initialization function with
571 * several DoFHandler arguments.
572 */
573 template <typename QuadratureType, typename number2, typename MappingType>
574 void
575 reinit(const MappingType &mapping,
576 const DoFHandler<dim> &dof_handler,
577 const AffineConstraints<number2> &constraint,
578 const QuadratureType &quad,
579 const AdditionalData &additional_data = AdditionalData());
580
581 /**
582 * Extracts the information needed to perform loops over cells. The
583 * DoFHandler and AffineConstraints objects describe the layout of degrees of
584 * freedom, the DoFHandler and the mapping describe the transformations from
585 * unit to real cell, and the finite element underlying the DoFHandler
586 * together with the quadrature formula describe the local operations. As
587 * opposed to the scalar case treated with the other initialization
588 * functions, this function allows for problems with two or more different
589 * finite elements. The DoFHandlers to each element must be passed as
590 * pointers to the initialization function. Alternatively, a system of
591 * several components may also be represented by a single DoFHandler with an
592 * FESystem element. The prerequisite for this case is that each base
593 * element of the FESystem must be compatible with the present class, such
594 * as the FE_Q or FE_DGQ classes.
595 *
596 * This function also allows for using several quadrature formulas, e.g.
597 * when the description contains independent integrations of elements of
598 * different degrees. However, the number of different quadrature formulas
599 * can be sets independently from the number of DoFHandlers, when several
600 * elements are always integrated with the same quadrature formula.
601 */
602 template <typename QuadratureType, typename number2, typename MappingType>
603 void
604 reinit(const MappingType &mapping,
605 const std::vector<const DoFHandler<dim> *> &dof_handler,
606 const std::vector<const AffineConstraints<number2> *> &constraint,
607 const std::vector<QuadratureType> &quad,
608 const AdditionalData &additional_data = AdditionalData());
609
610 /**
611 * Initializes the data structures. Same as before, but now the index set
612 * description of the locally owned range of degrees of freedom is taken
613 * from the DoFHandler. Moreover, only a single quadrature formula is used,
614 * as might be necessary when several components in a vector-valued problem
615 * are integrated together based on the same quadrature formula.
616 */
617 template <typename QuadratureType, typename number2, typename MappingType>
618 void
619 reinit(const MappingType &mapping,
620 const std::vector<const DoFHandler<dim> *> &dof_handler,
621 const std::vector<const AffineConstraints<number2> *> &constraint,
622 const QuadratureType &quad,
623 const AdditionalData &additional_data = AdditionalData());
624
625 /**
626 * Copy function. Creates a deep copy of all data structures. It is usually
627 * enough to keep the data for different operations once, so this function
628 * should not be needed very often.
629 */
630 void
631 copy_from(
632 const MatrixFree<dim, Number, VectorizedArrayType> &matrix_free_base);
633
634 /**
635 * Refreshes the geometry data stored in the MappingInfo fields when the
636 * underlying geometry has changed (e.g. by a mapping that can deform
637 * through a change in the spatial configuration like MappingFEField)
638 * whereas the topology of the mesh and unknowns have remained the
639 * same. Compared to reinit(), this operation only has to re-generate the
640 * geometry arrays and can thus be significantly cheaper (depending on the
641 * cost to evaluate the geometry).
642 */
643 void
644 update_mapping(const Mapping<dim> &mapping);
645
646 /**
647 * Same as above but with hp::MappingCollection.
648 */
649 void
650 update_mapping(const std::shared_ptr<hp::MappingCollection<dim>> &mapping);
651
652 /**
653 * Clear all data fields and brings the class into a condition similar to
654 * after having called the default constructor.
655 */
656 void
657 clear();
658
659 /** @} */
660
661 /**
662 * This class defines the type of data access for face integrals in loop ()
663 * that is passed on to the `update_ghost_values` and `compress` functions
664 * of the parallel vectors, with the purpose of being able to reduce the
665 * amount of data that must be exchanged. The data exchange is a real
666 * bottleneck in particular for high-degree DG methods, therefore a more
667 * restrictive way of exchange is clearly beneficial. Note that this
668 * selection applies to FEFaceEvaluation objects assigned to the exterior
669 * side of cells accessing `FaceToCellTopology::exterior_cells` only; all
670 * <i>interior</i> objects are available in any case.
671 */
672 enum class DataAccessOnFaces
673 {
674 /**
675 * The loop does not involve any FEFaceEvaluation access into neighbors,
676 * as is the case with only boundary integrals (but no interior face
677 * integrals) or when doing mass matrices in a MatrixFree::cell_loop()
678 * like setup.
679 */
680 none,
681
682 /**
683 * The loop does only involve FEFaceEvaluation access into neighbors by
684 * function values, such as FEFaceEvaluation::gather_evaluate() with
685 * argument EvaluationFlags::values, but no access to shape function
686 * derivatives (which typically need to access more data). For FiniteElement
687 * types where only some of the shape functions have support on a face, such
688 * as an FE_DGQ element with Lagrange polynomials with nodes on the element
689 * surface, the data exchange is reduced from `(k+1)^dim` to
690 * `(k+1)^(dim-1)`.
691 */
692 values,
693
694 /**
695 * Same as above. To be used if data has to be accessed from exterior faces
696 * if FEFaceEvaluation was reinitialized by providing the cell batch number
697 * and a face number. This configuration is useful in the context of
698 * cell-centric loops.
699 *
700 * @pre AdditionalData::hold_all_faces_to_owned_cells has to enabled.
701 */
702 values_all_faces,
703
704 /**
705 * The loop does involve FEFaceEvaluation access into neighbors by
706 * function values and gradients, but no second derivatives, such as
707 * FEFaceEvaluation::gather_evaluate() with EvaluationFlags::values and
708 * EvaluationFlags::gradients set. For FiniteElement types where only some
709 * of the shape functions have non-zero value and first derivative on a
710 * face, such as an FE_DGQHermite element, the data exchange is reduced,
711 * e.g. from `(k+1)^dim` to `2(k+1)^(dim-1)`. Note that for bases that do
712 * not have this special property, the full neighboring data is sent anyway.
713 */
714 gradients,
715
716 /**
717 * Same as above. To be used if data has to be accessed from exterior faces
718 * if FEFaceEvaluation was reinitialized by providing the cell batch number
719 * and a face number. This configuration is useful in the context of
720 * cell-centric loops.
721 *
722 * @pre AdditionalData::hold_all_faces_to_owned_cells has to enabled.
723 */
724 gradients_all_faces,
725
726 /**
727 * General setup where the user does not want to make a restriction. This
728 * is typically more expensive than the other options, but also the most
729 * conservative one because the full data of elements behind the faces to
730 * be computed locally will be exchanged.
731 */
732 unspecified
733 };
734
735 /**
736 * @name Matrix-free loops
737 */
738 /** @{ */
739 /**
740 * This method runs the loop over all cells (in parallel) and performs the
741 * MPI data exchange on the source vector and destination vector.
742 *
743 * @param cell_operation `std::function` with the signature <tt>cell_operation
744 * (const MatrixFree<dim,Number> &, OutVector &, InVector &,
745 * std::pair<unsigned int,unsigned int> &)</tt> where the first argument
746 * passes the data of the calling class and the last argument defines the
747 * range of cells which should be worked on (typically more than one cell
748 * should be worked on in order to reduce overheads). One can pass a pointer
749 * to an object in this place if it has an `operator()` with the correct set
750 * of arguments since such a pointer can be converted to the function object.
751 *
752 * @param dst Destination vector holding the result. If the vector is of
753 * type LinearAlgebra::distributed::Vector (or composite objects thereof
754 * such as LinearAlgebra::distributed::BlockVector), the loop calls
755 * LinearAlgebra::distributed::Vector::compress() at the end of the call
756 * internally. For other vectors, including parallel Trilinos or PETSc
757 * vectors, no such call is issued. Note that Trilinos/Epetra or PETSc
758 * vectors do currently not work in parallel because the present class uses
759 * MPI-local index addressing, as opposed to the global addressing implied
760 * by those external libraries.
761 *
762 * @param src Input vector. If the vector is of type
763 * LinearAlgebra::distributed::Vector (or composite objects thereof such as
764 * LinearAlgebra::distributed::BlockVector), the loop calls
765 * LinearAlgebra::distributed::Vector::update_ghost_values() at the start of
766 * the call internally to make sure all necessary data is locally
767 * available. Note, however, that the vector is reset to its original state
768 * at the end of the loop, i.e., if the vector was not ghosted upon entry of
769 * the loop, it will not be ghosted upon finishing the loop.
770 *
771 * @param zero_dst_vector If this flag is set to `true`, the vector `dst`
772 * will be set to zero inside the loop. Use this case in case you perform a
773 * typical `vmult()` operation on a matrix object, as it will typically be
774 * faster than calling `dst = 0;` before the loop separately. This is
775 * because the vector entries are set to zero only on subranges of the
776 * vector, making sure that the vector entries stay in caches as much as
777 * possible.
778 */
779 template <typename OutVector, typename InVector>
780 void
781 cell_loop(const std::function<void(
782 const MatrixFree<dim, Number, VectorizedArrayType> &,
783 OutVector &,
784 const InVector &,
785 const std::pair<unsigned int, unsigned int> &)> &cell_operation,
786 OutVector &dst,
787 const InVector &src,
788 const bool zero_dst_vector = false) const;
789
790 /**
791 * This is the second variant to run the loop over all cells, now providing
792 * a function pointer to a member function of class `CLASS`. This method
793 * obviates the need to define a lambda function or to call std::bind to bind
794 * the class into the given
795 * function in case the local function needs to access data in the class
796 * (i.e., it is a non-static member function).
797 *
798 * @param cell_operation Pointer to member function of `CLASS` with the
799 * signature <tt>cell_operation (const MatrixFree<dim,Number> &, OutVector &,
800 * InVector &, std::pair<unsigned int,unsigned int> &)</tt> where the first
801 * argument passes the data of the calling class and the last argument
802 * defines the range of cells which should be worked on (typically more than
803 * one cell should be worked on in order to reduce overheads).
804 *
805 * @param owning_class The object which provides the `cell_operation`
806 * call. To be compatible with this interface, the class must allow to call
807 * `owning_class->cell_operation(...)`.
808 *
809 * @param dst Destination vector holding the result. If the vector is of
810 * type LinearAlgebra::distributed::Vector (or composite objects thereof
811 * such as LinearAlgebra::distributed::BlockVector), the loop calls
812 * LinearAlgebra::distributed::Vector::compress() at the end of the call
813 * internally. For other vectors, including parallel Trilinos or PETSc
814 * vectors, no such call is issued. Note that Trilinos/Epetra or PETSc
815 * vectors do currently not work in parallel because the present class uses
816 * MPI-local index addressing, as opposed to the global addressing implied
817 * by those external libraries.
818 *
819 * @param src Input vector. If the vector is of type
820 * LinearAlgebra::distributed::Vector (or composite objects thereof such as
821 * LinearAlgebra::distributed::BlockVector), the loop calls
822 * LinearAlgebra::distributed::Vector::update_ghost_values() at the start of
823 * the call internally to make sure all necessary data is locally
824 * available. Note, however, that the vector is reset to its original state
825 * at the end of the loop, i.e., if the vector was not ghosted upon entry of
826 * the loop, it will not be ghosted upon finishing the loop.
827 *
828 * @param zero_dst_vector If this flag is set to `true`, the vector `dst`
829 * will be set to zero inside the loop. Use this case in case you perform a
830 * typical `vmult()` operation on a matrix object, as it will typically be
831 * faster than calling `dst = 0;` before the loop separately. This is
832 * because the vector entries are set to zero only on subranges of the
833 * vector, making sure that the vector entries stay in caches as much as
834 * possible.
835 */
836 template <typename CLASS, typename OutVector, typename InVector>
837 void
838 cell_loop(void (CLASS::*cell_operation)(
839 const MatrixFree &,
840 OutVector &,
841 const InVector &,
842 const std::pair<unsigned int, unsigned int> &) const,
843 const CLASS *owning_class,
844 OutVector &dst,
845 const InVector &src,
846 const bool zero_dst_vector = false) const;
847
848 /**
849 * Same as above, but for class member functions which are non-const.
850 */
851 template <typename CLASS, typename OutVector, typename InVector>
852 void
853 cell_loop(void (CLASS::*cell_operation)(
854 const MatrixFree &,
855 OutVector &,
856 const InVector &,
857 const std::pair<unsigned int, unsigned int> &),
858 CLASS *owning_class,
859 OutVector &dst,
860 const InVector &src,
861 const bool zero_dst_vector = false) const;
862
863 /**
864 * This function is similar to the cell_loop with an std::function object to
865 * specify to operation to be performed on cells, but adds two additional
866 * functors to execute some additional work before and after the cell
867 * integrals are computed.
868 *
869 * The two additional functors work on a range of degrees of freedom,
870 * expressed in terms of the degree-of-freedom numbering of the selected
871 * DoFHandler `dof_handler_index_pre_post` in MPI-local indices. The
872 * arguments to the functors represent a range of degrees of freedom at a
873 * granularity of
874 * internal::MatrixFreeFunctions::DoFInfo::chunk_size_zero_vector entries
875 * (except for the last chunk which is set to the number of locally owned
876 * entries) in the form `[first, last)`. The idea of these functors is to
877 * bring operations on vectors closer to the point where they accessed in a
878 * matrix-free loop, with the goal to increase cache hits by temporal
879 * locality. This loop guarantees that the `operation_before_loop` hits all
880 * relevant unknowns before they are first touched in the cell_operation
881 * (including the MPI data exchange), allowing to execute some vector update
882 * that the `src` vector depends upon. The `operation_after_loop` is similar
883 * - it starts to execute on a range of DoFs once all DoFs in that range
884 * have been touched for the last time by the `cell_operation`
885 * (including the MPI data exchange), allowing e.g. to compute some vector
886 * operations that depend on the result of the current cell loop in `dst` or
887 * want to modify `src`. The efficiency of caching depends on the numbering
888 * of the degrees of freedom because of the granularity of the ranges.
889 *
890 * @param cell_operation Pointer to member function of `CLASS` with the
891 * signature <tt>cell_operation (const MatrixFree<dim,Number> &, OutVector &,
892 * InVector &, std::pair<unsigned int,unsigned int> &)</tt> where the first
893 * argument passes the data of the calling class and the last argument
894 * defines the range of cells which should be worked on (typically more than
895 * one cell should be worked on in order to reduce overheads).
896 *
897 * @param owning_class The object which provides the `cell_operation`
898 * call. To be compatible with this interface, the class must allow to call
899 * `owning_class->cell_operation(...)`.
900 *
901 * @param dst Destination vector holding the result. If the vector is of
902 * type LinearAlgebra::distributed::Vector (or composite objects thereof
903 * such as LinearAlgebra::distributed::BlockVector), the loop calls
904 * LinearAlgebra::distributed::Vector::compress() at the end of the call
905 * internally. For other vectors, including parallel Trilinos or PETSc
906 * vectors, no such call is issued. Note that Trilinos/Epetra or PETSc
907 * vectors do currently not work in parallel because the present class uses
908 * MPI-local index addressing, as opposed to the global addressing implied
909 * by those external libraries.
910 *
911 * @param src Input vector. If the vector is of type
912 * LinearAlgebra::distributed::Vector (or composite objects thereof such as
913 * LinearAlgebra::distributed::BlockVector), the loop calls
914 * LinearAlgebra::distributed::Vector::update_ghost_values() at the start of
915 * the call internally to make sure all necessary data is locally
916 * available. Note, however, that the vector is reset to its original state
917 * at the end of the loop, i.e., if the vector was not ghosted upon entry of
918 * the loop, it will not be ghosted upon finishing the loop.
919 *
920 * @param operation_before_loop This functor can be used to perform an
921 * operation on entries of the `src` and `dst` vectors (or other vectors)
922 * before the operation on cells first touches a particular DoF according to
923 * the general description in the text above. This function is passed a
924 * range of the locally owned degrees of freedom on the selected
925 * `dof_handler_index_pre_post` (in MPI-local numbering).
926 *
927 * @param operation_after_loop This functor can be used to perform an
928 * operation on entries of the `src` and `dst` vectors (or other vectors)
929 * after the operation on cells last touches a particular DoF according to
930 * the general description in the text above. This function is passed a
931 * range of the locally owned degrees of freedom on the selected
932 * `dof_handler_index_pre_post` (in MPI-local numbering).
933 *
934 * @param dof_handler_index_pre_post Since MatrixFree can be initialized
935 * with a vector of DoFHandler objects, each of them will in general have
936 * different vector sizes and thus different ranges returned to
937 * `operation_before_loop` and `operation_after_loop`. Use this variable to
938 * specify which one of the DoFHandler objects the index range should be
939 * associated to. Defaults to the `dof_handler_index` 0.
940 *
941 * @note The close locality of the `operation_before_loop` and
942 * `operation_after_loop` is currently only implemented for the MPI-only
943 * case. In case threading is enabled, the complete `operation_before_loop`
944 * is scheduled before the parallel loop, and `operation_after_loop` is
945 * scheduled strictly afterwards, due to the complicated dependencies.
946 */
947 template <typename CLASS, typename OutVector, typename InVector>
948 void
949 cell_loop(void (CLASS::*cell_operation)(
950 const MatrixFree &,
951 OutVector &,
952 const InVector &,
953 const std::pair<unsigned int, unsigned int> &) const,
954 const CLASS *owning_class,
955 OutVector &dst,
956 const InVector &src,
957 const std::function<void(const unsigned int, const unsigned int)>
958 &operation_before_loop,
959 const std::function<void(const unsigned int, const unsigned int)>
960 &operation_after_loop,
961 const unsigned int dof_handler_index_pre_post = 0) const;
962
963 /**
964 * Same as above, but for class member functions which are non-const.
965 */
966 template <typename CLASS, typename OutVector, typename InVector>
967 void
968 cell_loop(void (CLASS::*cell_operation)(
969 const MatrixFree &,
970 OutVector &,
971 const InVector &,
972 const std::pair<unsigned int, unsigned int> &),
973 CLASS *owning_class,
974 OutVector &dst,
975 const InVector &src,
976 const std::function<void(const unsigned int, const unsigned int)>
977 &operation_before_loop,
978 const std::function<void(const unsigned int, const unsigned int)>
979 &operation_after_loop,
980 const unsigned int dof_handler_index_pre_post = 0) const;
981
982 /**
983 * Same as above, but taking an `std::function` as the `cell_operation`
984 * rather than a class member function.
985 */
986 template <typename OutVector, typename InVector>
987 void
988 cell_loop(const std::function<void(
989 const MatrixFree<dim, Number, VectorizedArrayType> &,
990 OutVector &,
991 const InVector &,
992 const std::pair<unsigned int, unsigned int> &)> &cell_operation,
993 OutVector &dst,
994 const InVector &src,
995 const std::function<void(const unsigned int, const unsigned int)>
996 &operation_before_loop,
997 const std::function<void(const unsigned int, const unsigned int)>
998 &operation_after_loop,
999 const unsigned int dof_handler_index_pre_post = 0) const;
1000
1001 /**
1002 * This method runs a loop over all cells (in parallel) and performs the MPI
1003 * data exchange on the source vector and destination vector. As opposed to
1004 * the other variants that only runs a function on cells, this method also
1005 * takes as arguments a function for the interior faces and for the boundary
1006 * faces, respectively.
1007 *
1008 * @param cell_operation `std::function` with the signature <tt>cell_operation
1009 * (const MatrixFree<dim,Number> &, OutVector &, InVector &,
1010 * std::pair<unsigned int,unsigned int> &)</tt> where the first argument
1011 * passes the data of the calling class and the last argument defines the
1012 * range of cells which should be worked on (typically more than one cell
1013 * should be worked on in order to reduce overheads). One can pass a pointer
1014 * to an object in this place if it has an <code>operator()</code> with the
1015 * correct set of arguments since such a pointer can be converted to the
1016 * function object.
1017 *
1018 * @param inner_face_operation `std::function` with the signature <tt>inner_face_operation
1019 * (const MatrixFree<dim,Number> &, OutVector &, InVector &,
1020 * std::pair<unsigned int,unsigned int> &)</tt> in analogy to
1021 * `cell_operation`, but now the part associated to the work on interior
1022 * faces. Note that the MatrixFree framework treats periodic faces as interior
1023 * ones, so they will be assigned their correct neighbor after applying
1024 * periodicity constraints within the inner_face_operation calls.
1025 *
1026 * @param boundary_face_operation `std::function` with the signature
1027 * <tt>boundary_face_operation (const MatrixFree<dim,Number> &, OutVector &,
1028 * InVector &, std::pair<unsigned int,unsigned int> &)</tt> in analogy to
1029 * `cell_operation` and `inner_face_operation`, but now the part
1030 * associated to the work on boundary faces. Boundary faces are separated by
1031 * their `boundary_id` and it is possible to query that id using
1032 * MatrixFree::get_boundary_id(). Note that both interior and faces use the
1033 * same numbering, and faces in the interior are assigned lower numbers than
1034 * the boundary faces.
1035 *
1036 * @param dst Destination vector holding the result. If the vector is of
1037 * type LinearAlgebra::distributed::Vector (or composite objects thereof
1038 * such as LinearAlgebra::distributed::BlockVector), the loop calls
1039 * LinearAlgebra::distributed::Vector::compress() at the end of the call
1040 * internally.
1041 *
1042 * @param src Input vector. If the vector is of type
1043 * LinearAlgebra::distributed::Vector (or composite objects thereof such as
1044 * LinearAlgebra::distributed::BlockVector), the loop calls
1045 * LinearAlgebra::distributed::Vector::update_ghost_values() at the start of
1046 * the call internally to make sure all necessary data is locally
1047 * available. Note, however, that the vector is reset to its original state
1048 * at the end of the loop, i.e., if the vector was not ghosted upon entry of
1049 * the loop, it will not be ghosted upon finishing the loop.
1050 *
1051 * @param zero_dst_vector If this flag is set to `true`, the vector `dst`
1052 * will be set to zero inside the loop. Use this case in case you perform a
1053 * typical `vmult()` operation on a matrix object, as it will typically be
1054 * faster than calling `dst = 0;` before the loop separately. This is
1055 * because the vector entries are set to zero only on subranges of the
1056 * vector, making sure that the vector entries stay in caches as much as
1057 * possible.
1058 *
1059 * @param dst_vector_face_access Set the type of access into the vector
1060 * `dst` that will happen inside the body of the @p inner_face_operation
1061 * function. As explained in the description of the DataAccessOnFaces
1062 * struct, the purpose of this selection is to reduce the amount of data
1063 * that must be exchanged over the MPI network (or via `memcpy` if within
1064 * the shared memory region of a node) to gain performance. Note that there
1065 * is no way to communicate this setting with the FEFaceEvaluation class,
1066 * therefore this selection must be made at this site in addition to what is
1067 * implemented inside the `inner_face_operation` function. As a
1068 * consequence, there is also no way to check that the setting passed to this
1069 * call is consistent with what is later done by `FEFaceEvaluation`, and it is
1070 * the user's responsibility to ensure correctness of data.
1071 *
1072 * @param src_vector_face_access Set the type of access into the vector
1073 * `src` that will happen inside the body of the @p inner_face_operation function,
1074 * in analogy to `dst_vector_face_access`.
1075 */
1076 template <typename OutVector, typename InVector>
1077 void
1078 loop(
1079 const std::function<
1080 void(const MatrixFree<dim, Number, VectorizedArrayType> &,
1081 OutVector &,
1082 const InVector &,
1083 const std::pair<unsigned int, unsigned int> &)> &cell_operation,
1084 const std::function<void(
1085 const MatrixFree<dim, Number, VectorizedArrayType> &,
1086 OutVector &,
1087 const InVector &,
1088 const std::pair<unsigned int, unsigned int> &)> &inner_face_operation,
1089 const std::function<void(
1090 const MatrixFree<dim, Number, VectorizedArrayType> &,
1091 OutVector &,
1092 const InVector &,
1093 const std::pair<unsigned int, unsigned int> &)> &boundary_face_operation,
1094 OutVector &dst,
1095 const InVector &src,
1096 const bool zero_dst_vector = false,
1097 const DataAccessOnFaces dst_vector_face_access =
1098 DataAccessOnFaces::unspecified,
1099 const DataAccessOnFaces src_vector_face_access =
1100 DataAccessOnFaces::unspecified) const;
1101
1102 /**
1103 * This is the second variant to run the loop over all cells, interior
1104 * faces, and boundary faces, now providing three function pointers to
1105 * member functions of class @p CLASS with the signature <code>operation
1106 * (const MatrixFree<dim,Number> &, OutVector &, InVector &,
1107 * std::pair<unsigned int,unsigned int>&)const</code>. This method obviates
1108 * the need to define a lambda function or to call std::bind to bind
1109 * the class into the given
1110 * function in case the local function needs to access data in the class
1111 * (i.e., it is a non-static member function).
1112 *
1113 * @param cell_operation Pointer to member function of `CLASS` with the
1114 * signature <tt>cell_operation (const MatrixFree<dim,Number> &, OutVector &,
1115 * InVector &, std::pair<unsigned int,unsigned int> &)</tt> where the first
1116 * argument passes the data of the calling class and the last argument
1117 * defines the range of cells which should be worked on (typically more than
1118 * one cell should be worked on in order to reduce overheads). Note that the
1119 * loop will typically split the `cell_range` into smaller pieces and work
1120 * on `cell_operation`, `inner_face_operation`, and
1121 * `boundary_face_operation` alternately, in order to increase the potential
1122 * reuse of vector entries in caches.
1123 *
1124 * @param inner_face_operation Pointer to member function of `CLASS` with the
1125 * signature <tt>inner_face_operation (const MatrixFree<dim,Number> &,
1126 * OutVector &, InVector &, std::pair<unsigned int,unsigned int> &)</tt> in
1127 * analogy to `cell_operation`, but now the part associated to the work on
1128 * interior faces. Note that the MatrixFree framework treats periodic faces as
1129 * interior ones, so they will be assigned their correct neighbor after
1130 * applying periodicity constraints within the inner_face_operation
1131 * calls.
1132 *
1133 * @param boundary_face_operation Pointer to member function of `CLASS` with the
1134 * signature <tt>boundary_face_operation (const MatrixFree<dim,Number> &,
1135 * OutVector
1136 * &, InVector &, std::pair<unsigned int,unsigned int> &)</tt> in analogy to
1137 * `cell_operation` and `inner_face_operation`, but now the part
1138 * associated to the work on boundary faces. Boundary faces are separated by
1139 * their `boundary_id` and it is possible to query that id using
1140 * MatrixFree::get_boundary_id(). Note that both interior and faces use the
1141 * same numbering, and faces in the interior are assigned lower numbers than
1142 * the boundary faces.
1143 *
1144 * @param owning_class The object which provides the `cell_operation`
1145 * call. To be compatible with this interface, the class must allow to call
1146 * `owning_class->cell_operation(...)`,
1147 * `owning_class->inner_face_operation(...)`, and
1148 * `owning_class->boundary_face_operation(...)`.
1149 *
1150 * @param dst Destination vector holding the result. If the vector is of
1151 * type LinearAlgebra::distributed::Vector (or composite objects thereof
1152 * such as LinearAlgebra::distributed::BlockVector), the loop calls
1153 * LinearAlgebra::distributed::Vector::compress() at the end of the call
1154 * internally.
1155 *
1156 * @param src Input vector. If the vector is of type
1157 * LinearAlgebra::distributed::Vector (or composite objects thereof such as
1158 * LinearAlgebra::distributed::BlockVector), the loop calls
1159 * LinearAlgebra::distributed::Vector::update_ghost_values() at the start of
1160 * the call internally to make sure all necessary data is locally
1161 * available. Note, however, that the vector is reset to its original state
1162 * at the end of the loop, i.e., if the vector was not ghosted upon entry of
1163 * the loop, it will not be ghosted upon finishing the loop.
1164 *
1165 * @param zero_dst_vector If this flag is set to `true`, the vector `dst`
1166 * will be set to zero inside the loop. Use this case in case you perform a
1167 * typical `vmult()` operation on a matrix object, as it will typically be
1168 * faster than calling `dst = 0;` before the loop separately. This is
1169 * because the vector entries are set to zero only on subranges of the
1170 * vector, making sure that the vector entries stay in caches as much as
1171 * possible.
1172 *
1173 * @param dst_vector_face_access Set the type of access into the vector
1174 * `dst` that will happen inside the body of the @p inner_face_operation
1175 * function. As explained in the description of the DataAccessOnFaces
1176 * struct, the purpose of this selection is to reduce the amount of data
1177 * that must be exchanged over the MPI network (or via `memcpy` if within
1178 * the shared memory region of a node) to gain performance. Note that there
1179 * is no way to communicate this setting with the FEFaceEvaluation class,
1180 * therefore this selection must be made at this site in addition to what is
1181 * implemented inside the `inner_face_operation` function. As a
1182 * consequence, there is also no way to check that the setting passed to this
1183 * call is consistent with what is later done by `FEFaceEvaluation`, and it is
1184 * the user's responsibility to ensure correctness of data.
1185 *
1186 * @param src_vector_face_access Set the type of access into the vector
1187 * `src` that will happen inside the body of the @p inner_face_operation function,
1188 * in analogy to `dst_vector_face_access`.
1189 */
1190 template <typename CLASS, typename OutVector, typename InVector>
1191 void
1192 loop(void (CLASS::*cell_operation)(
1193 const MatrixFree &,
1194 OutVector &,
1195 const InVector &,
1196 const std::pair<unsigned int, unsigned int> &) const,
1197 void (CLASS::*inner_face_operation)(
1198 const MatrixFree &,
1199 OutVector &,
1200 const InVector &,
1201 const std::pair<unsigned int, unsigned int> &) const,
1202 void (CLASS::*boundary_face_operation)(
1203 const MatrixFree &,
1204 OutVector &,
1205 const InVector &,
1206 const std::pair<unsigned int, unsigned int> &) const,
1207 const CLASS *owning_class,
1208 OutVector &dst,
1209 const InVector &src,
1210 const bool zero_dst_vector = false,
1211 const DataAccessOnFaces dst_vector_face_access =
1212 DataAccessOnFaces::unspecified,
1213 const DataAccessOnFaces src_vector_face_access =
1214 DataAccessOnFaces::unspecified) const;
1215
1216 /**
1217 * Same as above, but for class member functions which are non-const.
1218 */
1219 template <typename CLASS, typename OutVector, typename InVector>
1220 void
1221 loop(void (CLASS::*cell_operation)(
1222 const MatrixFree &,
1223 OutVector &,
1224 const InVector &,
1225 const std::pair<unsigned int, unsigned int> &),
1226 void (CLASS::*inner_face_operation)(
1227 const MatrixFree &,
1228 OutVector &,
1229 const InVector &,
1230 const std::pair<unsigned int, unsigned int> &),
1231 void (CLASS::*boundary_face_operation)(
1232 const MatrixFree &,
1233 OutVector &,
1234 const InVector &,
1235 const std::pair<unsigned int, unsigned int> &),
1236 CLASS *owning_class,
1237 OutVector &dst,
1238 const InVector &src,
1239 const bool zero_dst_vector = false,
1240 const DataAccessOnFaces dst_vector_face_access =
1241 DataAccessOnFaces::unspecified,
1242 const DataAccessOnFaces src_vector_face_access =
1243 DataAccessOnFaces::unspecified) const;
1244
1245 /**
1246 * This function is similar to the loop method above, but adds two additional
1247 * functors to execute some additional work before and after the cell, face
1248 * and boundary integrals are computed.
1249 *
1250 * The two additional functors work on a range of degrees of freedom,
1251 * expressed in terms of the degree-of-freedom numbering of the selected
1252 * DoFHandler `dof_handler_index_pre_post` in MPI-local indices. The
1253 * arguments to the functors represent a range of degrees of freedom at a
1254 * granularity of
1255 * internal::MatrixFreeFunctions::DoFInfo::chunk_size_zero_vector entries
1256 * (except for the last chunk which is set to the number of locally owned
1257 * entries) in the form `[first, last)`. The idea of these functors is to
1258 * bring operations on vectors closer to the point where they accessed in a
1259 * matrix-free loop, with the goal to increase cache hits by temporal
1260 * locality. This loop guarantees that the `operation_before_loop` hits all
1261 * relevant unknowns before they are first touched by any of the cell, face or
1262 * boundary operations (including the MPI data exchange), allowing to execute
1263 * some vector update that the `src` vector depends upon. The
1264 * `operation_after_loop` is similar - it starts to execute on a range of DoFs
1265 * once all DoFs in that range have been touched for the last time by the
1266 * cell, face and boundary operations (including the MPI data exchange),
1267 * allowing e.g. to compute some vector operations that depend on the result
1268 * of the current cell loop in `dst` or want to modify `src`. The efficiency
1269 * of caching depends on the numbering of the degrees of freedom because of
1270 * the granularity of the ranges.
1271 *
1272 * @param cell_operation Pointer to member function of `CLASS` with the
1273 * signature <tt>cell_operation (const MatrixFree<dim,Number> &, OutVector &,
1274 * InVector &, std::pair<unsigned int,unsigned int> &)</tt> where the first
1275 * argument passes the data of the calling class and the last argument
1276 * defines the range of cells which should be worked on (typically more than
1277 * one cell should be worked on in order to reduce overheads).
1278 *
1279 * @param inner_face_operation Pointer to member function of `CLASS` with the
1280 * signature <tt>inner_face_operation (const MatrixFree<dim,Number> &,
1281 * OutVector &, InVector &, std::pair<unsigned int,unsigned int> &)</tt> in
1282 * analogy to `cell_operation`, but now the part associated to the work on
1283 * interior faces. Note that the MatrixFree framework treats periodic faces as
1284 * interior ones, so they will be assigned their correct neighbor after
1285 * applying periodicity constraints within the inner_face_operation
1286 * calls.
1287 *
1288 * @param boundary_face_operation Pointer to member function of `CLASS` with the
1289 * signature <tt>boundary_face_operation (const MatrixFree<dim,Number> &,
1290 * OutVector
1291 * &, InVector &, std::pair<unsigned int,unsigned int> &)</tt> in analogy to
1292 * `cell_operation` and `inner_face_operation`, but now the part
1293 * associated to the work on boundary faces. Boundary faces are separated by
1294 * their `boundary_id` and it is possible to query that id using
1295 * MatrixFree::get_boundary_id(). Note that both interior and faces use the
1296 * same numbering, and faces in the interior are assigned lower numbers than
1297 * the boundary faces.
1298 *
1299 * @param owning_class The object which provides the `cell_operation`
1300 * call. To be compatible with this interface, the class must allow to call
1301 * `owning_class->cell_operation(...)`.
1302 *
1303 * @param dst Destination vector holding the result. If the vector is of
1304 * type LinearAlgebra::distributed::Vector (or composite objects thereof
1305 * such as LinearAlgebra::distributed::BlockVector), the loop calls
1306 * LinearAlgebra::distributed::Vector::compress() at the end of the call
1307 * internally. For other vectors, including parallel Trilinos or PETSc
1308 * vectors, no such call is issued. Note that Trilinos/Epetra or PETSc
1309 * vectors do currently not work in parallel because the present class uses
1310 * MPI-local index addressing, as opposed to the global addressing implied
1311 * by those external libraries.
1312 *
1313 * @param src Input vector. If the vector is of type
1314 * LinearAlgebra::distributed::Vector (or composite objects thereof such as
1315 * LinearAlgebra::distributed::BlockVector), the loop calls
1316 * LinearAlgebra::distributed::Vector::update_ghost_values() at the start of
1317 * the call internally to make sure all necessary data is locally
1318 * available. Note, however, that the vector is reset to its original state
1319 * at the end of the loop, i.e., if the vector was not ghosted upon entry of
1320 * the loop, it will not be ghosted upon finishing the loop.
1321 *
1322 * @param operation_before_loop This functor can be used to perform an
1323 * operation on entries of the `src` and `dst` vectors (or other vectors)
1324 * before the operation on cells first touches a particular DoF according to
1325 * the general description in the text above. This function is passed a
1326 * range of the locally owned degrees of freedom on the selected
1327 * `dof_handler_index_pre_post` (in MPI-local numbering).
1328 *
1329 * @param operation_after_loop This functor can be used to perform an
1330 * operation on entries of the `src` and `dst` vectors (or other vectors)
1331 * after the operation on cells last touches a particular DoF according to
1332 * the general description in the text above. This function is passed a
1333 * range of the locally owned degrees of freedom on the selected
1334 * `dof_handler_index_pre_post` (in MPI-local numbering).
1335 *
1336 * @param dof_handler_index_pre_post Since MatrixFree can be initialized
1337 * with a vector of DoFHandler objects, each of them will in general have
1338 * different vector sizes and thus different ranges returned to
1339 * `operation_before_loop` and `operation_after_loop`. Use this variable to
1340 * specify which one of the DoFHandler objects the index range should be
1341 * associated to. Defaults to the `dof_handler_index` 0.
1342 *
1343 * @param dst_vector_face_access Set the type of access into the vector
1344 * `dst` that will happen inside the body of the @p inner_face_operation
1345 * function. As explained in the description of the DataAccessOnFaces
1346 * struct, the purpose of this selection is to reduce the amount of data
1347 * that must be exchanged over the MPI network (or via `memcpy` if within
1348 * the shared memory region of a node) to gain performance. Note that there
1349 * is no way to communicate this setting with the FEFaceEvaluation class,
1350 * therefore this selection must be made at this site in addition to what is
1351 * implemented inside the `inner_face_operation` function. As a
1352 * consequence, there is also no way to check that the setting passed to this
1353 * call is consistent with what is later done by `FEFaceEvaluation`, and it is
1354 * the user's responsibility to ensure correctness of data.
1355 *
1356 * @param src_vector_face_access Set the type of access into the vector
1357 * `src` that will happen inside the body of the @p inner_face_operation function,
1358 * in analogy to `dst_vector_face_access`.
1359 *
1360 * @note The close locality of the `operation_before_loop` and
1361 * `operation_after_loop` is currently only implemented for the MPI-only
1362 * case. In case threading is enabled, the complete `operation_before_loop`
1363 * is scheduled before the parallel loop, and `operation_after_loop` is
1364 * scheduled strictly afterwards, due to the complicated dependencies.
1365 */
1366 template <typename CLASS, typename OutVector, typename InVector>
1367 void
1368 loop(void (CLASS::*cell_operation)(
1369 const MatrixFree &,
1370 OutVector &,
1371 const InVector &,
1372 const std::pair<unsigned int, unsigned int> &) const,
1373 void (CLASS::*inner_face_operation)(
1374 const MatrixFree &,
1375 OutVector &,
1376 const InVector &,
1377 const std::pair<unsigned int, unsigned int> &) const,
1378 void (CLASS::*boundary_face_operation)(
1379 const MatrixFree &,
1380 OutVector &,
1381 const InVector &,
1382 const std::pair<unsigned int, unsigned int> &) const,
1383 const CLASS *owning_class,
1384 OutVector &dst,
1385 const InVector &src,
1386 const std::function<void(const unsigned int, const unsigned int)>
1387 &operation_before_loop,
1388 const std::function<void(const unsigned int, const unsigned int)>
1389 &operation_after_loop,
1390 const unsigned int dof_handler_index_pre_post = 0,
1391 const DataAccessOnFaces dst_vector_face_access =
1392 DataAccessOnFaces::unspecified,
1393 const DataAccessOnFaces src_vector_face_access =
1394 DataAccessOnFaces::unspecified) const;
1395
1396 /**
1397 * Same as above, but for class member functions which are non-const.
1398 */
1399 template <typename CLASS, typename OutVector, typename InVector>
1400 void
1401 loop(void (CLASS::*cell_operation)(
1402 const MatrixFree &,
1403 OutVector &,
1404 const InVector &,
1405 const std::pair<unsigned int, unsigned int> &),
1406 void (CLASS::*inner_face_operation)(
1407 const MatrixFree &,
1408 OutVector &,
1409 const InVector &,
1410 const std::pair<unsigned int, unsigned int> &),
1411 void (CLASS::*boundary_face_operation)(
1412 const MatrixFree &,
1413 OutVector &,
1414 const InVector &,
1415 const std::pair<unsigned int, unsigned int> &),
1416 const CLASS *owning_class,
1417 OutVector &dst,
1418 const InVector &src,
1419 const std::function<void(const unsigned int, const unsigned int)>
1420 &operation_before_loop,
1421 const std::function<void(const unsigned int, const unsigned int)>
1422 &operation_after_loop,
1423 const unsigned int dof_handler_index_pre_post = 0,
1424 const DataAccessOnFaces dst_vector_face_access =
1425 DataAccessOnFaces::unspecified,
1426 const DataAccessOnFaces src_vector_face_access =
1427 DataAccessOnFaces::unspecified) const;
1428
1429 /**
1430 * Same as above, but taking an `std::function` as the `cell_operation`,
1431 * `inner_face_operation` and `boundary_face_operation` rather than a
1432 * class member function.
1433 */
1434 template <typename OutVector, typename InVector>
1435 void
1436 loop(
1437 const std::function<
1438 void(const MatrixFree<dim, Number, VectorizedArrayType> &,
1439 OutVector &,
1440 const InVector &,
1441 const std::pair<unsigned int, unsigned int> &)> &cell_operation,
1442 const std::function<void(
1443 const MatrixFree<dim, Number, VectorizedArrayType> &,
1444 OutVector &,
1445 const InVector &,
1446 const std::pair<unsigned int, unsigned int> &)> &inner_face_operation,
1447 const std::function<void(
1448 const MatrixFree<dim, Number, VectorizedArrayType> &,
1449 OutVector &,
1450 const InVector &,
1451 const std::pair<unsigned int, unsigned int> &)> &boundary_face_operation,
1452 OutVector &dst,
1453 const InVector &src,
1454 const std::function<void(const unsigned int, const unsigned int)>
1455 &operation_before_loop,
1456 const std::function<void(const unsigned int, const unsigned int)>
1457 &operation_after_loop,
1458 const unsigned int dof_handler_index_pre_post = 0,
1459 const DataAccessOnFaces dst_vector_face_access =
1460 DataAccessOnFaces::unspecified,
1461 const DataAccessOnFaces src_vector_face_access =
1462 DataAccessOnFaces::unspecified) const;
1463
1464 /**
1465 * This method runs the loop over all cells (in parallel) similarly as
1466 * cell_loop() does. However, this function is intended to be used
1467 * for the case if face and boundary integrals should be also
1468 * evaluated. In contrast to loop(), the user provides only a single function
1469 * that should contain the cell integral over a cell (or batch of cells when
1470 * vectorizing) and the face and boundary integrals over all its faces. This
1471 * is referred to in the literature as `element-centric loop` or `cell-centric
1472 * loop`.
1473 *
1474 * To be able to evaluate all face integrals (with values or gradients
1475 * from the neighboring cells), all ghost values from neighboring cells are
1476 * updated. Use
1477 * FEFaceEvaluation::reinit(cell, face_no) to access quantities on arbitrary
1478 * faces of a cell and the respective neighbors.
1479 *
1480 * @param cell_operation Pointer to member function of `CLASS` with the
1481 * signature <tt>cell_operation (const MatrixFree<dim,Number> &, OutVector &,
1482 * InVector &, std::pair<unsigned int,unsigned int> &)</tt> where the first
1483 * argument passes the data of the calling class and the last argument
1484 * defines the range of cells which should be worked on (typically more than
1485 * one cell is passed in from the loop in order to reduce overheads).
1486 *
1487 * @param owning_class The object which provides the `cell_operation`
1488 * call. To be compatible with this interface, the class must allow to call
1489 * `owning_class->cell_operation(...)`.
1490 *
1491 * @param dst Destination vector holding the result. If the vector is of
1492 * type LinearAlgebra::distributed::Vector (or composite objects thereof
1493 * such as LinearAlgebra::distributed::BlockVector), the loop calls
1494 * LinearAlgebra::distributed::Vector::compress() at the end of the call
1495 * internally.
1496 *
1497 * @param src Input vector. If the vector is of type
1498 * LinearAlgebra::distributed::Vector (or composite objects thereof such as
1499 * LinearAlgebra::distributed::BlockVector), the loop calls
1500 * LinearAlgebra::distributed::Vector::update_ghost_values() at the start of
1501 * the call internally to make sure all necessary data is locally
1502 * available. Note, however, that the vector is reset to its original state
1503 * at the end of the loop, i.e., if the vector was not ghosted upon entry of
1504 * the loop, it will not be ghosted upon finishing the loop.
1505 *
1506 * @param zero_dst_vector If this flag is set to `true`, the vector `dst`
1507 * will be set to zero inside the loop. Use this case in case you perform a
1508 * typical `vmult()` operation on a matrix object, as it will typically be
1509 * faster than calling `dst = 0;` before the loop separately. This is
1510 * because the vector entries are set to zero only on subranges of the
1511 * vector, making sure that the vector entries stay in caches as much as
1512 * possible.
1513 *
1514 * @param src_vector_face_access Set the type of access into the vector
1515 * `src` that will happen inside the body of the @p cell_operation function
1516 * during face integrals.
1517 * As explained in the description of the DataAccessOnFaces
1518 * struct, the purpose of this selection is to reduce the amount of data
1519 * that must be exchanged over the MPI network (or via `memcpy` if within
1520 * the shared memory region of a node) to gain performance. Note that there
1521 * is no way to communicate this setting with the FEFaceEvaluation class,
1522 * therefore this selection must be made at this site in addition to what is
1523 * implemented inside the `inner_face_operation` function. As a
1524 * consequence, there is also no way to check that the setting passed to this
1525 * call is consistent with what is later done by `FEFaceEvaluation`, and it is
1526 * the user's responsibility to ensure correctness of data.
1527 */
1528 template <typename CLASS, typename OutVector, typename InVector>
1529 void
1530 loop_cell_centric(void (CLASS::*cell_operation)(
1531 const MatrixFree &,
1532 OutVector &,
1533 const InVector &,
1534 const std::pair<unsigned int, unsigned int> &) const,
1535 const CLASS *owning_class,
1536 OutVector &dst,
1537 const InVector &src,
1538 const bool zero_dst_vector = false,
1539 const DataAccessOnFaces src_vector_face_access =
1540 DataAccessOnFaces::unspecified) const;
1541
1542 /**
1543 * Same as above, but for the class member function which is non-const.
1544 */
1545 template <typename CLASS, typename OutVector, typename InVector>
1546 void
1547 loop_cell_centric(void (CLASS::*cell_operation)(
1548 const MatrixFree &,
1549 OutVector &,
1550 const InVector &,
1551 const std::pair<unsigned int, unsigned int> &),
1552 CLASS *owning_class,
1553 OutVector &dst,
1554 const InVector &src,
1555 const bool zero_dst_vector = false,
1556 const DataAccessOnFaces src_vector_face_access =
1557 DataAccessOnFaces::unspecified) const;
1558
1559 /**
1560 * Same as above, but with std::function.
1561 */
1562 template <typename OutVector, typename InVector>
1563 void
1564 loop_cell_centric(
1565 const std::function<void(const MatrixFree &,
1566 OutVector &,
1567 const InVector &,
1568 const std::pair<unsigned int, unsigned int> &)>
1569 &cell_operation,
1570 OutVector &dst,
1571 const InVector &src,
1572 const bool zero_dst_vector = false,
1573 const DataAccessOnFaces src_vector_face_access =
1574 DataAccessOnFaces::unspecified) const;
1575
1576 /**
1577 * In the hp-adaptive case, a subrange of cells as computed during the cell
1578 * loop might contain elements of different degrees. Use this function to
1579 * compute what the subrange for an individual finite element degree is. The
1580 * finite element degree is associated to the vector component given in the
1581 * function call.
1582 */
1583 std::pair<unsigned int, unsigned int>
1584 create_cell_subrange_hp(const std::pair<unsigned int, unsigned int> &range,
1585 const unsigned int fe_degree,
1586 const unsigned int dof_handler_index = 0) const;
1587
1588 /**
1589 * In the hp-adaptive case, a subrange of cells as computed during the cell
1590 * loop might contain elements of different degrees. Use this function to
1591 * compute what the subrange for a given index the hp-finite element, as
1592 * opposed to the finite element degree in the other function.
1593 */
1594 std::pair<unsigned int, unsigned int>
1595 create_cell_subrange_hp_by_index(
1596 const std::pair<unsigned int, unsigned int> &range,
1597 const unsigned int fe_index,
1598 const unsigned int dof_handler_index = 0) const;
1599
1600 /**
1601 * In the hp-adaptive case, return number of active FE indices.
1602 */
1603 unsigned int
1604 n_active_fe_indices() const;
1605
1606 /**
1607 * In the hp-adaptive case, return the active FE index of a cell range.
1608 */
1609 unsigned int
1610 get_cell_active_fe_index(
1611 const std::pair<unsigned int, unsigned int> range,
1612 const unsigned int dof_handler_index = numbers::invalid_unsigned_int) const;
1613
1614 /**
1615 * In the hp-adaptive case, return the active FE index of a face range.
1616 */
1617 unsigned int
1618 get_face_active_fe_index(
1619 const std::pair<unsigned int, unsigned int> range,
1620 const bool is_interior_face = true,
1621 const unsigned int dof_handler_index = numbers::invalid_unsigned_int) const;
1622
1623 /** @} */
1624
1625 /**
1626 * @name Initialization of vectors
1627 */
1628 /** @{ */
1629 /**
1630 * Initialize function for a vector with each entry associated with a cell
1631 * batch (cell data). For reading and writing the vector use:
1632 * FEEvaluationBase::read_cell_data() and FEEvaluationBase::set_cell_data().
1633 */
1634 template <typename T>
1635 void
1636 initialize_cell_data_vector(AlignedVector<T> &vec) const;
1637
1638 /**
1639 * Initialize function for a vector with each entry associated with a face
1640 * batch (face data). For reading and writing the vector use:
1641 * FEEvaluationBase::read_face_data() and FEEvaluationBase::set_face_data().
1642 */
1643 template <typename T>
1644 void
1645 initialize_face_data_vector(AlignedVector<T> &vec) const;
1646
1647 /**
1648 * Initialize function for a general serial non-block vector.
1649 * After a call to this function, the length of the vector is equal to the
1650 * total number of degrees of freedom in the DoFHandler. Vector entries are
1651 * initialized with zero.
1652 *
1653 * If MatrixFree was set up with several DoFHandler objects, the parameter
1654 * @p dof_handler_index defines which component is to be used.
1655 *
1656 * @note Serial vectors also include Trilinos and PETSc vectors; however
1657 * in these cases, MatrixFree has to be used in a serial context, i.e., the
1658 * size of the communicator has to be exactly one.
1659 */
1660 template <typename VectorType>
1661 void
1662 initialize_dof_vector(VectorType &vec,
1663 const unsigned int dof_handler_index = 0) const;
1664
1665 /**
1666 * Specialization of the method initialize_dof_vector() for the
1667 * class LinearAlgebra::distributed::Vector@<Number@>.
1668 * See the other function with the same name for the general descriptions.
1669 *
1670 * @note For the parallel vectors used with MatrixFree and in FEEvaluation, a
1671 * vector needs to hold all
1672 * @ref GlossLocallyActiveDof "locally active DoFs"
1673 * and also some of the
1674 * @ref GlossLocallyRelevantDof "locally relevant DoFs".
1675 * The selection of DoFs is such that one can read all degrees of freedom on
1676 * all locally relevant elements (locally active) plus the degrees of freedom
1677 * that constraints expand into from the locally owned cells. However, not
1678 * all locally relevant DoFs are stored because most of them would never be
1679 * accessed in matrix-vector products and result in too much data sent
1680 * around which impacts the performance.
1681 */
1682 template <typename Number2>
1683 void
1684 initialize_dof_vector(LinearAlgebra::distributed::Vector<Number2> &vec,
1685 const unsigned int dof_handler_index = 0) const;
1686
1687 /**
1688 * Return the partitioner that represents the locally owned data and the
1689 * ghost indices where access is needed to for the cell loop. The
1690 * partitioner is constructed from the locally owned dofs and ghost dofs
1691 * given by the respective fields. If you want to have specific information
1692 * about these objects, you can query them with the respective access
1693 * functions. If you just want to initialize a (parallel) vector, you should
1694 * usually prefer this data structure as the data exchange information can
1695 * be reused from one vector to another.
1696 */
1697 const std::shared_ptr<const Utilities::MPI::Partitioner> &
1698 get_vector_partitioner(const unsigned int dof_handler_index = 0) const;
1699
1700 /**
1701 * Return the set of cells that are owned by the processor.
1702 */
1703 const IndexSet &
1704 get_locally_owned_set(const unsigned int dof_handler_index = 0) const;
1705
1706 /**
1707 * Return the set of ghost cells needed but not owned by the processor.
1708 */
1709 const IndexSet &
1710 get_ghost_set(const unsigned int dof_handler_index = 0) const;
1711
1712 /**
1713 * Return a list of all degrees of freedom that are constrained. The list
1714 * is returned in MPI-local index space for the locally owned range of the
1715 * vector, not in global MPI index space that spans all MPI processors. To
1716 * get numbers in global index space, call
1717 * <tt>get_vector_partitioner()->local_to_global</tt> on an entry of the
1718 * vector. In addition, it only returns the indices for degrees of freedom
1719 * that are owned locally, not for ghosts.
1720 */
1721 const std::vector<unsigned int> &
1722 get_constrained_dofs(const unsigned int dof_handler_index = 0) const;
1723
1724 /**
1725 * Computes a renumbering of degrees of freedom that better fits with the
1726 * data layout in MatrixFree according to the given layout of data. Note that
1727 * this function does not re-arrange the information stored in this class,
1728 * but rather creates a renumbering for consumption of
1729 * DoFHandler::renumber_dofs. To have any effect a MatrixFree object must be
1730 * set up again using the renumbered DoFHandler and AffineConstraints. Note
1731 * that if a DoFHandler calls DoFHandler::renumber_dofs, all information in
1732 * MatrixFree becomes invalid.
1733 */
1734 void
1735 renumber_dofs(std::vector<types::global_dof_index> &renumbering,
1736 const unsigned int dof_handler_index = 0);
1737
1738 /** @} */
1739
1740 /**
1741 * @name General information
1742 */
1743 /** @{ */
1744 /**
1745 * Return whether a given FiniteElement @p fe is supported by this class.
1746 */
1747 template <int spacedim>
1748 static bool
1749 is_supported(const FiniteElement<dim, spacedim> &fe);
1750
1751 /**
1752 * Return the number of different DoFHandlers specified at initialization.
1753 */
1754 unsigned int
1755 n_components() const;
1756
1757 /**
1758 * For the finite element underlying the DoFHandler specified by @p
1759 * dof_handler_index, return the number of base elements.
1760 */
1761 unsigned int
1762 n_base_elements(const unsigned int dof_handler_index) const;
1763
1764 /**
1765 * Return the number of cells this structure is based on. If you are using a
1766 * usual DoFHandler, it corresponds to the number of (locally owned) active
1767 * cells. Note that most data structures in this class do not directly act
1768 * on this number but rather on n_cell_batches() which gives the number of
1769 * cells as seen when lumping several cells together with vectorization.
1770 */
1771 unsigned int
1772 n_physical_cells() const;
1773
1774 /**
1775 * Return the number of cell batches that this structure works on. The
1776 * batches are formed by application of vectorization over several cells in
1777 * general. The cell range in @p cell_loop runs from zero to
1778 * n_cell_batches() (exclusive), so this is the appropriate size if you want
1779 * to store arrays of data for all cells to be worked on. This number is
1780 * approximately `n_physical_cells()/VectorizedArray::%size()`
1781 * (depending on how many cell batches that do not get filled up completely).
1782 */
1783 unsigned int
1784 n_cell_batches() const;
1785
1786 /**
1787 * Return the number of additional cell batches that this structure keeps
1788 * for face integration. Note that not all cells that are ghosted in the
1789 * triangulation are kept in this data structure, but only the ones which
1790 * are necessary for evaluating face integrals from both sides.
1791 */
1792 unsigned int
1793 n_ghost_cell_batches() const;
1794
1795 /**
1796 * Return the number of interior face batches that this structure works on.
1797 * The batches are formed by application of vectorization over several faces
1798 * in general. The face range in @p loop runs from zero to
1799 * n_inner_face_batches() (exclusive), so this is the appropriate size if
1800 * you want to store arrays of data for all interior faces to be worked on.
1801 * Note that it returns 0 unless mapping_update_flags_inner_faces is set
1802 * to a value different from UpdateFlags::update_default.
1803 */
1804 unsigned int
1805 n_inner_face_batches() const;
1806
1807 /**
1808 * Return the number of boundary face batches that this structure works on.
1809 * The batches are formed by application of vectorization over several faces
1810 * in general. The face range in @p loop runs from n_inner_face_batches() to
1811 * n_inner_face_batches()+n_boundary_face_batches() (exclusive), so if you
1812 * need to store arrays that hold data for all boundary faces but not the
1813 * interior ones, this number gives the appropriate size.
1814 * Note that it returns 0 unless mapping_update_flags_boundary_faces is set
1815 * to a value different from UpdateFlags::update_default.
1816 */
1817 unsigned int
1818 n_boundary_face_batches() const;
1819
1820 /**
1821 * Return the number of faces that are not processed locally but belong to
1822 * locally owned faces.
1823 */
1824 unsigned int
1825 n_ghost_inner_face_batches() const;
1826
1827 /**
1828 * In order to apply different operators to different parts of the boundary,
1829 * this method can be used to query the boundary id of a given face in the
1830 * faces' own sorting by lanes in a VectorizedArray. Only valid for an index
1831 * indicating a boundary face.
1832 *
1833 * @note Alternatively to this function, you can use
1834 * FEFaceEvaluation::boundary_id() to get the same information if a
1835 * FEFaceEvaluation object has been set up already.
1836 */
1837 types::boundary_id
1838 get_boundary_id(const unsigned int face_batch_index) const;
1839
1840 /**
1841 * Return the boundary ids for the faces within a cell, using the cells'
1842 * sorting by lanes in the VectorizedArray.
1843 */
1844 std::array<types::boundary_id, VectorizedArrayType::size()>
1845 get_faces_by_cells_boundary_id(const unsigned int cell_batch_index,
1846 const unsigned int face_number) const;
1847
1848 /**
1849 * Return the DoFHandler with the index as given to the respective
1850 * `std::vector` argument in the reinit() function.
1851 */
1852 const DoFHandler<dim> &
1853 get_dof_handler(const unsigned int dof_handler_index = 0) const;
1854
1855 /**
1856 * Return the AffineConstraints with the index as given to the
1857 * respective `std::vector` argument in the reinit() function. Only available
1858 * if the AffineConstraints objects have the same template parameter Number as
1859 * MatrixFree. Throws an exception otherwise.
1860 */
1861 const AffineConstraints<Number> &
1862 get_affine_constraints(const unsigned int dof_handler_index = 0) const;
1863
1864 /**
1865 * Return the cell iterator in deal.II speak to a given cell batch
1866 * (populating several lanes in a VectorizedArray) and the lane index within
1867 * the vectorization across cells in the renumbering of this structure.
1868 *
1869 * Note that the cell iterators in deal.II go through cells differently to
1870 * what the cell loop of this class does. This is because several cells are
1871 * processed together (vectorization across cells), and since cells with
1872 * neighbors on different MPI processors need to be accessed at a certain
1873 * time when accessing remote data and overlapping communication with
1874 * computation.
1875 */
1876 typename DoFHandler<dim>::cell_iterator
1877 get_cell_iterator(const unsigned int cell_batch_index,
1878 const unsigned int lane_index,
1879 const unsigned int dof_handler_index = 0) const;
1880
1881 /**
1882 * This returns the level and index for the cell that would be returned by
1883 * get_cell_iterator() for the same arguments `cell_batch_index` and
1884 * `lane_index`.
1885 */
1886 std::pair<int, int>
1887 get_cell_level_and_index(const unsigned int cell_batch_index,
1888 const unsigned int lane_index) const;
1889
1890 /**
1891 * Get MatrixFree index associated to a deal.II @p cell. To get
1892 * the actual cell batch index and lane, do the postprocessing
1893 * `index / VectorizedArrayType::size()` and `index %
1894 * VectorizedArrayType::size()`.
1895 */
1896 unsigned int
1897 get_matrix_free_cell_index(
1898 const typename Triangulation<dim>::cell_iterator &cell) const;
1899
1900 /**
1901 * Return the cell iterator in deal.II speak to an interior/exterior cell of
1902 * a face in a pair of a face batch and lane index. The second element of
1903 * the pair is the face number so that the face iterator can be accessed:
1904 * `pair.first->face(pair.second);`
1905 *
1906 * Note that the face iterators in deal.II go through cells differently to
1907 * what the face/boundary loop of this class does. This is because several
1908 * faces are worked on together (vectorization), and since faces with neighbor
1909 * cells on different MPI processors need to be accessed at a certain time
1910 * when accessing remote data and overlapping communication with computation.
1911 */
1912 std::pair<typename DoFHandler<dim>::cell_iterator, unsigned int>
1913 get_face_iterator(const unsigned int face_batch_index,
1914 const unsigned int lane_index,
1915 const bool interior = true,
1916 const unsigned int fe_component = 0) const;
1917
1918 /**
1919 * Since this class uses vectorized data types with usually more than one
1920 * value in the data field, a situation might occur when some components of
1921 * the vector type do not correspond to an actual cell in the mesh. When
1922 * using only this class, one usually does not need to bother about that
1923 * fact since the values are padded with zeros. However, when this class is
1924 * mixed with deal.II access to cells, care needs to be taken. This function
1925 * returns @p true if not all `n_lanes` cells for the given
1926 * `cell_batch_index` correspond to actual cells of the mesh and some are
1927 * merely present for padding reasons. To find out how many cells are
1928 * actually used, use the function n_active_entries_per_cell_batch().
1929 */
1930 bool
1931 at_irregular_cell(const unsigned int cell_batch_index) const;
1932
1933 /**
1934 * This query returns how many cells among the `VectorizedArrayType::size()`
1935 * many cells within a cell batch to actual cells in the mesh, rather than
1936 * being present for padding reasons. For most given cell batches in
1937 * n_cell_batches(), this number is equal to `VectorizedArrayType::size()`,
1938 * but there might be one or a few cell batches in the mesh (where the
1939 * numbers do not add up) where only some of the cells within a batch are
1940 * used, indicated by the function at_irregular_cell().
1941 */
1942 unsigned int
1943 n_active_entries_per_cell_batch(const unsigned int cell_batch_index) const;
1944
1945 /**
1946 * Use this function to find out how many faces over the length of
1947 * vectorization data types correspond to real faces (both interior and
1948 * boundary faces, as those use the same indexing but with different ranges)
1949 * in the mesh. For most given indices in n_inner_faces_batches() and
1950 * n_boundary_face_batches(), this is just @p vectorization_length many, but
1951 * there might be one or a few meshes (where the numbers do not add up)
1952 * where there are less such lanes filled.
1953 */
1954 unsigned int
1955 n_active_entries_per_face_batch(const unsigned int face_batch_index) const;
1956
1957 /**
1958 * Return the number of degrees of freedom per cell for a given hp-index.
1959 */
1960 unsigned int
1961 get_dofs_per_cell(const unsigned int dof_handler_index = 0,
1962 const unsigned int hp_active_fe_index = 0) const;
1963
1964 /**
1965 * Return the number of quadrature points per cell for a given hp-index.
1966 */
1967 unsigned int
1968 get_n_q_points(const unsigned int quad_index = 0,
1969 const unsigned int hp_active_fe_index = 0) const;
1970
1971 /**
1972 * Return the number of degrees of freedom on each face of the cell for
1973 * given hp-index.
1974 */
1975 unsigned int
1976 get_dofs_per_face(const unsigned int dof_handler_index = 0,
1977 const unsigned int hp_active_fe_index = 0) const;
1978
1979 /**
1980 * Return the number of quadrature points on each face of the cell for
1981 * given hp-index.
1982 */
1983 unsigned int
1984 get_n_q_points_face(const unsigned int quad_index = 0,
1985 const unsigned int hp_active_fe_index = 0) const;
1986
1987 /**
1988 * Return the quadrature rule for given hp-index.
1989 */
1990 const Quadrature<dim> &
1991 get_quadrature(const unsigned int quad_index = 0,
1992 const unsigned int hp_active_fe_index = 0) const;
1993
1994 /**
1995 * Return the quadrature rule for given hp-index.
1996 */
1997 const Quadrature<dim - 1> &
1998 get_face_quadrature(const unsigned int quad_index = 0,
1999 const unsigned int hp_active_fe_index = 0) const;
2000
2001 /**
2002 * Return the category the current batch range of cells was assigned to.
2003 * Categories run between the given values in the field
2004 * AdditionalData::cell_vectorization_category for the non-hp case
2005 * and return the active FE index in the hp-adaptive case.
2006 *
2007 * @note Following the behaviour of get_cell_category(), we return the
2008 * maximum category of any cell batch. In the hp case, it is
2009 * guaranteed that all cells and as a consequence all cell batches in a range
2010 * have the same category. Otherwise, there may be different categories in
2011 * different cell batches.
2012 */
2013 unsigned int
2014 get_cell_range_category(
2015 const std::pair<unsigned int, unsigned int> cell_batch_range,
2016 const unsigned int dof_handler_index = numbers::invalid_unsigned_int) const;
2017
2018 /**
2019 * Return the category of the cells on the two sides of the current batch
2020 * range of faces.
2021 */
2022 std::pair<unsigned int, unsigned int>
2023 get_face_range_category(
2024 const std::pair<unsigned int, unsigned int> face_batch_range,
2025 const unsigned int dof_handler_index = numbers::invalid_unsigned_int) const;
2026
2027 /**
2028 * Return the category the current batch of cells was assigned to. Categories
2029 * run between the given values in the field
2030 * AdditionalData::cell_vectorization_category for the non-hp case
2031 * and return the active FE index in the hp-adaptive case.
2032 *
2033 * @note In the non-hp case, a category of a cell batch is given
2034 * as the maximum category of any of its cell. In the hp case or the case that
2035 * MatrixFree::AdditionalData::cell_vectorization_categories_strict was
2036 * enabled, it is guaranteed that all cells have the same category.
2037 */
2038 unsigned int
2039 get_cell_category(
2040 const unsigned int cell_batch_index,
2041 const unsigned int dof_handler_index = numbers::invalid_unsigned_int) const;
2042
2043 /**
2044 * Return the category of the cells on the two sides of the current batch of
2045 * faces.
2046 */
2047 std::pair<unsigned int, unsigned int>
2048 get_face_category(
2049 const unsigned int face_batch_index,
2050 const unsigned int dof_handler_index = numbers::invalid_unsigned_int) const;
2051
2052 /**
2053 * Queries whether or not the indexation has been set.
2054 */
2055 bool
2056 indices_initialized() const;
2057
2058 /**
2059 * Queries whether or not the geometry-related information for the cells has
2060 * been set.
2061 */
2062 bool
2063 mapping_initialized() const;
2064
2065 /**
2066 * Return the level of the mesh to be worked on. Returns
2067 * numbers::invalid_unsigned_int if working on active cells.
2068 */
2069 unsigned int
2070 get_mg_level() const;
2071
2072 /**
2073 * Return an approximation of the memory consumption of this class in
2074 * bytes.
2075 */
2076 std::size_t
2077 memory_consumption() const;
2078
2079 /**
2080 * Prints a detailed summary of memory consumption in the different
2081 * structures of this class to the given output stream.
2082 */
2083 template <typename StreamType>
2084 void
2085 print_memory_consumption(StreamType &out) const;
2086
2087 /**
2088 * Prints a summary of this class to the given output stream. It is focused
2089 * on the indices, and does not print all the data stored.
2090 */
2091 void
2092 print(std::ostream &out) const;
2093
2094 /** @} */
2095
2096 /**
2097 * @name Access of internal data structure
2098 *
2099 * Note: Expert mode, interface not stable between releases.
2100 */
2101 /** @{ */
2102 /**
2103 * Return information on task graph.
2104 */
2105 const internal::MatrixFreeFunctions::TaskInfo &
2106 get_task_info() const;
2107
2108 /*
2109 * Return geometry-dependent information on the cells.
2110 */
2111 const internal::MatrixFreeFunctions::
2112 MappingInfo<dim, Number, VectorizedArrayType> &
2113 get_mapping_info() const;
2114
2115 /**
2116 * Return information on indexation degrees of freedom.
2117 */
2118 const internal::MatrixFreeFunctions::DoFInfo &
2119 get_dof_info(const unsigned int dof_handler_index_component = 0) const;
2120
2121 /**
2122 * Return the number of weights in the constraint pool.
2123 */
2124 unsigned int
2125 n_constraint_pool_entries() const;
2126
2127 /**
2128 * Return a pointer to the first number in the constraint pool data with
2129 * index @p pool_index (to be used together with @p constraint_pool_end()).
2130 */
2131 const Number *
2132 constraint_pool_begin(const unsigned int pool_index) const;
2133
2134 /**
2135 * Return a pointer to one past the last number in the constraint pool data
2136 * with index @p pool_index (to be used together with @p
2137 * constraint_pool_begin()).
2138 */
2139 const Number *
2140 constraint_pool_end(const unsigned int pool_index) const;
2141
2142 /**
2143 * Return the unit cell information for given hp-index.
2144 */
2145 const internal::MatrixFreeFunctions::ShapeInfo<Number> &
2146 get_shape_info(const unsigned int dof_handler_index_component = 0,
2147 const unsigned int quad_index = 0,
2148 const unsigned int fe_base_element = 0,
2149 const unsigned int hp_active_fe_index = 0,
2150 const unsigned int hp_active_quad_index = 0) const;
2151
2152 /**
2153 * Return the connectivity information of a face.
2154 */
2155 const internal::MatrixFreeFunctions::FaceToCellTopology<
2156 VectorizedArrayType::size()> &
2157 get_face_info(const unsigned int face_batch_index) const;
2158
2159
2160 /**
2161 * Return the table that translates a triple of the cell-batch number,
2162 * the index of a face within a cell and the index within the cell batch of
2163 * vectorization into the index within the faces array.
2164 */
2165 const Table<3, unsigned int> &
2166 get_cell_and_face_to_plain_faces() const;
2167
2168 /**
2169 * Obtains a scratch data object for internal use. Make sure to release it
2170 * afterwards by passing the pointer you obtain from this object to the
2171 * release_scratch_data() function. This interface is used by FEEvaluation
2172 * objects for storing their data structures.
2173 *
2174 * The organization of the internal data structure is a thread-local storage
2175 * of a list of vectors. Multiple threads will each get a separate storage
2176 * field and separate vectors, ensuring thread safety. The mechanism to
2177 * acquire and release objects is similar to the mechanisms used for the
2178 * local contributions of WorkStream, see
2179 * @ref workstream_paper "the WorkStream paper".
2180 */
2181 AlignedVector<VectorizedArrayType> *
2182 acquire_scratch_data() const;
2183
2184 /**
2185 * Makes the object of the scratchpad available again.
2186 */
2187 void
2188 release_scratch_data(const AlignedVector<VectorizedArrayType> *memory) const;
2189
2190 /**
2191 * Obtains a scratch data object for internal use. Make sure to release it
2192 * afterwards by passing the pointer you obtain from this object to the
2193 * release_scratch_data_non_threadsafe() function. Note that, as opposed to
2194 * acquire_scratch_data(), this method can only be called by a single thread
2195 * at a time, but opposed to the acquire_scratch_data() it is also possible
2196 * that the thread releasing the scratch data can be different than the one
2197 * that acquired it.
2198 */
2199 AlignedVector<Number> *
2200 acquire_scratch_data_non_threadsafe() const;
2201
2202 /**
2203 * Makes the object of the scratch data available again.
2204 */
2205 void
2206 release_scratch_data_non_threadsafe(
2207 const AlignedVector<Number> *memory) const;
2208
2209 /** @} */
2210
2211 private:
2212 /**
2213 * This is the actual reinit function that sets up the indices for the
2214 * DoFHandler case.
2215 */
2216 template <typename number2, int q_dim>
2217 void
2218 internal_reinit(
2219 const std::shared_ptr<hp::MappingCollection<dim>> &mapping,
2220 const std::vector<const DoFHandler<dim, dim> *> &dof_handlers,
2221 const std::vector<const AffineConstraints<number2> *> &constraint,
2222 const std::vector<IndexSet> &locally_owned_set,
2223 const std::vector<hp::QCollection<q_dim>> &quad,
2224 const AdditionalData &additional_data);
2225
2226 /**
2227 * Initializes the fields in DoFInfo together with the constraint pool that
2228 * holds all different weights in the constraints (not part of DoFInfo
2229 * because several DoFInfo classes can have the same weights which
2230 * consequently only need to be stored once).
2231 */
2232 template <typename number2>
2233 void
2234 initialize_indices(
2235 const std::vector<const AffineConstraints<number2> *> &constraint,
2236 const std::vector<IndexSet> &locally_owned_set,
2237 const AdditionalData &additional_data);
2238
2239 /**
2240 * Initializes the DoFHandlers based on a DoFHandler<dim> argument.
2241 */
2242 void
2243 initialize_dof_handlers(
2244 const std::vector<const DoFHandler<dim, dim> *> &dof_handlers,
2245 const AdditionalData &additional_data);
2246
2247 /**
2248 * Pointers to the DoFHandlers underlying the current problem.
2249 */
2250 std::vector<ObserverPointer<const DoFHandler<dim>>> dof_handlers;
2251
2252 /**
2253 * Pointers to the AffineConstraints underlying the current problem. Only
2254 * filled with an AffineConstraints object if objects of the same `Number`
2255 * template parameter as the `Number` template of MatrixFree is passed to
2256 * reinit(). Filled with nullptr otherwise.
2257 */
2258 std::vector<ObserverPointer<const AffineConstraints<Number>>>
2259 affine_constraints;
2260
2261 /**
2262 * Contains the information about degrees of freedom on the individual cells
2263 * and constraints.
2264 */
2265 std::vector<internal::MatrixFreeFunctions::DoFInfo> dof_info;
2266
2267 /**
2268 * Contains the weights for constraints stored in DoFInfo. Filled into a
2269 * separate field since several vector components might share similar
2270 * weights, which reduces memory consumption. Moreover, it obviates template
2271 * arguments on DoFInfo and keeps it a plain field of indices only.
2272 */
2273 std::vector<Number> constraint_pool_data;
2274
2275 /**
2276 * Contains an indicator to the start of the ith index in the constraint
2277 * pool data.
2278 */
2279 std::vector<unsigned int> constraint_pool_row_index;
2280
2281 /**
2282 * Holds information on transformation of cells from reference cell to real
2283 * cell that is needed for evaluating integrals.
2284 */
2285 internal::MatrixFreeFunctions::MappingInfo<dim, Number, VectorizedArrayType>
2286 mapping_info;
2287
2288 /**
2289 * Contains shape value information on the unit cell.
2290 */
2291 Table<4, internal::MatrixFreeFunctions::ShapeInfo<Number>> shape_info;
2292
2293 /**
2294 * Describes how the cells are gone through. With the cell level (first
2295 * index in this field) and the index within the level, one can reconstruct
2296 * a deal.II cell iterator and use all the traditional things deal.II offers
2297 * to do with cell iterators.
2298 */
2299 std::vector<std::pair<unsigned int, unsigned int>> cell_level_index;
2300
2301 /**
2302 * Conversion from deal.II index (active or level index) to MatrixFree index
2303 * (inverse of cell_level_index).
2304 */
2305 std::vector<unsigned int> mf_cell_indices;
2306
2307 /**
2308 * For discontinuous Galerkin, the cell_level_index includes cells that are
2309 * not on the local processor but that are needed to evaluate the cell
2310 * integrals. In cell_level_index_end_local, we store the number of local
2311 * cells.
2312 */
2313 unsigned int cell_level_index_end_local;
2314
2315 /**
2316 * Stores the basic layout of the cells and faces to be treated, including
2317 * the task layout for the shared memory parallelization and possible
2318 * overlaps between communications and computations with MPI.
2319 */
2320 internal::MatrixFreeFunctions::TaskInfo task_info;
2321
2322 /**
2323 * Vector holding face information. Only initialized if
2324 * build_face_info=true.
2325 */
2326 internal::MatrixFreeFunctions::FaceInfo<VectorizedArrayType::size()>
2327 face_info;
2328
2329 /**
2330 * Stores whether indices have been initialized.
2331 */
2332 bool indices_are_initialized;
2333
2334 /**
2335 * Stores whether indices have been initialized.
2336 */
2337 bool mapping_is_initialized;
2338
2339 /**
2340 * Scratchpad memory for use in evaluation. We allow more than one
2341 * evaluation object to attach to this field (this, the outer
2342 * std::vector), so we need to keep tracked of whether a certain data
2343 * field is already used (first part of pair) and keep a list of
2344 * objects.
2345 */
2346 mutable Threads::ThreadLocalStorage<
2347 std::list<std::pair<bool, AlignedVector<VectorizedArrayType>>>>
2348 scratch_pad;
2349
2350 /**
2351 * Scratchpad memory for use in evaluation and other contexts, non-thread
2352 * safe variant.
2353 */
2354 mutable std::list<std::pair<bool, AlignedVector<Number>>>
2355 scratch_pad_non_threadsafe;
2356
2357 /**
2358 * Stored the level of the mesh to be worked on.
2359 */
2360 unsigned int mg_level;
2361
2362 /**
2363 * Stores the index of the first DoFHandler that is in hp-mode. If no
2364 * DoFHandler is in hp-mode, the value is 0.
2365 */
2366 unsigned int first_hp_dof_handler_index;
2367 };
2368
2369
2370
2371 /*----------------------- Inline functions ----------------------------------*/
2372
2373 #ifndef DOXYGEN
2374
2375
2376
2377 template <int dim, typename Number, typename VectorizedArrayType>
2378 template <typename T>
2379 inline void
2380 MatrixFree<dim, Number, VectorizedArrayType>::initialize_cell_data_vector(
2381 AlignedVector<T> &vec) const
2382 {
2383 vec.resize(this->n_cell_batches() + this->n_ghost_cell_batches());
2384 }
2385
2386
2387
2388 template <int dim, typename Number, typename VectorizedArrayType>
2389 template <typename T>
2390 inline void
2391 MatrixFree<dim, Number, VectorizedArrayType>::initialize_face_data_vector(
2392 AlignedVector<T> &vec) const
2393 {
2394 vec.resize(this->n_inner_face_batches() + this->n_boundary_face_batches() +
2395 this->n_ghost_inner_face_batches());
2396 }
2397
2398
2399
2400 template <int dim, typename Number, typename VectorizedArrayType>
2401 template <typename VectorType>
2402 inline void
2403 MatrixFree<dim, Number, VectorizedArrayType>::initialize_dof_vector(
2404 VectorType &vec,
2405 const unsigned int comp) const
2406 {
2407 static_assert(IsBlockVector<VectorType>::value == false,
2408 "This function is not supported for block vectors.");
2409
2410 Assert(task_info.n_procs == 1,
2411 ExcMessage("This function can only be used in serial."));
2412
2413 AssertIndexRange(comp, n_components());
2414 vec.reinit(dof_info[comp].vector_partitioner->size());
2415 }
2416
2417
2418
2419 template <int dim, typename Number, typename VectorizedArrayType>
2420 template <typename Number2>
2421 inline void
2422 MatrixFree<dim, Number, VectorizedArrayType>::initialize_dof_vector(
2423 LinearAlgebra::distributed::Vector<Number2> &vec,
2424 const unsigned int comp) const
2425 {
2426 AssertIndexRange(comp, n_components());
2427 vec.reinit(dof_info[comp].vector_partitioner, task_info.communicator_sm);
2428 }
2429
2430
2431
2432 template <int dim, typename Number, typename VectorizedArrayType>
2433 inline const std::shared_ptr<const Utilities::MPI::Partitioner> &
2434 MatrixFree<dim, Number, VectorizedArrayType>::get_vector_partitioner(
2435 const unsigned int comp) const
2436 {
2437 AssertIndexRange(comp, n_components());
2438 return dof_info[comp].vector_partitioner;
2439 }
2440
2441
2442
2443 template <int dim, typename Number, typename VectorizedArrayType>
2444 inline const std::vector<unsigned int> &
2445 MatrixFree<dim, Number, VectorizedArrayType>::get_constrained_dofs(
2446 const unsigned int comp) const
2447 {
2448 AssertIndexRange(comp, n_components());
2449 return dof_info[comp].constrained_dofs;
2450 }
2451
2452
2453
2454 template <int dim, typename Number, typename VectorizedArrayType>
2455 inline unsigned int
2456 MatrixFree<dim, Number, VectorizedArrayType>::n_components() const
2457 {
2458 AssertDimension(dof_handlers.size(), dof_info.size());
2459 return dof_handlers.size();
2460 }
2461
2462
2463
2464 template <int dim, typename Number, typename VectorizedArrayType>
2465 inline unsigned int
2466 MatrixFree<dim, Number, VectorizedArrayType>::n_base_elements(
2467 const unsigned int dof_no) const
2468 {
2469 AssertDimension(dof_handlers.size(), dof_info.size());
2470 AssertIndexRange(dof_no, dof_handlers.size());
2471 return dof_handlers[dof_no]->get_fe().n_base_elements();
2472 }
2473
2474
2475
2476 template <int dim, typename Number, typename VectorizedArrayType>
2477 inline const internal::MatrixFreeFunctions::TaskInfo &
2478 MatrixFree<dim, Number, VectorizedArrayType>::get_task_info() const
2479 {
2480 return task_info;
2481 }
2482
2483
2484
2485 template <int dim, typename Number, typename VectorizedArrayType>
2486 inline unsigned int
2487 MatrixFree<dim, Number, VectorizedArrayType>::n_physical_cells() const
2488 {
2489 return task_info.n_active_cells;
2490 }
2491
2492
2493
2494 template <int dim, typename Number, typename VectorizedArrayType>
2495 inline unsigned int
2496 MatrixFree<dim, Number, VectorizedArrayType>::n_cell_batches() const
2497 {
2498 return *(task_info.cell_partition_data.end() - 2);
2499 }
2500
2501
2502
2503 template <int dim, typename Number, typename VectorizedArrayType>
2504 inline unsigned int
2505 MatrixFree<dim, Number, VectorizedArrayType>::n_ghost_cell_batches() const
2506 {
2507 return *(task_info.cell_partition_data.end() - 1) -
2508 *(task_info.cell_partition_data.end() - 2);
2509 }
2510
2511
2512
2513 template <int dim, typename Number, typename VectorizedArrayType>
2514 inline unsigned int
2515 MatrixFree<dim, Number, VectorizedArrayType>::n_inner_face_batches() const
2516 {
2517 if (task_info.face_partition_data.empty())
2518 return 0;
2519 return task_info.face_partition_data.back();
2520 }
2521
2522
2523
2524 template <int dim, typename Number, typename VectorizedArrayType>
2525 inline unsigned int
2526 MatrixFree<dim, Number, VectorizedArrayType>::n_boundary_face_batches() const
2527 {
2528 if (task_info.face_partition_data.empty())
2529 return 0;
2530 return task_info.boundary_partition_data.back() -
2531 task_info.face_partition_data.back();
2532 }
2533
2534
2535
2536 template <int dim, typename Number, typename VectorizedArrayType>
2537 inline unsigned int
2538 MatrixFree<dim, Number, VectorizedArrayType>::n_ghost_inner_face_batches() const
2539 {
2540 if (task_info.face_partition_data.empty())
2541 return 0;
2542 return face_info.faces.size() - task_info.boundary_partition_data.back();
2543 }
2544
2545
2546
2547 template <int dim, typename Number, typename VectorizedArrayType>
2548 inline types::boundary_id
2549 MatrixFree<dim, Number, VectorizedArrayType>::get_boundary_id(
2550 const unsigned int face_batch_index) const
2551 {
2552 Assert(face_batch_index >= task_info.boundary_partition_data[0] &&
2553 face_batch_index < task_info.boundary_partition_data.back(),
2554 ExcIndexRange(face_batch_index,
2555 task_info.boundary_partition_data[0],
2556 task_info.boundary_partition_data.back()));
2557 return types::boundary_id(face_info.faces[face_batch_index].exterior_face_no);
2558 }
2559
2560
2561
2562 template <int dim, typename Number, typename VectorizedArrayType>
2563 inline std::array<types::boundary_id, VectorizedArrayType::size()>
2564 MatrixFree<dim, Number, VectorizedArrayType>::get_faces_by_cells_boundary_id(
2565 const unsigned int cell_batch_index,
2566 const unsigned int face_number) const
2567 {
2568 AssertIndexRange(cell_batch_index, n_cell_batches());
2569 AssertIndexRange(face_number, ReferenceCells::max_n_faces<dim>());
2570 Assert(face_info.cell_and_face_boundary_id.size(0) >= n_cell_batches(),
2571 ExcNotInitialized());
2572 std::array<types::boundary_id, VectorizedArrayType::size()> result;
2573 result.fill(numbers::invalid_boundary_id);
2574 for (unsigned int v = 0;
2575 v < n_active_entries_per_cell_batch(cell_batch_index);
2576 ++v)
2577 result[v] =
2578 face_info.cell_and_face_boundary_id(cell_batch_index, face_number, v);
2579 return result;
2580 }
2581
2582
2583
2584 template <int dim, typename Number, typename VectorizedArrayType>
2585 inline const internal::MatrixFreeFunctions::
2586 MappingInfo<dim, Number, VectorizedArrayType> &
2587 MatrixFree<dim, Number, VectorizedArrayType>::get_mapping_info() const
2588 {
2589 return mapping_info;
2590 }
2591
2592
2593
2594 template <int dim, typename Number, typename VectorizedArrayType>
2595 inline const internal::MatrixFreeFunctions::DoFInfo &
2596 MatrixFree<dim, Number, VectorizedArrayType>::get_dof_info(
2597 const unsigned int dof_index) const
2598 {
2599 AssertIndexRange(dof_index, n_components());
2600 return dof_info[dof_index];
2601 }
2602
2603
2604
2605 template <int dim, typename Number, typename VectorizedArrayType>
2606 inline unsigned int
2607 MatrixFree<dim, Number, VectorizedArrayType>::n_constraint_pool_entries() const
2608 {
2609 return constraint_pool_row_index.size() - 1;
2610 }
2611
2612
2613
2614 template <int dim, typename Number, typename VectorizedArrayType>
2615 inline const Number *
2616 MatrixFree<dim, Number, VectorizedArrayType>::constraint_pool_begin(
2617 const unsigned int row) const
2618 {
2619 AssertIndexRange(row, constraint_pool_row_index.size() - 1);
2620 return constraint_pool_data.empty() ?
2621 nullptr :
2622 constraint_pool_data.data() + constraint_pool_row_index[row];
2623 }
2624
2625
2626
2627 template <int dim, typename Number, typename VectorizedArrayType>
2628 inline const Number *
2629 MatrixFree<dim, Number, VectorizedArrayType>::constraint_pool_end(
2630 const unsigned int row) const
2631 {
2632 AssertIndexRange(row, constraint_pool_row_index.size() - 1);
2633 return constraint_pool_data.empty() ?
2634 nullptr :
2635 constraint_pool_data.data() + constraint_pool_row_index[row + 1];
2636 }
2637
2638
2639
2640 template <int dim, typename Number, typename VectorizedArrayType>
2641 inline std::pair<unsigned int, unsigned int>
2642 MatrixFree<dim, Number, VectorizedArrayType>::create_cell_subrange_hp(
2643 const std::pair<unsigned int, unsigned int> &range,
2644 const unsigned int degree,
2645 const unsigned int dof_handler_component) const
2646 {
2647 if (dof_info[dof_handler_component].cell_active_fe_index.empty())
2648 {
2649 AssertDimension(
2650 dof_info[dof_handler_component].fe_index_conversion.size(), 1);
2651 AssertDimension(
2652 dof_info[dof_handler_component].fe_index_conversion[0].size(), 1);
2653 if (dof_info[dof_handler_component].fe_index_conversion[0][0] == degree)
2654 return range;
2655 else
2656 return {range.second, range.second};
2657 }
2658
2659 const unsigned int fe_index =
2660 dof_info[dof_handler_component].fe_index_from_degree(0, degree);
2661 if (fe_index >= dof_info[dof_handler_component].max_fe_index)
2662 return {range.second, range.second};
2663 else
2664 return create_cell_subrange_hp_by_index(range,
2665 fe_index,
2666 dof_handler_component);
2667 }
2668
2669
2670
2671 template <int dim, typename Number, typename VectorizedArrayType>
2672 inline bool
2673 MatrixFree<dim, Number, VectorizedArrayType>::at_irregular_cell(
2674 const unsigned int cell_batch_index) const
2675 {
2676 AssertIndexRange(cell_batch_index, task_info.cell_partition_data.back());
2677 return VectorizedArrayType::size() > 1 &&
2678 cell_level_index[(cell_batch_index + 1) * VectorizedArrayType::size() -
2679 1] == cell_level_index[(cell_batch_index + 1) *
2680 VectorizedArrayType::size() -
2681 2];
2682 }
2683
2684
2685
2686 template <int dim, typename Number, typename VectorizedArrayType>
2687 unsigned int
2688 MatrixFree<dim, Number, VectorizedArrayType>::n_active_fe_indices() const
2689 {
2690 return shape_info.size(2);
2691 }
2692
2693
2694 template <int dim, typename Number, typename VectorizedArrayType>
2695 unsigned int
2696 MatrixFree<dim, Number, VectorizedArrayType>::get_cell_active_fe_index(
2697 const std::pair<unsigned int, unsigned int> range,
2698 const unsigned int dof_handler_index) const
2699 {
2700 const unsigned int dof_no =
2701 dof_handler_index == numbers::invalid_unsigned_int ?
2702 first_hp_dof_handler_index :
2703 dof_handler_index;
2704
2705 const auto &fe_indices = dof_info[dof_no].cell_active_fe_index;
2706
2707 if (fe_indices.empty() == true ||
2708 dof_handlers[dof_no]->get_fe_collection().size() == 1)
2709 return 0;
2710
2711 const auto index = fe_indices[range.first];
2712
2713 for (unsigned int i = range.first; i < range.second; ++i)
2714 AssertDimension(index, fe_indices[i]);
2715
2716 return index;
2717 }
2718
2719
2720
2721 template <int dim, typename Number, typename VectorizedArrayType>
2722 unsigned int
2723 MatrixFree<dim, Number, VectorizedArrayType>::get_face_active_fe_index(
2724 const std::pair<unsigned int, unsigned int> range,
2725 const bool is_interior_face,
2726 const unsigned int dof_handler_index) const
2727 {
2728 const unsigned int dof_no =
2729 dof_handler_index == numbers::invalid_unsigned_int ?
2730 first_hp_dof_handler_index :
2731 dof_handler_index;
2732
2733 const auto &fe_indices = dof_info[dof_no].cell_active_fe_index;
2734
2735 if (fe_indices.empty() == true)
2736 return 0;
2737
2738 if (is_interior_face)
2739 {
2740 const unsigned int index =
2741 fe_indices[face_info.faces[range.first].cells_interior[0] /
2742 VectorizedArrayType::size()];
2743
2744 for (unsigned int i = range.first; i < range.second; ++i)
2745 AssertDimension(index,
2746 fe_indices[face_info.faces[i].cells_interior[0] /
2747 VectorizedArrayType::size()]);
2748
2749 return index;
2750 }
2751 else
2752 {
2753 const unsigned int index =
2754 fe_indices[face_info.faces[range.first].cells_exterior[0] /
2755 VectorizedArrayType::size()];
2756
2757 for (unsigned int i = range.first; i < range.second; ++i)
2758 AssertDimension(index,
2759 fe_indices[face_info.faces[i].cells_exterior[0] /
2760 VectorizedArrayType::size()]);
2761
2762 return index;
2763 }
2764 }
2765
2766
2767
2768 template <int dim, typename Number, typename VectorizedArrayType>
2769 inline unsigned int
2770 MatrixFree<dim, Number, VectorizedArrayType>::n_active_entries_per_cell_batch(
2771 const unsigned int cell_batch_index) const
2772 {
2773 Assert(!dof_info.empty(), ExcNotInitialized());
2774 AssertIndexRange(cell_batch_index, task_info.cell_partition_data.back());
2775 const std::vector<unsigned char> &n_lanes_filled =
2776 dof_info[0].n_vectorization_lanes_filled
2777 [internal::MatrixFreeFunctions::DoFInfo::dof_access_cell];
2778 AssertIndexRange(cell_batch_index, n_lanes_filled.size());
2779
2780 return n_lanes_filled[cell_batch_index];
2781 }
2782
2783
2784
2785 template <int dim, typename Number, typename VectorizedArrayType>
2786 inline unsigned int
2787 MatrixFree<dim, Number, VectorizedArrayType>::n_active_entries_per_face_batch(
2788 const unsigned int face_batch_index) const
2789 {
2790 AssertIndexRange(face_batch_index, face_info.faces.size());
2791 Assert(!dof_info.empty(), ExcNotInitialized());
2792 const std::vector<unsigned char> &n_lanes_filled =
2793 dof_info[0].n_vectorization_lanes_filled
2794 [internal::MatrixFreeFunctions::DoFInfo::dof_access_face_interior];
2795 AssertIndexRange(face_batch_index, n_lanes_filled.size());
2796 return n_lanes_filled[face_batch_index];
2797 }
2798
2799
2800
2801 template <int dim, typename Number, typename VectorizedArrayType>
2802 inline unsigned int
2803 MatrixFree<dim, Number, VectorizedArrayType>::get_dofs_per_cell(
2804 const unsigned int dof_handler_index,
2805 const unsigned int active_fe_index) const
2806 {
2807 return dof_info[dof_handler_index].dofs_per_cell[active_fe_index];
2808 }
2809
2810
2811
2812 template <int dim, typename Number, typename VectorizedArrayType>
2813 inline unsigned int
2814 MatrixFree<dim, Number, VectorizedArrayType>::get_n_q_points(
2815 const unsigned int quad_index,
2816 const unsigned int active_fe_index) const
2817 {
2818 AssertIndexRange(quad_index, mapping_info.cell_data.size());
2819 return mapping_info.cell_data[quad_index]
2820 .descriptor[active_fe_index]
2821 .n_q_points;
2822 }
2823
2824
2825
2826 template <int dim, typename Number, typename VectorizedArrayType>
2827 inline unsigned int
2828 MatrixFree<dim, Number, VectorizedArrayType>::get_dofs_per_face(
2829 const unsigned int dof_handler_index,
2830 const unsigned int active_fe_index) const
2831 {
2832 return dof_info[dof_handler_index].dofs_per_face[active_fe_index];
2833 }
2834
2835
2836
2837 template <int dim, typename Number, typename VectorizedArrayType>
2838 inline unsigned int
2839 MatrixFree<dim, Number, VectorizedArrayType>::get_n_q_points_face(
2840 const unsigned int quad_index,
2841 const unsigned int active_fe_index) const
2842 {
2843 AssertIndexRange(quad_index, mapping_info.face_data.size());
2844 return mapping_info.face_data[quad_index]
2845 .descriptor[active_fe_index]
2846 .n_q_points;
2847 }
2848
2849
2850
2851 template <int dim, typename Number, typename VectorizedArrayType>
2852 inline const IndexSet &
2853 MatrixFree<dim, Number, VectorizedArrayType>::get_locally_owned_set(
2854 const unsigned int dof_handler_index) const
2855 {
2856 return dof_info[dof_handler_index].vector_partitioner->locally_owned_range();
2857 }
2858
2859
2860
2861 template <int dim, typename Number, typename VectorizedArrayType>
2862 inline const IndexSet &
2863 MatrixFree<dim, Number, VectorizedArrayType>::get_ghost_set(
2864 const unsigned int dof_handler_index) const
2865 {
2866 return dof_info[dof_handler_index].vector_partitioner->ghost_indices();
2867 }
2868
2869
2870
2871 template <int dim, typename Number, typename VectorizedArrayType>
2872 inline const internal::MatrixFreeFunctions::ShapeInfo<Number> &
2873 MatrixFree<dim, Number, VectorizedArrayType>::get_shape_info(
2874 const unsigned int dof_handler_index,
2875 const unsigned int index_quad,
2876 const unsigned int index_fe,
2877 const unsigned int active_fe_index,
2878 const unsigned int active_quad_index) const
2879 {
2880 AssertIndexRange(dof_handler_index, dof_info.size());
2881 const unsigned int ind =
2882 dof_info[dof_handler_index].global_base_element_offset + index_fe;
2883 AssertIndexRange(ind, shape_info.size(0));
2884 AssertIndexRange(index_quad, shape_info.size(1));
2885 AssertIndexRange(active_fe_index, shape_info.size(2));
2886 AssertIndexRange(active_quad_index, shape_info.size(3));
2887 return shape_info(ind, index_quad, active_fe_index, active_quad_index);
2888 }
2889
2890
2891
2892 template <int dim, typename Number, typename VectorizedArrayType>
2893 inline const internal::MatrixFreeFunctions::FaceToCellTopology<
2894 VectorizedArrayType::size()> &
2895 MatrixFree<dim, Number, VectorizedArrayType>::get_face_info(
2896 const unsigned int face_batch_index) const
2897 {
2898 AssertIndexRange(face_batch_index, face_info.faces.size());
2899 return face_info.faces[face_batch_index];
2900 }
2901
2902
2903
2904 template <int dim, typename Number, typename VectorizedArrayType>
2905 inline const Table<3, unsigned int> &
2906 MatrixFree<dim, Number, VectorizedArrayType>::get_cell_and_face_to_plain_faces()
2907 const
2908 {
2909 return face_info.cell_and_face_to_plain_faces;
2910 }
2911
2912
2913
2914 template <int dim, typename Number, typename VectorizedArrayType>
2915 inline const Quadrature<dim> &
2916 MatrixFree<dim, Number, VectorizedArrayType>::get_quadrature(
2917 const unsigned int quad_index,
2918 const unsigned int active_fe_index) const
2919 {
2920 AssertIndexRange(quad_index, mapping_info.cell_data.size());
2921 return mapping_info.cell_data[quad_index]
2922 .descriptor[active_fe_index]
2923 .quadrature;
2924 }
2925
2926
2927
2928 template <int dim, typename Number, typename VectorizedArrayType>
2929 inline const Quadrature<dim - 1> &
2930 MatrixFree<dim, Number, VectorizedArrayType>::get_face_quadrature(
2931 const unsigned int quad_index,
2932 const unsigned int active_fe_index) const
2933 {
2934 AssertIndexRange(quad_index, mapping_info.face_data.size());
2935 return mapping_info.face_data[quad_index]
2936 .descriptor[active_fe_index]
2937 .quadrature;
2938 }
2939
2940
2941
2942 template <int dim, typename Number, typename VectorizedArrayType>
2943 inline unsigned int
2944 MatrixFree<dim, Number, VectorizedArrayType>::get_cell_range_category(
2945 const std::pair<unsigned int, unsigned int> range,
2946 const unsigned int dof_handler_index) const
2947 {
2948 auto result = get_cell_category(range.first, dof_handler_index);
2949
2950 for (unsigned int i = range.first; i < range.second; ++i)
2951 result = std::max(result, get_cell_category(i, dof_handler_index));
2952
2953 return result;
2954 }
2955
2956
2957
2958 template <int dim, typename Number, typename VectorizedArrayType>
2959 inline std::pair<unsigned int, unsigned int>
2960 MatrixFree<dim, Number, VectorizedArrayType>::get_face_range_category(
2961 const std::pair<unsigned int, unsigned int> range,
2962 const unsigned int dof_handler_index) const
2963 {
2964 auto result = get_face_category(range.first, dof_handler_index);
2965
2966 for (unsigned int i = range.first; i < range.second; ++i)
2967 {
2968 result.first =
2969 std::max(result.first, get_face_category(i, dof_handler_index).first);
2970 result.second =
2971 std::max(result.second, get_face_category(i, dof_handler_index).second);
2972 }
2973
2974 return result;
2975 }
2976
2977
2978
2979 template <int dim, typename Number, typename VectorizedArrayType>
2980 inline unsigned int
2981 MatrixFree<dim, Number, VectorizedArrayType>::get_cell_category(
2982 const unsigned int cell_batch_index,
2983 const unsigned int dof_handler_index) const
2984 {
2985 AssertIndexRange(0, dof_info.size());
2986
2987 const unsigned int dof_no =
2988 dof_handler_index == numbers::invalid_unsigned_int ?
2989 first_hp_dof_handler_index :
2990 dof_handler_index;
2991
2992 AssertIndexRange(cell_batch_index,
2993 dof_info[dof_no].cell_active_fe_index.size());
2994 if (dof_info[dof_no].cell_active_fe_index.empty())
2995 return 0;
2996 else
2997 return dof_info[dof_no].cell_active_fe_index[cell_batch_index];
2998 }
2999
3000
3001
3002 template <int dim, typename Number, typename VectorizedArrayType>
3003 inline std::pair<unsigned int, unsigned int>
3004 MatrixFree<dim, Number, VectorizedArrayType>::get_face_category(
3005 const unsigned int face_batch_index,
3006 const unsigned int dof_handler_index) const
3007 {
3008 const unsigned int dof_no =
3009 dof_handler_index == numbers::invalid_unsigned_int ?
3010 first_hp_dof_handler_index :
3011 dof_handler_index;
3012
3013 AssertIndexRange(face_batch_index, face_info.faces.size());
3014 if (dof_info[dof_no].cell_active_fe_index.empty())
3015 return std::make_pair(0U, 0U);
3016
3017 std::pair<unsigned int, unsigned int> result = std::make_pair(0U, 0U);
3018 for (unsigned int v = 0;
3019 v < VectorizedArrayType::size() &&
3020 face_info.faces[face_batch_index].cells_interior[v] !=
3021 numbers::invalid_unsigned_int;
3022 ++v)
3023 result.first = std::max(
3024 result.first,
3025 dof_info[dof_no].cell_active_fe_index[face_info.faces[face_batch_index]
3026 .cells_interior[v] /
3027 VectorizedArrayType::size()]);
3028 if (face_info.faces[face_batch_index].cells_exterior[0] !=
3029 numbers::invalid_unsigned_int)
3030 for (unsigned int v = 0;
3031 v < VectorizedArrayType::size() &&
3032 face_info.faces[face_batch_index].cells_exterior[v] !=
3033 numbers::invalid_unsigned_int;
3034 ++v)
3035 result.second = std::max(
3036 result.second,
3037 dof_info[dof_no].cell_active_fe_index[face_info.faces[face_batch_index]
3038 .cells_exterior[v] /
3039 VectorizedArrayType::size()]);
3040 else
3041 result.second = numbers::invalid_unsigned_int;
3042 return result;
3043 }
3044
3045
3046
3047 template <int dim, typename Number, typename VectorizedArrayType>
3048 inline bool
3049 MatrixFree<dim, Number, VectorizedArrayType>::indices_initialized() const
3050 {
3051 return indices_are_initialized;
3052 }
3053
3054
3055
3056 template <int dim, typename Number, typename VectorizedArrayType>
3057 inline bool
3058 MatrixFree<dim, Number, VectorizedArrayType>::mapping_initialized() const
3059 {
3060 return mapping_is_initialized;
3061 }
3062
3063
3064 template <int dim, typename Number, typename VectorizedArrayType>
3065 inline unsigned int
3066 MatrixFree<dim, Number, VectorizedArrayType>::get_mg_level() const
3067 {
3068 return mg_level;
3069 }
3070
3071
3072
3073 template <int dim, typename Number, typename VectorizedArrayType>
3074 AlignedVector<VectorizedArrayType> *
3075 MatrixFree<dim, Number, VectorizedArrayType>::acquire_scratch_data() const
3076 {
3077 using list_type =
3078 std::list<std::pair<bool, AlignedVector<VectorizedArrayType>>>;
3079 list_type &data = scratch_pad.get();
3080 for (typename list_type::iterator it = data.begin(); it != data.end(); ++it)
3081 if (it->first == false)
3082 {
3083 it->first = true;
3084 return &it->second;
3085 }
3086 data.emplace_front(true, AlignedVector<VectorizedArrayType>());
3087 return &data.front().second;
3088 }
3089
3090
3091
3092 template <int dim, typename Number, typename VectorizedArrayType>
3093 void
3094 MatrixFree<dim, Number, VectorizedArrayType>::release_scratch_data(
3095 const AlignedVector<VectorizedArrayType> *scratch) const
3096 {
3097 using list_type =
3098 std::list<std::pair<bool, AlignedVector<VectorizedArrayType>>>;
3099 list_type &data = scratch_pad.get();
3100 for (typename list_type::iterator it = data.begin(); it != data.end(); ++it)
3101 if (&it->second == scratch)
3102 {
3103 Assert(it->first == true, ExcInternalError());
3104 it->first = false;
3105 return;
3106 }
3107 AssertThrow(false, ExcMessage("Tried to release invalid scratch pad"));
3108 }
3109
3110
3111
3112 template <int dim, typename Number, typename VectorizedArrayType>
3113 AlignedVector<Number> *
3114 MatrixFree<dim, Number, VectorizedArrayType>::
3115 acquire_scratch_data_non_threadsafe() const
3116 {
3117 for (typename std::list<std::pair<bool, AlignedVector<Number>>>::iterator it =
3118 scratch_pad_non_threadsafe.begin();
3119 it != scratch_pad_non_threadsafe.end();
3120 ++it)
3121 if (it->first == false)
3122 {
3123 it->first = true;
3124 return &it->second;
3125 }
3126 scratch_pad_non_threadsafe.push_front(
3127 std::make_pair(true, AlignedVector<Number>()));
3128 return &scratch_pad_non_threadsafe.front().second;
3129 }
3130
3131
3132
3133 template <int dim, typename Number, typename VectorizedArrayType>
3134 void
3135 MatrixFree<dim, Number, VectorizedArrayType>::
3136 release_scratch_data_non_threadsafe(
3137 const AlignedVector<Number> *scratch) const
3138 {
3139 for (typename std::list<std::pair<bool, AlignedVector<Number>>>::iterator it =
3140 scratch_pad_non_threadsafe.begin();
3141 it != scratch_pad_non_threadsafe.end();
3142 ++it)
3143 if (&it->second == scratch)
3144 {
3145 Assert(it->first == true, ExcInternalError());
3146 it->first = false;
3147 return;
3148 }
3149 AssertThrow(false, ExcMessage("Tried to release invalid scratch pad"));
3150 }
3151
3152
3153
3154 // ------------------------------ reinit functions ---------------------------
3155
3156 namespace internal
3157 {
3158 namespace MatrixFreeImplementation
3159 {
3160 template <int dim, int spacedim>
3161 inline std::vector<IndexSet>
3162 extract_locally_owned_index_sets(
3163 const std::vector<const DoFHandler<dim, spacedim> *> &dofh,
3164 const unsigned int level)
3165 {
3166 std::vector<IndexSet> locally_owned_set;
3167 locally_owned_set.reserve(dofh.size());
3168 for (unsigned int j = 0; j < dofh.size(); ++j)
3169 if (level == numbers::invalid_unsigned_int)
3170 locally_owned_set.push_back(dofh[j]->locally_owned_dofs());
3171 else
3172 locally_owned_set.push_back(dofh[j]->locally_owned_mg_dofs(level));
3173 return locally_owned_set;
3174 }
3175 } // namespace MatrixFreeImplementation
3176 } // namespace internal
3177
3178
3179
3180 template <int dim, typename Number, typename VectorizedArrayType>
3181 template <typename QuadratureType, typename number2, typename MappingType>
3182 void
3183 MatrixFree<dim, Number, VectorizedArrayType>::reinit(
3184 const MappingType &mapping,
3185 const DoFHandler<dim> &dof_handler,
3186 const AffineConstraints<number2> &constraints_in,
3187 const QuadratureType &quad,
3188 const typename MatrixFree<dim, Number, VectorizedArrayType>::AdditionalData
3189 &additional_data)
3190 {
3191 std::vector<const DoFHandler<dim, dim> *> dof_handlers;
3192 std::vector<const AffineConstraints<number2> *> constraints;
3193
3194 dof_handlers.push_back(&dof_handler);
3195 constraints.push_back(&constraints_in);
3196
3197 std::vector<IndexSet> locally_owned_sets =
3198 internal::MatrixFreeImplementation::extract_locally_owned_index_sets(
3199 dof_handlers, additional_data.mg_level);
3200
3201 std::vector<hp::QCollection<dim>> quad_hp;
3202 quad_hp.emplace_back(quad);
3203
3204 internal_reinit(std::make_shared<hp::MappingCollection<dim>>(mapping),
3205 dof_handlers,
3206 constraints,
3207 locally_owned_sets,
3208 quad_hp,
3209 additional_data);
3210 }
3211
3212
3213
3214 template <int dim, typename Number, typename VectorizedArrayType>
3215 template <typename QuadratureType, typename number2, typename MappingType>
3216 void
3217 MatrixFree<dim, Number, VectorizedArrayType>::reinit(
3218 const MappingType &mapping,
3219 const std::vector<const DoFHandler<dim> *> &dof_handler,
3220 const std::vector<const AffineConstraints<number2> *> &constraint,
3221 const QuadratureType &quad,
3222 const typename MatrixFree<dim, Number, VectorizedArrayType>::AdditionalData
3223 &additional_data)
3224 {
3225 std::vector<IndexSet> locally_owned_set =
3226 internal::MatrixFreeImplementation::extract_locally_owned_index_sets(
3227 dof_handler, additional_data.mg_level);
3228 std::vector<hp::QCollection<dim>> quad_hp;
3229 quad_hp.emplace_back(quad);
3230
3231 internal_reinit(std::make_shared<hp::MappingCollection<dim>>(mapping),
3232 dof_handler,
3233 constraint,
3234 locally_owned_set,
3235 quad_hp,
3236 additional_data);
3237 }
3238
3239
3240
3241 template <int dim, typename Number, typename VectorizedArrayType>
3242 template <typename QuadratureType, typename number2, typename MappingType>
3243 void
3244 MatrixFree<dim, Number, VectorizedArrayType>::reinit(
3245 const MappingType &mapping,
3246 const std::vector<const DoFHandler<dim> *> &dof_handler,
3247 const std::vector<const AffineConstraints<number2> *> &constraint,
3248 const std::vector<QuadratureType> &quad,
3249 const typename MatrixFree<dim, Number, VectorizedArrayType>::AdditionalData
3250 &additional_data)
3251 {
3252 std::vector<IndexSet> locally_owned_set =
3253 internal::MatrixFreeImplementation::extract_locally_owned_index_sets(
3254 dof_handler, additional_data.mg_level);
3255 std::vector<hp::QCollection<dim>> quad_hp;
3256 for (unsigned int q = 0; q < quad.size(); ++q)
3257 quad_hp.emplace_back(quad[q]);
3258
3259 internal_reinit(std::make_shared<hp::MappingCollection<dim>>(mapping),
3260 dof_handler,
3261 constraint,
3262 locally_owned_set,
3263 quad_hp,
3264 additional_data);
3265 }
3266
3267
3268
3269 // ------------------------------ implementation of loops --------------------
3270
3271 // internal helper functions that define how to call MPI data exchange
3272 // functions: for generic vectors, do nothing at all. For distributed vectors,
3273 // call update_ghost_values_start function and so on. If we have collections
3274 // of vectors, just do the individual functions of the components. In order to
3275 // keep ghost values consistent (whether we are in read or write mode), we
3276 // also reset the values at the end. the whole situation is a bit complicated
3277 // by the fact that we need to treat block vectors differently, which use some
3278 // additional helper functions to select the blocks and template magic.
3279 namespace internal
3280 {
3281 /**
3282 * Internal class for exchanging data between vectors.
3283 */
3284 template <int dim, typename Number, typename VectorizedArrayType>
3285 struct VectorDataExchange
3286 {
3287 // A shift for the MPI messages to reduce the risk for accidental
3288 // interaction with other open communications that a user program might
3289 // set up (parallel vectors support unfinished communication). We let
3290 // the other vectors use the first 20 assigned numbers and start the
3291 // matrix-free communication.
3292 static constexpr unsigned int channel_shift = 20;
3293
3294
3295
3296 /**
3297 * Constructor. Takes MF data, flag for face access in DG and
3298 * number of components.
3299 */
3300 VectorDataExchange(
3301 const dealii::MatrixFree<dim, Number, VectorizedArrayType> &matrix_free,
3302 const typename dealii::MatrixFree<dim, Number, VectorizedArrayType>::
3303 DataAccessOnFaces vector_face_access,
3304 const unsigned int n_components)
3305 : matrix_free(matrix_free)
3306 , vector_face_access(
3307 matrix_free.get_task_info().face_partition_data.empty() ?
3308 dealii::MatrixFree<dim, Number, VectorizedArrayType>::
3309 DataAccessOnFaces::unspecified :
3310 vector_face_access)
3311 , ghosts_were_set(false)
3312 # ifdef DEAL_II_WITH_MPI
3313 , tmp_data(n_components)
3314 , requests(n_components)
3315 # endif
3316 {
3317 (void)n_components;
3318 if (this->vector_face_access !=
3319 dealii::MatrixFree<dim, Number, VectorizedArrayType>::
3320 DataAccessOnFaces::unspecified)
3321 for (unsigned int c = 0; c < matrix_free.n_components(); ++c)
3322 AssertDimension(
3323 matrix_free.get_dof_info(c).vector_exchanger_face_variants.size(),
3324 5);
3325 }
3326
3327
3328
3329 /**
3330 * Destructor.
3331 */
3332 ~VectorDataExchange() // NOLINT
3333 {
3334 # ifdef DEAL_II_WITH_MPI
3335 for (unsigned int i = 0; i < tmp_data.size(); ++i)
3336 if (tmp_data[i] != nullptr)
3337 matrix_free.release_scratch_data_non_threadsafe(tmp_data[i]);
3338 # endif
3339 }
3340
3341
3342
3343 /**
3344 * Go through all components in MF object and choose the one
3345 * whose partitioner is compatible with the Partitioner in this component.
3346 */
3347 template <typename VectorType>
3348 unsigned int
3349 find_vector_in_mf(const VectorType &vec,
3350 const bool check_global_compatibility = true) const
3351 {
3352 // case 1: vector was set up with MatrixFree::initialize_dof_vector()
3353 for (unsigned int c = 0; c < matrix_free.n_components(); ++c)
3354 if (vec.get_partitioner().get() ==
3355 matrix_free.get_dof_info(c).vector_partitioner.get())
3356 return c;
3357
3358 // case 2: user provided own partitioner (compatibility mode)
3359 for (unsigned int c = 0; c < matrix_free.n_components(); ++c)
3360 if (check_global_compatibility ?
3361 vec.get_partitioner()->is_globally_compatible(
3362 *matrix_free.get_dof_info(c).vector_partitioner) :
3363 vec.get_partitioner()->is_compatible(
3364 *matrix_free.get_dof_info(c).vector_partitioner))
3365 return c;
3366
3367 Assert(false,
3368 ExcNotImplemented("Could not find partitioner that fits vector"));
3369
3370 return numbers::invalid_unsigned_int;
3371 }
3372
3373
3374
3375 /**
3376 * Get partitioner for the given @p mf_component taking into
3377 * account vector_face_access set in constructor.
3378 */
3379 const internal::MatrixFreeFunctions::VectorDataExchange::Base &
3380 get_partitioner(const unsigned int mf_component) const
3381 {
3382 AssertDimension(matrix_free.get_dof_info(mf_component)
3383 .vector_exchanger_face_variants.size(),
3384 5);
3385 if (vector_face_access ==
3386 dealii::MatrixFree<dim, Number, VectorizedArrayType>::
3387 DataAccessOnFaces::none)
3388 return *matrix_free.get_dof_info(mf_component)
3389 .vector_exchanger_face_variants[0];
3390 else if (vector_face_access ==
3391 dealii::MatrixFree<dim, Number, VectorizedArrayType>::
3392 DataAccessOnFaces::values)
3393 return *matrix_free.get_dof_info(mf_component)
3394 .vector_exchanger_face_variants[1];
3395 else if (vector_face_access ==
3396 dealii::MatrixFree<dim, Number, VectorizedArrayType>::
3397 DataAccessOnFaces::gradients)
3398 return *matrix_free.get_dof_info(mf_component)
3399 .vector_exchanger_face_variants[2];
3400 else if (vector_face_access ==
3401 dealii::MatrixFree<dim, Number, VectorizedArrayType>::
3402 DataAccessOnFaces::values_all_faces)
3403 return *matrix_free.get_dof_info(mf_component)
3404 .vector_exchanger_face_variants[3];
3405 else if (vector_face_access ==
3406 dealii::MatrixFree<dim, Number, VectorizedArrayType>::
3407 DataAccessOnFaces::gradients_all_faces)
3408 return *matrix_free.get_dof_info(mf_component)
3409 .vector_exchanger_face_variants[4];
3410 else
3411 return *matrix_free.get_dof_info(mf_component).vector_exchanger.get();
3412 }
3413
3414
3415
3416 /**
3417 * Start update_ghost_value for serial vectors
3418 */
3419 template <typename VectorType,
3420 std::enable_if_t<is_not_parallel_vector<VectorType>, VectorType>
3421 * = nullptr>
3422 void
3423 update_ghost_values_start(const unsigned int /*component_in_block_vector*/,
3424 const VectorType & /*vec*/)
3425 {}
3426
3427
3428 /**
3429 * Start update_ghost_value for vectors that do not support
3430 * the split into _start() and finish() stages
3431 */
3432 template <typename VectorType,
3433 std::enable_if_t<!has_update_ghost_values_start<VectorType> &&
3434 !is_not_parallel_vector<VectorType>,
3435 VectorType> * = nullptr>
3436 void
3437 update_ghost_values_start(const unsigned int component_in_block_vector,
3438 const VectorType &vec)
3439 {
3440 (void)component_in_block_vector;
3441 const bool ghosts_set = vec.has_ghost_elements();
3442
3443 Assert(matrix_free.get_task_info().allow_ghosted_vectors_in_loops ||
3444 ghosts_set == false,
3445 ExcNotImplemented());
3446
3447 if (ghosts_set)
3448 {
3449 ghosts_were_set = true;
3450 return;
3451 }
3452
3453 vec.update_ghost_values();
3454 }
3455
3456
3457
3458 /**
3459 * Start update_ghost_value for vectors that _do_ support
3460 * the split into _start() and finish() stages, but don't support
3461 * exchange on a subset of DoFs
3462 */
3463 template <typename VectorType,
3464 std::enable_if_t<has_update_ghost_values_start<VectorType> &&
3465 !has_exchange_on_subset<VectorType>,
3466 VectorType> * = nullptr>
3467 void
3468 update_ghost_values_start(const unsigned int component_in_block_vector,
3469 const VectorType &vec)
3470 {
3471 (void)component_in_block_vector;
3472 const bool ghosts_set = vec.has_ghost_elements();
3473
3474 Assert(matrix_free.get_task_info().allow_ghosted_vectors_in_loops ||
3475 ghosts_set == false,
3476 ExcNotImplemented());
3477
3478 if (ghosts_set)
3479 {
3480 ghosts_were_set = true;
3481 return;
3482 }
3483
3484 vec.update_ghost_values_start(component_in_block_vector + channel_shift);
3485 }
3486
3487
3488
3489 /**
3490 * Finally, start update_ghost_value for vectors that _do_ support
3491 * the split into _start() and finish() stages and also support
3492 * exchange on a subset of DoFs,
3493 * i.e. LinearAlgebra::distributed::Vector
3494 */
3495 template <typename VectorType,
3496 std::enable_if_t<has_update_ghost_values_start<VectorType> &&
3497 has_exchange_on_subset<VectorType>,
3498 VectorType> * = nullptr>
3499 void
3500 update_ghost_values_start(const unsigned int component_in_block_vector,
3501 const VectorType &vec)
3502 {
3503 static_assert(std::is_same_v<Number, typename VectorType::value_type>,
3504 "Type mismatch between VectorType and VectorDataExchange");
3505 (void)component_in_block_vector;
3506 const bool ghosts_set = vec.has_ghost_elements();
3507
3508 Assert(matrix_free.get_task_info().allow_ghosted_vectors_in_loops ||
3509 ghosts_set == false,
3510 ExcNotImplemented());
3511
3512 if (ghosts_set)
3513 {
3514 ghosts_were_set = true;
3515 return;
3516 }
3517
3518 if (vec.size() != 0)
3519 {
3520 # ifdef DEAL_II_WITH_MPI
3521 const unsigned int mf_component = find_vector_in_mf(vec);
3522
3523 const auto &part = get_partitioner(mf_component);
3524
3525 if (part.n_ghost_indices() == 0 && part.n_import_indices() == 0 &&
3526 part.n_import_sm_procs() == 0)
3527 return;
3528
3529 tmp_data[component_in_block_vector] =
3530 matrix_free.acquire_scratch_data_non_threadsafe();
3531 tmp_data[component_in_block_vector]->resize_fast(
3532 part.n_import_indices());
3533 AssertDimension(requests.size(), tmp_data.size());
3534
3535 part.export_to_ghosted_array_start(
3536 component_in_block_vector * 2 + channel_shift,
3537 ArrayView<const Number>(vec.begin(), part.locally_owned_size()),
3538 vec.shared_vector_data(),
3539 ArrayView<Number>(const_cast<Number *>(vec.begin()) +
3540 part.locally_owned_size(),
3541 matrix_free.get_dof_info(mf_component)
3542 .vector_partitioner->n_ghost_indices()),
3543 ArrayView<Number>(tmp_data[component_in_block_vector]->begin(),
3544 part.n_import_indices()),
3545 this->requests[component_in_block_vector]);
3546 # endif
3547 }
3548 }
3549
3550
3551
3552 /**
3553 * Finish update_ghost_value for vectors that do not support
3554 * the split into _start() and finish() stages and serial vectors
3555 */
3556 template <typename VectorType,
3557 std::enable_if_t<!has_update_ghost_values_start<VectorType>,
3558 VectorType> * = nullptr>
3559 void
3560 update_ghost_values_finish(const unsigned int /*component_in_block_vector*/,
3561 const VectorType & /*vec*/)
3562 {}
3563
3564
3565
3566 /**
3567 * Finish update_ghost_value for vectors that _do_ support
3568 * the split into _start() and finish() stages, but don't support
3569 * exchange on a subset of DoFs
3570 */
3571 template <typename VectorType,
3572 std::enable_if_t<has_update_ghost_values_start<VectorType> &&
3573 !has_exchange_on_subset<VectorType>,
3574 VectorType> * = nullptr>
3575 void
3576 update_ghost_values_finish(const unsigned int component_in_block_vector,
3577 const VectorType &vec)
3578 {
3579 (void)component_in_block_vector;
3580
3581 if (ghosts_were_set)
3582 return;
3583
3584 vec.update_ghost_values_finish();
3585 }
3586
3587
3588
3589 /**
3590 * Finish update_ghost_value for vectors that _do_ support
3591 * the split into _start() and finish() stages and also support
3592 * exchange on a subset of DoFs,
3593 * i.e. LinearAlgebra::distributed::Vector
3594 */
3595 template <typename VectorType,
3596 std::enable_if_t<has_update_ghost_values_start<VectorType> &&
3597 has_exchange_on_subset<VectorType>,
3598 VectorType> * = nullptr>
3599 void
3600 update_ghost_values_finish(const unsigned int component_in_block_vector,
3601 const VectorType &vec)
3602 {
3603 static_assert(std::is_same_v<Number, typename VectorType::value_type>,
3604 "Type mismatch between VectorType and VectorDataExchange");
3605 (void)component_in_block_vector;
3606
3607 if (ghosts_were_set)
3608 return;
3609
3610 if (vec.size() != 0)
3611 {
3612 # ifdef DEAL_II_WITH_MPI
3613 AssertIndexRange(component_in_block_vector, tmp_data.size());
3614 AssertDimension(requests.size(), tmp_data.size());
3615
3616 const unsigned int mf_component = find_vector_in_mf(vec);
3617
3618 const auto &part = get_partitioner(mf_component);
3619
3620 if (part.n_ghost_indices() != 0 || part.n_import_indices() != 0 ||
3621 part.n_import_sm_procs() != 0)
3622 {
3623 part.export_to_ghosted_array_finish(
3624 ArrayView<const Number>(vec.begin(), part.locally_owned_size()),
3625 vec.shared_vector_data(),
3626 ArrayView<Number>(const_cast<Number *>(vec.begin()) +
3627 part.locally_owned_size(),
3628 matrix_free.get_dof_info(mf_component)
3629 .vector_partitioner->n_ghost_indices()),
3630 this->requests[component_in_block_vector]);
3631
3632 matrix_free.release_scratch_data_non_threadsafe(
3633 tmp_data[component_in_block_vector]);
3634 tmp_data[component_in_block_vector] = nullptr;
3635 }
3636 # endif
3637 }
3638 // let vector know that ghosts are being updated and we can read from
3639 // them
3640 vec.set_ghost_state(true);
3641 }
3642
3643
3644
3645 /**
3646 * Start compress for serial vectors
3647 */
3648 template <typename VectorType,
3649 std::enable_if_t<is_not_parallel_vector<VectorType>, VectorType>
3650 * = nullptr>
3651 void
3652 compress_start(const unsigned int /*component_in_block_vector*/,
3653 VectorType & /*vec*/)
3654 {}
3655
3656
3657
3658 /**
3659 * Start compress for vectors that do not support
3660 * the split into _start() and finish() stages
3661 */
3662 template <typename VectorType,
3663 std::enable_if_t<!has_compress_start<VectorType> &&
3664 !is_not_parallel_vector<VectorType>,
3665 VectorType> * = nullptr>
3666 void
3667 compress_start(const unsigned int component_in_block_vector,
3668 VectorType &vec)
3669 {
3670 (void)component_in_block_vector;
3671 Assert(vec.has_ghost_elements() == false, ExcNotImplemented());
3672 vec.compress(VectorOperation::add);
3673 }
3674
3675
3676
3677 /**
3678 * Start compress for vectors that _do_ support
3679 * the split into _start() and finish() stages, but don't support
3680 * exchange on a subset of DoFs
3681 */
3682 template <typename VectorType,
3683 std::enable_if_t<has_compress_start<VectorType> &&
3684 !has_exchange_on_subset<VectorType>,
3685 VectorType> * = nullptr>
3686 void
3687 compress_start(const unsigned int component_in_block_vector,
3688 VectorType &vec)
3689 {
3690 (void)component_in_block_vector;
3691 Assert(vec.has_ghost_elements() == false, ExcNotImplemented());
3692 vec.compress_start(component_in_block_vector + channel_shift);
3693 }
3694
3695
3696
3697 /**
3698 * Start compress for vectors that _do_ support
3699 * the split into _start() and finish() stages and also support
3700 * exchange on a subset of DoFs,
3701 * i.e. LinearAlgebra::distributed::Vector
3702 */
3703 template <typename VectorType,
3704 std::enable_if_t<has_compress_start<VectorType> &&
3705 has_exchange_on_subset<VectorType>,
3706 VectorType> * = nullptr>
3707 void
3708 compress_start(const unsigned int component_in_block_vector,
3709 VectorType &vec)
3710 {
3711 static_assert(std::is_same_v<Number, typename VectorType::value_type>,
3712 "Type mismatch between VectorType and VectorDataExchange");
3713 (void)component_in_block_vector;
3714 Assert(vec.has_ghost_elements() == false, ExcNotImplemented());
3715
3716 if (vec.size() != 0)
3717 {
3718 # ifdef DEAL_II_WITH_MPI
3719 const unsigned int mf_component = find_vector_in_mf(vec);
3720
3721 const auto &part = get_partitioner(mf_component);
3722
3723 if (part.n_ghost_indices() == 0 && part.n_import_indices() == 0 &&
3724 part.n_import_sm_procs() == 0)
3725 return;
3726
3727 tmp_data[component_in_block_vector] =
3728 matrix_free.acquire_scratch_data_non_threadsafe();
3729 tmp_data[component_in_block_vector]->resize_fast(
3730 part.n_import_indices());
3731 AssertDimension(requests.size(), tmp_data.size());
3732
3733 part.import_from_ghosted_array_start(
3734 VectorOperation::add,
3735 component_in_block_vector * 2 + channel_shift,
3736 ArrayView<Number>(vec.begin(), part.locally_owned_size()),
3737 vec.shared_vector_data(),
3738 ArrayView<Number>(vec.begin() + part.locally_owned_size(),
3739 matrix_free.get_dof_info(mf_component)
3740 .vector_partitioner->n_ghost_indices()),
3741 ArrayView<Number>(tmp_data[component_in_block_vector]->begin(),
3742 part.n_import_indices()),
3743 this->requests[component_in_block_vector]);
3744 # endif
3745 }
3746 }
3747
3748
3749
3750 /**
3751 * Finish compress for vectors that do not support
3752 * the split into _start() and finish() stages and serial vectors
3753 */
3754 template <
3755 typename VectorType,
3756 std::enable_if_t<!has_compress_start<VectorType>, VectorType> * = nullptr>
3757 void
3758 compress_finish(const unsigned int /*component_in_block_vector*/,
3759 VectorType & /*vec*/)
3760 {}
3761
3762
3763
3764 /**
3765 * Finish compress for vectors that _do_ support
3766 * the split into _start() and finish() stages, but don't support
3767 * exchange on a subset of DoFs
3768 */
3769 template <typename VectorType,
3770 std::enable_if_t<has_compress_start<VectorType> &&
3771 !has_exchange_on_subset<VectorType>,
3772 VectorType> * = nullptr>
3773 void
3774 compress_finish(const unsigned int component_in_block_vector,
3775 VectorType &vec)
3776 {
3777 (void)component_in_block_vector;
3778 vec.compress_finish(VectorOperation::add);
3779 }
3780
3781
3782
3783 /**
3784 * Start compress for vectors that _do_ support
3785 * the split into _start() and finish() stages and also support
3786 * exchange on a subset of DoFs,
3787 * i.e. LinearAlgebra::distributed::Vector
3788 */
3789 template <typename VectorType,
3790 std::enable_if_t<has_compress_start<VectorType> &&
3791 has_exchange_on_subset<VectorType>,
3792 VectorType> * = nullptr>
3793 void
3794 compress_finish(const unsigned int component_in_block_vector,
3795 VectorType &vec)
3796 {
3797 static_assert(std::is_same_v<Number, typename VectorType::value_type>,
3798 "Type mismatch between VectorType and VectorDataExchange");
3799 (void)component_in_block_vector;
3800 if (vec.size() != 0)
3801 {
3802 # ifdef DEAL_II_WITH_MPI
3803 AssertIndexRange(component_in_block_vector, tmp_data.size());
3804 AssertDimension(requests.size(), tmp_data.size());
3805
3806 const unsigned int mf_component = find_vector_in_mf(vec);
3807
3808 const auto &part = get_partitioner(mf_component);
3809
3810 if (part.n_ghost_indices() != 0 || part.n_import_indices() != 0 ||
3811 part.n_import_sm_procs() != 0)
3812 {
3813 part.import_from_ghosted_array_finish(
3814 VectorOperation::add,
3815 ArrayView<Number>(vec.begin(), part.locally_owned_size()),
3816 vec.shared_vector_data(),
3817 ArrayView<Number>(vec.begin() + part.locally_owned_size(),
3818 matrix_free.get_dof_info(mf_component)
3819 .vector_partitioner->n_ghost_indices()),
3820 ArrayView<const Number>(
3821 tmp_data[component_in_block_vector]->begin(),
3822 part.n_import_indices()),
3823 this->requests[component_in_block_vector]);
3824
3825 matrix_free.release_scratch_data_non_threadsafe(
3826 tmp_data[component_in_block_vector]);
3827 tmp_data[component_in_block_vector] = nullptr;
3828 }
3829
3830 if (Utilities::MPI::job_supports_mpi())
3831 {
3832 const int ierr =
3833 MPI_Barrier(matrix_free.get_task_info().communicator_sm);
3834 AssertThrowMPI(ierr);
3835 }
3836 # endif
3837 }
3838 }
3839
3840
3841
3842 /**
3843 * Reset all ghost values for serial vectors
3844 */
3845 template <typename VectorType,
3846 std::enable_if_t<is_not_parallel_vector<VectorType>, VectorType>
3847 * = nullptr>
3848 void
3849 reset_ghost_values(const VectorType & /*vec*/) const
3850 {}
3851
3852
3853
3854 /**
3855 * Reset all ghost values for vector that don't support
3856 * exchange on a subset of DoFs
3857 */
3858 template <typename VectorType,
3859 std::enable_if_t<!has_exchange_on_subset<VectorType> &&
3860 !is_not_parallel_vector<VectorType>,
3861 VectorType> * = nullptr>
3862 void
3863 reset_ghost_values(const VectorType &vec) const
3864 {
3865 if (ghosts_were_set == true)
3866 return;
3867
3868 vec.zero_out_ghost_values();
3869 }
3870
3871
3872
3873 /**
3874 * Reset all ghost values for vector that _do_ support
3875 * exchange on a subset of DoFs, i.e.
3876 * LinearAlgebra::distributed::Vector
3877 */
3878 template <typename VectorType,
3879 std::enable_if_t<has_exchange_on_subset<VectorType>, VectorType>
3880 * = nullptr>
3881 void
3882 reset_ghost_values(const VectorType &vec) const
3883 {
3884 static_assert(std::is_same_v<Number, typename VectorType::value_type>,
3885 "Type mismatch between VectorType and VectorDataExchange");
3886 if (ghosts_were_set == true)
3887 return;
3888
3889 if (vec.size() != 0)
3890 {
3891 # ifdef DEAL_II_WITH_MPI
3892 AssertDimension(requests.size(), tmp_data.size());
3893
3894 const unsigned int mf_component = find_vector_in_mf(vec);
3895
3896 const auto &part = get_partitioner(mf_component);
3897
3898 if (part.n_ghost_indices() > 0)
3899 {
3900 part.reset_ghost_values(
3901 ArrayView<Number>(const_cast<VectorType &>(vec).begin() +
3902 part.locally_owned_size(),
3903 matrix_free.get_dof_info(mf_component)
3904 .vector_partitioner->n_ghost_indices()));
3905 }
3906
3907 # endif
3908 }
3909 // let vector know that it's not ghosted anymore
3910 vec.set_ghost_state(false);
3911 }
3912
3913
3914
3915 /**
3916 * Zero out vector region for vector that _do_ support
3917 * exchange on a subset of DoFs <==> begin() + ind == local_element(ind),
3918 * i.e. LinearAlgebra::distributed::Vector
3919 */
3920 template <typename VectorType,
3921 std::enable_if_t<has_exchange_on_subset<VectorType>, VectorType>
3922 * = nullptr>
3923 void
3924 zero_vector_region(const unsigned int range_index, VectorType &vec) const
3925 {
3926 static_assert(std::is_same_v<Number, typename VectorType::value_type>,
3927 "Type mismatch between VectorType and VectorDataExchange");
3928 if (range_index == numbers::invalid_unsigned_int)
3929 vec = Number();
3930 else
3931 {
3932 const unsigned int mf_component = find_vector_in_mf(vec, false);
3933 const internal::MatrixFreeFunctions::DoFInfo &dof_info =
3934 matrix_free.get_dof_info(mf_component);
3935 Assert(dof_info.vector_zero_range_list_index.empty() == false,
3936 ExcNotInitialized());
3937
3938 Assert(vec.partitioners_are_compatible(*dof_info.vector_partitioner),
3939 ExcInternalError());
3940 AssertIndexRange(range_index,
3941 dof_info.vector_zero_range_list_index.size() - 1);
3942 for (unsigned int id =
3943 dof_info.vector_zero_range_list_index[range_index];
3944 id != dof_info.vector_zero_range_list_index[range_index + 1];
3945 ++id)
3946 std::memset(vec.begin() + dof_info.vector_zero_range_list[id].first,
3947 0,
3948 (dof_info.vector_zero_range_list[id].second -
3949 dof_info.vector_zero_range_list[id].first) *
3950 sizeof(Number));
3951 }
3952 }
3953
3954
3955
3956 /**
3957 * Zero out vector region for vector that do _not_ support exchange on a
3958 * subset of DoFs <==> begin() + ind == local_element(ind) but are still a
3959 * vector type
3960 */
3961 template <typename VectorType,
3962 std::enable_if_t<!has_exchange_on_subset<VectorType>, VectorType>
3963 * = nullptr,
3964 std::enable_if_t<has_assignment_operator<VectorType>, VectorType>
3965 * = nullptr>
3966 void
3967 zero_vector_region(const unsigned int range_index, VectorType &vec) const
3968 {
3969 if (range_index == numbers::invalid_unsigned_int || range_index == 0)
3970 {
3971 if constexpr (std::is_same_v<
3972 ArrayView<typename VectorType::value_type>,
3973 VectorType>)
3974 {
3975 for (unsigned int i = 0; i < vec.size(); ++i)
3976 vec[i] = typename VectorType::value_type();
3977 }
3978 else
3979 vec = typename VectorType::value_type();
3980 }
3981 }
3982
3983
3984
3985 /**
3986 * Zero out vector region for non-vector types, i.e., classes that do not
3987 * have operator=(const VectorType::value_type)
3988 */
3989 void
3990 zero_vector_region(const unsigned int, ...) const
3991 {
3992 Assert(false,
3993 ExcNotImplemented(
3994 "Zeroing is only implemented for vector types "
3995 "which provide operator=(const VectorType::value_type)"));
3996 }
3997
3998
3999
4000 const dealii::MatrixFree<dim, Number, VectorizedArrayType> &matrix_free;
4001 const typename dealii::MatrixFree<dim, Number, VectorizedArrayType>::
4002 DataAccessOnFaces vector_face_access;
4003 bool ghosts_were_set;
4004 # ifdef DEAL_II_WITH_MPI
4005 std::vector<AlignedVector<Number> *> tmp_data;
4006 std::vector<std::vector<MPI_Request>> requests;
4007 # endif
4008 }; // VectorDataExchange
4009
4010 template <typename VectorStruct>
4011 unsigned int
4012 n_components(const VectorStruct &vec);
4013
4014 template <typename VectorStruct>
4015 unsigned int
4016 n_components_block(const VectorStruct &vec, const std::bool_constant<true>)
4017 {
4018 unsigned int components = 0;
4019 for (unsigned int bl = 0; bl < vec.n_blocks(); ++bl)
4020 components += n_components(vec.block(bl));
4021 return components;
4022 }
4023
4024 template <typename VectorStruct>
4025 unsigned int
4026 n_components_block(const VectorStruct &, const std::bool_constant<false>)
4027 {
4028 return 1;
4029 }
4030
4031 template <typename VectorStruct>
4032 unsigned int
4033 n_components(const VectorStruct &vec)
4034 {
4035 return n_components_block(
4036 vec, std::bool_constant<IsBlockVector<VectorStruct>::value>());
4037 }
4038
4039 template <typename VectorStruct>
4040 inline unsigned int
4041 n_components(const std::vector<VectorStruct> &vec)
4042 {
4043 unsigned int components = 0;
4044 for (unsigned int comp = 0; comp < vec.size(); ++comp)
4045 components += n_components_block(
4046 vec[comp], std::bool_constant<IsBlockVector<VectorStruct>::value>());
4047 return components;
4048 }
4049
4050 template <typename VectorStruct>
4051 inline unsigned int
4052 n_components(const std::vector<VectorStruct *> &vec)
4053 {
4054 unsigned int components = 0;
4055 for (unsigned int comp = 0; comp < vec.size(); ++comp)
4056 components += n_components_block(
4057 *vec[comp], std::bool_constant<IsBlockVector<VectorStruct>::value>());
4058 return components;
4059 }
4060
4061
4062
4063 // A helper function to identify block vectors with many components where we
4064 // should not try to overlap computations and communication because there
4065 // would be too many outstanding communication requests.
4066
4067 // default value for vectors that do not have communication_block_size
4068 template <typename VectorStruct,
4069 std::enable_if_t<!has_communication_block_size<VectorStruct>,
4070 VectorStruct> * = nullptr>
4071 constexpr unsigned int
4072 get_communication_block_size(const VectorStruct &)
4073 {
4074 return numbers::invalid_unsigned_int;
4075 }
4076
4077
4078
4079 template <typename VectorStruct,
4080 std::enable_if_t<has_communication_block_size<VectorStruct>,
4081 VectorStruct> * = nullptr>
4082 constexpr unsigned int
4083 get_communication_block_size(const VectorStruct &)
4084 {
4085 return VectorStruct::communication_block_size;
4086 }
4087
4088
4089
4090 template <typename VectorType,
4091 std::enable_if_t<is_not_parallel_vector<VectorType>, VectorType> * =
4092 nullptr>
4093 bool
4094 has_ghost_elements(const VectorType &vec)
4095 {
4096 (void)vec;
4097 return false;
4098 }
4099
4100
4101
4102 template <typename VectorType,
4103 std::enable_if_t<!is_not_parallel_vector<VectorType>, VectorType>
4104 * = nullptr>
4105 bool
4106 has_ghost_elements(const VectorType &vec)
4107 {
4108 return vec.has_ghost_elements();
4109 }
4110
4111
4112
4113 // --------------------------------------------------------------------------
4114 // below we have wrappers to distinguish between block and non-block vectors.
4115 // --------------------------------------------------------------------------
4116
4117 //
4118 // update_ghost_values_start
4119 //
4120
4121 // update_ghost_values for block vectors
4122 template <int dim,
4123 typename VectorStruct,
4124 typename Number,
4125 typename VectorizedArrayType,
4126 std::enable_if_t<IsBlockVector<VectorStruct>::value, VectorStruct>
4127 * = nullptr>
4128 void
4129 update_ghost_values_start(
4130 const VectorStruct &vec,
4131 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger,
4132 const unsigned int channel = 0)
4133 {
4134 if (get_communication_block_size(vec) < vec.n_blocks())
4135 {
4136 const bool ghosts_set = vec.has_ghost_elements();
4137
4138 Assert(exchanger.matrix_free.get_task_info()
4139 .allow_ghosted_vectors_in_loops ||
4140 ghosts_set == false,
4141 ExcNotImplemented());
4142
4143 if (ghosts_set)
4144 {
4145 exchanger.ghosts_were_set = true;
4146 return;
4147 }
4148
4149 vec.update_ghost_values();
4150 }
4151 else
4152 {
4153 for (unsigned int i = 0; i < vec.n_blocks(); ++i)
4154 update_ghost_values_start(vec.block(i), exchanger, channel + i);
4155 }
4156 }
4157
4158
4159
4160 // update_ghost_values for non-block vectors
4161 template <int dim,
4162 typename VectorStruct,
4163 typename Number,
4164 typename VectorizedArrayType,
4165 std::enable_if_t<!IsBlockVector<VectorStruct>::value, VectorStruct>
4166 * = nullptr>
4167 void
4168 update_ghost_values_start(
4169 const VectorStruct &vec,
4170 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger,
4171 const unsigned int channel = 0)
4172 {
4173 exchanger.update_ghost_values_start(channel, vec);
4174 }
4175
4176
4177
4178 // update_ghost_values_start() for vector of vectors
4179 template <int dim,
4180 typename VectorStruct,
4181 typename Number,
4182 typename VectorizedArrayType>
4183 inline void
4184 update_ghost_values_start(
4185 const std::vector<VectorStruct> &vec,
4186 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger)
4187 {
4188 unsigned int component_index = 0;
4189 for (unsigned int comp = 0; comp < vec.size(); ++comp)
4190 {
4191 update_ghost_values_start(vec[comp], exchanger, component_index);
4192 component_index += n_components(vec[comp]);
4193 }
4194 }
4195
4196
4197
4198 // update_ghost_values_start() for vector of pointers to vectors
4199 template <int dim,
4200 typename VectorStruct,
4201 typename Number,
4202 typename VectorizedArrayType>
4203 inline void
4204 update_ghost_values_start(
4205 const std::vector<VectorStruct *> &vec,
4206 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger)
4207 {
4208 unsigned int component_index = 0;
4209 for (unsigned int comp = 0; comp < vec.size(); ++comp)
4210 {
4211 update_ghost_values_start(*vec[comp], exchanger, component_index);
4212 component_index += n_components(*vec[comp]);
4213 }
4214 }
4215
4216
4217
4218 //
4219 // update_ghost_values_finish
4220 //
4221
4222 // for block vectors
4223 template <int dim,
4224 typename VectorStruct,
4225 typename Number,
4226 typename VectorizedArrayType,
4227 std::enable_if_t<IsBlockVector<VectorStruct>::value, VectorStruct>
4228 * = nullptr>
4229 void
4230 update_ghost_values_finish(
4231 const VectorStruct &vec,
4232 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger,
4233 const unsigned int channel = 0)
4234 {
4235 if (get_communication_block_size(vec) < vec.n_blocks())
4236 {
4237 // do nothing, everything has already been completed in the _start()
4238 // call
4239 }
4240 else
4241 for (unsigned int i = 0; i < vec.n_blocks(); ++i)
4242 update_ghost_values_finish(vec.block(i), exchanger, channel + i);
4243 }
4244
4245
4246
4247 // for non-block vectors
4248 template <int dim,
4249 typename VectorStruct,
4250 typename Number,
4251 typename VectorizedArrayType,
4252 std::enable_if_t<!IsBlockVector<VectorStruct>::value, VectorStruct>
4253 * = nullptr>
4254 void
4255 update_ghost_values_finish(
4256 const VectorStruct &vec,
4257 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger,
4258 const unsigned int channel = 0)
4259 {
4260 exchanger.update_ghost_values_finish(channel, vec);
4261 }
4262
4263
4264
4265 // for vector of vectors
4266 template <int dim,
4267 typename VectorStruct,
4268 typename Number,
4269 typename VectorizedArrayType>
4270 inline void
4271 update_ghost_values_finish(
4272 const std::vector<VectorStruct> &vec,
4273 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger)
4274 {
4275 unsigned int component_index = 0;
4276 for (unsigned int comp = 0; comp < vec.size(); ++comp)
4277 {
4278 update_ghost_values_finish(vec[comp], exchanger, component_index);
4279 component_index += n_components(vec[comp]);
4280 }
4281 }
4282
4283
4284
4285 // for vector of pointers to vectors
4286 template <int dim,
4287 typename VectorStruct,
4288 typename Number,
4289 typename VectorizedArrayType>
4290 inline void
4291 update_ghost_values_finish(
4292 const std::vector<VectorStruct *> &vec,
4293 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger)
4294 {
4295 unsigned int component_index = 0;
4296 for (unsigned int comp = 0; comp < vec.size(); ++comp)
4297 {
4298 update_ghost_values_finish(*vec[comp], exchanger, component_index);
4299 component_index += n_components(*vec[comp]);
4300 }
4301 }
4302
4303
4304
4305 //
4306 // compress_start
4307 //
4308
4309 // for block vectors
4310 template <int dim,
4311 typename VectorStruct,
4312 typename Number,
4313 typename VectorizedArrayType,
4314 std::enable_if_t<IsBlockVector<VectorStruct>::value, VectorStruct>
4315 * = nullptr>
4316 inline void
4317 compress_start(
4318 VectorStruct &vec,
4319 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger,
4320 const unsigned int channel = 0)
4321 {
4322 if (get_communication_block_size(vec) < vec.n_blocks())
4323 vec.compress(VectorOperation::add);
4324 else
4325 for (unsigned int i = 0; i < vec.n_blocks(); ++i)
4326 compress_start(vec.block(i), exchanger, channel + i);
4327 }
4328
4329
4330
4331 // for non-block vectors
4332 template <int dim,
4333 typename VectorStruct,
4334 typename Number,
4335 typename VectorizedArrayType,
4336 std::enable_if_t<!IsBlockVector<VectorStruct>::value, VectorStruct>
4337 * = nullptr>
4338 inline void
4339 compress_start(
4340 VectorStruct &vec,
4341 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger,
4342 const unsigned int channel = 0)
4343 {
4344 exchanger.compress_start(channel, vec);
4345 }
4346
4347
4348
4349 // for std::vector of vectors
4350 template <int dim,
4351 typename VectorStruct,
4352 typename Number,
4353 typename VectorizedArrayType>
4354 inline void
4355 compress_start(
4356 std::vector<VectorStruct> &vec,
4357 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger)
4358 {
4359 unsigned int component_index = 0;
4360 for (unsigned int comp = 0; comp < vec.size(); ++comp)
4361 {
4362 compress_start(vec[comp], exchanger, component_index);
4363 component_index += n_components(vec[comp]);
4364 }
4365 }
4366
4367
4368
4369 // for std::vector of pointer to vectors
4370 template <int dim,
4371 typename VectorStruct,
4372 typename Number,
4373 typename VectorizedArrayType>
4374 inline void
4375 compress_start(
4376 std::vector<VectorStruct *> &vec,
4377 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger)
4378 {
4379 unsigned int component_index = 0;
4380 for (unsigned int comp = 0; comp < vec.size(); ++comp)
4381 {
4382 compress_start(*vec[comp], exchanger, component_index);
4383 component_index += n_components(*vec[comp]);
4384 }
4385 }
4386
4387
4388
4389 //
4390 // compress_finish
4391 //
4392
4393 // for block vectors
4394 template <int dim,
4395 typename VectorStruct,
4396 typename Number,
4397 typename VectorizedArrayType,
4398 std::enable_if_t<IsBlockVector<VectorStruct>::value, VectorStruct>
4399 * = nullptr>
4400 inline void
4401 compress_finish(
4402 VectorStruct &vec,
4403 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger,
4404 const unsigned int channel = 0)
4405 {
4406 if (get_communication_block_size(vec) < vec.n_blocks())
4407 {
4408 // do nothing, everything has already been completed in the _start()
4409 // call
4410 }
4411 else
4412 for (unsigned int i = 0; i < vec.n_blocks(); ++i)
4413 compress_finish(vec.block(i), exchanger, channel + i);
4414 }
4415
4416
4417
4418 // for non-block vectors
4419 template <int dim,
4420 typename VectorStruct,
4421 typename Number,
4422 typename VectorizedArrayType,
4423 std::enable_if_t<!IsBlockVector<VectorStruct>::value, VectorStruct>
4424 * = nullptr>
4425 inline void
4426 compress_finish(
4427 VectorStruct &vec,
4428 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger,
4429 const unsigned int channel = 0)
4430 {
4431 exchanger.compress_finish(channel, vec);
4432 }
4433
4434
4435
4436 // for std::vector of vectors
4437 template <int dim,
4438 typename VectorStruct,
4439 typename Number,
4440 typename VectorizedArrayType>
4441 inline void
4442 compress_finish(
4443 std::vector<VectorStruct> &vec,
4444 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger)
4445 {
4446 unsigned int component_index = 0;
4447 for (unsigned int comp = 0; comp < vec.size(); ++comp)
4448 {
4449 compress_finish(vec[comp], exchanger, component_index);
4450 component_index += n_components(vec[comp]);
4451 }
4452 }
4453
4454
4455
4456 // for std::vector of pointer to vectors
4457 template <int dim,
4458 typename VectorStruct,
4459 typename Number,
4460 typename VectorizedArrayType>
4461 inline void
4462 compress_finish(
4463 std::vector<VectorStruct *> &vec,
4464 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger)
4465 {
4466 unsigned int component_index = 0;
4467 for (unsigned int comp = 0; comp < vec.size(); ++comp)
4468 {
4469 compress_finish(*vec[comp], exchanger, component_index);
4470 component_index += n_components(*vec[comp]);
4471 }
4472 }
4473
4474
4475
4476 //
4477 // reset_ghost_values:
4478 //
4479 // if the input vector did not have ghosts imported, clear them here again
4480 // in order to avoid subsequent operations e.g. in linear solvers to work
4481 // with ghosts all the time
4482 //
4483
4484 // for block vectors
4485 template <int dim,
4486 typename VectorStruct,
4487 typename Number,
4488 typename VectorizedArrayType,
4489 std::enable_if_t<IsBlockVector<VectorStruct>::value, VectorStruct>
4490 * = nullptr>
4491 inline void
4492 reset_ghost_values(
4493 const VectorStruct &vec,
4494 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger)
4495 {
4496 // return immediately if there is nothing to do.
4497 if (exchanger.ghosts_were_set == true)
4498 return;
4499
4500 for (unsigned int i = 0; i < vec.n_blocks(); ++i)
4501 reset_ghost_values(vec.block(i), exchanger);
4502 }
4503
4504
4505
4506 // for non-block vectors
4507 template <int dim,
4508 typename VectorStruct,
4509 typename Number,
4510 typename VectorizedArrayType,
4511 std::enable_if_t<!IsBlockVector<VectorStruct>::value, VectorStruct>
4512 * = nullptr>
4513 inline void
4514 reset_ghost_values(
4515 const VectorStruct &vec,
4516 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger)
4517 {
4518 // return immediately if there is nothing to do.
4519 if (exchanger.ghosts_were_set == true)
4520 return;
4521
4522 exchanger.reset_ghost_values(vec);
4523 }
4524
4525
4526
4527 // for std::vector of vectors
4528 template <int dim,
4529 typename VectorStruct,
4530 typename Number,
4531 typename VectorizedArrayType>
4532 inline void
4533 reset_ghost_values(
4534 const std::vector<VectorStruct> &vec,
4535 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger)
4536 {
4537 // return immediately if there is nothing to do.
4538 if (exchanger.ghosts_were_set == true)
4539 return;
4540
4541 for (unsigned int comp = 0; comp < vec.size(); ++comp)
4542 reset_ghost_values(vec[comp], exchanger);
4543 }
4544
4545
4546
4547 // for std::vector of pointer to vectors
4548 template <int dim,
4549 typename VectorStruct,
4550 typename Number,
4551 typename VectorizedArrayType>
4552 inline void
4553 reset_ghost_values(
4554 const std::vector<VectorStruct *> &vec,
4555 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger)
4556 {
4557 // return immediately if there is nothing to do.
4558 if (exchanger.ghosts_were_set == true)
4559 return;
4560
4561 for (unsigned int comp = 0; comp < vec.size(); ++comp)
4562 reset_ghost_values(*vec[comp], exchanger);
4563 }
4564
4565
4566
4567 //
4568 // zero_vector_region
4569 //
4570
4571 // for block vectors
4572 template <int dim,
4573 typename VectorStruct,
4574 typename Number,
4575 typename VectorizedArrayType,
4576 std::enable_if_t<IsBlockVector<VectorStruct>::value, VectorStruct>
4577 * = nullptr>
4578 inline void
4579 zero_vector_region(
4580 const unsigned int range_index,
4581 VectorStruct &vec,
4582 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger)
4583 {
4584 for (unsigned int i = 0; i < vec.n_blocks(); ++i)
4585 exchanger.zero_vector_region(range_index, vec.block(i));
4586 }
4587
4588
4589
4590 // for non-block vectors
4591 template <int dim,
4592 typename VectorStruct,
4593 typename Number,
4594 typename VectorizedArrayType,
4595 std::enable_if_t<!IsBlockVector<VectorStruct>::value, VectorStruct>
4596 * = nullptr>
4597 inline void
4598 zero_vector_region(
4599 const unsigned int range_index,
4600 VectorStruct &vec,
4601 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger)
4602 {
4603 exchanger.zero_vector_region(range_index, vec);
4604 }
4605
4606
4607
4608 // for std::vector of vectors
4609 template <int dim,
4610 typename VectorStruct,
4611 typename Number,
4612 typename VectorizedArrayType>
4613 inline void
4614 zero_vector_region(
4615 const unsigned int range_index,
4616 std::vector<VectorStruct> &vec,
4617 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger)
4618 {
4619 for (unsigned int comp = 0; comp < vec.size(); ++comp)
4620 zero_vector_region(range_index, vec[comp], exchanger);
4621 }
4622
4623
4624
4625 // for std::vector of pointers to vectors
4626 template <int dim,
4627 typename VectorStruct,
4628 typename Number,
4629 typename VectorizedArrayType>
4630 inline void
4631 zero_vector_region(
4632 const unsigned int range_index,
4633 std::vector<VectorStruct *> &vec,
4634 VectorDataExchange<dim, Number, VectorizedArrayType> &exchanger)
4635 {
4636 for (unsigned int comp = 0; comp < vec.size(); ++comp)
4637 zero_vector_region(range_index, *vec[comp], exchanger);
4638 }
4639
4640
4641
4642 // Apply a unit matrix operation to constrained DoFs: Default cases where we
4643 // cannot detect a LinearAlgebra::distributed::Vector, we do not do
4644 // anything, else we apply the constraints as a unit operation
4645 template <typename VectorStruct1, typename VectorStruct2>
4646 inline void
4647 apply_operation_to_constrained_dofs(const std::vector<unsigned int> &,
4648 const VectorStruct1 &,
4649 VectorStruct2 &)
4650 {}
4651
4652 template <typename Number>
4653 inline void
4654 apply_operation_to_constrained_dofs(
4655 const std::vector<unsigned int> &constrained_dofs,
4656 const LinearAlgebra::distributed::Vector<Number> &src,
4657 LinearAlgebra::distributed::Vector<Number> &dst)
4658 {
4659 for (const unsigned int i : constrained_dofs)
4660 dst.local_element(i) = src.local_element(i);
4661 }
4662
4663
4664 namespace MatrixFreeFunctions
4665 {
4666 // struct to select between a const interface and a non-const interface
4667 // for MFWorker
4668 template <typename, typename, typename, typename, bool>
4669 struct InterfaceSelector
4670 {};
4671
4672 // Version for constant functions
4673 template <typename MF,
4674 typename InVector,
4675 typename OutVector,
4676 typename Container>
4677 struct InterfaceSelector<MF, InVector, OutVector, Container, true>
4678 {
4679 using function_type = void (Container::*)(
4680 const MF &,
4681 OutVector &,
4682 const InVector &,
4683 const std::pair<unsigned int, unsigned int> &) const;
4684 };
4685
4686 // Version for non-constant functions
4687 template <typename MF,
4688 typename InVector,
4689 typename OutVector,
4690 typename Container>
4691 struct InterfaceSelector<MF, InVector, OutVector, Container, false>
4692 {
4693 using function_type =
4694 void (Container::*)(const MF &,
4695 OutVector &,
4696 const InVector &,
4697 const std::pair<unsigned int, unsigned int> &);
4698 };
4699 } // namespace MatrixFreeFunctions
4700
4701
4702
4703 // A implementation class for the worker object that runs the various
4704 // operations we want to perform during the matrix-free loop
4705 template <typename MF,
4706 typename InVector,
4707 typename OutVector,
4708 typename Container,
4709 bool is_constant>
4710 class MFWorker : public MFWorkerInterface
4711 {
4712 public:
4713 // An alias to make the arguments further down more readable
4714 using function_type = typename MatrixFreeFunctions::
4715 InterfaceSelector<MF, InVector, OutVector, Container, is_constant>::
4716 function_type;
4717
4718 // constructor, binds all the arguments to this class
4719 MFWorker(const MF &matrix_free,
4720 const InVector &src,
4721 OutVector &dst,
4722 const bool zero_dst_vector_setting,
4723 const Container &container,
4724 function_type cell_function,
4725 function_type face_function,
4726 function_type boundary_function,
4727 const typename MF::DataAccessOnFaces src_vector_face_access =
4728 MF::DataAccessOnFaces::none,
4729 const typename MF::DataAccessOnFaces dst_vector_face_access =
4730 MF::DataAccessOnFaces::none,
4731 const std::function<void(const unsigned int, const unsigned int)>
4732 &operation_before_loop = {},
4733 const std::function<void(const unsigned int, const unsigned int)>
4734 &operation_after_loop = {},
4735 const unsigned int dof_handler_index_pre_post = 0)
4736 : matrix_free(matrix_free)
4737 , container(const_cast<Container &>(container))
4738 , cell_function(cell_function)
4739 , face_function(face_function)
4740 , boundary_function(boundary_function)
4741 , src(src)
4742 , dst(dst)
4743 , src_data_exchanger(matrix_free,
4744 src_vector_face_access,
4745 n_components(src))
4746 , dst_data_exchanger(matrix_free,
4747 dst_vector_face_access,
4748 n_components(dst))
4749 , src_and_dst_are_same(PointerComparison::equal(&src, &dst))
4750 , zero_dst_vector_setting(zero_dst_vector_setting &&
4751 !src_and_dst_are_same)
4752 , operation_before_loop(operation_before_loop)
4753 , operation_after_loop(operation_after_loop)
4754 , dof_handler_index_pre_post(dof_handler_index_pre_post)
4755 {
4756 Assert(!has_ghost_elements(dst),
4757 ExcMessage("The destination vector passed to the matrix-free "
4758 "loop is ghosted. This is not allowed."));
4759 }
4760
4761 // Runs the cell work. If no function is given, nothing is done
4762 virtual void
4763 cell(const std::pair<unsigned int, unsigned int> &cell_range) override
4764 {
4765 if (cell_function != nullptr && cell_range.second > cell_range.first)
4766 for (unsigned int i = 0; i < matrix_free.n_active_fe_indices(); ++i)
4767 {
4768 const auto cell_subrange =
4769 matrix_free.create_cell_subrange_hp_by_index(cell_range, i);
4770
4771 if (cell_subrange.second <= cell_subrange.first)
4772 continue;
4773
4774 (container.*
4775 cell_function)(matrix_free, this->dst, this->src, cell_subrange);
4776 }
4777 }
4778
4779 virtual void
4780 cell(const unsigned int range_index) override
4781 {
4782 process_range(cell_function,
4783 matrix_free.get_task_info().cell_partition_data_hp_ptr,
4784 matrix_free.get_task_info().cell_partition_data_hp,
4785 range_index);
4786 }
4787
4788 virtual void
4789 face(const unsigned int range_index) override
4790 {
4791 process_range(face_function,
4792 matrix_free.get_task_info().face_partition_data_hp_ptr,
4793 matrix_free.get_task_info().face_partition_data_hp,
4794 range_index);
4795 }
4796
4797 virtual void
4798 boundary(const unsigned int range_index) override
4799 {
4800 process_range(boundary_function,
4801 matrix_free.get_task_info().boundary_partition_data_hp_ptr,
4802 matrix_free.get_task_info().boundary_partition_data_hp,
4803 range_index);
4804 }
4805
4806 private:
4807 void
4808 process_range(const function_type &fu,
4809 const std::vector<unsigned int> &ptr,
4810 const std::vector<unsigned int> &data,
4811 const unsigned int range_index)
4812 {
4813 if (fu == nullptr)
4814 return;
4815
4816 AssertIndexRange(range_index + 1, ptr.size());
4817 for (unsigned int i = ptr[range_index]; i < ptr[range_index + 1]; ++i)
4818 {
4819 AssertIndexRange(2 * i + 1, data.size());
4820 (container.*fu)(matrix_free,
4821 this->dst,
4822 this->src,
4823 std::make_pair(data[2 * i], data[2 * i + 1]));
4824 }
4825 }
4826
4827 public:
4828 // Starts the communication for the update ghost values operation. We
4829 // cannot call this update if ghost and destination are the same because
4830 // that would introduce spurious entries in the destination (there is also
4831 // the problem that reading from a vector that we also write to is usually
4832 // not intended in case there is overlap, but this is up to the
4833 // application code to decide and we cannot catch this case here).
4834 virtual void
4835 vector_update_ghosts_start() override
4836 {
4837 if (!src_and_dst_are_same)
4838 internal::update_ghost_values_start(src, src_data_exchanger);
4839 }
4840
4841 // Finishes the communication for the update ghost values operation
4842 virtual void
4843 vector_update_ghosts_finish() override
4844 {
4845 if (!src_and_dst_are_same)
4846 internal::update_ghost_values_finish(src, src_data_exchanger);
4847 }
4848
4849 // Starts the communication for the vector compress operation
4850 virtual void
4851 vector_compress_start() override
4852 {
4853 internal::compress_start(dst, dst_data_exchanger);
4854 }
4855
4856 // Finishes the communication for the vector compress operation
4857 virtual void
4858 vector_compress_finish() override
4859 {
4860 internal::compress_finish(dst, dst_data_exchanger);
4861 if (!src_and_dst_are_same)
4862 internal::reset_ghost_values(src, src_data_exchanger);
4863 }
4864
4865 // Zeros the given input vector
4866 virtual void
4867 zero_dst_vector_range(const unsigned int range_index) override
4868 {
4869 if (zero_dst_vector_setting)
4870 internal::zero_vector_region(range_index, dst, dst_data_exchanger);
4871 }
4872
4873 virtual void
4874 cell_loop_pre_range(const unsigned int range_index) override
4875 {
4876 if (operation_before_loop)
4877 {
4878 const internal::MatrixFreeFunctions::DoFInfo &dof_info =
4879 matrix_free.get_dof_info(dof_handler_index_pre_post);
4880 if (range_index == numbers::invalid_unsigned_int)
4881 {
4882 // Case with threaded loop -> currently no overlap implemented
4883 dealii::parallel::apply_to_subranges(
4884 0U,
4885 dof_info.vector_partitioner->locally_owned_size(),
4886 operation_before_loop,
4887 internal::VectorImplementation::minimum_parallel_grain_size);
4888 }
4889 else
4890 {
4891 AssertIndexRange(range_index,
4892 dof_info.cell_loop_pre_list_index.size() - 1);
4893 for (unsigned int id =
4894 dof_info.cell_loop_pre_list_index[range_index];
4895 id != dof_info.cell_loop_pre_list_index[range_index + 1];
4896 ++id)
4897 operation_before_loop(dof_info.cell_loop_pre_list[id].first,
4898 dof_info.cell_loop_pre_list[id].second);
4899 }
4900 }
4901 }
4902
4903 virtual void
4904 cell_loop_post_range(const unsigned int range_index) override
4905 {
4906 if (operation_after_loop)
4907 {
4908 // Run unit matrix operation on constrained dofs if we are at the
4909 // last range
4910 const std::vector<unsigned int> &partition_row_index =
4911 matrix_free.get_task_info().partition_row_index;
4912 if (range_index ==
4913 partition_row_index[partition_row_index.size() - 2] - 1)
4914 apply_operation_to_constrained_dofs(
4915 matrix_free.get_constrained_dofs(dof_handler_index_pre_post),
4916 src,
4917 dst);
4918
4919 const internal::MatrixFreeFunctions::DoFInfo &dof_info =
4920 matrix_free.get_dof_info(dof_handler_index_pre_post);
4921 if (range_index == numbers::invalid_unsigned_int)
4922 {
4923 // Case with threaded loop -> currently no overlap implemented
4924 dealii::parallel::apply_to_subranges(
4925 0U,
4926 dof_info.vector_partitioner->locally_owned_size(),
4927 operation_after_loop,
4928 internal::VectorImplementation::minimum_parallel_grain_size);
4929 }
4930 else
4931 {
4932 AssertIndexRange(range_index,
4933 dof_info.cell_loop_post_list_index.size() - 1);
4934 for (unsigned int id =
4935 dof_info.cell_loop_post_list_index[range_index];
4936 id != dof_info.cell_loop_post_list_index[range_index + 1];
4937 ++id)
4938 operation_after_loop(dof_info.cell_loop_post_list[id].first,
4939 dof_info.cell_loop_post_list[id].second);
4940 }
4941 }
4942 }
4943
4944 private:
4945 const MF &matrix_free;
4946 Container &container;
4947 function_type cell_function;
4948 function_type face_function;
4949 function_type boundary_function;
4950
4951 const InVector &src;
4952 OutVector &dst;
4953 VectorDataExchange<MF::dimension,
4954 typename MF::value_type,
4955 typename MF::vectorized_value_type>
4956 src_data_exchanger;
4957 VectorDataExchange<MF::dimension,
4958 typename MF::value_type,
4959 typename MF::vectorized_value_type>
4960 dst_data_exchanger;
4961 const bool src_and_dst_are_same;
4962 const bool zero_dst_vector_setting;
4963 const std::function<void(const unsigned int, const unsigned int)>
4964 operation_before_loop;
4965 const std::function<void(const unsigned int, const unsigned int)>
4966 operation_after_loop;
4967 const unsigned int dof_handler_index_pre_post;
4968 };
4969
4970
4971
4972 /**
4973 * An internal class to convert three function pointers to the
4974 * scheme with virtual functions above.
4975 */
4976 template <class MF, typename InVector, typename OutVector>
4977 struct MFClassWrapper
4978 {
4979 using function_type =
4980 std::function<void(const MF &,
4981 OutVector &,
4982 const InVector &,
4983 const std::pair<unsigned int, unsigned int> &)>;
4984
4985 MFClassWrapper(const function_type cell,
4986 const function_type face,
4987 const function_type boundary)
4988 : cell(cell)
4989 , face(face)
4990 , boundary(boundary)
4991 {}
4992
4993 void
4994 cell_integrator(const MF &mf,
4995 OutVector &dst,
4996 const InVector &src,
4997 const std::pair<unsigned int, unsigned int> &range) const
4998 {
4999 if (cell)
5000 cell(mf, dst, src, range);
5001 }
5002
5003 void
5004 face_integrator(const MF &mf,
5005 OutVector &dst,
5006 const InVector &src,
5007 const std::pair<unsigned int, unsigned int> &range) const
5008 {
5009 if (face)
5010 face(mf, dst, src, range);
5011 }
5012
5013 void
5014 boundary_integrator(
5015 const MF &mf,
5016 OutVector &dst,
5017 const InVector &src,
5018 const std::pair<unsigned int, unsigned int> &range) const
5019 {
5020 if (boundary)
5021 boundary(mf, dst, src, range);
5022 }
5023
5024 const function_type cell;
5025 const function_type face;
5026 const function_type boundary;
5027 };
5028
5029 } // end of namespace internal
5030
5031
5032
5033 template <int dim, typename Number, typename VectorizedArrayType>
5034 template <typename OutVector, typename InVector>
5035 inline void
5036 MatrixFree<dim, Number, VectorizedArrayType>::cell_loop(
5037 const std::function<void(const MatrixFree<dim, Number, VectorizedArrayType> &,
5038 OutVector &,
5039 const InVector &,
5040 const std::pair<unsigned int, unsigned int> &)>
5041 &cell_operation,
5042 OutVector &dst,
5043 const InVector &src,
5044 const bool zero_dst_vector) const
5045 {
5046 using Wrapper =
5047 internal::MFClassWrapper<MatrixFree<dim, Number, VectorizedArrayType>,
5048 InVector,
5049 OutVector>;
5050 Wrapper wrap(cell_operation, nullptr, nullptr);
5051 internal::MFWorker<MatrixFree<dim, Number, VectorizedArrayType>,
5052 InVector,
5053 OutVector,
5054 Wrapper,
5055 true>
5056 worker(*this,
5057 src,
5058 dst,
5059 zero_dst_vector,
5060 wrap,
5061 &Wrapper::cell_integrator,
5062 &Wrapper::face_integrator,
5063 &Wrapper::boundary_integrator);
5064
5065 task_info.loop(worker);
5066 }
5067
5068
5069
5070 template <int dim, typename Number, typename VectorizedArrayType>
5071 template <typename OutVector, typename InVector>
5072 inline void
5073 MatrixFree<dim, Number, VectorizedArrayType>::cell_loop(
5074 const std::function<void(const MatrixFree<dim, Number, VectorizedArrayType> &,
5075 OutVector &,
5076 const InVector &,
5077 const std::pair<unsigned int, unsigned int> &)>
5078 &cell_operation,
5079 OutVector &dst,
5080 const InVector &src,
5081 const std::function<void(const unsigned int, const unsigned int)>
5082 &operation_before_loop,
5083 const std::function<void(const unsigned int, const unsigned int)>
5084 &operation_after_loop,
5085 const unsigned int dof_handler_index_pre_post) const
5086 {
5087 using Wrapper =
5088 internal::MFClassWrapper<MatrixFree<dim, Number, VectorizedArrayType>,
5089 InVector,
5090 OutVector>;
5091 Wrapper wrap(cell_operation, nullptr, nullptr);
5092 internal::MFWorker<MatrixFree<dim, Number, VectorizedArrayType>,
5093 InVector,
5094 OutVector,
5095 Wrapper,
5096 true>
5097 worker(*this,
5098 src,
5099 dst,
5100 false,
5101 wrap,
5102 &Wrapper::cell_integrator,
5103 &Wrapper::face_integrator,
5104 &Wrapper::boundary_integrator,
5105 DataAccessOnFaces::none,
5106 DataAccessOnFaces::none,
5107 operation_before_loop,
5108 operation_after_loop,
5109 dof_handler_index_pre_post);
5110
5111 task_info.loop(worker);
5112 }
5113
5114
5115
5116 template <int dim, typename Number, typename VectorizedArrayType>
5117 template <typename OutVector, typename InVector>
5118 inline void
5119 MatrixFree<dim, Number, VectorizedArrayType>::loop(
5120 const std::function<void(const MatrixFree<dim, Number, VectorizedArrayType> &,
5121 OutVector &,
5122 const InVector &,
5123 const std::pair<unsigned int, unsigned int> &)>
5124 &cell_operation,
5125 const std::function<void(const MatrixFree<dim, Number, VectorizedArrayType> &,
5126 OutVector &,
5127 const InVector &,
5128 const std::pair<unsigned int, unsigned int> &)>
5129 &inner_face_operation,
5130 const std::function<void(const MatrixFree<dim, Number, VectorizedArrayType> &,
5131 OutVector &,
5132 const InVector &,
5133 const std::pair<unsigned int, unsigned int> &)>
5134 &boundary_face_operation,
5135 OutVector &dst,
5136 const InVector &src,
5137 const bool zero_dst_vector,
5138 const DataAccessOnFaces dst_vector_face_access,
5139 const DataAccessOnFaces src_vector_face_access) const
5140 {
5141 using Wrapper =
5142 internal::MFClassWrapper<MatrixFree<dim, Number, VectorizedArrayType>,
5143 InVector,
5144 OutVector>;
5145 Wrapper wrap(cell_operation, inner_face_operation, boundary_face_operation);
5146 internal::MFWorker<MatrixFree<dim, Number, VectorizedArrayType>,
5147 InVector,
5148 OutVector,
5149 Wrapper,
5150 true>
5151 worker(*this,
5152 src,
5153 dst,
5154 zero_dst_vector,
5155 wrap,
5156 &Wrapper::cell_integrator,
5157 &Wrapper::face_integrator,
5158 &Wrapper::boundary_integrator,
5159 src_vector_face_access,
5160 dst_vector_face_access);
5161
5162 task_info.loop(worker);
5163 }
5164
5165
5166
5167 template <int dim, typename Number, typename VectorizedArrayType>
5168 template <typename CLASS, typename OutVector, typename InVector>
5169 inline void
5170 MatrixFree<dim, Number, VectorizedArrayType>::cell_loop(
5171 void (CLASS::*function_pointer)(
5172 const MatrixFree<dim, Number, VectorizedArrayType> &,
5173 OutVector &,
5174 const InVector &,
5175 const std::pair<unsigned int, unsigned int> &) const,
5176 const CLASS *owning_class,
5177 OutVector &dst,
5178 const InVector &src,
5179 const bool zero_dst_vector) const
5180 {
5181 internal::MFWorker<MatrixFree<dim, Number, VectorizedArrayType>,
5182 InVector,
5183 OutVector,
5184 CLASS,
5185 true>
5186 worker(*this,
5187 src,
5188 dst,
5189 zero_dst_vector,
5190 *owning_class,
5191 function_pointer,
5192 nullptr,
5193 nullptr);
5194 task_info.loop(worker);
5195 }
5196
5197
5198
5199 template <int dim, typename Number, typename VectorizedArrayType>
5200 template <typename CLASS, typename OutVector, typename InVector>
5201 inline void
5202 MatrixFree<dim, Number, VectorizedArrayType>::cell_loop(
5203 void (CLASS::*function_pointer)(
5204 const MatrixFree<dim, Number, VectorizedArrayType> &,
5205 OutVector &,
5206 const InVector &,
5207 const std::pair<unsigned int, unsigned int> &) const,
5208 const CLASS *owning_class,
5209 OutVector &dst,
5210 const InVector &src,
5211 const std::function<void(const unsigned int, const unsigned int)>
5212 &operation_before_loop,
5213 const std::function<void(const unsigned int, const unsigned int)>
5214 &operation_after_loop,
5215 const unsigned int dof_handler_index_pre_post) const
5216 {
5217 internal::MFWorker<MatrixFree<dim, Number, VectorizedArrayType>,
5218 InVector,
5219 OutVector,
5220 CLASS,
5221 true>
5222 worker(*this,
5223 src,
5224 dst,
5225 false,
5226 *owning_class,
5227 function_pointer,
5228 nullptr,
5229 nullptr,
5230 DataAccessOnFaces::none,
5231 DataAccessOnFaces::none,
5232 operation_before_loop,
5233 operation_after_loop,
5234 dof_handler_index_pre_post);
5235 task_info.loop(worker);
5236 }
5237
5238
5239
5240 template <int dim, typename Number, typename VectorizedArrayType>
5241 template <typename CLASS, typename OutVector, typename InVector>
5242 inline void
5243 MatrixFree<dim, Number, VectorizedArrayType>::loop(
5244 void (CLASS::*cell_operation)(
5245 const MatrixFree<dim, Number, VectorizedArrayType> &,
5246 OutVector &,
5247 const InVector &,
5248 const std::pair<unsigned int, unsigned int> &) const,
5249 void (CLASS::*inner_face_operation)(
5250 const MatrixFree<dim, Number, VectorizedArrayType> &,
5251 OutVector &,
5252 const InVector &,
5253 const std::pair<unsigned int, unsigned int> &) const,
5254 void (CLASS::*boundary_face_operation)(
5255 const MatrixFree<dim, Number, VectorizedArrayType> &,
5256 OutVector &,
5257 const InVector &,
5258 const std::pair<unsigned int, unsigned int> &) const,
5259 const CLASS *owning_class,
5260 OutVector &dst,
5261 const InVector &src,
5262 const bool zero_dst_vector,
5263 const DataAccessOnFaces dst_vector_face_access,
5264 const DataAccessOnFaces src_vector_face_access) const
5265 {
5266 internal::MFWorker<MatrixFree<dim, Number, VectorizedArrayType>,
5267 InVector,
5268 OutVector,
5269 CLASS,
5270 true>
5271 worker(*this,
5272 src,
5273 dst,
5274 zero_dst_vector,
5275 *owning_class,
5276 cell_operation,
5277 inner_face_operation,
5278 boundary_face_operation,
5279 src_vector_face_access,
5280 dst_vector_face_access);
5281 task_info.loop(worker);
5282 }
5283
5284
5285
5286 template <int dim, typename Number, typename VectorizedArrayType>
5287 template <typename CLASS, typename OutVector, typename InVector>
5288 inline void
5289 MatrixFree<dim, Number, VectorizedArrayType>::cell_loop(
5290 void (CLASS::*function_pointer)(
5291 const MatrixFree<dim, Number, VectorizedArrayType> &,
5292 OutVector &,
5293 const InVector &,
5294 const std::pair<unsigned int, unsigned int> &),
5295 CLASS *owning_class,
5296 OutVector &dst,
5297 const InVector &src,
5298 const bool zero_dst_vector) const
5299 {
5300 internal::MFWorker<MatrixFree<dim, Number, VectorizedArrayType>,
5301 InVector,
5302 OutVector,
5303 CLASS,
5304 false>
5305 worker(*this,
5306 src,
5307 dst,
5308 zero_dst_vector,
5309 *owning_class,
5310 function_pointer,
5311 nullptr,
5312 nullptr);
5313 task_info.loop(worker);
5314 }
5315
5316
5317
5318 template <int dim, typename Number, typename VectorizedArrayType>
5319 template <typename CLASS, typename OutVector, typename InVector>
5320 inline void
5321 MatrixFree<dim, Number, VectorizedArrayType>::cell_loop(
5322 void (CLASS::*function_pointer)(
5323 const MatrixFree<dim, Number, VectorizedArrayType> &,
5324 OutVector &,
5325 const InVector &,
5326 const std::pair<unsigned int, unsigned int> &),
5327 CLASS *owning_class,
5328 OutVector &dst,
5329 const InVector &src,
5330 const std::function<void(const unsigned int, const unsigned int)>
5331 &operation_before_loop,
5332 const std::function<void(const unsigned int, const unsigned int)>
5333 &operation_after_loop,
5334 const unsigned int dof_handler_index_pre_post) const
5335 {
5336 internal::MFWorker<MatrixFree<dim, Number, VectorizedArrayType>,
5337 InVector,
5338 OutVector,
5339 CLASS,
5340 false>
5341 worker(*this,
5342 src,
5343 dst,
5344 false,
5345 *owning_class,
5346 function_pointer,
5347 nullptr,
5348 nullptr,
5349 DataAccessOnFaces::none,
5350 DataAccessOnFaces::none,
5351 operation_before_loop,
5352 operation_after_loop,
5353 dof_handler_index_pre_post);
5354 task_info.loop(worker);
5355 }
5356
5357
5358
5359 template <int dim, typename Number, typename VectorizedArrayType>
5360 template <typename CLASS, typename OutVector, typename InVector>
5361 inline void
5362 MatrixFree<dim, Number, VectorizedArrayType>::loop(
5363 void (CLASS::*cell_operation)(
5364 const MatrixFree<dim, Number, VectorizedArrayType> &,
5365 OutVector &,
5366 const InVector &,
5367 const std::pair<unsigned int, unsigned int> &),
5368 void (CLASS::*inner_face_operation)(
5369 const MatrixFree<dim, Number, VectorizedArrayType> &,
5370 OutVector &,
5371 const InVector &,
5372 const std::pair<unsigned int, unsigned int> &),
5373 void (CLASS::*boundary_face_operation)(
5374 const MatrixFree<dim, Number, VectorizedArrayType> &,
5375 OutVector &,
5376 const InVector &,
5377 const std::pair<unsigned int, unsigned int> &),
5378 CLASS *owning_class,
5379 OutVector &dst,
5380 const InVector &src,
5381 const bool zero_dst_vector,
5382 const DataAccessOnFaces dst_vector_face_access,
5383 const DataAccessOnFaces src_vector_face_access) const
5384 {
5385 internal::MFWorker<MatrixFree<dim, Number, VectorizedArrayType>,
5386 InVector,
5387 OutVector,
5388 CLASS,
5389 false>
5390 worker(*this,
5391 src,
5392 dst,
5393 zero_dst_vector,
5394 *owning_class,
5395 cell_operation,
5396 inner_face_operation,
5397 boundary_face_operation,
5398 src_vector_face_access,
5399 dst_vector_face_access);
5400 task_info.loop(worker);
5401 }
5402
5403
5404
5405 template <int dim, typename Number, typename VectorizedArrayType>
5406 template <typename OutVector, typename InVector>
5407 inline void
5408 MatrixFree<dim, Number, VectorizedArrayType>::loop(
5409 const std::function<void(const MatrixFree<dim, Number, VectorizedArrayType> &,
5410 OutVector &,
5411 const InVector &,
5412 const std::pair<unsigned int, unsigned int> &)>
5413 &cell_operation,
5414 const std::function<void(const MatrixFree<dim, Number, VectorizedArrayType> &,
5415 OutVector &,
5416 const InVector &,
5417 const std::pair<unsigned int, unsigned int> &)>
5418 &inner_face_operation,
5419 const std::function<void(const MatrixFree<dim, Number, VectorizedArrayType> &,
5420 OutVector &,
5421 const InVector &,
5422 const std::pair<unsigned int, unsigned int> &)>
5423 &boundary_face_operation,
5424 OutVector &dst,
5425 const InVector &src,
5426 const std::function<void(const unsigned int, const unsigned int)>
5427 &operation_before_loop,
5428 const std::function<void(const unsigned int, const unsigned int)>
5429 &operation_after_loop,
5430 const unsigned int dof_handler_index_pre_post,
5431 const DataAccessOnFaces dst_vector_face_access,
5432 const DataAccessOnFaces src_vector_face_access) const
5433 {
5434 using Wrapper =
5435 internal::MFClassWrapper<MatrixFree<dim, Number, VectorizedArrayType>,
5436 InVector,
5437 OutVector>;
5438 Wrapper wrap(cell_operation, inner_face_operation, boundary_face_operation);
5439 internal::MFWorker<MatrixFree<dim, Number, VectorizedArrayType>,
5440 InVector,
5441 OutVector,
5442 Wrapper,
5443 true>
5444 worker(*this,
5445 src,
5446 dst,
5447 false,
5448 wrap,
5449 &Wrapper::cell_integrator,
5450 &Wrapper::face_integrator,
5451 &Wrapper::boundary_integrator,
5452 src_vector_face_access,
5453 dst_vector_face_access,
5454 operation_before_loop,
5455 operation_after_loop,
5456 dof_handler_index_pre_post);
5457
5458 task_info.loop(worker);
5459 }
5460
5461
5462
5463 template <int dim, typename Number, typename VectorizedArrayType>
5464 template <typename CLASS, typename OutVector, typename InVector>
5465 inline void
5466 MatrixFree<dim, Number, VectorizedArrayType>::loop(
5467 void (CLASS::*cell_operation)(const MatrixFree &,
5468 OutVector &,
5469 const InVector &,
5470 const std::pair<unsigned int, unsigned int> &)
5471 const,
5472 void (CLASS::*inner_face_operation)(
5473 const MatrixFree &,
5474 OutVector &,
5475 const InVector &,
5476 const std::pair<unsigned int, unsigned int> &) const,
5477 void (CLASS::*boundary_face_operation)(
5478 const MatrixFree &,
5479 OutVector &,
5480 const InVector &,
5481 const std::pair<unsigned int, unsigned int> &) const,
5482 const CLASS *owning_class,
5483 OutVector &dst,
5484 const InVector &src,
5485 const std::function<void(const unsigned int, const unsigned int)>
5486 &operation_before_loop,
5487 const std::function<void(const unsigned int, const unsigned int)>
5488 &operation_after_loop,
5489 const unsigned int dof_handler_index_pre_post,
5490 const DataAccessOnFaces dst_vector_face_access,
5491 const DataAccessOnFaces src_vector_face_access) const
5492 {
5493 internal::MFWorker<MatrixFree<dim, Number, VectorizedArrayType>,
5494 InVector,
5495 OutVector,
5496 CLASS,
5497 true>
5498 worker(*this,
5499 src,
5500 dst,
5501 false,
5502 *owning_class,
5503 cell_operation,
5504 inner_face_operation,
5505 boundary_face_operation,
5506 src_vector_face_access,
5507 dst_vector_face_access,
5508 operation_before_loop,
5509 operation_after_loop,
5510 dof_handler_index_pre_post);
5511 task_info.loop(worker);
5512 }
5513
5514
5515
5516 template <int dim, typename Number, typename VectorizedArrayType>
5517 template <typename CLASS, typename OutVector, typename InVector>
5518 inline void
5519 MatrixFree<dim, Number, VectorizedArrayType>::loop(
5520 void (CLASS::*cell_operation)(const MatrixFree &,
5521 OutVector &,
5522 const InVector &,
5523 const std::pair<unsigned int, unsigned int> &),
5524 void (CLASS::*inner_face_operation)(
5525 const MatrixFree &,
5526 OutVector &,
5527 const InVector &,
5528 const std::pair<unsigned int, unsigned int> &),
5529 void (CLASS::*boundary_face_operation)(
5530 const MatrixFree &,
5531 OutVector &,
5532 const InVector &,
5533 const std::pair<unsigned int, unsigned int> &),
5534 const CLASS *owning_class,
5535 OutVector &dst,
5536 const InVector &src,
5537 const std::function<void(const unsigned int, const unsigned int)>
5538 &operation_before_loop,
5539 const std::function<void(const unsigned int, const unsigned int)>
5540 &operation_after_loop,
5541 const unsigned int dof_handler_index_pre_post,
5542 const DataAccessOnFaces dst_vector_face_access,
5543 const DataAccessOnFaces src_vector_face_access) const
5544 {
5545 internal::MFWorker<MatrixFree<dim, Number, VectorizedArrayType>,
5546 InVector,
5547 OutVector,
5548 CLASS,
5549 false>
5550 worker(*this,
5551 src,
5552 dst,
5553 false,
5554 *owning_class,
5555 cell_operation,
5556 inner_face_operation,
5557 boundary_face_operation,
5558 src_vector_face_access,
5559 dst_vector_face_access,
5560 operation_before_loop,
5561 operation_after_loop,
5562 dof_handler_index_pre_post);
5563 task_info.loop(worker);
5564 }
5565
5566
5567
5568 template <int dim, typename Number, typename VectorizedArrayType>
5569 template <typename CLASS, typename OutVector, typename InVector>
5570 inline void
5571 MatrixFree<dim, Number, VectorizedArrayType>::loop_cell_centric(
5572 void (CLASS::*function_pointer)(
5573 const MatrixFree<dim, Number, VectorizedArrayType> &,
5574 OutVector &,
5575 const InVector &,
5576 const std::pair<unsigned int, unsigned int> &) const,
5577 const CLASS *owning_class,
5578 OutVector &dst,
5579 const InVector &src,
5580 const bool zero_dst_vector,
5581 const DataAccessOnFaces src_vector_face_access) const
5582 {
5583 auto src_vector_face_access_temp = src_vector_face_access;
5584 if (DataAccessOnFaces::gradients == src_vector_face_access_temp)
5585 src_vector_face_access_temp = DataAccessOnFaces::gradients_all_faces;
5586 else if (DataAccessOnFaces::values == src_vector_face_access_temp)
5587 src_vector_face_access_temp = DataAccessOnFaces::values_all_faces;
5588
5589 internal::MFWorker<MatrixFree<dim, Number, VectorizedArrayType>,
5590 InVector,
5591 OutVector,
5592 CLASS,
5593 true>
5594 worker(*this,
5595 src,
5596 dst,
5597 zero_dst_vector,
5598 *owning_class,
5599 function_pointer,
5600 nullptr,
5601 nullptr,
5602 src_vector_face_access_temp,
5603 DataAccessOnFaces::none);
5604 task_info.loop(worker);
5605 }
5606
5607
5608
5609 template <int dim, typename Number, typename VectorizedArrayType>
5610 template <typename CLASS, typename OutVector, typename InVector>
5611 inline void
5612 MatrixFree<dim, Number, VectorizedArrayType>::loop_cell_centric(
5613 void (CLASS::*function_pointer)(
5614 const MatrixFree<dim, Number, VectorizedArrayType> &,
5615 OutVector &,
5616 const InVector &,
5617 const std::pair<unsigned int, unsigned int> &),
5618 CLASS *owning_class,
5619 OutVector &dst,
5620 const InVector &src,
5621 const bool zero_dst_vector,
5622 const DataAccessOnFaces src_vector_face_access) const
5623 {
5624 auto src_vector_face_access_temp = src_vector_face_access;
5625 if (DataAccessOnFaces::gradients == src_vector_face_access_temp)
5626 src_vector_face_access_temp = DataAccessOnFaces::gradients_all_faces;
5627 else if (DataAccessOnFaces::values == src_vector_face_access_temp)
5628 src_vector_face_access_temp = DataAccessOnFaces::values_all_faces;
5629
5630 internal::MFWorker<MatrixFree<dim, Number, VectorizedArrayType>,
5631 InVector,
5632 OutVector,
5633 CLASS,
5634 false>
5635 worker(*this,
5636 src,
5637 dst,
5638 zero_dst_vector,
5639 *owning_class,
5640 function_pointer,
5641 nullptr,
5642 nullptr,
5643 src_vector_face_access_temp,
5644 DataAccessOnFaces::none);
5645 task_info.loop(worker);
5646 }
5647
5648
5649
5650 template <int dim, typename Number, typename VectorizedArrayType>
5651 template <typename OutVector, typename InVector>
5652 inline void
5653 MatrixFree<dim, Number, VectorizedArrayType>::loop_cell_centric(
5654 const std::function<void(const MatrixFree<dim, Number, VectorizedArrayType> &,
5655 OutVector &,
5656 const InVector &,
5657 const std::pair<unsigned int, unsigned int> &)>
5658 &cell_operation,
5659 OutVector &dst,
5660 const InVector &src,
5661 const bool zero_dst_vector,
5662 const DataAccessOnFaces src_vector_face_access) const
5663 {
5664 auto src_vector_face_access_temp = src_vector_face_access;
5665 if (DataAccessOnFaces::gradients == src_vector_face_access_temp)
5666 src_vector_face_access_temp = DataAccessOnFaces::gradients_all_faces;
5667 else if (DataAccessOnFaces::values == src_vector_face_access_temp)
5668 src_vector_face_access_temp = DataAccessOnFaces::values_all_faces;
5669
5670 using Wrapper =
5671 internal::MFClassWrapper<MatrixFree<dim, Number, VectorizedArrayType>,
5672 InVector,
5673 OutVector>;
5674 Wrapper wrap(cell_operation, nullptr, nullptr);
5675
5676 internal::MFWorker<MatrixFree<dim, Number, VectorizedArrayType>,
5677 InVector,
5678 OutVector,
5679 Wrapper,
5680 true>
5681 worker(*this,
5682 src,
5683 dst,
5684 zero_dst_vector,
5685 wrap,
5686 &Wrapper::cell_integrator,
5687 &Wrapper::face_integrator,
5688 &Wrapper::boundary_integrator,
5689 src_vector_face_access_temp,
5690 DataAccessOnFaces::none);
5691 task_info.loop(worker);
5692 }
5693
5694
5695 #endif // ifndef DOXYGEN
5696
5697
5698
5699 DEAL_II_NAMESPACE_CLOSE
5700
5701 #endif