Debian Codegraph @

include/deal.II/matrix_free/portable_matrix_free.h

raw
1 // ------------------------------------------------------------------------
2 //
3 // SPDX-License-Identifier: LGPL-2.1-or-later
4 // Copyright (C) 2017 - 2025 by the deal.II authors
5 //
6 // This file is part of the deal.II library.
7 //
8 // Part of the source code is dual licensed under Apache-2.0 WITH
9 // LLVM-exception OR LGPL-2.1-or-later. Detailed license information
10 // governing the source code and code contributions can be found in
11 // LICENSE.md and CONTRIBUTING.md at the top level directory of deal.II.
12 //
13 // ------------------------------------------------------------------------
14
15
16 #ifndef dealii_portable_matrix_free_h
17 #define dealii_portable_matrix_free_h
18
19 #include <deal.II/base/config.h>
20
21 #include <deal.II/base/exceptions.h>
22 #include <deal.II/base/memory_space.h>
23 #include <deal.II/base/mpi_stub.h>
24 #include <deal.II/base/partitioner.h>
25 #include <deal.II/base/quadrature.h>
26 #include <deal.II/base/tensor.h>
27 #include <deal.II/base/utilities.h>
28
29 #include <deal.II/dofs/dof_handler.h>
30
31 #include <deal.II/fe/fe_update_flags.h>
32 #include <deal.II/fe/mapping.h>
33
34 #include <deal.II/grid/filtered_iterator.h>
35
36 #include <deal.II/lac/affine_constraints.h>
37 #include <deal.II/lac/la_parallel_vector.h>
38
39 #include <deal.II/matrix_free/shape_info.h>
40
41 #include <Kokkos_Array.hpp>
42 #include <Kokkos_Core.hpp>
43
44
45
46 DEAL_II_NAMESPACE_OPEN
47
48 // Forward declaration
49 namespace internal
50 {
51 namespace MatrixFreeFunctions
52 {
53 enum class ConstraintKinds : std::uint16_t;
54 }
55 } // namespace internal
56
57 namespace Portable
58 {
59 // forward declaration
60 #ifndef DOXYGEN
61 namespace internal
62 {
63 template <int dim, typename Number>
64 class ReinitHelper;
65 }
66 template <int dim, typename Number>
67 struct SharedData;
68 #endif
69
70 /**
71 * Maximum number of DofHandler supported at the same time in
72 * Portable::MatrixFree computations. This limit also applies for the number
73 * of blocks in a BlockVector and the number of FEEvaluation objects that can
74 * be active in a single cell_loop().
75 */
76 inline constexpr unsigned int n_max_dof_handlers = 5;
77
78 /**
79 * Type for source and destination vectors in device functions like
80 * MatrixFree::cell_loop().
81 *
82 * This is a type alias to a Kokkos::View to a chunk of memory, typically
83 * pointing to the local elements in the LinearAlgebra::distributed::Vector.
84 */
85 template <typename Number>
86 using DeviceVector =
87 Kokkos::View<Number *, MemorySpace::Default::kokkos_space>;
88
89 /**
90 * A block vector used for source and destination vectors in device functions
91 * like MatrixFree::cell_loop().
92 *
93 * The maximum number of block is limited by the constant @p n_max_dof_handlers.
94 */
95 template <typename Number>
96 class DeviceBlockVector
97 {
98 public:
99 /**
100 * Constructor.
101 */
102 DeviceBlockVector()
103 : n_blocks(0)
104 {}
105
106 /**
107 * Constructor.
108 */
109 DeviceBlockVector(const DeviceBlockVector &other) = default;
110
111 /**
112 * Constructor from a DeviceVector. Creates a DeviceBlockVector
113 * with a single block.
114 */
115 explicit DeviceBlockVector(const DeviceVector<Number> &src)
116 : n_blocks(1)
117 , blocks{src}
118 {}
119
120 /**
121 * Constructor from a LinearAlgebra::distributed::BlockVector. Creates
122 * a DeviceVector from each block and stores it.
123 */
124 template <typename MemorySpace>
125 DeviceBlockVector(
126 const LinearAlgebra::distributed::BlockVector<Number, MemorySpace> &src)
127 : n_blocks(src.n_blocks())
128 {
129 Assert(src.n_blocks() <= n_max_dof_handlers,
130 ExcMessage("Portable::MatrixFree is configured with " +
131 Utilities::to_string(n_max_dof_handlers) +
132 " but you are passing a BlockVector with " +
133 Utilities::to_string(src.n_blocks()) + " blocks."));
134
135 for (int b = 0; b < src.n_blocks(); ++b)
136 blocks[b] = DeviceVector<Number>(src.block(b).get_values(),
137 src.block(b).locally_owned_size());
138 }
139
140 /**
141 * Access block @p index.
142 */
143 DEAL_II_HOST_DEVICE
144 DeviceVector<Number> &
145 block(unsigned int index)
146 {
147 AssertIndexRange(index, n_blocks);
148 return blocks[index];
149 }
150
151 /**
152 * Access block @p index.
153 */
154 DEAL_II_HOST_DEVICE const DeviceVector<Number> &
155 block(unsigned int index) const
156 {
157 AssertIndexRange(index, n_blocks);
158 return blocks[index];
159 }
160
161 private:
162 /**
163 * The number of components / blocks
164 */
165 unsigned int n_blocks;
166
167 /**
168 * Storage for the blocks
169 */
170 Kokkos::Array<DeviceVector<Number>, n_max_dof_handlers> blocks;
171 };
172
173 /**
174 * This class collects all the data that is stored for the matrix free
175 * implementation. The storage scheme is tailored towards several loops
176 * performed with the same data, i.e., typically doing many matrix-vector
177 * products or residual computations on the same mesh.
178 *
179 * This class does not implement any operations involving finite element basis
180 * functions, i.e., regarding the operation performed on the cells. For these
181 * operations, the class FEEvaluation is designed to use the data collected in
182 * this class.
183 *
184 * This class implements a loop over all cells (cell_loop()). This loop is
185 * scheduled in such a way that cells that share degrees of freedom
186 * are not worked on simultaneously, which implies that it is possible to
187 * write to vectors in parallel without having to explicitly synchronize
188 * access to these vectors and matrices. This class does not implement any
189 * shape values, all it does is to cache the respective data. To implement
190 * finite element operations, use the class Portable::FEEvaluation.
191 *
192 * This class traverse the cells in a different order than the usual
193 * Triangulation class in deal.II.
194 *
195 * @note Only float and double are supported.
196 */
197 template <int dim, typename Number = double>
198 class MatrixFree : public EnableObserverPointer
199 {
200 public:
201 using jacobian_type = Tensor<2, dim, Tensor<1, dim, Number>>;
202 using point_type = Point<dim, Number>;
203 using CellFilter =
204 FilteredIterator<typename DoFHandler<dim>::active_cell_iterator>;
205
206 /**
207 * Standardized data struct to pipe additional data to MatrixFree.
208 */
209 struct AdditionalData
210 {
211 /**
212 * Constructor.
213 */
214 AdditionalData(const UpdateFlags mapping_update_flags =
215 update_gradients | update_JxW_values |
216 update_quadrature_points,
217 const bool use_coloring = false,
218 const bool overlap_communication_computation = false)
219 : mapping_update_flags(mapping_update_flags)
220 , use_coloring(use_coloring)
221 , overlap_communication_computation(overlap_communication_computation)
222 {
223 #ifndef DEAL_II_MPI_WITH_DEVICE_SUPPORT
224 AssertThrow(
225 overlap_communication_computation == false,
226 ExcMessage(
227 "Overlapping communication and computation requires device-aware MPI."));
228 #endif
229 if (overlap_communication_computation == true)
230 AssertThrow(
231 use_coloring == false || overlap_communication_computation == false,
232 ExcMessage(
233 "Overlapping communication and coloring are incompatible options. Only one of them can be enabled."));
234 }
235 /**
236 * This flag is used to determine which quantities should be cached. This
237 * class can cache data needed for gradient computations (inverse
238 * Jacobians), Jacobian determinants (JxW), quadrature points as well as
239 * data for Hessians (derivative of Jacobians). By default, only data for
240 * gradients and Jacobian determinants times quadrature weights, JxW, are
241 * cached. If quadrature points of second derivatives are needed, they
242 * must be specified by this field.
243 */
244 UpdateFlags mapping_update_flags;
245
246 /**
247 * If true, use graph coloring. Otherwise, use atomic operations. Graph
248 * coloring ensures bitwise reproducibility but is slower on Pascal and
249 * newer architectures.
250 */
251 bool use_coloring;
252
253 /**
254 * Overlap MPI communications with computation. This requires device-aware
255 * MPI and use_coloring must be false.
256 */
257 bool overlap_communication_computation;
258 };
259
260 /**
261 * Structure which is passed to the kernel. It is used to pass all the
262 * necessary information from the CPU to the GPU and is precomputed
263 * on the CPU. This data is read-only once we run on the GPU.
264 */
265 struct PrecomputedData
266 {
267 /**
268 * Kokkos::View of the quadrature points.
269 */
270 Kokkos::View<point_type **, MemorySpace::Default::kokkos_space> q_points;
271
272 /**
273 * Map the position in the local vector to the position in the global
274 * vector.
275 */
276 Kokkos::View<types::global_dof_index **,
277 MemorySpace::Default::kokkos_space>
278 local_to_global;
279
280 /**
281 * Kokkos::View of the inverse Jacobian.
282 */
283 Kokkos::View<Number **[dim][dim], MemorySpace::Default::kokkos_space>
284 inv_jacobian;
285
286 /**
287 * Kokkos::View of the Jacobian times the weights.
288 */
289 Kokkos::View<Number **, MemorySpace::Default::kokkos_space> JxW;
290
291 /**
292 * Mask deciding where constraints are set on a given cell.
293 */
294 Kokkos::View<dealii::internal::MatrixFreeFunctions::ConstraintKinds *,
295 MemorySpace::Default::kokkos_space>
296 constraint_mask;
297
298 /**
299 * Values of the shape functions.
300 */
301 Kokkos::View<Number *, MemorySpace::Default::kokkos_space> shape_values;
302
303 /**
304 * Gradients of the shape functions.
305 */
306 Kokkos::View<Number *, MemorySpace::Default::kokkos_space>
307 shape_gradients;
308
309 /**
310 * Gradients of the shape functions for collocation methods.
311 */
312 Kokkos::View<Number *, MemorySpace::Default::kokkos_space>
313 co_shape_gradients;
314
315 /**
316 * Weights used when resolving hanging nodes.
317 */
318 Kokkos::View<Number *, MemorySpace::Default::kokkos_space>
319 constraint_weights;
320
321 /**
322 * Number of cells.
323 */
324 unsigned int n_cells;
325
326 /**
327 * Number of components.
328 */
329 unsigned int n_components;
330
331 /**
332 * Length of the padding.
333 */
334 unsigned int padding_length;
335
336 /**
337 * Row start (including padding).
338 */
339 unsigned int row_start;
340
341 /**
342 * If true, use graph coloring has been used and we can simply add into
343 * the destingation vector. Otherwise, use atomic operations.
344 */
345 bool use_coloring;
346
347 /**
348 * Encodes the type of element detected at construction. FEEvaluation
349 * will select the most efficient algorithm based on the given element
350 * type.
351 */
352 ::dealii::internal::MatrixFreeFunctions::ElementType element_type;
353
354 /**
355 * Size of the scratch pad for temporary storage in shared memory.
356 */
357 unsigned int scratch_pad_size;
358 };
359
360
361 /**
362 * A pointer to this data structure is passed to the user code in
363 * device code that computes operations in quadrature points. It
364 * can be used to construct Portable::FEEvaluation objects and contains
365 * necessary precomputed data and scratch memory space to perform
366 * matrix-free evaluations.
367 */
368 struct Data
369 {
370 using TeamHandle = Kokkos::TeamPolicy<
371 MemorySpace::Default::kokkos_space::execution_space>::member_type;
372
373 /**
374 * TeamPolicy handle.
375 */
376 TeamHandle team_member;
377
378 const unsigned int n_dofhandler;
379 const int cell_index;
380 const PrecomputedData *precomputed_data;
381 SharedData<dim, Number> *shared_data;
382
383 /**
384 * Return the quadrature point index local. The index is
385 * only unique for a given MPI process.
386 */
387 DEAL_II_HOST_DEVICE unsigned int
388 local_q_point_id(const unsigned int cell,
389 const unsigned int n_q_points,
390 const unsigned int q_point) const
391 {
392 return (precomputed_data->row_start / precomputed_data->padding_length +
393 cell) *
394 n_q_points +
395 q_point;
396 }
397
398
399 /**
400 * Return the quadrature point.
401 */
402 DEAL_II_HOST_DEVICE
403 typename Portable::MatrixFree<dim, Number>::point_type &
404 get_quadrature_point(const unsigned int cell,
405 const unsigned int q_point) const
406 {
407 return precomputed_data->q_points(q_point, cell);
408 }
409 };
410
411 /**
412 * Default constructor.
413 */
414 MatrixFree();
415
416 /**
417 * Return the length of the padding.
418 */
419 unsigned int
420 get_padding_length() const;
421
422 /**
423 * Extracts the information needed to perform loops over cells. The
424 * DoFHandler and AffineConstraints objects describe the layout of
425 * degrees of freedom, the DoFHandler and the mapping describe the
426 * transformation from unit to real cell, and the finite element
427 * underlying the DoFHandler together with the quadrature formula
428 * describe the local operations. This function takes an IteratorFilters
429 * object (predicate) to loop over a subset of the active cells. When using
430 * MPI, the predicate should filter out non locally owned cells.
431 */
432 template <typename IteratorFiltersType>
433 void
434 reinit(const Mapping<dim> &mapping,
435 const DoFHandler<dim> &dof_handler,
436 const AffineConstraints<Number> &constraints,
437 const Quadrature<1> &quad,
438 const IteratorFiltersType &iterator_filter,
439 const AdditionalData &additional_data = AdditionalData());
440
441 /**
442 * Same as above using Iterators::LocallyOwnedCell() as predicate.
443 */
444 void
445 reinit(const Mapping<dim> &mapping,
446 const DoFHandler<dim> &dof_handler,
447 const AffineConstraints<Number> &constraints,
448 const Quadrature<1> &quad,
449 const AdditionalData &additional_data = AdditionalData());
450
451 /**
452 * Initializes the data structures. Same as above but using a Q1 mapping.
453 */
454 void
455 reinit(const DoFHandler<dim> &dof_handler,
456 const AffineConstraints<Number> &constraints,
457 const Quadrature<1> &quad,
458 const AdditionalData &additional_data = AdditionalData());
459
460 /**
461 * Return the Data structure associated with @p color.
462 */
463 PrecomputedData
464 get_data(unsigned int color) const;
465
466 // clang-format off
467 /**
468 * This method runs the loop over all cells and apply the local operation on
469 * each element in parallel. @p func is a functor which is applied on each color.
470 *
471 * @p func needs to define
472 * \code
473 * DEAL_II_HOST_DEVICE void operator()(
474 * const typename Portable::MatrixFree<dim, Number>::Data *data,
475 * const DeviceVector<Number> &src,
476 * DeviceVector<Number> &dst) const;
477 * static const unsigned int n_local_dofs;
478 * static const unsigned int n_q_points;
479 * \endcode
480 */
481 // clang-format on
482 template <typename Functor, typename VectorType>
483 void
484 cell_loop(const Functor &func,
485 const VectorType &src,
486 VectorType &dst) const;
487
488 /**
489 * This method runs the loop over all cells and apply the local operation on
490 * each element in parallel. This function is very similar to cell_loop()
491 * but it uses a simpler functor.
492 *
493 * @p func needs to define
494 * \code
495 * DEAL_II_HOST_DEVICE void operator()(
496 * const typename Portable::MatrixFree<dim, Number>::Data *data);
497 * static const unsigned int n_local_dofs;
498 * static const unsigned int n_q_points;
499 * \endcode
500 */
501 template <typename Functor>
502 void
503 evaluate_coefficients(Functor func) const;
504
505 /**
506 * Copy the values of the constrained entries from @p src to @p dst. This is
507 * used to impose zero Dirichlet boundary condition.
508 */
509 template <typename VectorType>
510 void
511 copy_constrained_values(const VectorType &src, VectorType &dst) const;
512
513 /**
514 * Set the entries in @p dst corresponding to constrained values to @p val.
515 * The main purpose of this function is to set the constrained entries of
516 * the source vector used in cell_loop() to zero.
517 */
518 template <typename VectorType>
519 void
520 set_constrained_values(const Number val, VectorType &dst) const;
521
522 /**
523 * Initialize a distributed vector. The local elements correspond to the
524 * locally owned degrees of freedom and the ghost elements correspond to the
525 * (additional) locally relevant dofs.
526 */
527 template <typename MemorySpaceType>
528 void
529 initialize_dof_vector(
530 LinearAlgebra::distributed::Vector<Number, MemorySpaceType> &vec) const;
531
532 /**
533 * Return the colored graph of locally owned active cells.
534 */
535 const std::vector<std::vector<CellFilter>> &
536 get_colored_graph() const;
537
538 /**
539 * Return the partitioner that represents the locally owned data and the
540 * ghost indices where access is needed to for the cell loop. The
541 * partitioner is constructed from the locally owned dofs and ghost dofs
542 * given by the respective fields. If you want to have specific information
543 * about these objects, you can query them with the respective access
544 * functions. If you just want to initialize a (parallel) vector, you should
545 * usually prefer this data structure as the data exchange information can
546 * be reused from one vector to another.
547 */
548 const std::shared_ptr<const Utilities::MPI::Partitioner> &
549 get_vector_partitioner() const;
550
551 /**
552 * Return the DoFHandler.
553 */
554 const DoFHandler<dim> &
555 get_dof_handler() const;
556
557 /**
558 * Return an approximation of the memory consumption of this class in bytes.
559 */
560 std::size_t
561 memory_consumption() const;
562
563 private:
564 /**
565 * Initializes the data structures.
566 */
567 template <typename IteratorFiltersType>
568 void
569 internal_reinit(const Mapping<dim> &mapping,
570 const DoFHandler<dim> &dof_handler,
571 const AffineConstraints<Number> &constraints,
572 const Quadrature<1> &quad,
573 const IteratorFiltersType &iterator_filter,
574 const std::shared_ptr<const MPI_Comm> &comm,
575 const AdditionalData additional_data);
576
577 /**
578 * Helper function. Loop over all the cells and apply the functor on each
579 * element in parallel. This function is used when MPI is not used.
580 */
581 template <typename Functor, typename VectorType>
582 void
583 serial_cell_loop(const Functor &func,
584 const VectorType &src,
585 VectorType &dst) const;
586
587 /**
588 * Helper function. Loop over all the cells and apply the functor on each
589 * element in parallel. This function is used when MPI is used.
590 */
591 template <typename Functor>
592 void
593 distributed_cell_loop(
594 const Functor &func,
595 const LinearAlgebra::distributed::Vector<Number, MemorySpace::Default>
596 &src,
597 LinearAlgebra::distributed::Vector<Number, MemorySpace::Default> &dst)
598 const;
599
600 /**
601 * Same as above but for BlockVector.
602 */
603 template <typename Functor>
604 void
605 distributed_cell_loop(
606 const Functor &func,
607 const LinearAlgebra::distributed::BlockVector<Number,
608 MemorySpace::Default> &src,
609 LinearAlgebra::distributed::BlockVector<Number, MemorySpace::Default>
610 &dst) const;
611
612
613 /**
614 * Unique ID associated with the object.
615 */
616 int my_id;
617
618 /**
619 * If true, use graph coloring. Otherwise, use atomic operations. Graph
620 * coloring ensures bitwise reproducibility but is slower on Pascal and
621 * newer architectures.
622 */
623 bool use_coloring;
624
625 /**
626 * Overlap MPI communications with computation. This requires device-aware
627 * MPI and use_coloring must be false.
628 */
629 bool overlap_communication_computation;
630
631 /**
632 * Total number of degrees of freedom.
633 */
634 types::global_dof_index n_dofs;
635
636 /**
637 * Encodes the type of element detected at construction. FEEvaluation
638 * will select the most efficient algorithm based on the given element
639 * type.
640 */
641 ::dealii::internal::MatrixFreeFunctions::ElementType element_type;
642
643 /**
644 * Size of the scratch pad for temporary storage in shared memory.
645 */
646 unsigned int scratch_pad_size;
647
648 /**
649 * Degree of the finite element used.
650 */
651 unsigned int fe_degree;
652
653 /**
654 * Number of components.
655 */
656 unsigned int n_components;
657
658 /**
659 * Number of scalar degrees of freedom per cell.
660 */
661 unsigned int scalar_dofs_per_cell;
662
663 /**
664 * Number of degrees of freedom per cell.
665 */
666 unsigned int dofs_per_cell;
667
668 /**
669 * Number of constrained degrees of freedom.
670 */
671 unsigned int n_constrained_dofs;
672
673 /**
674 * Number of quadrature points per cells.
675 */
676 unsigned int q_points_per_cell;
677
678 /**
679 * Number of colors produced by the graph coloring algorithm.
680 */
681 unsigned int n_colors;
682
683 /**
684 * Number of cells in each color.
685 */
686 std::vector<unsigned int> n_cells;
687
688 /**
689 * Vector of Kokkos::View to the quadrature points associated to the cells
690 * of each color.
691 */
692 std::vector<Kokkos::View<point_type **, MemorySpace::Default::kokkos_space>>
693 q_points;
694
695 /**
696 * Map the position in the local vector to the position in the global
697 * vector.
698 */
699 std::vector<Kokkos::View<types::global_dof_index **,
700 MemorySpace::Default::kokkos_space>>
701 local_to_global;
702
703 /**
704 * Vector of Kokkos::View of the inverse Jacobian associated to the cells of
705 * each color.
706 */
707 std::vector<
708 Kokkos::View<Number **[dim][dim], MemorySpace::Default::kokkos_space>>
709 inv_jacobian;
710
711 /**
712 * Vector of Kokkos::View to the Jacobian times the weights associated to
713 * the cells of each color.
714 */
715 std::vector<Kokkos::View<Number **, MemorySpace::Default::kokkos_space>>
716 JxW;
717
718 /**
719 * Kokkos::View to the constrained degrees of freedom.
720 */
721 Kokkos::View<types::global_dof_index *, MemorySpace::Default::kokkos_space>
722 constrained_dofs;
723
724 /**
725 * Mask deciding where constraints are set on a given cell.
726 */
727 std::vector<
728 Kokkos::View<dealii::internal::MatrixFreeFunctions::ConstraintKinds *,
729 MemorySpace::Default::kokkos_space>>
730 constraint_mask;
731
732 /**
733 * Values of the shape functions.
734 */
735 Kokkos::View<Number *, MemorySpace::Default::kokkos_space> shape_values;
736
737 /**
738 * Gradients of the shape functions.
739 */
740 Kokkos::View<Number *, MemorySpace::Default::kokkos_space> shape_gradients;
741
742 /**
743 * Gradients of the shape functions for collocation methods.
744 */
745 Kokkos::View<Number *, MemorySpace::Default::kokkos_space>
746 co_shape_gradients;
747
748 /**
749 * Weights used when resolving hanging nodes.
750 */
751 Kokkos::View<Number *, MemorySpace::Default::kokkos_space>
752 constraint_weights;
753
754 /**
755 * Shared pointer to a Partitioner for distributed Vectors used in
756 * cell_loop. When MPI is not used the pointer is null.
757 */
758 std::shared_ptr<const Utilities::MPI::Partitioner> partitioner;
759
760
761 /**
762 * Length of the padding (closest power of two larger than or equal to
763 * the number of thread).
764 */
765 unsigned int padding_length;
766
767 /**
768 * Row start of each color.
769 */
770 std::vector<unsigned int> row_start;
771
772 /**
773 * Pointer to the DoFHandler associated with the object.
774 */
775 const DoFHandler<dim> *dof_handler;
776
777 /**
778 * Colored graphed of locally owned active cells.
779 */
780 std::vector<std::vector<CellFilter>> graph;
781
782 friend class internal::ReinitHelper<dim, Number>;
783 };
784
785
786
787 template <int dim, typename Number>
788 struct SharedData
789 {
790 using SharedViewValues = Kokkos::View<
791 Number **,
792 MemorySpace::Default::kokkos_space::execution_space::scratch_memory_space,
793 Kokkos::MemoryTraits<Kokkos::Unmanaged>>;
794 using SharedViewGradients = Kokkos::View<
795 Number ***,
796 MemorySpace::Default::kokkos_space::execution_space::scratch_memory_space,
797 Kokkos::MemoryTraits<Kokkos::Unmanaged>>;
798 using SharedViewScratchPad = Kokkos::View<
799 Number *,
800 MemorySpace::Default::kokkos_space::execution_space::scratch_memory_space,
801 Kokkos::MemoryTraits<Kokkos::Unmanaged>>;
802
803 DEAL_II_HOST_DEVICE
804 SharedData(const SharedViewValues &values,
805 const SharedViewGradients &gradients,
806 const SharedViewScratchPad &scratch_pad)
807 : values(values)
808 , gradients(gradients)
809 , scratch_pad(scratch_pad)
810 {}
811
812 /**
813 * Memory for dof and quad values.
814 */
815 SharedViewValues values;
816
817 /**
818 * Memory for computed gradients in reference coordinate system.
819 */
820 SharedViewGradients gradients;
821
822 /**
823 * Memory for temporary arrays required by evaluation and integration.
824 */
825 SharedViewScratchPad scratch_pad;
826 };
827
828
829
830 /**
831 * Structure which is passed to the kernel. It is used to pass all the
832 * necessary information from the CPU to the GPU.
833 */
834 template <int dim, typename Number>
835 struct DataHost
836 {
837 /**
838 * Kokkos::View of quadrature points on the host.
839 */
840 typename Kokkos::View<Point<dim, Number> **,
841 MemorySpace::Default::kokkos_space>::HostMirror
842 q_points;
843
844 /**
845 * Map the position in the local vector to the position in the global
846 * vector.
847 */
848 typename Kokkos::View<types::global_dof_index **,
849 MemorySpace::Default::kokkos_space>::HostMirror
850 local_to_global;
851
852 /**
853 * Kokkos::View of inverse Jacobians on the host.
854 */
855 typename Kokkos::View<Number **[dim][dim],
856 MemorySpace::Default::kokkos_space>::HostMirror
857 inv_jacobian;
858
859 /**
860 * Kokkos::View of Jacobian times the weights on the host.
861 */
862 typename Kokkos::View<Number **,
863 MemorySpace::Default::kokkos_space>::HostMirror JxW;
864
865 /**
866 * Number of cells.
867 */
868 unsigned int n_cells;
869
870 /**
871 * Length of the padding.
872 */
873 unsigned int padding_length;
874
875 /**
876 * Row start (including padding).
877 */
878 unsigned int row_start;
879
880 /**
881 * Mask deciding where constraints are set on a given cell.
882 */
883 typename Kokkos::View<
884 dealii::internal::MatrixFreeFunctions::ConstraintKinds *,
885 MemorySpace::Default::kokkos_space>::HostMirror constraint_mask;
886
887 /**
888 * If true, use graph coloring has been used and we can simply add into
889 * the destingation vector. Otherwise, use atomic operations.
890 */
891 bool use_coloring;
892
893
894
895 /**
896 * This function is the host version of local_q_point_id().
897 */
898 unsigned int
899 local_q_point_id(const unsigned int cell,
900 const unsigned int n_q_points,
901 const unsigned int q_point) const
902 {
903 return (row_start / padding_length + cell) * n_q_points + q_point;
904 }
905
906
907
908 /**
909 * This function is the host version of get_quadrature_point().
910 */
911 Point<dim, Number>
912 get_quadrature_point(const unsigned int cell,
913 const unsigned int q_point) const
914 {
915 return q_points(q_point, cell);
916 }
917 };
918
919
920
921 /**
922 * Copy @p data from the @ref GlossDevice "device" to the host. @p update_flags should be
923 * identical to the one used in MatrixFree::AdditionalData.
924 *
925 * @relates Portable::MatrixFree
926 */
927 template <int dim, typename Number>
928 DataHost<dim, Number>
929 copy_mf_data_to_host(
930 const typename dealii::Portable::MatrixFree<dim, Number>::PrecomputedData
931 &data,
932 const UpdateFlags &update_flags)
933 {
934 DataHost<dim, Number> data_host;
935
936 data_host.n_cells = data.n_cells;
937 data_host.padding_length = data.padding_length;
938 data_host.row_start = data.row_start;
939 data_host.use_coloring = data.use_coloring;
940
941 if (update_flags & update_quadrature_points)
942 {
943 data_host.q_points = Kokkos::create_mirror(data.q_points);
944 Kokkos::deep_copy(data_host.q_points, data.q_points);
945 }
946
947 data_host.local_to_global = Kokkos::create_mirror(data.local_to_global);
948 Kokkos::deep_copy(data_host.local_to_global, data.local_to_global);
949
950 if (update_flags & update_gradients)
951 {
952 data_host.inv_jacobian = Kokkos::create_mirror(data.inv_jacobian);
953 Kokkos::deep_copy(data_host.inv_jacobian, data.inv_jacobian);
954 }
955
956 if (update_flags & update_JxW_values)
957 {
958 data_host.JxW = Kokkos::create_mirror(data.JxW);
959 Kokkos::deep_copy(data_host.JxW, data.JxW);
960 }
961
962 data_host.constraint_mask = Kokkos::create_mirror(data.constraint_mask);
963 Kokkos::deep_copy(data_host.constraint_mask, data.constraint_mask);
964
965 return data_host;
966 }
967
968
969 /*----------------------- Inline functions ---------------------------------*/
970
971 #ifndef DOXYGEN
972
973 template <int dim, typename Number>
974 inline const std::vector<std::vector<
975 FilteredIterator<typename DoFHandler<dim>::active_cell_iterator>>> &
976 MatrixFree<dim, Number>::get_colored_graph() const
977 {
978 return graph;
979 }
980
981
982
983 template <int dim, typename Number>
984 inline const std::shared_ptr<const Utilities::MPI::Partitioner> &
985 MatrixFree<dim, Number>::get_vector_partitioner() const
986 {
987 return partitioner;
988 }
989
990
991
992 template <int dim, typename Number>
993 inline const DoFHandler<dim> &
994 MatrixFree<dim, Number>::get_dof_handler() const
995 {
996 Assert(dof_handler != nullptr, ExcNotInitialized());
997
998 return *dof_handler;
999 }
1000
1001 #endif
1002
1003 } // namespace Portable
1004
1005 DEAL_II_NAMESPACE_CLOSE
1006
1007 #endif