Debian Codegraph @

include/deal.II/matrix_free/portable_matrix_free.templates.h

raw
1 // ------------------------------------------------------------------------
2 //
3 // SPDX-License-Identifier: LGPL-2.1-or-later
4 // Copyright (C) 2017 - 2025 by the deal.II authors
5 //
6 // This file is part of the deal.II library.
7 //
8 // Part of the source code is dual licensed under Apache-2.0 WITH
9 // LLVM-exception OR LGPL-2.1-or-later. Detailed license information
10 // governing the source code and code contributions can be found in
11 // LICENSE.md and CONTRIBUTING.md at the top level directory of deal.II.
12 //
13 // ------------------------------------------------------------------------
14
15
16 #ifndef dealii_portable_matrix_free_templates_h
17 #define dealii_portable_matrix_free_templates_h
18
19 #include <deal.II/base/config.h>
20
21 #include <deal.II/base/exception_macros.h>
22 #include <deal.II/base/exceptions.h>
23 #include <deal.II/base/graph_coloring.h>
24 #include <deal.II/base/memory_space.h>
25
26 #include <deal.II/dofs/dof_tools.h>
27
28 #include <deal.II/fe/fe_dgq.h>
29 #include <deal.II/fe/fe_values.h>
30 #include <deal.II/fe/mapping_q1.h>
31
32 #include <deal.II/lac/block_vector_base.h>
33
34 #include <deal.II/matrix_free/portable_hanging_nodes_internal.h>
35 #include <deal.II/matrix_free/portable_matrix_free.h>
36 #include <deal.II/matrix_free/shape_info.h>
37
38 #include <Kokkos_Core.hpp>
39
40 #include <cmath>
41 #include <functional>
42 #include <string>
43
44
45 DEAL_II_NAMESPACE_OPEN
46
47 namespace Portable
48 {
49 namespace internal
50 {
51 /**
52 * Helper class to (re)initialize MatrixFree object.
53 */
54 template <int dim, typename Number>
55 class ReinitHelper
56 {
57 public:
58 ReinitHelper(
59 MatrixFree<dim, Number> *data,
60 const Mapping<dim> &mapping,
61 const FiniteElement<dim, dim> &fe,
62 const Quadrature<1> &quad,
63 const ::dealii::internal::MatrixFreeFunctions::ShapeInfo<Number>
64 &shape_info,
65 const DoFHandler<dim> &dof_handler,
66 const UpdateFlags &update_flags);
67
68 void
69 resize(const unsigned int n_colors);
70
71 template <typename CellFilter>
72 void
73 fill_data(
74 const unsigned int color,
75 const std::vector<CellFilter> &graph,
76 const std::shared_ptr<const Utilities::MPI::Partitioner> &partitioner);
77
78 private:
79 MatrixFree<dim, Number> *data;
80 // Local buffer
81 std::vector<types::global_dof_index> local_dof_indices;
82 FEValues<dim> fe_values;
83 // Convert the default dof numbering to a lexicographic one
84 const std::vector<unsigned int> &lexicographic_inv;
85 std::vector<types::global_dof_index> lexicographic_dof_indices;
86 const unsigned int fe_degree;
87 const unsigned int n_components;
88 const unsigned int scalar_dofs_per_cell;
89 const unsigned int dofs_per_cell;
90 const unsigned int q_points_per_cell;
91 const UpdateFlags &update_flags;
92 const unsigned int padding_length;
93 dealii::internal::MatrixFreeFunctions::HangingNodes<dim> hanging_nodes;
94 };
95
96
97
98 template <int dim, typename Number>
99 ReinitHelper<dim, Number>::ReinitHelper(
100 MatrixFree<dim, Number> *data,
101 const Mapping<dim> &mapping,
102 const FiniteElement<dim> &fe,
103 const Quadrature<1> &quad,
104 const ::dealii::internal::MatrixFreeFunctions::ShapeInfo<Number>
105 &shape_info,
106 const DoFHandler<dim> &dof_handler,
107 const UpdateFlags &update_flags)
108 : data(data)
109 , fe_values(mapping,
110 fe,
111 Quadrature<dim>(quad),
112 update_inverse_jacobians | update_quadrature_points |
113 update_values | update_gradients | update_JxW_values)
114 , lexicographic_inv(shape_info.lexicographic_numbering)
115 , fe_degree(data->fe_degree)
116 , n_components(data->n_components)
117 , scalar_dofs_per_cell(data->scalar_dofs_per_cell)
118 , dofs_per_cell(data->dofs_per_cell)
119 , q_points_per_cell(data->q_points_per_cell)
120 , update_flags(update_flags)
121 , padding_length(data->get_padding_length())
122 , hanging_nodes(dof_handler.get_triangulation())
123 {
124 local_dof_indices.resize(data->dofs_per_cell);
125 lexicographic_dof_indices.resize(dofs_per_cell);
126 fe_values.always_allow_check_for_cell_similarity(true);
127 }
128
129
130
131 template <int dim, typename Number>
132 void
133 ReinitHelper<dim, Number>::resize(const unsigned int n_colors)
134 {
135 // We need at least three colors when we are using device-aware MPI and
136 // overlapping the communication
137 data->n_cells.resize(std::max(n_colors, 3U), 0);
138 data->local_to_global.resize(n_colors);
139 data->constraint_mask.resize(n_colors);
140
141 data->row_start.resize(n_colors);
142
143 if (update_flags & update_quadrature_points)
144 data->q_points.resize(n_colors);
145
146 if (update_flags & update_JxW_values)
147 data->JxW.resize(n_colors);
148
149 if (update_flags & update_gradients)
150 data->inv_jacobian.resize(n_colors);
151 }
152
153
154
155 template <int dim, typename Number>
156 template <typename CellFilter>
157 void
158 ReinitHelper<dim, Number>::fill_data(
159 const unsigned int color,
160 const std::vector<CellFilter> &graph,
161 const std::shared_ptr<const Utilities::MPI::Partitioner> &partitioner)
162 {
163 const unsigned int n_cells = data->n_cells[color];
164
165 // Create the Views
166 data->local_to_global[color] =
167 Kokkos::View<types::global_dof_index **,
168 MemorySpace::Default::kokkos_space>(
169 Kokkos::view_alloc("local_to_global_" + std::to_string(color),
170 Kokkos::WithoutInitializing),
171 dofs_per_cell,
172 n_cells);
173
174 if (update_flags & update_quadrature_points)
175 data->q_points[color] =
176 Kokkos::View<Point<dim, Number> **,
177 MemorySpace::Default::kokkos_space>(
178 Kokkos::view_alloc("q_points_" + std::to_string(color),
179 Kokkos::WithoutInitializing),
180 q_points_per_cell,
181 n_cells);
182
183 if (update_flags & update_JxW_values)
184 data->JxW[color] =
185 Kokkos::View<Number **, MemorySpace::Default::kokkos_space>(
186 Kokkos::view_alloc("JxW_" + std::to_string(color),
187 Kokkos::WithoutInitializing),
188 q_points_per_cell,
189 n_cells);
190
191 if (update_flags & update_gradients)
192 data->inv_jacobian[color] =
193 Kokkos::View<Number **[dim][dim], MemorySpace::Default::kokkos_space>(
194 Kokkos::view_alloc("inv_jacobian_" + std::to_string(color),
195 Kokkos::WithoutInitializing),
196 q_points_per_cell,
197 n_cells);
198
199 // Initialize to zero, i.e., unconstrained cell
200 data->constraint_mask[color] =
201 Kokkos::View<dealii::internal::MatrixFreeFunctions::ConstraintKinds *,
202 MemorySpace::Default::kokkos_space>(
203 "constraint_mask_" + std::to_string(color), n_cells * n_components);
204
205 // Create the host mirrow Views and fill them
206 auto constraint_mask_host =
207 Kokkos::create_mirror_view(data->constraint_mask[color]);
208
209 typename std::remove_reference_t<
210 decltype(data->q_points[color])>::HostMirror q_points_host;
211 typename std::remove_reference_t<decltype(data->JxW[color])>::HostMirror
212 JxW_host;
213 typename std::remove_reference_t<
214 decltype(data->inv_jacobian[color])>::HostMirror inv_jacobian_host;
215 #if DEAL_II_KOKKOS_VERSION_GTE(3, 6, 0)
216 auto local_to_global_host =
217 Kokkos::create_mirror_view(Kokkos::WithoutInitializing,
218 data->local_to_global[color]);
219 if (update_flags & update_quadrature_points)
220 q_points_host = Kokkos::create_mirror_view(Kokkos::WithoutInitializing,
221 data->q_points[color]);
222 if (update_flags & update_JxW_values)
223 JxW_host = Kokkos::create_mirror_view(Kokkos::WithoutInitializing,
224 data->JxW[color]);
225 if (update_flags & update_gradients)
226 inv_jacobian_host =
227 Kokkos::create_mirror_view(Kokkos::WithoutInitializing,
228 data->inv_jacobian[color]);
229 #else
230 auto local_to_global_host =
231 Kokkos::create_mirror_view(data->local_to_global[color]);
232 if (update_flags & update_quadrature_points)
233 q_points_host = Kokkos::create_mirror_view(data->q_points[color]);
234 if (update_flags & update_JxW_values)
235 JxW_host = Kokkos::create_mirror_view(data->JxW[color]);
236 if (update_flags & update_gradients)
237 inv_jacobian_host =
238 Kokkos::create_mirror_view(data->inv_jacobian[color]);
239 #endif
240
241 auto cell = graph.cbegin(), end_cell = graph.cend();
242 for (unsigned int cell_id = 0; cell != end_cell; ++cell, ++cell_id)
243 {
244 (*cell)->get_dof_indices(local_dof_indices);
245 // When using MPI, we need to transform the local_dof_indices, which
246 // contain global numbers of dof indices in the MPI universe, to get
247 // local (to the current MPI process) dof indices.
248 if (partitioner)
249 for (auto &index : local_dof_indices)
250 index = partitioner->global_to_local(index);
251
252 for (unsigned int i = 0; i < dofs_per_cell; ++i)
253 lexicographic_dof_indices[i] =
254 local_dof_indices[lexicographic_inv[i]];
255
256 const ArrayView<
257 dealii::internal::MatrixFreeFunctions::ConstraintKinds>
258 cell_id_view(&constraint_mask_host[cell_id * n_components],
259 n_components);
260
261 hanging_nodes.setup_constraints(*cell,
262 partitioner,
263 {lexicographic_inv},
264 lexicographic_dof_indices,
265 cell_id_view);
266
267 for (unsigned int i = 0; i < dofs_per_cell; ++i)
268 local_to_global_host(i, cell_id) = lexicographic_dof_indices[i];
269
270 fe_values.reinit(*cell);
271
272 // Quadrature points
273 if (update_flags & update_quadrature_points)
274 {
275 for (unsigned int i = 0; i < q_points_per_cell; ++i)
276 q_points_host(i, cell_id) = fe_values.quadrature_point(i);
277 }
278
279 if (update_flags & update_JxW_values)
280 {
281 for (unsigned int i = 0; i < q_points_per_cell; ++i)
282 JxW_host(i, cell_id) = fe_values.JxW(i);
283 }
284
285 if (update_flags & update_gradients)
286 {
287 for (unsigned int i = 0; i < q_points_per_cell; ++i)
288 for (unsigned int d = 0; d < dim; ++d)
289 for (unsigned int e = 0; e < dim; ++e)
290 inv_jacobian_host(i, cell_id, d, e) =
291 fe_values.inverse_jacobian(i)[d][e];
292 }
293 }
294
295 // Copy the data to the device
296 Kokkos::deep_copy(data->constraint_mask[color], constraint_mask_host);
297 Kokkos::deep_copy(data->local_to_global[color], local_to_global_host);
298 if (update_flags & update_quadrature_points)
299 Kokkos::deep_copy(data->q_points[color], q_points_host);
300 if (update_flags & update_JxW_values)
301 Kokkos::deep_copy(data->JxW[color], JxW_host);
302 if (update_flags & update_gradients)
303 Kokkos::deep_copy(data->inv_jacobian[color], inv_jacobian_host);
304 }
305
306
307
308 template <int dim, typename number>
309 std::vector<types::global_dof_index>
310 get_conflict_indices(
311 const FilteredIterator<typename DoFHandler<dim>::active_cell_iterator>
312 &cell,
313 const AffineConstraints<number> &constraints)
314 {
315 std::vector<types::global_dof_index> local_dof_indices(
316 cell->get_fe().n_dofs_per_cell());
317 cell->get_dof_indices(local_dof_indices);
318 constraints.resolve_indices(local_dof_indices);
319
320 return local_dof_indices;
321 }
322
323
324
325 template <typename VectorType>
326 struct VectorLocalSize
327 {
328 static unsigned int
329 get(const VectorType &vec)
330 {
331 return vec.locally_owned_size();
332 }
333 };
334
335
336
337 template <int dim, typename Number, typename Functor, bool IsBlock>
338 struct ApplyKernel
339 {
340 using TeamHandle = Kokkos::TeamPolicy<
341 MemorySpace::Default::kokkos_space::execution_space>::member_type;
342 using SharedViewValues =
343 Kokkos::View<Number **,
344 MemorySpace::Default::kokkos_space::execution_space::
345 scratch_memory_space,
346 Kokkos::MemoryTraits<Kokkos::Unmanaged>>;
347 using SharedViewGradients =
348 Kokkos::View<Number ***,
349 MemorySpace::Default::kokkos_space::execution_space::
350 scratch_memory_space,
351 Kokkos::MemoryTraits<Kokkos::Unmanaged>>;
352 using SharedViewScratchPad =
353 Kokkos::View<Number *,
354 MemorySpace::Default::kokkos_space::execution_space::
355 scratch_memory_space,
356 Kokkos::MemoryTraits<Kokkos::Unmanaged>>;
357
358 ApplyKernel(
359 Functor func,
360 const typename MatrixFree<dim, Number>::PrecomputedData gpu_data,
361 const LinearAlgebra::distributed::Vector<Number, MemorySpace::Default>
362 &src,
363 LinearAlgebra::distributed::Vector<Number, MemorySpace::Default> &dst)
364 : func(func)
365 , gpu_data(gpu_data)
366 , src(DeviceVector<Number>(src.get_values(), src.locally_owned_size()))
367 , dst(DeviceVector<Number>(dst.get_values(), dst.locally_owned_size()))
368 {}
369
370 ApplyKernel(
371 Functor func,
372 const typename MatrixFree<dim, Number>::PrecomputedData gpu_data,
373 const LinearAlgebra::distributed::BlockVector<Number,
374 MemorySpace::Default>
375 &src,
376 LinearAlgebra::distributed::BlockVector<Number, MemorySpace::Default>
377 &dst)
378 : func(func)
379 , gpu_data(gpu_data)
380 , src(src)
381 , dst(dst)
382 {}
383
384 Functor func;
385 const typename MatrixFree<dim, Number>::PrecomputedData gpu_data;
386 const DeviceBlockVector<Number> src;
387 DeviceBlockVector<Number> dst;
388
389
390 // Provide the shared memory capacity. This function takes the team_size
391 // as an argument, which allows team_size dependent allocations.
392 std::size_t
393 team_shmem_size(int /*team_size*/) const
394 {
395 return SharedViewValues::shmem_size(Functor::n_q_points,
396 gpu_data.n_components) +
397 SharedViewGradients::shmem_size(Functor::n_q_points,
398 dim,
399 gpu_data.n_components) +
400 SharedViewScratchPad::shmem_size(gpu_data.scratch_pad_size);
401 }
402
403
404 DEAL_II_HOST_DEVICE
405 void
406 operator()(const TeamHandle &team_member) const
407 {
408 // Get the scratch memory
409 SharedViewValues values(team_member.team_shmem(),
410 Functor::n_q_points,
411 gpu_data.n_components);
412 SharedViewGradients gradients(team_member.team_shmem(),
413 Functor::n_q_points,
414 dim,
415 gpu_data.n_components);
416 SharedViewScratchPad scratch_pad(team_member.team_shmem(),
417 gpu_data.scratch_pad_size);
418
419 SharedData<dim, Number> shared_data(values, gradients, scratch_pad);
420
421 const int cell_index = team_member.league_rank();
422
423 typename MatrixFree<dim, Number>::Data data{team_member,
424 /* n_dofhandler */ 1,
425 cell_index,
426 &gpu_data,
427 &shared_data};
428
429 if constexpr (IsBlock)
430 {
431 DeviceBlockVector<Number> nonconstdst = dst;
432 func(&data, src, nonconstdst);
433 }
434 else
435 {
436 DeviceVector<Number> nonconstdst = dst.block(0);
437 func(&data, src.block(0), nonconstdst);
438 }
439 }
440 };
441 } // namespace internal
442
443
444
445 template <int dim, typename Number>
446 MatrixFree<dim, Number>::MatrixFree()
447 : my_id(-1)
448 , n_dofs(0)
449 , padding_length(0)
450 , dof_handler(nullptr)
451 {}
452
453
454
455 template <int dim, typename Number>
456 template <typename IteratorFiltersType>
457 void
458 MatrixFree<dim, Number>::reinit(const Mapping<dim> &mapping,
459 const DoFHandler<dim> &dof_handler,
460 const AffineConstraints<Number> &constraints,
461 const Quadrature<1> &quad,
462 const IteratorFiltersType &iterator_filter,
463 const AdditionalData &additional_data)
464 {
465 const auto &triangulation = dof_handler.get_triangulation();
466 if (const auto parallel_triangulation =
467 dynamic_cast<const parallel::TriangulationBase<dim> *>(
468 &triangulation))
469 internal_reinit(mapping,
470 dof_handler,
471 constraints,
472 quad,
473 iterator_filter,
474 std::make_shared<const MPI_Comm>(
475 parallel_triangulation->get_mpi_communicator()),
476 additional_data);
477 else
478 internal_reinit(mapping,
479 dof_handler,
480 constraints,
481 quad,
482 iterator_filter,
483 nullptr,
484 additional_data);
485 }
486
487
488
489 template <int dim, typename Number>
490 void
491 MatrixFree<dim, Number>::reinit(const Mapping<dim> &mapping,
492 const DoFHandler<dim> &dof_handler,
493 const AffineConstraints<Number> &constraints,
494 const Quadrature<1> &quad,
495 const AdditionalData &additional_data)
496 {
497 IteratorFilters::LocallyOwnedCell locally_owned_cell_filter;
498 reinit(mapping,
499 dof_handler,
500 constraints,
501 quad,
502 locally_owned_cell_filter,
503 additional_data);
504 }
505
506
507
508 template <int dim, typename Number>
509 void
510 MatrixFree<dim, Number>::reinit(const DoFHandler<dim> &dof_handler,
511 const AffineConstraints<Number> &constraints,
512 const Quadrature<1> &quad,
513 const AdditionalData &additional_data)
514 {
515 reinit(StaticMappingQ1<dim>::mapping,
516 dof_handler,
517 constraints,
518 quad,
519 additional_data);
520 }
521
522
523
524 template <int dim, typename Number>
525 typename MatrixFree<dim, Number>::PrecomputedData
526 MatrixFree<dim, Number>::get_data(unsigned int color) const
527 {
528 PrecomputedData data_copy;
529 if (q_points.size() > 0)
530 data_copy.q_points = q_points[color];
531 if (inv_jacobian.size() > 0)
532 data_copy.inv_jacobian = inv_jacobian[color];
533 if (JxW.size() > 0)
534 data_copy.JxW = JxW[color];
535 data_copy.local_to_global = local_to_global[color];
536 data_copy.constraint_mask = constraint_mask[color];
537 data_copy.shape_values = shape_values;
538 data_copy.shape_gradients = shape_gradients;
539 data_copy.co_shape_gradients = co_shape_gradients;
540 data_copy.constraint_weights = constraint_weights;
541 data_copy.n_cells = n_cells[color];
542 data_copy.n_components = n_components;
543 data_copy.padding_length = padding_length;
544 data_copy.row_start = row_start[color];
545 data_copy.use_coloring = use_coloring;
546 data_copy.element_type = element_type;
547 data_copy.scratch_pad_size = scratch_pad_size;
548
549 return data_copy;
550 }
551
552
553
554 template <int dim, typename Number>
555 template <typename VectorType>
556 void
557 MatrixFree<dim, Number>::copy_constrained_values(const VectorType &src,
558 VectorType &dst) const
559 {
560 static_assert(
561 std::is_same_v<Number, typename VectorType::value_type>,
562 "VectorType::value_type and Number should be of the same type.");
563 Assert(src.size() == dst.size(),
564 ExcMessage("src and dst vectors have different size."));
565 // FIXME When using C++17, we can use KOKKOS_CLASS_LAMBDA and this
566 // work-around can be removed.
567 auto constr_dofs = constrained_dofs;
568 const unsigned int size = internal::VectorLocalSize<VectorType>::get(dst);
569 const Number *src_ptr = src.get_values();
570 Number *dst_ptr = dst.get_values();
571 Kokkos::parallel_for(
572 "dealii::copy_constrained_values",
573 Kokkos::RangePolicy<MemorySpace::Default::kokkos_space::execution_space>(
574 0, n_constrained_dofs),
575 KOKKOS_LAMBDA(int dof) {
576 // When working with distributed vectors, the constrained dofs are
577 // computed for ghosted vectors but we want to copy the values of the
578 // constrained dofs of non-ghosted vectors.
579 const auto constrained_dof = constr_dofs[dof];
580 if (constrained_dof < size)
581 dst_ptr[constrained_dof] = src_ptr[constrained_dof];
582 });
583 }
584
585
586
587 template <int dim, typename Number>
588 template <typename VectorType>
589 void
590 MatrixFree<dim, Number>::set_constrained_values(Number val,
591 VectorType &dst) const
592 {
593 static_assert(
594 std::is_same_v<Number, typename VectorType::value_type>,
595 "VectorType::value_type and Number should be of the same type.");
596 Number *dst_ptr = dst.get_values();
597 // FIXME When using C++17, we can use KOKKOS_CLASS_LAMBDA and this
598 // work-around can be removed.
599 auto constr_dofs = constrained_dofs;
600 // When working with distributed vectors, the constrained dofs are
601 // computed for ghosted vectors but we want to set the values of the
602 // constrained dofs of non-ghosted vectors.
603 const unsigned int size =
604 partitioner ? dst.locally_owned_size() : dst.size();
605 Kokkos::parallel_for(
606 "dealii::set_constrained_values",
607 Kokkos::RangePolicy<MemorySpace::Default::kokkos_space::execution_space>(
608 0, n_constrained_dofs),
609 KOKKOS_LAMBDA(int dof) {
610 if (constr_dofs[dof] < size)
611 dst_ptr[constr_dofs[dof]] = val;
612 });
613 }
614
615
616
617 template <int dim, typename Number>
618 template <typename MemorySpaceType>
619 void
620 MatrixFree<dim, Number>::initialize_dof_vector(
621 LinearAlgebra::distributed::Vector<Number, MemorySpaceType> &vec) const
622 {
623 if (partitioner)
624 vec.reinit(partitioner);
625 else
626 vec.reinit(n_dofs);
627 }
628
629
630
631 template <int dim, typename Number>
632 unsigned int
633 MatrixFree<dim, Number>::get_padding_length() const
634 {
635 return padding_length;
636 }
637
638
639
640 template <int dim, typename Number>
641 template <typename Functor, typename VectorType>
642 void
643 MatrixFree<dim, Number>::cell_loop(const Functor &func,
644 const VectorType &src,
645 VectorType &dst) const
646 {
647 if (partitioner)
648 distributed_cell_loop(func, src, dst);
649 else
650 serial_cell_loop(func, src, dst);
651 }
652
653
654
655 template <int dim, typename Number>
656 template <typename Functor>
657 void
658 MatrixFree<dim, Number>::evaluate_coefficients(Functor func) const
659 {
660 const unsigned int n_q_points = Functor::n_q_points;
661
662 for (unsigned int i = 0; i < n_colors; ++i)
663 if (n_cells[i] > 0)
664 {
665 auto color_data = get_data(i);
666
667 MemorySpace::Default::kokkos_space::execution_space exec;
668 Kokkos::TeamPolicy<
669 MemorySpace::Default::kokkos_space::execution_space>
670 team_policy(exec, n_cells[i], Kokkos::AUTO);
671
672 Kokkos::parallel_for(
673 "dealii::MatrixFree::evaluate_coeff_cell_loop",
674 team_policy,
675 KOKKOS_LAMBDA(const Kokkos::TeamPolicy<
676 MemorySpace::Default::kokkos_space::execution_space>::
677 member_type &team_member) {
678 Kokkos::parallel_for(Kokkos::TeamVectorRange(team_member,
679 n_q_points),
680 [&](const int q_point) {
681 const int cell_index =
682 team_member.league_rank();
683
684 Data data{team_member,
685 /* n_dofhandler */ 1,
686 cell_index,
687 &color_data,
688 /* shared_data */ nullptr};
689
690 func(&data, cell_index, q_point);
691 });
692 });
693 }
694 }
695
696
697
698 template <int dim, typename Number>
699 std::size_t
700 MatrixFree<dim, Number>::memory_consumption() const
701 {
702 // First compute the size of n_cells, row_starts, kernel launch parameters,
703 // and constrained_dofs
704 std::size_t bytes = n_cells.size() * sizeof(unsigned int) * 2 +
705 n_constrained_dofs * sizeof(unsigned int);
706
707 // For each color, add local_to_global, inv_jacobian, JxW, and q_points.
708 // FIXME
709 for (unsigned int i = 0; i < n_colors; ++i)
710 {
711 bytes += n_cells[i] * padding_length * sizeof(unsigned int) +
712 n_cells[i] * padding_length * dim * dim * sizeof(Number) +
713 n_cells[i] * padding_length * sizeof(Number) +
714 n_cells[i] * padding_length * sizeof(point_type) +
715 n_cells[i] * sizeof(unsigned int);
716 }
717
718 return bytes;
719 }
720
721
722
723 namespace internal
724 {
725 /**
726 * Helper function for determining the scratch pad size.
727 */
728 inline unsigned int
729 compute_scratch_pad_size(
730 const ::dealii::internal::MatrixFreeFunctions::ElementType element_type,
731 const int dim,
732 const int fe_degree,
733 const int n_q_points_1d)
734 {
735 using ElementType = ::dealii::internal::MatrixFreeFunctions::ElementType;
736
737 if (fe_degree >= 0 && element_type <= ElementType::tensor_symmetric)
738 {
739 // evaluate/integrate with FEEvaluationImplCollocation or
740 // FEEvaluationImplTransformToCollocation
741 return Utilities::pow(n_q_points_1d, dim);
742 }
743 else if (fe_degree >= 0 &&
744 element_type <= ElementType::tensor_symmetric_no_collocation)
745 {
746 // evaluate/integrate with FEEvaluationImpl
747 if (dim == 1)
748 return n_q_points_1d;
749 else if (dim == 2)
750 return (fe_degree + 1) * n_q_points_1d;
751 else if (dim == 3)
752 return (fe_degree + 1) * n_q_points_1d *
753 (fe_degree + 1 + n_q_points_1d);
754 else
755 AssertThrow(false, ExcNotImplemented());
756 }
757 else
758 AssertThrow(false, ExcNotImplemented());
759
760 return numbers::invalid_unsigned_int;
761 }
762 } // namespace internal
763
764
765
766 template <int dim, typename Number>
767 template <typename IteratorFiltersType>
768 void
769 MatrixFree<dim, Number>::internal_reinit(
770 const Mapping<dim> &mapping,
771 const DoFHandler<dim> &dof_handler_,
772 const AffineConstraints<Number> &constraints,
773 const Quadrature<1> &quad,
774 const IteratorFiltersType &iterator_filter,
775 const std::shared_ptr<const MPI_Comm> &comm,
776 const AdditionalData additional_data)
777 {
778 dof_handler = &dof_handler_;
779
780 UpdateFlags update_flags = additional_data.mapping_update_flags;
781 if (update_flags & update_gradients)
782 update_flags |= update_JxW_values;
783
784 this->use_coloring = additional_data.use_coloring;
785 this->overlap_communication_computation =
786 additional_data.overlap_communication_computation;
787
788 n_dofs = dof_handler->n_dofs();
789
790 const FiniteElement<dim> &fe = dof_handler->get_fe();
791
792 fe_degree = fe.degree;
793 // TODO this should be a templated parameter
794 const unsigned int n_dofs_1d = fe_degree + 1;
795 const unsigned int n_q_points_1d = quad.size();
796
797 // TODO remove the limitation in the future
798 AssertThrow(n_dofs_1d <= n_q_points_1d,
799 ExcMessage("n_q_points_1d must be greater than or equal to "
800 "fe_degree + 1."));
801
802 // Set padding length to the closest power of two larger than or equal to
803 // the number of threads.
804 padding_length = 1 << static_cast<unsigned int>(std::ceil(
805 dim * std::log2(static_cast<float>(n_q_points_1d))));
806
807 dofs_per_cell = fe.n_dofs_per_cell();
808 n_components = fe.n_components();
809 scalar_dofs_per_cell = dofs_per_cell / n_components;
810 q_points_per_cell = Utilities::fixed_power<dim>(n_q_points_1d);
811
812 ::dealii::internal::MatrixFreeFunctions::ShapeInfo<Number> shape_info(quad,
813 fe);
814
815 this->element_type = shape_info.element_type;
816 this->scratch_pad_size = internal::compute_scratch_pad_size(
817 shape_info.element_type, dim, fe_degree, n_q_points_1d);
818
819 unsigned int size_shape_values = n_dofs_1d * n_q_points_1d;
820
821 shape_values = Kokkos::View<Number *, MemorySpace::Default::kokkos_space>(
822 Kokkos::view_alloc("shape_values", Kokkos::WithoutInitializing),
823 size_shape_values);
824 Kokkos::deep_copy(shape_values,
825 Kokkos::View<Number *, Kokkos::HostSpace>(
826 shape_info.data.front().shape_values.data(),
827 size_shape_values));
828
829 if (update_flags & update_gradients)
830 {
831 shape_gradients =
832 Kokkos::View<Number *, MemorySpace::Default::kokkos_space>(
833 Kokkos::view_alloc("shape_gradients", Kokkos::WithoutInitializing),
834 size_shape_values);
835 Kokkos::deep_copy(shape_gradients,
836 Kokkos::View<Number *, Kokkos::HostSpace>(
837 shape_info.data.front().shape_gradients.data(),
838 size_shape_values));
839
840
841 co_shape_gradients =
842 Kokkos::View<Number *, MemorySpace::Default::kokkos_space>(
843 Kokkos::view_alloc("co_shape_gradients",
844 Kokkos::WithoutInitializing),
845 n_q_points_1d * n_q_points_1d);
846 Kokkos::deep_copy(
847 co_shape_gradients,
848 Kokkos::View<Number *, Kokkos::HostSpace>(
849 shape_info.data.front().shape_gradients_collocation.data(),
850 n_q_points_1d * n_q_points_1d));
851 }
852
853 internal::ReinitHelper<dim, Number> helper(
854 this, mapping, fe, quad, shape_info, *dof_handler, update_flags);
855
856 const unsigned int constraint_weights_size =
857 shape_info.data.front().subface_interpolation_matrices[0].size();
858 constraint_weights =
859 Kokkos::View<Number *, MemorySpace::Default::kokkos_space>(
860 Kokkos::view_alloc("constraint_weights", Kokkos::WithoutInitializing),
861 constraint_weights_size);
862 auto constraint_weights_host =
863 Kokkos::create_mirror_view(constraint_weights);
864 for (unsigned int i = 0; i < constraint_weights_size; ++i)
865 {
866 constraint_weights_host[i] =
867 shape_info.data.front().subface_interpolation_matrices[0][i];
868 }
869 Kokkos::deep_copy(constraint_weights, constraint_weights_host);
870
871 // Create a graph coloring
872 CellFilter begin(iterator_filter, dof_handler->begin_active());
873 CellFilter end(iterator_filter, dof_handler->end());
874
875 if (begin != end)
876 {
877 if (additional_data.use_coloring)
878 {
879 const auto fun = [&](const CellFilter &filter) {
880 return internal::get_conflict_indices<dim, Number>(filter,
881 constraints);
882 };
883 graph = GraphColoring::make_graph_coloring(begin, end, fun);
884 }
885 else
886 {
887 graph.clear();
888 if (additional_data.overlap_communication_computation)
889 {
890 // We create one color (1) with the cells on the boundary of the
891 // local domain and two colors (0 and 2) with the interior
892 // cells.
893 graph.resize(3, std::vector<CellFilter>());
894
895 std::vector<bool> ghost_vertices(
896 dof_handler->get_triangulation().n_vertices(), false);
897
898 for (const auto &cell :
899 dof_handler->get_triangulation().active_cell_iterators())
900 if (cell->is_ghost())
901 for (unsigned int i = 0;
902 i < GeometryInfo<dim>::vertices_per_cell;
903 i++)
904 ghost_vertices[cell->vertex_index(i)] = true;
905
906 std::vector<dealii::FilteredIterator<dealii::TriaActiveIterator<
907 dealii::DoFCellAccessor<dim, dim, false>>>>
908 inner_cells;
909
910 for (auto cell = begin; cell != end; ++cell)
911 {
912 bool ghost_vertex = false;
913
914 for (unsigned int i = 0;
915 i < GeometryInfo<dim>::vertices_per_cell;
916 i++)
917 if (ghost_vertices[cell->vertex_index(i)])
918 {
919 ghost_vertex = true;
920 break;
921 }
922
923 if (ghost_vertex)
924 graph[1].emplace_back(cell);
925 else
926 inner_cells.emplace_back(cell);
927 }
928 for (unsigned i = 0; i < inner_cells.size(); ++i)
929 if (i < inner_cells.size() / 2)
930 graph[0].emplace_back(inner_cells[i]);
931 else
932 graph[2].emplace_back(inner_cells[i]);
933 }
934 else
935 {
936 // If we are not using coloring, all the cells belong to the
937 // same color.
938 graph.resize(1, std::vector<CellFilter>());
939 for (auto cell = begin; cell != end; ++cell)
940 graph[0].emplace_back(cell);
941 }
942 }
943 }
944 n_colors = graph.size();
945
946 helper.resize(n_colors);
947
948 IndexSet locally_relevant_dofs;
949 if (comm)
950 {
951 locally_relevant_dofs =
952 DoFTools::extract_locally_relevant_dofs(*dof_handler);
953 partitioner = std::make_shared<Utilities::MPI::Partitioner>(
954 dof_handler->locally_owned_dofs(), locally_relevant_dofs, *comm);
955 }
956 for (unsigned int i = 0; i < n_colors; ++i)
957 {
958 n_cells[i] = graph[i].size();
959 helper.fill_data(i, graph[i], partitioner);
960 }
961
962 // Setup row starts
963 if (n_colors > 0)
964 row_start[0] = 0;
965 for (unsigned int i = 1; i < n_colors; ++i)
966 row_start[i] = row_start[i - 1] + n_cells[i - 1] * get_padding_length();
967
968 // Constrained indices
969 n_constrained_dofs = constraints.n_constraints();
970
971 if (n_constrained_dofs != 0)
972 {
973 std::vector<dealii::types::global_dof_index> constrained_dofs_host(
974 n_constrained_dofs);
975
976 if (partitioner)
977 {
978 const unsigned int n_local_dofs =
979 locally_relevant_dofs.n_elements();
980 unsigned int i_constraint = 0;
981 for (unsigned int i = 0; i < n_local_dofs; ++i)
982 {
983 // is_constrained uses a global dof id but
984 // constrained_dofs_host works on the local id
985 if (constraints.is_constrained(partitioner->local_to_global(i)))
986 {
987 constrained_dofs_host[i_constraint] = i;
988 ++i_constraint;
989 }
990 }
991 }
992 else
993 {
994 const unsigned int n_local_dofs = dof_handler->n_dofs();
995 unsigned int i_constraint = 0;
996 for (unsigned int i = 0; i < n_local_dofs; ++i)
997 {
998 if (constraints.is_constrained(i))
999 {
1000 constrained_dofs_host[i_constraint] = i;
1001 ++i_constraint;
1002 }
1003 }
1004 }
1005
1006 constrained_dofs = Kokkos::View<types::global_dof_index *,
1007 MemorySpace::Default::kokkos_space>(
1008 Kokkos::view_alloc("constrained_dofs", Kokkos::WithoutInitializing),
1009 n_constrained_dofs);
1010
1011 Kokkos::View<types::global_dof_index *,
1012 MemorySpace::Default::kokkos_space,
1013 Kokkos::MemoryTraits<Kokkos::Unmanaged>>
1014 constrained_dofs_host_view(constrained_dofs_host.data(),
1015 constrained_dofs_host.size());
1016 Kokkos::deep_copy(constrained_dofs, constrained_dofs_host_view);
1017 }
1018 }
1019
1020
1021
1022 template <int dim, typename Number>
1023 template <typename Functor, typename VectorType>
1024 void
1025 MatrixFree<dim, Number>::serial_cell_loop(const Functor &func,
1026 const VectorType &src,
1027 VectorType &dst) const
1028 {
1029 // Execute the loop on the cells
1030 for (unsigned int color = 0; color < n_colors; ++color)
1031 if (n_cells[color] > 0)
1032 {
1033 MemorySpace::Default::kokkos_space::execution_space exec;
1034 Kokkos::TeamPolicy<
1035 MemorySpace::Default::kokkos_space::execution_space>
1036 team_policy(exec, n_cells[color], Kokkos::AUTO);
1037
1038
1039 internal::
1040 ApplyKernel<dim, Number, Functor, IsBlockVector<VectorType>::value>
1041 apply_kernel(func, get_data(color), src, dst);
1042
1043 Kokkos::parallel_for("dealii::MatrixFree::serial_cell_loop",
1044 team_policy,
1045 apply_kernel);
1046 }
1047 Kokkos::fence();
1048 }
1049
1050
1051
1052 template <int dim, typename Number>
1053 template <typename Functor>
1054 void
1055 MatrixFree<dim, Number>::distributed_cell_loop(
1056 const Functor &,
1057 const LinearAlgebra::distributed::BlockVector<Number, MemorySpace::Default>
1058 &,
1059 LinearAlgebra::distributed::BlockVector<Number, MemorySpace::Default> &)
1060 const
1061 {
1062 Assert(false, ExcNotImplemented());
1063 }
1064
1065
1066
1067 template <int dim, typename Number>
1068 template <typename Functor>
1069 void
1070 MatrixFree<dim, Number>::distributed_cell_loop(
1071 const Functor &func,
1072 const LinearAlgebra::distributed::Vector<Number, MemorySpace::Default> &src,
1073 LinearAlgebra::distributed::Vector<Number, MemorySpace::Default> &dst) const
1074 {
1075 MemorySpace::Default::kokkos_space::execution_space exec;
1076
1077 // in case we have compatible partitioners, we can simply use the provided
1078 // vectors
1079 if (src.get_partitioner().get() == partitioner.get() &&
1080 dst.get_partitioner().get() == partitioner.get())
1081 {
1082 // This code is inspired to the code in TaskInfo::loop.
1083 if (overlap_communication_computation)
1084 {
1085 src.update_ghost_values_start(0);
1086
1087 // In parallel, it's possible that some processors do not own any
1088 // cells.
1089 if (n_cells[0] > 0)
1090 {
1091 Kokkos::TeamPolicy<
1092 MemorySpace::Default::kokkos_space::execution_space>
1093 team_policy(exec, n_cells[0], Kokkos::AUTO);
1094
1095 internal::ApplyKernel<dim, Number, Functor, false> apply_kernel(
1096 func, get_data(0), src, dst);
1097
1098 Kokkos::parallel_for(
1099 "dealii::MatrixFree::distributed_cell_loop_0",
1100 team_policy,
1101 apply_kernel);
1102 }
1103 src.update_ghost_values_finish();
1104
1105 // In serial this color does not exist because there are no ghost
1106 // cells
1107 if (n_cells[1] > 0)
1108 {
1109 Kokkos::TeamPolicy<
1110 MemorySpace::Default::kokkos_space::execution_space>
1111 team_policy(exec, n_cells[1], Kokkos::AUTO);
1112
1113 internal::ApplyKernel<dim, Number, Functor, false> apply_kernel(
1114 func, get_data(1), src, dst);
1115
1116 Kokkos::parallel_for(
1117 "dealii::MatrixFree::distributed_cell_loop_1",
1118 team_policy,
1119 apply_kernel);
1120
1121 // We need a synchronization point because we don't want
1122 // device-aware MPI to start the MPI communication until the
1123 // kernel is done.
1124 Kokkos::fence();
1125 }
1126
1127 dst.compress_start(0, VectorOperation::add);
1128 // When the mesh is coarse it is possible that some processors do
1129 // not own any cells
1130 if (n_cells[2] > 0)
1131 {
1132 Kokkos::TeamPolicy<
1133 MemorySpace::Default::kokkos_space::execution_space>
1134 team_policy(exec, n_cells[2], Kokkos::AUTO);
1135
1136 internal::ApplyKernel<dim, Number, Functor, false> apply_kernel(
1137 func, get_data(2), src, dst);
1138
1139 Kokkos::parallel_for(
1140 "dealii::MatrixFree::distributed_cell_loop_2",
1141 team_policy,
1142 apply_kernel);
1143 }
1144 dst.compress_finish(VectorOperation::add);
1145 }
1146 else
1147 {
1148 src.update_ghost_values();
1149 std::vector<
1150 Kokkos::View<Number *, MemorySpace::Default::kokkos_space>>
1151 values_colors(n_colors);
1152 std::vector<
1153 Kokkos::View<Number *[dim], MemorySpace::Default::kokkos_space>>
1154 gradients_colors(n_colors);
1155
1156 // Execute the loop on the cells
1157 for (unsigned int i = 0; i < n_colors; ++i)
1158 if (n_cells[i] > 0)
1159 {
1160 Kokkos::TeamPolicy<
1161 MemorySpace::Default::kokkos_space::execution_space>
1162 team_policy(exec, n_cells[i], Kokkos::AUTO);
1163
1164 internal::ApplyKernel<dim, Number, Functor, false>
1165 apply_kernel(func, get_data(i), src, dst);
1166
1167 Kokkos::parallel_for(
1168 "dealii::MatrixFree::distributed_cell_loop_" +
1169 std::to_string(i),
1170 team_policy,
1171 apply_kernel);
1172 }
1173 dst.compress(VectorOperation::add);
1174 }
1175 src.zero_out_ghost_values();
1176 }
1177 else
1178 {
1179 // Create the ghosted source and the ghosted destination
1180 LinearAlgebra::distributed::Vector<Number, MemorySpace::Default>
1181 ghosted_src(partitioner);
1182 LinearAlgebra::distributed::Vector<Number, MemorySpace::Default>
1183 ghosted_dst(ghosted_src);
1184 ghosted_src = src;
1185 ghosted_dst = dst;
1186 ghosted_dst.zero_out_ghost_values();
1187
1188 // Execute the loop on the cells
1189 for (unsigned int i = 0; i < n_colors; ++i)
1190 if (n_cells[i] > 0)
1191 {
1192 Kokkos::TeamPolicy<
1193 MemorySpace::Default::kokkos_space::execution_space>
1194 team_policy(exec, n_cells[i], Kokkos::AUTO);
1195
1196 internal::ApplyKernel<dim, Number, Functor, false> apply_kernel(
1197 func, get_data(i), ghosted_src, ghosted_dst);
1198
1199 Kokkos::parallel_for(
1200 "dealii::MatrixFree::distributed_cell_loop_" +
1201 std::to_string(i),
1202 team_policy,
1203 apply_kernel);
1204 }
1205
1206 // Add the ghosted values
1207 ghosted_dst.compress(VectorOperation::add);
1208 dst = ghosted_dst;
1209 }
1210 }
1211
1212 } // namespace Portable
1213
1214 DEAL_II_NAMESPACE_CLOSE
1215
1216 #endif