Debian Codegraph @

include/deal.II/matrix_free/portable_fe_evaluation.h

raw
1 // ------------------------------------------------------------------------
2 //
3 // SPDX-License-Identifier: LGPL-2.1-or-later
4 // Copyright (C) 2023 - 2025 by the deal.II authors
5 //
6 // This file is part of the deal.II library.
7 //
8 // Part of the source code is dual licensed under Apache-2.0 WITH
9 // LLVM-exception OR LGPL-2.1-or-later. Detailed license information
10 // governing the source code and code contributions can be found in
11 // LICENSE.md and CONTRIBUTING.md at the top level directory of deal.II.
12 //
13 // ------------------------------------------------------------------------
14
15 #ifndef dealii_portable_fe_evaluation_h
16 #define dealii_portable_fe_evaluation_h
17
18 #include <deal.II/base/config.h>
19
20 #include <deal.II/base/memory_space.h>
21 #include <deal.II/base/tensor.h>
22 #include <deal.II/base/utilities.h>
23
24 #include <deal.II/matrix_free/evaluation_flags.h>
25 #include <deal.II/matrix_free/portable_evaluation_kernels.h>
26 #include <deal.II/matrix_free/portable_hanging_nodes_internal.h>
27 #include <deal.II/matrix_free/portable_matrix_free.h>
28 #include <deal.II/matrix_free/portable_matrix_free.templates.h>
29 #include <deal.II/matrix_free/portable_tensor_product_kernels.h>
30
31 #include <Kokkos_Core.hpp>
32
33 DEAL_II_NAMESPACE_OPEN
34
35 /**
36 * Namespace for portable capabilities
37 */
38 namespace Portable
39 {
40 /**
41 * This class provides all the functions necessary to evaluate functions at
42 * quadrature points and cell integrations. In functionality, this class is
43 * similar to FEValues<dim>.
44 *
45 * This class has five template arguments:
46 *
47 * @tparam dim Dimension in which this class is to be used
48 *
49 * @tparam fe_degree Degree of the tensor prodict finite element with fe_degree+1
50 * degrees of freedom per coordinate direction
51 *
52 * @tparam n_q_points_1d Number of points in the quadrature formular in 1d,
53 * defaults to fe_degree+1
54 *
55 * @tparam n_components Number of vector components when solving a system of
56 * PDEs. If the same operation is applied to several components of a PDE (e.g.
57 * a vector Laplace equation), they can be applied simultaneously with one
58 * call (and often more efficiently). Defaults to 1
59 *
60 * @tparam Number Number format, @p double or @p float. Defaults to @p
61 * double.
62 */
63 template <int dim,
64 int fe_degree,
65 int n_q_points_1d = fe_degree + 1,
66 int n_components_ = 1,
67 typename Number = double>
68 class FEEvaluation
69 {
70 public:
71 /**
72 * An alias for scalar quantities.
73 */
74 using value_type = std::conditional_t<(n_components_ == 1),
75 Number,
76 Tensor<1, n_components_, Number>>;
77
78 /**
79 * An alias for vectorial quantities.
80 */
81 using gradient_type = std::conditional_t<
82 n_components_ == 1,
83 Tensor<1, dim, Number>,
84 std::conditional_t<n_components_ == dim,
85 Tensor<2, dim, Number>,
86 Tensor<1, n_components_, Tensor<1, dim, Number>>>>;
87
88 /**
89 * An alias to kernel specific information.
90 */
91 using data_type = typename MatrixFree<dim, Number>::Data;
92
93 /**
94 * Dimension.
95 */
96 static constexpr unsigned int dimension = dim;
97
98 /**
99 * Number of components.
100 */
101 static constexpr unsigned int n_components = n_components_;
102
103 /**
104 * Number of quadrature points per cell.
105 */
106 static constexpr unsigned int n_q_points =
107 Utilities::pow(n_q_points_1d, dim);
108
109 /**
110 * Number of tensor degrees of freedom of a scalar component determined
111 * from the given template argument `fe_degree`.
112 */
113 static constexpr unsigned int tensor_dofs_per_component =
114 Utilities::pow(fe_degree + 1, dim);
115
116 /**
117 * Number of tensor degrees of freedom of all component determined from
118 * the given template argument `fe_degree`.
119 */
120 static constexpr unsigned int tensor_dofs_per_cell =
121 tensor_dofs_per_component * n_components;
122
123 /**
124 * Constructor. You will need to provide a pointer to the
125 * Portable::MatrixFree::Data object, which is typically provided to the
126 * functor inside the
127 * Portable::MatrixFree::cell_loop() and the index @p dof_index of the DoFHandler
128 * if more than one was provided when the Portable::MatrixFree object was
129 * initialized.
130 */
131 DEAL_II_HOST_DEVICE
132 explicit FEEvaluation(const data_type *data,
133 const unsigned int dof_index = 0);
134
135 /**
136 * Return the index of the current cell.
137 */
138 DEAL_II_HOST_DEVICE
139 int
140 get_current_cell_index();
141
142 /**
143 * Return a pointer to the MatrixFree<dim, Number>::Data object on device
144 * that contains necessary constraint, dof index, and shape function
145 * information for evaluation used in the matrix-free kernels.
146 */
147 DEAL_II_HOST_DEVICE
148 const data_type *
149 get_matrix_free_data();
150
151 /**
152 * For the vector @p src, read out the values on the degrees of freedom of
153 * the current cell, and store them internally. Similar functionality as
154 * the function DoFAccessor::get_interpolated_dof_values when no
155 * constraints are present, but it also includes constraints from hanging
156 * nodes, so once can see it as a similar function to
157 * AffineConstraints::read_dof_values() as well.
158 */
159 DEAL_II_HOST_DEVICE void
160 read_dof_values(const DeviceVector<Number> &src);
161
162 /**
163 * Take the value stored internally on dof values of the current cell and
164 * sum them into the vector @p dst. The function also applies constraints
165 * during the write operation. The functionality is hence similar to the
166 * function AffineConstraints::distribute_local_to_global.
167 */
168 DEAL_II_HOST_DEVICE void
169 distribute_local_to_global(DeviceVector<Number> &dst) const;
170
171 /**
172 * Evaluate the function values and the gradients of the FE function given
173 * at the DoF values in the input vector at the quadrature points on the
174 * unit cell. The function argument @p evaluate_flag specifies which parts
175 * shall actually be computed. This function needs to be called before the
176 * functions @p get_value() or @p get_gradient() give useful information.
177 */
178 DEAL_II_HOST_DEVICE void
179 evaluate(const EvaluationFlags::EvaluationFlags evaluate_flag);
180
181 /**
182 * This function takes the values and/or gradients that are stored on
183 * quadrature points, tests them by all the basis functions/gradients on
184 * the cell and performs the cell integration as specified by the
185 * @p integration_flag argument.
186 */
187 DEAL_II_HOST_DEVICE void
188 integrate(const EvaluationFlags::EvaluationFlags integration_flag);
189
190 /**
191 * Same as above, except that the quadrature point is computed from thread
192 * id.
193 */
194 DEAL_II_HOST_DEVICE value_type
195 get_value(int q_point) const;
196
197 /**
198 * Same as above, except that the local dof index is computed from the
199 * thread id.
200 */
201 DEAL_II_HOST_DEVICE value_type
202 get_dof_value(int q_point) const;
203
204 /**
205 * Same as above, except that the quadrature point is computed from the
206 * thread id.
207 */
208 DEAL_II_HOST_DEVICE void
209 submit_value(const value_type &val_in, int q_point);
210
211 /**
212 * Same as above, except that the local dof index is computed from the
213 * thread id.
214 */
215 DEAL_II_HOST_DEVICE void
216 submit_dof_value(const value_type &val_in, int q_point);
217
218 /**
219 * Same as above, except that the quadrature point is computed from the
220 * thread id.
221 */
222 DEAL_II_HOST_DEVICE gradient_type
223 get_gradient(int q_point) const;
224
225 /**
226 * Same as above, except that the quadrature point is computed from the
227 * thread id.
228 */
229 DEAL_II_HOST_DEVICE void
230 submit_gradient(const gradient_type &grad_in, int q_point);
231
232 // clang-format off
233 /**
234 * Same as above, except that the functor @p func only takes a single input
235 * argument (fe_eval) and computes the quadrature point from the thread id.
236 *
237 * @p func needs to define
238 * \code
239 * DEAL_II_HOST_DEVICE void operator()(
240 * Portable::FEEvaluation<dim, fe_degree, n_q_points_1d, n_components, Number> *fe_eval) const;
241 * \endcode
242 */
243 // clang-format on
244 template <typename Functor>
245 DEAL_II_HOST_DEVICE void
246 apply_for_each_quad_point(const Functor &func);
247
248 private:
249 const typename MatrixFree<dim, Number>::Data *data;
250 const typename MatrixFree<dim, Number>::PrecomputedData *precomputed_data;
251 SharedData<dim, Number> *shared_data;
252 int cell_id;
253 };
254
255
256
257 template <int dim,
258 int fe_degree,
259 int n_q_points_1d,
260 int n_components_,
261 typename Number>
262 DEAL_II_HOST_DEVICE
263 FEEvaluation<dim, fe_degree, n_q_points_1d, n_components_, Number>::
264 FEEvaluation(const data_type *data, const unsigned int dof_index)
265 : data(data)
266 , precomputed_data(data->precomputed_data)
267 , shared_data(data->shared_data)
268 , cell_id(data->team_member.league_rank())
269 {
270 AssertIndexRange(dof_index, data->n_dofhandler);
271 }
272
273
274
275 template <int dim,
276 int fe_degree,
277 int n_q_points_1d,
278 int n_components_,
279 typename Number>
280 DEAL_II_HOST_DEVICE int
281 FEEvaluation<dim, fe_degree, n_q_points_1d, n_components_, Number>::
282 get_current_cell_index()
283 {
284 return cell_id;
285 }
286
287
288
289 template <int dim,
290 int fe_degree,
291 int n_q_points_1d,
292 int n_components_,
293 typename Number>
294 DEAL_II_HOST_DEVICE const typename FEEvaluation<dim,
295 fe_degree,
296 n_q_points_1d,
297 n_components_,
298 Number>::data_type *
299 FEEvaluation<dim, fe_degree, n_q_points_1d, n_components_, Number>::
300 get_matrix_free_data()
301 {
302 return data;
303 }
304
305
306
307 template <int dim,
308 int fe_degree,
309 int n_q_points_1d,
310 int n_components_,
311 typename Number>
312 DEAL_II_HOST_DEVICE void
313 FEEvaluation<dim, fe_degree, n_q_points_1d, n_components_, Number>::
314 read_dof_values(const DeviceVector<Number> &src)
315 {
316 // Populate the scratch memory
317 Kokkos::parallel_for(Kokkos::TeamThreadRange(data->team_member,
318 tensor_dofs_per_component),
319 [&](const int &i) {
320 for (unsigned int c = 0; c < n_components_; ++c)
321 shared_data->values(i, c) =
322 src[precomputed_data->local_to_global(
323 i + tensor_dofs_per_component * c, cell_id)];
324 });
325 data->team_member.team_barrier();
326
327 for (unsigned int c = 0; c < n_components_; ++c)
328 {
329 if (precomputed_data->constraint_mask(cell_id * n_components + c) !=
330 dealii::internal::MatrixFreeFunctions::ConstraintKinds::
331 unconstrained)
332 internal::resolve_hanging_nodes<dim, fe_degree, false, Number>(
333 data->team_member,
334 precomputed_data->constraint_weights,
335 precomputed_data->constraint_mask(cell_id * n_components + c),
336 Kokkos::subview(shared_data->values, Kokkos::ALL, c));
337 }
338 }
339
340
341
342 template <int dim,
343 int fe_degree,
344 int n_q_points_1d,
345 int n_components_,
346 typename Number>
347 DEAL_II_HOST_DEVICE void
348 FEEvaluation<dim, fe_degree, n_q_points_1d, n_components_, Number>::
349 distribute_local_to_global(DeviceVector<Number> &dst) const
350 {
351 for (unsigned int c = 0; c < n_components_; ++c)
352 {
353 if (precomputed_data->constraint_mask(cell_id * n_components + c) !=
354 dealii::internal::MatrixFreeFunctions::ConstraintKinds::
355 unconstrained)
356 internal::resolve_hanging_nodes<dim, fe_degree, true, Number>(
357 data->team_member,
358 precomputed_data->constraint_weights,
359 precomputed_data->constraint_mask(cell_id * n_components + c),
360 Kokkos::subview(shared_data->values, Kokkos::ALL, c));
361 }
362
363 if (precomputed_data->use_coloring)
364 {
365 Kokkos::parallel_for(
366 Kokkos::TeamThreadRange(data->team_member, tensor_dofs_per_component),
367 [&](const int &i) {
368 for (unsigned int c = 0; c < n_components_; ++c)
369 dst[precomputed_data->local_to_global(
370 i + tensor_dofs_per_component * c, cell_id)] +=
371 shared_data->values(i, c);
372 });
373 }
374 else
375 {
376 Kokkos::parallel_for(
377 Kokkos::TeamThreadRange(data->team_member, tensor_dofs_per_component),
378 [&](const int &i) {
379 for (unsigned int c = 0; c < n_components_; ++c)
380 Kokkos::atomic_add(&dst[precomputed_data->local_to_global(
381 i + (tensor_dofs_per_component)*c, cell_id)],
382 shared_data->values(i, c));
383 });
384 }
385 }
386
387
388
389 template <int dim,
390 int fe_degree,
391 int n_q_points_1d,
392 int n_components,
393 typename Number>
394 DEAL_II_HOST_DEVICE void
395 FEEvaluation<dim, fe_degree, n_q_points_1d, n_components, Number>::evaluate(
396 const EvaluationFlags::EvaluationFlags evaluation_flag)
397 {
398 using ElementType = ::dealii::internal::MatrixFreeFunctions::ElementType;
399
400 if (fe_degree >= 0 && fe_degree + 1 == n_q_points_1d &&
401 precomputed_data->element_type ==
402 ElementType::tensor_symmetric_collocation)
403 {
404 internal::FEEvaluationImplCollocation<dim, fe_degree, Number>::evaluate(
405 n_components, evaluation_flag, data);
406 }
407 // '<=' on type means tensor_symmetric or tensor_symmetric_hermite, see
408 // shape_info.h for more details
409 else if (fe_degree >= 0 &&
410 internal::use_collocation_evaluation(fe_degree, n_q_points_1d) &&
411 precomputed_data->element_type <= ElementType::tensor_symmetric)
412 {
413 internal::FEEvaluationImplTransformToCollocation<
414 dim,
415 fe_degree,
416 n_q_points_1d,
417 Number>::evaluate(n_components, evaluation_flag, data);
418 }
419 else if (fe_degree >= 0 && precomputed_data->element_type <=
420 ElementType::tensor_symmetric_no_collocation)
421 {
422 internal::FEEvaluationImpl<dim, fe_degree, n_q_points_1d, Number>::
423 evaluate(n_components, evaluation_flag, data);
424 }
425 else
426 {
427 Kokkos::abort("The element type is not yet supported by the portable "
428 "matrix-free module.");
429 }
430 }
431
432
433
434 template <int dim,
435 int fe_degree,
436 int n_q_points_1d,
437 int n_components_,
438 typename Number>
439 DEAL_II_HOST_DEVICE void
440 FEEvaluation<dim, fe_degree, n_q_points_1d, n_components_, Number>::integrate(
441 const EvaluationFlags::EvaluationFlags integration_flag)
442 {
443 using ElementType = ::dealii::internal::MatrixFreeFunctions::ElementType;
444
445 if (fe_degree >= 0 && fe_degree + 1 == n_q_points_1d &&
446 precomputed_data->element_type ==
447 ElementType::tensor_symmetric_collocation)
448 {
449 internal::FEEvaluationImplCollocation<dim, fe_degree, Number>::
450 integrate(n_components, integration_flag, data);
451 }
452 // '<=' on type means tensor_symmetric or tensor_symmetric_hermite, see
453 // shape_info.h for more details
454 else if (fe_degree >= 0 &&
455 internal::use_collocation_evaluation(fe_degree, n_q_points_1d) &&
456 precomputed_data->element_type <= ElementType::tensor_symmetric)
457 {
458 internal::FEEvaluationImplTransformToCollocation<
459 dim,
460 fe_degree,
461 n_q_points_1d,
462 Number>::integrate(n_components, integration_flag, data);
463 }
464 else if (fe_degree >= 0 && precomputed_data->element_type <=
465 ElementType::tensor_symmetric_no_collocation)
466 {
467 internal::FEEvaluationImpl<dim, fe_degree, n_q_points_1d, Number>::
468 integrate(n_components, integration_flag, data);
469 }
470 else
471 {
472 Kokkos::abort("The element type is not yet supported by the portable "
473 "matrix-free module.");
474 }
475 }
476
477
478
479 template <int dim,
480 int fe_degree,
481 int n_q_points_1d,
482 int n_components_,
483 typename Number>
484 DEAL_II_HOST_DEVICE typename FEEvaluation<dim,
485 fe_degree,
486 n_q_points_1d,
487 n_components_,
488 Number>::value_type
489 FEEvaluation<dim, fe_degree, n_q_points_1d, n_components_, Number>::get_value(
490 int q_point) const
491 {
492 Assert(q_point >= 0 && q_point < n_q_points, ExcInternalError());
493 if constexpr (n_components_ == 1)
494 {
495 return shared_data->values(q_point, 0);
496 }
497 else
498 {
499 value_type result;
500 for (unsigned int c = 0; c < n_components; ++c)
501 result[c] = shared_data->values(q_point, c);
502 return result;
503 }
504 }
505
506
507
508 template <int dim,
509 int fe_degree,
510 int n_q_points_1d,
511 int n_components_,
512 typename Number>
513 DEAL_II_HOST_DEVICE typename FEEvaluation<dim,
514 fe_degree,
515 n_q_points_1d,
516 n_components_,
517 Number>::value_type
518 FEEvaluation<dim, fe_degree, n_q_points_1d, n_components_, Number>::
519 get_dof_value(int dof) const
520 {
521 Assert(dof >= 0 && dof < static_cast<int>(tensor_dofs_per_component),
522 ExcInternalError());
523 if constexpr (n_components_ == 1)
524 {
525 return shared_data->values(dof, 0);
526 }
527 else
528 {
529 value_type result;
530 for (unsigned int c = 0; c < n_components; ++c)
531 result[c] = shared_data->values(dof, c);
532 return result;
533 }
534 }
535
536
537
538 template <int dim,
539 int fe_degree,
540 int n_q_points_1d,
541 int n_components_,
542 typename Number>
543 DEAL_II_HOST_DEVICE void
544 FEEvaluation<dim, fe_degree, n_q_points_1d, n_components_, Number>::
545 submit_value(const value_type &val_in, int q_point)
546 {
547 Assert(q_point >= 0 && q_point < n_q_points, ExcInternalError());
548 if constexpr (n_components_ == 1)
549 {
550 shared_data->values(q_point, 0) =
551 val_in * precomputed_data->JxW(q_point, cell_id);
552 }
553 else
554 {
555 for (unsigned int c = 0; c < n_components; ++c)
556 shared_data->values(q_point, c) =
557 val_in[c] * precomputed_data->JxW(q_point, cell_id);
558 }
559 }
560
561
562
563 template <int dim,
564 int fe_degree,
565 int n_q_points_1d,
566 int n_components_,
567 typename Number>
568 DEAL_II_HOST_DEVICE void
569 FEEvaluation<dim, fe_degree, n_q_points_1d, n_components_, Number>::
570 submit_dof_value(const value_type &val_in, int dof)
571 {
572 Assert(dof >= 0 && dof < tensor_dofs_per_component, ExcInternalError());
573 if constexpr (n_components_ == 1)
574 {
575 shared_data->values(dof, 0) = val_in;
576 }
577 else
578 {
579 for (unsigned int c = 0; c < n_components; ++c)
580 shared_data->values(dof, c) = val_in[c];
581 }
582 }
583
584
585
586 template <int dim,
587 int fe_degree,
588 int n_q_points_1d,
589 int n_components_,
590 typename Number>
591 DEAL_II_HOST_DEVICE typename FEEvaluation<dim,
592 fe_degree,
593 n_q_points_1d,
594 n_components_,
595 Number>::gradient_type
596 FEEvaluation<dim, fe_degree, n_q_points_1d, n_components_, Number>::
597 get_gradient(int q_point) const
598 {
599 Assert(q_point >= 0 && q_point < n_q_points, ExcInternalError());
600 gradient_type grad;
601
602 if constexpr (n_components_ == 1)
603 {
604 for (unsigned int d_1 = 0; d_1 < dim; ++d_1)
605 {
606 Number tmp = 0.;
607 for (unsigned int d_2 = 0; d_2 < dim; ++d_2)
608 tmp +=
609 precomputed_data->inv_jacobian(q_point, cell_id, d_2, d_1) *
610 shared_data->gradients(q_point, d_2, 0);
611 grad[d_1] = tmp;
612 }
613 }
614 else
615 {
616 for (unsigned int c = 0; c < n_components; ++c)
617 for (unsigned int d_1 = 0; d_1 < dim; ++d_1)
618 {
619 Number tmp = 0.;
620 for (unsigned int d_2 = 0; d_2 < dim; ++d_2)
621 tmp +=
622 precomputed_data->inv_jacobian(q_point, cell_id, d_2, d_1) *
623 shared_data->gradients(q_point, d_2, c);
624 grad[c][d_1] = tmp;
625 }
626 }
627
628 return grad;
629 }
630
631
632
633 template <int dim,
634 int fe_degree,
635 int n_q_points_1d,
636 int n_components_,
637 typename Number>
638 DEAL_II_HOST_DEVICE void
639 FEEvaluation<dim, fe_degree, n_q_points_1d, n_components_, Number>::
640 submit_gradient(const gradient_type &grad_in, int q_point)
641 {
642 Assert(q_point >= 0 && q_point < n_q_points, ExcInternalError());
643 if constexpr (n_components_ == 1)
644 {
645 for (unsigned int d_1 = 0; d_1 < dim; ++d_1)
646 {
647 Number tmp = 0.;
648 for (unsigned int d_2 = 0; d_2 < dim; ++d_2)
649 tmp +=
650 precomputed_data->inv_jacobian(q_point, cell_id, d_1, d_2) *
651 grad_in[d_2];
652 shared_data->gradients(q_point, d_1, 0) =
653 tmp * precomputed_data->JxW(q_point, cell_id);
654 }
655 }
656 else
657 {
658 for (unsigned int c = 0; c < n_components; ++c)
659 for (unsigned int d_1 = 0; d_1 < dim; ++d_1)
660 {
661 Number tmp = 0.;
662 for (unsigned int d_2 = 0; d_2 < dim; ++d_2)
663 tmp +=
664 precomputed_data->inv_jacobian(q_point, cell_id, d_1, d_2) *
665 grad_in[c][d_2];
666 shared_data->gradients(q_point, d_1, c) =
667 tmp * precomputed_data->JxW(q_point, cell_id);
668 }
669 }
670 }
671
672
673
674 template <int dim,
675 int fe_degree,
676 int n_q_points_1d,
677 int n_components_,
678 typename Number>
679 template <typename Functor>
680 DEAL_II_HOST_DEVICE void
681 FEEvaluation<dim, fe_degree, n_q_points_1d, n_components_, Number>::
682 apply_for_each_quad_point(const Functor &func)
683 {
684 Kokkos::parallel_for(Kokkos::TeamThreadRange(data->team_member, n_q_points),
685 [&](const int &i) { func(this, i); });
686 data->team_member.team_barrier();
687 }
688
689
690
691 #ifndef DOXYGEN
692 template <int dim,
693 int fe_degree,
694 int n_q_points_1d,
695 int n_components_,
696 typename Number>
697 constexpr unsigned int
698 FEEvaluation<dim, fe_degree, n_q_points_1d, n_components_, Number>::
699 n_q_points;
700 #endif
701 } // namespace Portable
702
703 DEAL_II_NAMESPACE_CLOSE
704
705 #endif